]> git.proxmox.com Git - mirror_ovs.git/blame - datapath/datapath.c
projects / mirror_ovs.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
datapath: Use nla_nest_start_noflag()
[mirror_ovs.git] / datapath / datapath.c
CommitLineData
064af421 1/*
e23775f2 2 * Copyright (c) 2007-2015 Nicira, Inc.
a14bc59f 3 *
a9a29d22
JG		 4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of version 2 of the GNU General Public
6 * License as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program; if not, write to the Free Software
15 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
16 * 02110-1301, USA
064af421
BP		 17 */
18
dfffaef1
JP		 19#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
20
064af421
BP		 21#include <linux/init.h>
22#include <linux/module.h>
064af421 23#include <linux/if_arp.h>
064af421
BP		 24#include <linux/if_vlan.h>
25#include <linux/in.h>
26#include <linux/ip.h>
982b8810 27#include <linux/jhash.h>
064af421
BP		 28#include <linux/delay.h>
29#include <linux/time.h>
30#include <linux/etherdevice.h>
ed099e92 31#include <linux/genetlink.h>
064af421
BP		 32#include <linux/kernel.h>
33#include <linux/kthread.h>
064af421
BP		 34#include <linux/mutex.h>
35#include <linux/percpu.h>
36#include <linux/rcupdate.h>
37#include <linux/tcp.h>
38#include <linux/udp.h>
39#include <linux/version.h>
40#include <linux/ethtool.h>
064af421 41#include <linux/wait.h>
064af421 42#include <asm/div64.h>
656a0e37 43#include <linux/highmem.h>
064af421
BP		 44#include <linux/netfilter_bridge.h>
45#include <linux/netfilter_ipv4.h>
46#include <linux/inetdevice.h>
47#include <linux/list.h>
077257b8 48#include <linux/openvswitch.h>
064af421 49#include <linux/rculist.h>
064af421 50#include <linux/dmi.h>
36956a7d 51#include <net/genetlink.h>
2a4999f3
PS		 52#include <net/net_namespace.h>
53#include <net/netns/generic.h>
907c26a8 54#include <net/nsh.h>
064af421 55
064af421 56#include "datapath.h"
038e34ab 57#include "conntrack.h"
064af421 58#include "flow.h"
d103f479 59#include "flow_table.h"
a097c0b2 60#include "flow_netlink.h"
1cb57039 61#include "meter.h"
e23775f2 62#include "gso.h"
f2459fe7 63#include "vport-internal_dev.h"
d5de5b0d 64#include "vport-netdev.h"
064af421 65
f56f0b73 66unsigned int ovs_net_id __read_mostly;
2a4999f3 67
cb25142c
PS		 68static struct genl_family dp_packet_genl_family;
69static struct genl_family dp_flow_genl_family;
70static struct genl_family dp_datapath_genl_family;
71
bc619e29
JS		 72static const struct nla_policy flow_policy[];
73
18fd3a52
PS		 74static struct genl_multicast_group ovs_dp_flow_multicast_group = {
75 .name = OVS_FLOW_MCGROUP
cb25142c
PS		 76};
77
18fd3a52
PS		 78static struct genl_multicast_group ovs_dp_datapath_multicast_group = {
79 .name = OVS_DATAPATH_MCGROUP
cb25142c
PS		 80};
81
18fd3a52
PS		 82struct genl_multicast_group ovs_dp_vport_multicast_group = {
83 .name = OVS_VPORT_MCGROUP
cb25142c
PS		 84};
85
afad3556 86/* Check if need to build a reply message.
af465b67
PS		 87 * OVS userspace sets the NLM_F_ECHO flag if it needs the reply.
88 */
114fce23
SG		 89static bool ovs_must_notify(struct genl_family *family, struct genl_info *info,
90 unsigned int group)
afad3556
JR		 91{
92 return info->nlhdr->nlmsg_flags & NLM_F_ECHO ||
6233a1bd 93 genl_has_listeners(family, genl_info_net(info), group);
afad3556
JR		 94}
95
18fd3a52 96static void ovs_notify(struct genl_family *family, struct genl_multicast_group *grp,
cb25142c 97 struct sk_buff *skb, struct genl_info *info)
e297c6b7 98{
0643a78b 99 genl_notify(family, skb, info, GROUP_ID(grp), GFP_KERNEL);
e297c6b7
TG		 100}
101
ed099e92
BP		 102/**
103 * DOC: Locking:
064af421 104 *
cd2a59e9
PS		 105 * All writes e.g. Writes to device state (add/remove datapath, port, set
106 * operations on vports, etc.), Writes to other state (flow table
107 * modifications, set miscellaneous datapath parameters, etc.) are protected
108 * by ovs_lock.
ed099e92
BP		 109 *
110 * Reads are protected by RCU.
111 *
112 * There are a few special cases (mostly stats) that have their own
113 * synchronization but they nest under all of above and don't interact with
114 * each other.
cd2a59e9
PS		 115 *
116 * The RTNL lock nests inside ovs_mutex.
064af421 117 */
ed099e92 118
cd2a59e9
PS		 119static DEFINE_MUTEX(ovs_mutex);
120
121void ovs_lock(void)
122{
123 mutex_lock(&ovs_mutex);
124}
125
126void ovs_unlock(void)
127{
128 mutex_unlock(&ovs_mutex);
129}
130
131#ifdef CONFIG_LOCKDEP
132int lockdep_ovsl_is_held(void)
133{
134 if (debug_locks)
135 return lockdep_is_held(&ovs_mutex);
136 else
137 return 1;
138}
139#endif
140
5ae440c3 141static int queue_gso_packets(struct datapath *dp, struct sk_buff *,
f1f60b85 142 const struct sw_flow_key *,
4c7804f1
WT		 143 const struct dp_upcall_info *,
144 uint32_t cutlen);
5ae440c3 145static int queue_userspace_packet(struct datapath *dp, struct sk_buff *,
7d16c847 146 const struct sw_flow_key *,
4c7804f1
WT		 147 const struct dp_upcall_info *,
148 uint32_t cutlen);
064af421 149
cd2a59e9 150/* Must be called with rcu_read_lock or ovs_mutex. */
850b6b3b 151const char *ovs_dp_name(const struct datapath *dp)
f2459fe7 152{
cd2a59e9 153 struct vport *vport = ovs_vport_ovsl_rcu(dp, OVSP_LOCAL);
e23775f2 154 return ovs_vport_name(vport);
f2459fe7
JG		 155}
156
f1f60b85 157static int get_dpifindex(const struct datapath *dp)
99769a40
JG		 158{
159 struct vport *local;
160 int ifindex;
161
162 rcu_read_lock();
163
95b1d73a 164 local = ovs_vport_rcu(dp, OVSP_LOCAL);
99769a40 165 if (local)
e23775f2 166 ifindex = local->dev->ifindex;
99769a40
JG		 167 else
168 ifindex = 0;
169
170 rcu_read_unlock();
171
172 return ifindex;
173}
174
46c6a11d
JG		 175static void destroy_dp_rcu(struct rcu_head *rcu)
176{
177 struct datapath *dp = container_of(rcu, struct datapath, rcu);
46c6a11d 178
e379e4d1 179 ovs_flow_tbl_destroy(&dp->table);
46c6a11d 180 free_percpu(dp->stats_percpu);
95b1d73a 181 kfree(dp->ports);
1cb57039 182 ovs_meters_exit(dp);
5ca1ba48 183 kfree(dp);
46c6a11d
JG		 184}
185
95b1d73a
PS		 186static struct hlist_head *vport_hash_bucket(const struct datapath *dp,
187 u16 port_no)
188{
189 return &dp->ports[port_no & (DP_VPORT_HASH_BUCKETS - 1)];
190}
191
aa917006 192/* Called with ovs_mutex or RCU read lock. */
95b1d73a
PS		 193struct vport *ovs_lookup_vport(const struct datapath *dp, u16 port_no)
194{
195 struct vport *vport;
95b1d73a
PS		 196 struct hlist_head *head;
197
198 head = vport_hash_bucket(dp, port_no);
f8dfbcb7 199 hlist_for_each_entry_rcu(vport, head, dp_hash_node) {
95b1d73a
PS		 200 if (vport->port_no == port_no)
201 return vport;
202 }
203 return NULL;
204}
205
cd2a59e9 206/* Called with ovs_mutex. */
c19e6535 207static struct vport *new_vport(const struct vport_parms *parms)
064af421 208{
f2459fe7 209 struct vport *vport;
f2459fe7 210
850b6b3b 211 vport = ovs_vport_add(parms);
c19e6535
BP		 212 if (!IS_ERR(vport)) {
213 struct datapath *dp = parms->dp;
95b1d73a 214 struct hlist_head *head = vport_hash_bucket(dp, vport->port_no);
064af421 215
95b1d73a 216 hlist_add_head_rcu(&vport->dp_hash_node, head);
c19e6535 217 }
c19e6535 218 return vport;
064af421
BP		 219}
220
850b6b3b 221void ovs_dp_detach_port(struct vport *p)
064af421 222{
cd2a59e9 223 ASSERT_OVSL();
064af421 224
064af421 225 /* First drop references to device. */
95b1d73a 226 hlist_del_rcu(&p->dp_hash_node);
f2459fe7 227
7237e4f4 228 /* Then destroy it. */
850b6b3b 229 ovs_vport_del(p);
064af421
BP		 230}
231
fb66fbd1 232/* Must be called with rcu_read_lock. */
e74d4817 233void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key)
064af421 234{
a6059080 235 const struct vport *p = OVS_CB(skb)->input_vport;
064af421 236 struct datapath *dp = p->dp;
3544358a 237 struct sw_flow *flow;
ad50cb60 238 struct sw_flow_actions *sf_acts;
064af421 239 struct dp_stats_percpu *stats;
e9141eec 240 u64 *stats_counter;
4fa72a95 241 u32 n_mask_hit;
064af421 242
70dbc259 243 stats = this_cpu_ptr(dp->stats_percpu);
a063b0df 244
52a23d92 245 /* Look up flow. */
e74d4817 246 flow = ovs_flow_tbl_lookup_stats(&dp->table, key, skb_get_hash(skb),
5604935e 247 &n_mask_hit);
52a23d92
JG		 248 if (unlikely(!flow)) {
249 struct dp_upcall_info upcall;
a7d607c5 250 int error;
52a23d92 251
0e469d3b 252 memset(&upcall, 0, sizeof(upcall));
52a23d92 253 upcall.cmd = OVS_PACKET_CMD_MISS;
beb1c69a 254 upcall.portid = ovs_vport_find_upcall_portid(p, skb);
a94ebc39 255 upcall.mru = OVS_CB(skb)->mru;
4c7804f1 256 error = ovs_dp_upcall(dp, skb, key, &upcall, 0);
a7d607c5
LR		 257 if (unlikely(error))
258 kfree_skb(skb);
259 else
260 consume_skb(skb);
52a23d92
JG		 261 stats_counter = &stats->n_missed;
262 goto out;
263 }
264
e74d4817 265 ovs_flow_stats_update(flow, key->tp.flags, skb);
ad50cb60 266 sf_acts = rcu_dereference(flow->sf_acts);
7d16c847
PS		 267 ovs_execute_actions(dp, skb, sf_acts, key);
268
b0b906cc 269 stats_counter = &stats->n_hit;
55574bb0 270
8819fac7 271out:
55574bb0 272 /* Update datapath statistics. */
b81deb15 273 u64_stats_update_begin(&stats->syncp);
e9141eec 274 (*stats_counter)++;
4fa72a95 275 stats->n_mask_hit += n_mask_hit;
b81deb15 276 u64_stats_update_end(&stats->syncp);
064af421
BP		 277}
278
850b6b3b 279int ovs_dp_upcall(struct datapath *dp, struct sk_buff *skb,
f1f60b85 280 const struct sw_flow_key *key,
4c7804f1
WT		 281 const struct dp_upcall_info *upcall_info,
282 uint32_t cutlen)
aa5a8fdc
JG		 283{
284 struct dp_stats_percpu *stats;
285 int err;
286
28aea917 287 if (upcall_info->portid == 0) {
b063d9f0 288 err = -ENOTCONN;
b063d9f0
JG		 289 goto err;
290 }
291
7257b535 292 if (!skb_is_gso(skb))
4c7804f1 293 err = queue_userspace_packet(dp, skb, key, upcall_info, cutlen);
7257b535 294 else
4c7804f1 295 err = queue_gso_packets(dp, skb, key, upcall_info, cutlen);
d76195db
JG		 296 if (err)
297 goto err;
298
299 return 0;
aa5a8fdc 300
aa5a8fdc 301err:
70dbc259 302 stats = this_cpu_ptr(dp->stats_percpu);
aa5a8fdc 303
b81deb15 304 u64_stats_update_begin(&stats->syncp);
aa5a8fdc 305 stats->n_lost++;
b81deb15 306 u64_stats_update_end(&stats->syncp);
aa5a8fdc 307
aa5a8fdc 308 return err;
982b8810
BP		 309}
310
5ae440c3 311static int queue_gso_packets(struct datapath *dp, struct sk_buff *skb,
f1f60b85 312 const struct sw_flow_key *key,
4c7804f1
WT		 313 const struct dp_upcall_info *upcall_info,
314 uint32_t cutlen)
cb5087ca 315{
969b8e6b 316#ifdef HAVE_SKB_GSO_UDP
f54a7a5d 317 unsigned int gso_type = skb_shinfo(skb)->gso_type;
7257b535 318 struct sw_flow_key later_key;
969b8e6b 319#endif
7257b535 320 struct sk_buff *segs, *nskb;
b2a23c4e 321 struct ovs_skb_cb ovs_cb;
7257b535 322 int err;
cb5087ca 323
b2a23c4e 324 ovs_cb = *OVS_CB(skb);
1d04cd4e 325 segs = __skb_gso_segment(skb, NETIF_F_SG, false);
b2a23c4e 326 *OVS_CB(skb) = ovs_cb;
79089764
PS		 327 if (IS_ERR(segs))
328 return PTR_ERR(segs);
d1da7669
PS		 329 if (segs == NULL)
330 return -EINVAL;
969b8e6b 331#ifdef HAVE_SKB_GSO_UDP
9b277b39 332 if (gso_type & SKB_GSO_UDP) {
c135bba1 333 /* The initial flow key extracted by ovs_flow_key_extract()
9b277b39
PS		 334 * in this case is for a first fragment, so we need to
335 * properly mark later fragments.
336 */
e74d4817 337 later_key = *key;
9b277b39
PS		 338 later_key.ip.frag = OVS_FRAG_TYPE_LATER;
339 }
969b8e6b 340#endif
7257b535
BP		 341 /* Queue all of the segments. */
342 skb = segs;
cb5087ca 343 do {
b2a23c4e 344 *OVS_CB(skb) = ovs_cb;
969b8e6b 345#ifdef HAVE_SKB_GSO_UDP
9b277b39 346 if (gso_type & SKB_GSO_UDP && skb != segs)
e74d4817 347 key = &later_key;
969b8e6b 348#endif
4c7804f1 349 err = queue_userspace_packet(dp, skb, key, upcall_info, cutlen);
982b8810 350 if (err)
7257b535 351 break;
856081f6 352
36ce148c 353 } while ((skb = skb->next));
cb5087ca 354
7257b535
BP		 355 /* Free all of the segments. */
356 skb = segs;
357 do {
358 nskb = skb->next;
359 if (err)
360 kfree_skb(skb);
361 else
362 consume_skb(skb);
363 } while ((skb = nskb));
364 return err;
365}
366
8b7ea2d4 367static size_t upcall_msg_size(const struct dp_upcall_info *upcall_info,
6b330a60 368 unsigned int hdrlen, int actions_attrlen)
0afa2373
TG		 369{
370 size_t size = NLMSG_ALIGN(sizeof(struct ovs_header))
533bea51 371 + nla_total_size(hdrlen) /* OVS_PACKET_ATTR_PACKET */
039fb36c
WT		 372 + nla_total_size(ovs_key_attr_size()) /* OVS_PACKET_ATTR_KEY */
373 + nla_total_size(sizeof(unsigned int)); /* OVS_PACKET_ATTR_LEN */
0afa2373
TG		 374
375 /* OVS_PACKET_ATTR_USERDATA */
8b7ea2d4
WZ		 376 if (upcall_info->userdata)
377 size += NLA_ALIGN(upcall_info->userdata->nla_len);
378
379 /* OVS_PACKET_ATTR_EGRESS_TUN_KEY */
380 if (upcall_info->egress_tun_info)
381 size += nla_total_size(ovs_tun_key_attr_size());
0afa2373 382
0e469d3b
NM		 383 /* OVS_PACKET_ATTR_ACTIONS */
384 if (upcall_info->actions_len)
6b330a60 385 size += nla_total_size(actions_attrlen);
0e469d3b 386
a94ebc39
JS		 387 /* OVS_PACKET_ATTR_MRU */
388 if (upcall_info->mru)
389 size += nla_total_size(sizeof(upcall_info->mru));
390
0afa2373
TG		 391 return size;
392}
393
a94ebc39
JS		 394static void pad_packet(struct datapath *dp, struct sk_buff *skb)
395{
396 if (!(dp->user_features & OVS_DP_F_UNALIGNED)) {
397 size_t plen = NLA_ALIGN(skb->len) - skb->len;
398
399 if (plen > 0)
0ace0a26 400 skb_put_zero(skb, plen);
a94ebc39
JS		 401 }
402}
403
5ae440c3 404static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb,
f1f60b85 405 const struct sw_flow_key *key,
4c7804f1
WT		 406 const struct dp_upcall_info *upcall_info,
407 uint32_t cutlen)
7257b535
BP		 408{
409 struct ovs_header *upcall;
6161d3fd 410 struct sk_buff *nskb = NULL;
82706a6f 411 struct sk_buff *user_skb = NULL; /* to be queued to userspace */
7257b535 412 struct nlattr *nla;
978188b2 413 size_t len;
533bea51 414 unsigned int hlen;
5ae440c3
TG		 415 int err, dp_ifindex;
416
417 dp_ifindex = get_dpifindex(dp);
418 if (!dp_ifindex)
419 return -ENODEV;
7257b535 420
efd8a18e 421 if (skb_vlan_tag_present(skb)) {
6161d3fd
JG		 422 nskb = skb_clone(skb, GFP_ATOMIC);
423 if (!nskb)
424 return -ENOMEM;
07ac71ea 425
8063e095 426 nskb = __vlan_hwaccel_push_inside(nskb);
07ac71ea
PS		 427 if (!nskb)
428 return -ENOMEM;
429
6161d3fd
JG		 430 skb = nskb;
431 }
432
433 if (nla_attr_size(skb->len) > USHRT_MAX) {
434 err = -EFBIG;
435 goto out;
436 }
7257b535 437
533bea51
TG		 438 /* Complete checksum if needed */
439 if (skb->ip_summed == CHECKSUM_PARTIAL &&
a0c9fedc 440 (err = skb_csum_hwoffload_help(skb, 0)))
533bea51
TG		 441 goto out;
442
443 /* Older versions of OVS user space enforce alignment of the last
444 * Netlink attribute to NLA_ALIGNTO which would require extensive
445 * padding logic. Only perform zerocopy if padding is not required.
446 */
447 if (dp->user_features & OVS_DP_F_UNALIGNED)
448 hlen = skb_zerocopy_headlen(skb);
449 else
450 hlen = skb->len;
451
6b330a60
GR		 452 len = upcall_msg_size(upcall_info, hlen - cutlen,
453 OVS_CB(skb)->acts_origlen);
40c08cda 454 user_skb = genlmsg_new(len, GFP_ATOMIC);
6161d3fd
JG		 455 if (!user_skb) {
456 err = -ENOMEM;
457 goto out;
458 }
7257b535
BP		 459
460 upcall = genlmsg_put(user_skb, 0, 0, &dp_packet_genl_family,
461 0, upcall_info->cmd);
f463bdc9
KL		 462 if (!upcall) {
463 err = -EINVAL;
464 goto out;
465 }
7257b535
BP		 466 upcall->dp_ifindex = dp_ifindex;
467
db7f2238 468 err = ovs_nla_put_key(key, key, OVS_PACKET_ATTR_KEY, false, user_skb);
9a621f82 469 BUG_ON(err);
7257b535
BP		 470
471 if (upcall_info->userdata)
e995e3df 472 __nla_put(user_skb, OVS_PACKET_ATTR_USERDATA,
462a988b 473 nla_len(upcall_info->userdata),
e995e3df 474 nla_data(upcall_info->userdata));
7257b535 475
e23775f2 476
8b7ea2d4 477 if (upcall_info->egress_tun_info) {
09c33996
YHW		 478 nla = nla_nest_start_noflag(user_skb,
479 OVS_PACKET_ATTR_EGRESS_TUN_KEY);
eee4c8eb
KL		 480 if (!nla) {
481 err = -EMSGSIZE;
482 goto out;
483 }
aad7cb91
PS		 484 err = ovs_nla_put_tunnel_info(user_skb,
485 upcall_info->egress_tun_info);
8b7ea2d4
WZ		 486 BUG_ON(err);
487 nla_nest_end(user_skb, nla);
488 }
489
0e469d3b 490 if (upcall_info->actions_len) {
09c33996 491 nla = nla_nest_start_noflag(user_skb, OVS_PACKET_ATTR_ACTIONS);
eee4c8eb
KL		 492 if (!nla) {
493 err = -EMSGSIZE;
494 goto out;
495 }
0e469d3b
NM		 496 err = ovs_nla_put_actions(upcall_info->actions,
497 upcall_info->actions_len,
498 user_skb);
499 if (!err)
500 nla_nest_end(user_skb, nla);
501 else
502 nla_nest_cancel(user_skb, nla);
503 }
504
a94ebc39
JS		 505 /* Add OVS_PACKET_ATTR_MRU */
506 if (upcall_info->mru) {
507 if (nla_put_u16(user_skb, OVS_PACKET_ATTR_MRU,
508 upcall_info->mru)) {
509 err = -ENOBUFS;
510 goto out;
511 }
512 pad_packet(dp, user_skb);
513 }
514
039fb36c
WT		 515 /* Add OVS_PACKET_ATTR_LEN when packet is truncated */
516 if (cutlen > 0) {
517 if (nla_put_u32(user_skb, OVS_PACKET_ATTR_LEN,
518 skb->len)) {
519 err = -ENOBUFS;
520 goto out;
521 }
522 pad_packet(dp, user_skb);
523 }
524
533bea51 525 /* Only reserve room for attribute header, packet data is added
af465b67
PS		 526 * in skb_zerocopy()
527 */
533bea51
TG		 528 if (!(nla = nla_reserve(user_skb, OVS_PACKET_ATTR_PACKET, 0))) {
529 err = -ENOBUFS;
530 goto out;
531 }
4c7804f1 532 nla->nla_len = nla_attr_size(skb->len - cutlen);
bed53bd1 533
4c7804f1 534 err = skb_zerocopy(user_skb, skb, skb->len - cutlen, hlen);
2c272bd9
ZK		 535 if (err)
536 goto out;
7257b535 537
ef507cec 538 /* Pad OVS_PACKET_ATTR_PACKET if linear copy was performed */
a94ebc39 539 pad_packet(dp, user_skb);
ef507cec 540
533bea51 541 ((struct nlmsghdr *) user_skb->data)->nlmsg_len = user_skb->len;
6161d3fd 542
533bea51 543 err = genlmsg_unicast(ovs_dp_get_net(dp), user_skb, upcall_info->portid);
82706a6f 544 user_skb = NULL;
6161d3fd 545out:
2c272bd9
ZK		 546 if (err)
547 skb_tx_error(skb);
82706a6f 548 kfree_skb(user_skb);
6161d3fd
JG		 549 kfree_skb(nskb);
550 return err;
cb5087ca
BP		 551}
552
df2c07f4 553static int ovs_packet_cmd_execute(struct sk_buff *skb, struct genl_info *info)
064af421 554{
df2c07f4 555 struct ovs_header *ovs_header = info->userhdr;
a94ebc39 556 struct net *net = sock_net(skb->sk);
982b8810 557 struct nlattr **a = info->attrs;
e0e57990 558 struct sw_flow_actions *acts;
982b8810 559 struct sk_buff *packet;
e0e57990 560 struct sw_flow *flow;
ad50cb60 561 struct sw_flow_actions *sf_acts;
f7cd0081 562 struct datapath *dp;
a6059080 563 struct vport *input_vport;
a94ebc39 564 u16 mru = 0;
3f19d399 565 int len;
d6569377 566 int err;
2e460098 567 bool log = !a[OVS_PACKET_ATTR_PROBE];
064af421 568
f7cd0081 569 err = -EINVAL;
df2c07f4 570 if (!a[OVS_PACKET_ATTR_PACKET] || !a[OVS_PACKET_ATTR_KEY] ||
7c3072cc 571 !a[OVS_PACKET_ATTR_ACTIONS])
e5cad958 572 goto err;
064af421 573
df2c07f4 574 len = nla_len(a[OVS_PACKET_ATTR_PACKET]);
3f19d399 575 packet = __dev_alloc_skb(NET_IP_ALIGN + len, GFP_KERNEL);
f7cd0081
BP		 576 err = -ENOMEM;
577 if (!packet)
e5cad958 578 goto err;
3f19d399
BP		 579 skb_reserve(packet, NET_IP_ALIGN);
580
bf3d6fce 581 nla_memcpy(__skb_put(packet, len), a[OVS_PACKET_ATTR_PACKET], len);
8d5ebd83 582
a94ebc39
JS		 583 /* Set packet's mru */
584 if (a[OVS_PACKET_ATTR_MRU]) {
585 mru = nla_get_u16(a[OVS_PACKET_ATTR_MRU]);
586 packet->ignore_df = 1;
587 }
588 OVS_CB(packet)->mru = mru;
589
e0e57990 590 /* Build an sw_flow for sending this packet. */
df65fec1 591 flow = ovs_flow_alloc();
e0e57990
BP		 592 err = PTR_ERR(flow);
593 if (IS_ERR(flow))
e5cad958 594 goto err_kfree_skb;
064af421 595
038e34ab
JS		 596 err = ovs_flow_key_extract_userspace(net, a[OVS_PACKET_ATTR_KEY],
597 packet, &flow->key, log);
e0e57990 598 if (err)
9321954a 599 goto err_flow_free;
e0e57990 600
a94ebc39 601 err = ovs_nla_copy_actions(net, a[OVS_PACKET_ATTR_ACTIONS],
9233cef7 602 &flow->key, &acts, log);
9b405f1a
PS		 603 if (err)
604 goto err_flow_free;
e0e57990 605
ff27161e 606 rcu_assign_pointer(flow->sf_acts, acts);
abff858b 607 packet->priority = flow->key.phy.priority;
3025a772 608 packet->mark = flow->key.phy.skb_mark;
e0e57990 609
d6569377 610 rcu_read_lock();
a94ebc39 611 dp = get_dp_rcu(net, ovs_header->dp_ifindex);
f7cd0081 612 err = -ENODEV;
e5cad958
BP		 613 if (!dp)
614 goto err_unlock;
cc4015df 615
a6059080
AZ		 616 input_vport = ovs_vport_rcu(dp, flow->key.phy.in_port);
617 if (!input_vport)
618 input_vport = ovs_vport_rcu(dp, OVSP_LOCAL);
619
620 if (!input_vport)
621 goto err_unlock;
622
e23775f2 623 packet->dev = input_vport->dev;
a6059080 624 OVS_CB(packet)->input_vport = input_vport;
ad50cb60 625 sf_acts = rcu_dereference(flow->sf_acts);
a6059080 626
e9141eec 627 local_bh_disable();
7d16c847 628 err = ovs_execute_actions(dp, packet, sf_acts, &flow->key);
e9141eec 629 local_bh_enable();
d6569377 630 rcu_read_unlock();
e0e57990 631
a1c564be 632 ovs_flow_free(flow, false);
e5cad958 633 return err;
064af421 634
e5cad958
BP		 635err_unlock:
636 rcu_read_unlock();
9321954a 637err_flow_free:
a1c564be 638 ovs_flow_free(flow, false);
e5cad958
BP		 639err_kfree_skb:
640 kfree_skb(packet);
641err:
d6569377 642 return err;
064af421
BP		 643}
644
df2c07f4 645static const struct nla_policy packet_policy[OVS_PACKET_ATTR_MAX + 1] = {
7c3072cc 646 [OVS_PACKET_ATTR_PACKET] = { .len = ETH_HLEN },
df2c07f4
JP		 647 [OVS_PACKET_ATTR_KEY] = { .type = NLA_NESTED },
648 [OVS_PACKET_ATTR_ACTIONS] = { .type = NLA_NESTED },
2e460098 649 [OVS_PACKET_ATTR_PROBE] = { .type = NLA_FLAG },
a94ebc39 650 [OVS_PACKET_ATTR_MRU] = { .type = NLA_U16 },
982b8810
BP		 651};
652
18fd3a52 653static struct genl_ops dp_packet_genl_ops[] = {
df2c07f4 654 { .cmd = OVS_PACKET_CMD_EXECUTE,
a6a8674d 655 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
982b8810 656 .policy = packet_policy,
df2c07f4 657 .doit = ovs_packet_cmd_execute
982b8810
BP		 658 }
659};
660
ba63fe26 661static struct genl_family dp_packet_genl_family __ro_after_init = {
cb25142c
PS		 662 .hdrsize = sizeof(struct ovs_header),
663 .name = OVS_PACKET_FAMILY,
664 .version = OVS_PACKET_VERSION,
665 .maxattr = OVS_PACKET_ATTR_MAX,
666 .netnsok = true,
667 .parallel_ops = true,
668 .ops = dp_packet_genl_ops,
669 .n_ops = ARRAY_SIZE(dp_packet_genl_ops),
ba63fe26 670 .module = THIS_MODULE,
cb25142c
PS		 671};
672
f1f60b85 673static void get_dp_stats(const struct datapath *dp, struct ovs_dp_stats *stats,
4fa72a95 674 struct ovs_dp_megaflow_stats *mega_stats)
064af421 675{
d6569377 676 int i;
f180c2e2 677
4fa72a95
AZ		 678 memset(mega_stats, 0, sizeof(*mega_stats));
679
994dc286 680 stats->n_flows = ovs_flow_tbl_count(&dp->table);
4fa72a95 681 mega_stats->n_masks = ovs_flow_tbl_num_masks(&dp->table);
064af421 682
7257b535 683 stats->n_hit = stats->n_missed = stats->n_lost = 0;
4fa72a95 684
d6569377
BP		 685 for_each_possible_cpu(i) {
686 const struct dp_stats_percpu *percpu_stats;
687 struct dp_stats_percpu local_stats;
821cb9fa 688 unsigned int start;
44e05eca 689
d6569377 690 percpu_stats = per_cpu_ptr(dp->stats_percpu, i);
064af421 691
d6569377 692 do {
b81deb15 693 start = u64_stats_fetch_begin_irq(&percpu_stats->syncp);
d6569377 694 local_stats = *percpu_stats;
b81deb15 695 } while (u64_stats_fetch_retry_irq(&percpu_stats->syncp, start));
064af421 696
d6569377
BP		 697 stats->n_hit += local_stats.n_hit;
698 stats->n_missed += local_stats.n_missed;
699 stats->n_lost += local_stats.n_lost;
4fa72a95 700 mega_stats->n_mask_hit += local_stats.n_mask_hit;
d6569377
BP		 701 }
702}
064af421 703
bc619e29
JS		 704static bool should_fill_key(const struct sw_flow_id *sfid, uint32_t ufid_flags)
705{
706 return ovs_identifier_is_ufid(sfid) &&
707 !(ufid_flags & OVS_UFID_F_OMIT_KEY);
708}
709
710static bool should_fill_mask(uint32_t ufid_flags)
711{
712 return !(ufid_flags & OVS_UFID_F_OMIT_MASK);
713}
714
715static bool should_fill_actions(uint32_t ufid_flags)
0afa2373 716{
bc619e29
JS		 717 return !(ufid_flags & OVS_UFID_F_OMIT_ACTIONS);
718}
719
720static size_t ovs_flow_cmd_msg_size(const struct sw_flow_actions *acts,
721 const struct sw_flow_id *sfid,
722 uint32_t ufid_flags)
723{
724 size_t len = NLMSG_ALIGN(sizeof(struct ovs_header));
725
726 /* OVS_FLOW_ATTR_UFID */
727 if (sfid && ovs_identifier_is_ufid(sfid))
728 len += nla_total_size(sfid->ufid_len);
729
730 /* OVS_FLOW_ATTR_KEY */
731 if (!sfid || should_fill_key(sfid, ufid_flags))
732 len += nla_total_size(ovs_key_attr_size());
733
734 /* OVS_FLOW_ATTR_MASK */
735 if (should_fill_mask(ufid_flags))
736 len += nla_total_size(ovs_key_attr_size());
737
738 /* OVS_FLOW_ATTR_ACTIONS */
739 if (should_fill_actions(ufid_flags))
c3bb15b3 740 len += nla_total_size(acts->orig_len);
bc619e29
JS		 741
742 return len
91b37647 743 + nla_total_size_64bit(sizeof(struct ovs_flow_stats)) /* OVS_FLOW_ATTR_STATS */
0afa2373 744 + nla_total_size(1) /* OVS_FLOW_ATTR_TCP_FLAGS */
91b37647 745 + nla_total_size_64bit(8); /* OVS_FLOW_ATTR_USED */
0afa2373
TG		 746}
747
f1948bb9
JS		 748/* Called with ovs_mutex or RCU read lock. */
749static int ovs_flow_cmd_fill_stats(const struct sw_flow *flow,
750 struct sk_buff *skb)
751{
752 struct ovs_flow_stats stats;
753 __be16 tcp_flags;
754 unsigned long used;
755
b0f3a2fe 756 ovs_flow_stats_get(flow, &stats, &used, &tcp_flags);
f71db6b1 757
b0f3a2fe 758 if (used &&
89be7da8
PS		 759 nla_put_u64_64bit(skb, OVS_FLOW_ATTR_USED, ovs_flow_used_time(used),
760 OVS_FLOW_ATTR_PAD))
f1948bb9 761 return -EMSGSIZE;
d6569377 762
b0f3a2fe 763 if (stats.n_packets &&
91b37647
PS		 764 nla_put_64bit(skb, OVS_FLOW_ATTR_STATS,
765 sizeof(struct ovs_flow_stats), &stats,
766 OVS_FLOW_ATTR_PAD))
f1948bb9 767 return -EMSGSIZE;
b0b906cc 768
b0f3a2fe
PS		 769 if ((u8)ntohs(tcp_flags) &&
770 nla_put_u8(skb, OVS_FLOW_ATTR_TCP_FLAGS, (u8)ntohs(tcp_flags)))
f1948bb9
JS		 771 return -EMSGSIZE;
772
773 return 0;
774}
775
776/* Called with ovs_mutex or RCU read lock. */
777static int ovs_flow_cmd_fill_actions(const struct sw_flow *flow,
778 struct sk_buff *skb, int skb_orig_len)
779{
780 struct nlattr *start;
781 int err;
d6569377 782
df2c07f4 783 /* If OVS_FLOW_ATTR_ACTIONS doesn't fit, skip dumping the actions if
30053024
BP		 784 * this is the first flow to be dumped into 'skb'. This is unusual for
785 * Netlink but individual action lists can be longer than
786 * NLMSG_GOODSIZE and thus entirely undumpable if we didn't do this.
787 * The userspace caller can always fetch the actions separately if it
788 * really wants them. (Most userspace callers in fact don't care.)
789 *
790 * This can only fail for dump operations because the skb is always
791 * properly sized for single flows.
792 */
09c33996 793 start = nla_nest_start_noflag(skb, OVS_FLOW_ATTR_ACTIONS);
f6f481ee 794 if (start) {
f44ccce1
PS		 795 const struct sw_flow_actions *sf_acts;
796
780ec6ae 797 sf_acts = rcu_dereference_ovsl(flow->sf_acts);
a097c0b2
PS		 798 err = ovs_nla_put_actions(sf_acts->actions,
799 sf_acts->actions_len, skb);
f71db6b1 800
0a25b039
BP		 801 if (!err)
802 nla_nest_end(skb, start);
803 else {
804 if (skb_orig_len)
f1948bb9 805 return err;
0a25b039
BP		 806
807 nla_nest_cancel(skb, start);
808 }
f1948bb9
JS		 809 } else if (skb_orig_len) {
810 return -EMSGSIZE;
811 }
812
813 return 0;
814}
815
816/* Called with ovs_mutex or RCU read lock. */
2c622e5a 817static int ovs_flow_cmd_fill_info(const struct sw_flow *flow, int dp_ifindex,
f1948bb9 818 struct sk_buff *skb, u32 portid,
bc619e29 819 u32 seq, u32 flags, u8 cmd, u32 ufid_flags)
f1948bb9
JS		 820{
821 const int skb_orig_len = skb->len;
822 struct ovs_header *ovs_header;
823 int err;
824
7d16c847
PS		 825 ovs_header = genlmsg_put(skb, portid, seq, &dp_flow_genl_family,
826 flags, cmd);
f1948bb9
JS		 827 if (!ovs_header)
828 return -EMSGSIZE;
7d16c847 829
f1948bb9
JS		 830 ovs_header->dp_ifindex = dp_ifindex;
831
bc619e29 832 err = ovs_nla_put_identifier(flow, skb);
db7f2238
JS		 833 if (err)
834 goto error;
835
bc619e29
JS		 836 if (should_fill_key(&flow->id, ufid_flags)) {
837 err = ovs_nla_put_masked_key(flow, skb);
838 if (err)
839 goto error;
840 }
841
842 if (should_fill_mask(ufid_flags)) {
843 err = ovs_nla_put_mask(flow, skb);
844 if (err)
845 goto error;
846 }
f1948bb9
JS		 847
848 err = ovs_flow_cmd_fill_stats(flow, skb);
849 if (err)
850 goto error;
851
bc619e29
JS		 852 if (should_fill_actions(ufid_flags)) {
853 err = ovs_flow_cmd_fill_actions(flow, skb, skb_orig_len);
854 if (err)
855 goto error;
856 }
37a1300c 857
23b48dc1
TG		 858 genlmsg_end(skb, ovs_header);
859 return 0;
d6569377 860
37a1300c 861error:
df2c07f4 862 genlmsg_cancel(skb, ovs_header);
d6569377 863 return err;
44e05eca
BP		 864}
865
f71db6b1
JR		 866/* May not be called with RCU read lock. */
867static struct sk_buff *ovs_flow_cmd_alloc_info(const struct sw_flow_actions *acts,
bc619e29 868 const struct sw_flow_id *sfid,
afad3556 869 struct genl_info *info,
bc619e29
JS		 870 bool always,
871 uint32_t ufid_flags)
44e05eca 872{
afad3556 873 struct sk_buff *skb;
bc619e29 874 size_t len;
d6569377 875
114fce23
SG		 876 if (!always && !ovs_must_notify(&dp_flow_genl_family, info,
877 GROUP_ID(&ovs_dp_flow_multicast_group)))
afad3556
JR		 878 return NULL;
879
bc619e29 880 len = ovs_flow_cmd_msg_size(acts, sfid, ufid_flags);
40c08cda 881 skb = genlmsg_new(len, GFP_KERNEL);
afad3556
JR		 882 if (!skb)
883 return ERR_PTR(-ENOMEM);
884
885 return skb;
37a1300c 886}
8d5ebd83 887
f71db6b1 888/* Called with ovs_mutex. */
7d16c847 889static struct sk_buff *ovs_flow_cmd_build_info(const struct sw_flow *flow,
f71db6b1
JR		 890 int dp_ifindex,
891 struct genl_info *info, u8 cmd,
bc619e29 892 bool always, u32 ufid_flags)
37a1300c
BP		 893{
894 struct sk_buff *skb;
895 int retval;
d6569377 896
bc619e29
JS		 897 skb = ovs_flow_cmd_alloc_info(ovsl_dereference(flow->sf_acts),
898 &flow->id, info, always, ufid_flags);
a6ddcc9a 899 if (IS_ERR_OR_NULL(skb))
afad3556 900 return skb;
d6569377 901
2c622e5a 902 retval = ovs_flow_cmd_fill_info(flow, dp_ifindex, skb,
f71db6b1 903 info->snd_portid, info->snd_seq, 0,
bc619e29 904 cmd, ufid_flags);
37a1300c 905 BUG_ON(retval < 0);
d6569377 906 return skb;
064af421
BP		 907}
908
0c9fd022 909static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info)
064af421 910{
a94ebc39 911 struct net *net = sock_net(skb->sk);
37a1300c 912 struct nlattr **a = info->attrs;
df2c07f4 913 struct ovs_header *ovs_header = info->userhdr;
bc619e29 914 struct sw_flow *flow = NULL, *new_flow;
a1c564be 915 struct sw_flow_mask mask;
37a1300c 916 struct sk_buff *reply;
9c52546b 917 struct datapath *dp;
0c9fd022 918 struct sw_flow_actions *acts;
a1c564be 919 struct sw_flow_match match;
bc619e29 920 u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
bc4a05c6 921 int error;
9233cef7 922 bool log = !a[OVS_FLOW_ATTR_PROBE];
064af421 923
6740b721 924 /* Must have key and actions. */
37a1300c 925 error = -EINVAL;
a473df5b 926 if (!a[OVS_FLOW_ATTR_KEY]) {
7d16c847 927 OVS_NLERR(log, "Flow key attr not present in new flow.");
37a1300c 928 goto error;
a473df5b
JG		 929 }
930 if (!a[OVS_FLOW_ATTR_ACTIONS]) {
7d16c847 931 OVS_NLERR(log, "Flow actions attr not present in new flow.");
6740b721 932 goto error;
a473df5b 933 }
a1c564be 934
6740b721 935 /* Most of the time we need to allocate a new flow, do it before
af465b67
PS		 936 * locking.
937 */
6740b721
JR		 938 new_flow = ovs_flow_alloc();
939 if (IS_ERR(new_flow)) {
940 error = PTR_ERR(new_flow);
941 goto error;
942 }
943
944 /* Extract key. */
9b94fa6c 945 ovs_match_init(&match, &new_flow->key, false, &mask);
038e34ab 946 error = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY],
9233cef7 947 a[OVS_FLOW_ATTR_MASK], log);
37a1300c 948 if (error)
6740b721 949 goto err_kfree_flow;
064af421 950
bc619e29
JS		 951 /* Extract flow identifier. */
952 error = ovs_nla_get_identifier(&new_flow->id, a[OVS_FLOW_ATTR_UFID],
1d334d4f 953 &new_flow->key, log);
bc619e29
JS		 954 if (error)
955 goto err_kfree_flow;
9b405f1a 956
1d334d4f 957 /* unmasked key is needed to match when ufid is not used. */
958 if (ovs_identifier_is_key(&new_flow->id))
959 match.key = new_flow->id.unmasked_key;
960
961 ovs_flow_mask_key(&new_flow->key, &new_flow->key, true, &mask);
962
6740b721 963 /* Validate actions. */
a94ebc39
JS		 964 error = ovs_nla_copy_actions(net, a[OVS_FLOW_ATTR_ACTIONS],
965 &new_flow->key, &acts, log);
0c9fd022 966 if (error) {
7d16c847 967 OVS_NLERR(log, "Flow actions may not be safe on all matching packets.");
4f67b12a 968 goto err_kfree_flow;
6740b721
JR		 969 }
970
bc619e29
JS		 971 reply = ovs_flow_cmd_alloc_info(acts, &new_flow->id, info, false,
972 ufid_flags);
6740b721
JR		 973 if (IS_ERR(reply)) {
974 error = PTR_ERR(reply);
975 goto err_kfree_acts;
37a1300c
BP		 976 }
977
cd2a59e9 978 ovs_lock();
a94ebc39 979 dp = get_dp(net, ovs_header->dp_ifindex);
6740b721
JR		 980 if (unlikely(!dp)) {
981 error = -ENODEV;
cd2a59e9 982 goto err_unlock_ovs;
6740b721 983 }
bc619e29 984
a1c564be 985 /* Check if this is a duplicate flow */
bc619e29
JS		 986 if (ovs_identifier_is_ufid(&new_flow->id))
987 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &new_flow->id);
988 if (!flow)
1d334d4f 989 flow = ovs_flow_tbl_lookup(&dp->table, &new_flow->key);
6740b721
JR		 990 if (likely(!flow)) {
991 rcu_assign_pointer(new_flow->sf_acts, acts);
d6569377 992
d6569377 993 /* Put flow in bucket. */
6740b721
JR		 994 error = ovs_flow_tbl_insert(&dp->table, new_flow, &mask);
995 if (unlikely(error)) {
0585f7a8 996 acts = NULL;
6740b721
JR		 997 goto err_unlock_ovs;
998 }
999
1000 if (unlikely(reply)) {
2c622e5a 1001 error = ovs_flow_cmd_fill_info(new_flow,
6740b721
JR		 1002 ovs_header->dp_ifindex,
1003 reply, info->snd_portid,
1004 info->snd_seq, 0,
bc619e29
JS		 1005 OVS_FLOW_CMD_NEW,
1006 ufid_flags);
6740b721 1007 BUG_ON(error < 0);
0585f7a8 1008 }
6740b721 1009 ovs_unlock();
d6569377 1010 } else {
0c9fd022
JR		 1011 struct sw_flow_actions *old_acts;
1012
d6569377
BP		 1013 /* Bail out if we're not allowed to modify an existing flow.
1014 * We accept NLM_F_CREATE in place of the intended NLM_F_EXCL
1015 * because Generic Netlink treats the latter as a dump
1016 * request. We also accept NLM_F_EXCL in case that bug ever
1017 * gets fixed.
1018 */
6740b721
JR		 1019 if (unlikely(info->nlhdr->nlmsg_flags & (NLM_F_CREATE
1020 | NLM_F_EXCL))) {
1021 error = -EEXIST;
cd2a59e9 1022 goto err_unlock_ovs;
6740b721 1023 }
bc619e29
JS		 1024 /* The flow identifier has to be the same for flow updates.
1025 * Look for any overlapping flow.
1026 */
1027 if (unlikely(!ovs_flow_cmp(flow, &match))) {
1028 if (ovs_identifier_is_key(&flow->id))
1029 flow = ovs_flow_tbl_lookup_exact(&dp->table,
1030 &match);
1031 else /* UFID matches but key is different */
1032 flow = NULL;
3440e4bc
AW		 1033 if (!flow) {
1034 error = -ENOENT;
1035 goto err_unlock_ovs;
1036 }
6740b721 1037 }
0c9fd022
JR		 1038 /* Update actions. */
1039 old_acts = ovsl_dereference(flow->sf_acts);
1040 rcu_assign_pointer(flow->sf_acts, acts);
0c9fd022 1041
6740b721 1042 if (unlikely(reply)) {
2c622e5a 1043 error = ovs_flow_cmd_fill_info(flow,
6740b721
JR		 1044 ovs_header->dp_ifindex,
1045 reply, info->snd_portid,
1046 info->snd_seq, 0,
bc619e29
JS		 1047 OVS_FLOW_CMD_NEW,
1048 ufid_flags);
6740b721
JR		 1049 BUG_ON(error < 0);
1050 }
1051 ovs_unlock();
0c9fd022 1052
e23775f2 1053 ovs_nla_free_flow_actions_rcu(old_acts);
6740b721 1054 ovs_flow_free(new_flow, false);
0c9fd022 1055 }
6740b721
JR		 1056
1057 if (reply)
cb25142c 1058 ovs_notify(&dp_flow_genl_family, &ovs_dp_flow_multicast_group, reply, info);
0c9fd022
JR		 1059 return 0;
1060
0c9fd022
JR		 1061err_unlock_ovs:
1062 ovs_unlock();
6740b721
JR		 1063 kfree_skb(reply);
1064err_kfree_acts:
e23775f2 1065 ovs_nla_free_flow_actions(acts);
6740b721
JR		 1066err_kfree_flow:
1067 ovs_flow_free(new_flow, false);
0c9fd022
JR		 1068error:
1069 return error;
1070}
1071
cc561abf 1072/* Factor out action copy to avoid "Wframe-larger-than=1024" warning. */
19b6110d 1073static noinline_for_stack struct sw_flow_actions *get_flow_actions(struct net *net,
a94ebc39 1074 const struct nlattr *a,
cc561abf 1075 const struct sw_flow_key *key,
9233cef7
JR		 1076 const struct sw_flow_mask *mask,
1077 bool log)
cc561abf
PS		 1078{
1079 struct sw_flow_actions *acts;
1080 struct sw_flow_key masked_key;
1081 int error;
1082
ad4adec2 1083 ovs_flow_mask_key(&masked_key, key, true, mask);
a94ebc39 1084 error = ovs_nla_copy_actions(net, a, &masked_key, &acts, log);
cc561abf 1085 if (error) {
9233cef7 1086 OVS_NLERR(log,
7d16c847 1087 "Actions may not be safe on all matching packets");
cc561abf
PS		 1088 return ERR_PTR(error);
1089 }
1090
1091 return acts;
1092}
1093
850c2a4d
TZ		 1094/* Factor out match-init and action-copy to avoid
1095 * "Wframe-larger-than=1024" warning. Because mask is only
1096 * used to get actions, we new a function to save some
1097 * stack space.
1098 *
1099 * If there are not key and action attrs, we return 0
1100 * directly. In the case, the caller will also not use the
1101 * match as before. If there is action attr, we try to get
1102 * actions and save them to *acts. Before returning from
1103 * the function, we reset the match->mask pointer. Because
1104 * we should not to return match object with dangling reference
1105 * to mask.
1106 * */
19b6110d
AB		 1107static noinline_for_stack int
1108ovs_nla_init_match_and_action(struct net *net,
1109 struct sw_flow_match *match,
1110 struct sw_flow_key *key,
1111 struct nlattr **a,
1112 struct sw_flow_actions **acts,
1113 bool log)
850c2a4d
TZ		 1114{
1115 struct sw_flow_mask mask;
1116 int error = 0;
1117
1118 if (a[OVS_FLOW_ATTR_KEY]) {
1119 ovs_match_init(match, key, true, &mask);
1120 error = ovs_nla_get_match(net, match, a[OVS_FLOW_ATTR_KEY],
1121 a[OVS_FLOW_ATTR_MASK], log);
1122 if (error)
1123 goto error;
1124 }
1125
1126 if (a[OVS_FLOW_ATTR_ACTIONS]) {
1127 if (!a[OVS_FLOW_ATTR_KEY]) {
1128 OVS_NLERR(log,
1129 "Flow key attribute not present in set flow.");
497895ca
CJ		 1130 error = -EINVAL;
1131 goto error;
850c2a4d
TZ		 1132 }
1133
1134 *acts = get_flow_actions(net, a[OVS_FLOW_ATTR_ACTIONS], key,
1135 &mask, log);
1136 if (IS_ERR(*acts)) {
1137 error = PTR_ERR(*acts);
1138 goto error;
1139 }
1140 }
1141
1142 /* On success, error is 0. */
1143error:
1144 match->mask = NULL;
1145 return error;
1146}
1147
0c9fd022
JR		 1148static int ovs_flow_cmd_set(struct sk_buff *skb, struct genl_info *info)
1149{
a94ebc39 1150 struct net *net = sock_net(skb->sk);
0c9fd022
JR		 1151 struct nlattr **a = info->attrs;
1152 struct ovs_header *ovs_header = info->userhdr;
1d2a1b5f 1153 struct sw_flow_key key;
0c9fd022 1154 struct sw_flow *flow;
0c9fd022
JR		 1155 struct sk_buff *reply = NULL;
1156 struct datapath *dp;
6740b721 1157 struct sw_flow_actions *old_acts = NULL, *acts = NULL;
0c9fd022 1158 struct sw_flow_match match;
bc619e29
JS		 1159 struct sw_flow_id sfid;
1160 u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
b24baa1a 1161 int error = 0;
9233cef7 1162 bool log = !a[OVS_FLOW_ATTR_PROBE];
bc619e29 1163 bool ufid_present;
0c9fd022 1164
bc619e29 1165 ufid_present = ovs_nla_get_ufid(&sfid, a[OVS_FLOW_ATTR_UFID], log);
850c2a4d 1166 if (!a[OVS_FLOW_ATTR_KEY] && !ufid_present) {
b24baa1a
PS		 1167 OVS_NLERR(log,
1168 "Flow set message rejected, Key attribute missing.");
850c2a4d 1169 return -EINVAL;
b24baa1a 1170 }
850c2a4d
TZ		 1171
1172 error = ovs_nla_init_match_and_action(net, &match, &key, a,
1173 &acts, log);
0c9fd022
JR		 1174 if (error)
1175 goto error;
d6569377 1176
850c2a4d 1177 if (acts) {
ff27161e 1178 /* Can allocate before locking if have acts. */
bc619e29
JS		 1179 reply = ovs_flow_cmd_alloc_info(acts, &sfid, info, false,
1180 ufid_flags);
6740b721
JR		 1181 if (IS_ERR(reply)) {
1182 error = PTR_ERR(reply);
1183 goto err_kfree_acts;
90b8c2f7 1184 }
0c9fd022
JR		 1185 }
1186
1187 ovs_lock();
a94ebc39 1188 dp = get_dp(net, ovs_header->dp_ifindex);
6740b721
JR		 1189 if (unlikely(!dp)) {
1190 error = -ENODEV;
0c9fd022 1191 goto err_unlock_ovs;
6740b721 1192 }
0c9fd022 1193 /* Check that the flow exists. */
bc619e29
JS		 1194 if (ufid_present)
1195 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &sfid);
1196 else
1197 flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
6740b721
JR		 1198 if (unlikely(!flow)) {
1199 error = -ENOENT;
0c9fd022 1200 goto err_unlock_ovs;
6740b721 1201 }
3440e4bc 1202
0c9fd022 1203 /* Update actions, if present. */
6740b721 1204 if (likely(acts)) {
0c9fd022
JR		 1205 old_acts = ovsl_dereference(flow->sf_acts);
1206 rcu_assign_pointer(flow->sf_acts, acts);
6740b721
JR		 1207
1208 if (unlikely(reply)) {
2c622e5a 1209 error = ovs_flow_cmd_fill_info(flow,
6740b721
JR		 1210 ovs_header->dp_ifindex,
1211 reply, info->snd_portid,
1212 info->snd_seq, 0,
229486c2 1213 OVS_FLOW_CMD_SET,
bc619e29 1214 ufid_flags);
6740b721
JR		 1215 BUG_ON(error < 0);
1216 }
1217 } else {
1218 /* Could not alloc without acts before locking. */
7d16c847 1219 reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex,
229486c2 1220 info, OVS_FLOW_CMD_SET, false,
bc619e29
JS		 1221 ufid_flags);
1222
6740b721
JR		 1223 if (unlikely(IS_ERR(reply))) {
1224 error = PTR_ERR(reply);
1225 goto err_unlock_ovs;
1226 }
9c52546b 1227 }
0c9fd022 1228
0c9fd022
JR		 1229 /* Clear stats. */
1230 if (a[OVS_FLOW_ATTR_CLEAR])
1231 ovs_flow_stats_clear(flow);
cd2a59e9 1232 ovs_unlock();
37a1300c 1233
6740b721 1234 if (reply)
cb25142c 1235 ovs_notify(&dp_flow_genl_family, &ovs_dp_flow_multicast_group, reply, info);
6740b721 1236 if (old_acts)
e23775f2 1237 ovs_nla_free_flow_actions_rcu(old_acts);
7d16c847 1238
d6569377 1239 return 0;
704a1e09 1240
cd2a59e9
PS		 1241err_unlock_ovs:
1242 ovs_unlock();
6740b721
JR		 1243 kfree_skb(reply);
1244err_kfree_acts:
e23775f2 1245 ovs_nla_free_flow_actions(acts);
37a1300c 1246error:
9c52546b 1247 return error;
704a1e09
BP		 1248}
1249
df2c07f4 1250static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info)
704a1e09 1251{
37a1300c 1252 struct nlattr **a = info->attrs;
df2c07f4 1253 struct ovs_header *ovs_header = info->userhdr;
038e34ab 1254 struct net *net = sock_net(skb->sk);
37a1300c 1255 struct sw_flow_key key;
37a1300c 1256 struct sk_buff *reply;
704a1e09 1257 struct sw_flow *flow;
9c52546b 1258 struct datapath *dp;
a1c564be 1259 struct sw_flow_match match;
bc619e29
JS		 1260 struct sw_flow_id ufid;
1261 u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1262 int err = 0;
9233cef7 1263 bool log = !a[OVS_FLOW_ATTR_PROBE];
bc619e29 1264 bool ufid_present;
704a1e09 1265
bc619e29
JS		 1266 ufid_present = ovs_nla_get_ufid(&ufid, a[OVS_FLOW_ATTR_UFID], log);
1267 if (a[OVS_FLOW_ATTR_KEY]) {
9b94fa6c 1268 ovs_match_init(&match, &key, true, NULL);
038e34ab 1269 err = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY], NULL,
bc619e29
JS		 1270 log);
1271 } else if (!ufid_present) {
9233cef7
JR		 1272 OVS_NLERR(log,
1273 "Flow get message rejected, Key attribute missing.");
bc619e29 1274 err = -EINVAL;
1b936472 1275 }
37a1300c
BP		 1276 if (err)
1277 return err;
704a1e09 1278
cd2a59e9 1279 ovs_lock();
2a4999f3 1280 dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
cd2a59e9
PS		 1281 if (!dp) {
1282 err = -ENODEV;
1283 goto unlock;
1284 }
704a1e09 1285
bc619e29
JS		 1286 if (ufid_present)
1287 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid);
1288 else
1289 flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
3440e4bc 1290 if (!flow) {
cd2a59e9
PS		 1291 err = -ENOENT;
1292 goto unlock;
1293 }
d6569377 1294
7d16c847 1295 reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex, info,
229486c2 1296 OVS_FLOW_CMD_GET, true, ufid_flags);
cd2a59e9
PS		 1297 if (IS_ERR(reply)) {
1298 err = PTR_ERR(reply);
1299 goto unlock;
1300 }
36956a7d 1301
cd2a59e9 1302 ovs_unlock();
37a1300c 1303 return genlmsg_reply(reply, info);
cd2a59e9
PS		 1304unlock:
1305 ovs_unlock();
1306 return err;
d6569377 1307}
9c52546b 1308
df2c07f4 1309static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info)
d6569377 1310{
37a1300c 1311 struct nlattr **a = info->attrs;
df2c07f4 1312 struct ovs_header *ovs_header = info->userhdr;
038e34ab 1313 struct net *net = sock_net(skb->sk);
37a1300c 1314 struct sw_flow_key key;
37a1300c 1315 struct sk_buff *reply;
bc619e29 1316 struct sw_flow *flow = NULL;
d6569377 1317 struct datapath *dp;
a1c564be 1318 struct sw_flow_match match;
bc619e29
JS		 1319 struct sw_flow_id ufid;
1320 u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
d6569377 1321 int err;
9233cef7 1322 bool log = !a[OVS_FLOW_ATTR_PROBE];
bc619e29 1323 bool ufid_present;
36956a7d 1324
bc619e29
JS		 1325 ufid_present = ovs_nla_get_ufid(&ufid, a[OVS_FLOW_ATTR_UFID], log);
1326 if (a[OVS_FLOW_ATTR_KEY]) {
9b94fa6c 1327 ovs_match_init(&match, &key, true, NULL);
038e34ab
JS		 1328 err = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY],
1329 NULL, log);
cde7f3ba
JR		 1330 if (unlikely(err))
1331 return err;
1332 }
1333
cd2a59e9 1334 ovs_lock();
2a4999f3 1335 dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
cde7f3ba 1336 if (unlikely(!dp)) {
cd2a59e9
PS		 1337 err = -ENODEV;
1338 goto unlock;
1339 }
7d16c847 1340
bc619e29 1341 if (unlikely(!a[OVS_FLOW_ATTR_KEY] && !ufid_present)) {
994dc286 1342 err = ovs_flow_tbl_flush(&dp->table);
cd2a59e9
PS		 1343 goto unlock;
1344 }
7d16c847 1345
bc619e29
JS		 1346 if (ufid_present)
1347 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid);
1348 else
1349 flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
3440e4bc 1350 if (unlikely(!flow)) {
cd2a59e9
PS		 1351 err = -ENOENT;
1352 goto unlock;
1353 }
d6569377 1354
994dc286 1355 ovs_flow_tbl_remove(&dp->table, flow);
cde7f3ba 1356 ovs_unlock();
37a1300c 1357
46051cf8 1358 reply = ovs_flow_cmd_alloc_info(rcu_dereference_raw(flow->sf_acts),
bc619e29 1359 &flow->id, info, false, ufid_flags);
cde7f3ba
JR		 1360
1361 if (likely(reply)) {
1362 if (likely(!IS_ERR(reply))) {
7d16c847
PS		 1363 rcu_read_lock(); /*To keep RCU checker happy. */
1364 err = ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex,
cde7f3ba
JR		 1365 reply, info->snd_portid,
1366 info->snd_seq, 0,
bc619e29
JS		 1367 OVS_FLOW_CMD_DEL,
1368 ufid_flags);
cde7f3ba
JR		 1369 rcu_read_unlock();
1370 BUG_ON(err < 0);
cb25142c 1371 ovs_notify(&dp_flow_genl_family, &ovs_dp_flow_multicast_group, reply, info);
cde7f3ba 1372 } else {
cb25142c
PS		 1373 genl_set_err(&dp_flow_genl_family, sock_net(skb->sk), 0,
1374 GROUP_ID(&ovs_dp_flow_multicast_group), PTR_ERR(reply));
1375
cde7f3ba 1376 }
afad3556 1377 }
37a1300c 1378
a1c564be 1379 ovs_flow_free(flow, true);
37a1300c 1380 return 0;
cd2a59e9
PS		 1381unlock:
1382 ovs_unlock();
1383 return err;
37a1300c
BP		 1384}
1385
df2c07f4 1386static int ovs_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
37a1300c 1387{
bc619e29 1388 struct nlattr *a[__OVS_FLOW_ATTR_MAX];
df2c07f4 1389 struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
994dc286 1390 struct table_instance *ti;
37a1300c 1391 struct datapath *dp;
bc619e29
JS		 1392 u32 ufid_flags;
1393 int err;
1394
1395 err = genlmsg_parse(cb->nlh, &dp_flow_genl_family, a,
15702dc9 1396 OVS_FLOW_ATTR_MAX, flow_policy, NULL);
bc619e29
JS		 1397 if (err)
1398 return err;
1399 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
37a1300c 1400
f44ccce1 1401 rcu_read_lock();
01ac0970 1402 dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
cd2a59e9 1403 if (!dp) {
f44ccce1 1404 rcu_read_unlock();
37a1300c 1405 return -ENODEV;
cd2a59e9 1406 }
37a1300c 1407
994dc286 1408 ti = rcu_dereference(dp->table.ti);
37a1300c 1409 for (;;) {
37a1300c
BP		 1410 struct sw_flow *flow;
1411 u32 bucket, obj;
1412
1413 bucket = cb->args[0];
1414 obj = cb->args[1];
994dc286 1415 flow = ovs_flow_tbl_dump_next(ti, &bucket, &obj);
3544358a 1416 if (!flow)
37a1300c
BP		 1417 break;
1418
2c622e5a 1419 if (ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex, skb,
28aea917 1420 NETLINK_CB(cb->skb).portid,
37a1300c 1421 cb->nlh->nlmsg_seq, NLM_F_MULTI,
229486c2 1422 OVS_FLOW_CMD_GET, ufid_flags) < 0)
37a1300c
BP		 1423 break;
1424
1425 cb->args[0] = bucket;
1426 cb->args[1] = obj;
1427 }
f44ccce1 1428 rcu_read_unlock();
37a1300c 1429 return skb->len;
704a1e09
BP		 1430}
1431
cb25142c
PS		 1432static const struct nla_policy flow_policy[OVS_FLOW_ATTR_MAX + 1] = {
1433 [OVS_FLOW_ATTR_KEY] = { .type = NLA_NESTED },
9233cef7 1434 [OVS_FLOW_ATTR_MASK] = { .type = NLA_NESTED },
cb25142c
PS		 1435 [OVS_FLOW_ATTR_ACTIONS] = { .type = NLA_NESTED },
1436 [OVS_FLOW_ATTR_CLEAR] = { .type = NLA_FLAG },
9233cef7 1437 [OVS_FLOW_ATTR_PROBE] = { .type = NLA_FLAG },
bc619e29
JS		 1438 [OVS_FLOW_ATTR_UFID] = { .type = NLA_UNSPEC, .len = 1 },
1439 [OVS_FLOW_ATTR_UFID_FLAGS] = { .type = NLA_U32 },
cb25142c
PS		 1440};
1441
18fd3a52 1442static struct genl_ops dp_flow_genl_ops[] = {
df2c07f4 1443 { .cmd = OVS_FLOW_CMD_NEW,
a6a8674d 1444 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
37a1300c 1445 .policy = flow_policy,
0c9fd022 1446 .doit = ovs_flow_cmd_new
37a1300c 1447 },
df2c07f4 1448 { .cmd = OVS_FLOW_CMD_DEL,
a6a8674d 1449 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
37a1300c 1450 .policy = flow_policy,
df2c07f4 1451 .doit = ovs_flow_cmd_del
37a1300c 1452 },
df2c07f4 1453 { .cmd = OVS_FLOW_CMD_GET,
37a1300c
BP		 1454 .flags = 0, /* OK for unprivileged users. */
1455 .policy = flow_policy,
df2c07f4
JP		 1456 .doit = ovs_flow_cmd_get,
1457 .dumpit = ovs_flow_cmd_dump
37a1300c 1458 },
df2c07f4 1459 { .cmd = OVS_FLOW_CMD_SET,
a6a8674d 1460 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
37a1300c 1461 .policy = flow_policy,
0c9fd022 1462 .doit = ovs_flow_cmd_set,
37a1300c
BP		 1463 },
1464};
1465
ba63fe26 1466static struct genl_family dp_flow_genl_family __ro_after_init = {
df2c07f4 1467 .hdrsize = sizeof(struct ovs_header),
cb25142c
PS		 1468 .name = OVS_FLOW_FAMILY,
1469 .version = OVS_FLOW_VERSION,
1470 .maxattr = OVS_FLOW_ATTR_MAX,
b3dcb73c 1471 .netnsok = true,
cb25142c
PS		 1472 .parallel_ops = true,
1473 .ops = dp_flow_genl_ops,
1474 .n_ops = ARRAY_SIZE(dp_flow_genl_ops),
1475 .mcgrps = &ovs_dp_flow_multicast_group,
1476 .n_mcgrps = 1,
ba63fe26 1477 .module = THIS_MODULE,
aaff4b55
BP		 1478};
1479
0afa2373
TG		 1480static size_t ovs_dp_cmd_msg_size(void)
1481{
1482 size_t msgsize = NLMSG_ALIGN(sizeof(struct ovs_header));
1483
1484 msgsize += nla_total_size(IFNAMSIZ);
91b37647
PS		 1485 msgsize += nla_total_size_64bit(sizeof(struct ovs_dp_stats));
1486 msgsize += nla_total_size_64bit(sizeof(struct ovs_dp_megaflow_stats));
300af20a 1487 msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_USER_FEATURES */
0afa2373
TG		 1488
1489 return msgsize;
1490}
1491
d637497c 1492/* Called with ovs_mutex. */
df2c07f4 1493static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,
28aea917 1494 u32 portid, u32 seq, u32 flags, u8 cmd)
064af421 1495{
df2c07f4 1496 struct ovs_header *ovs_header;
e926dfe3 1497 struct ovs_dp_stats dp_stats;
4fa72a95 1498 struct ovs_dp_megaflow_stats dp_megaflow_stats;
064af421
BP		 1499 int err;
1500
28aea917 1501 ovs_header = genlmsg_put(skb, portid, seq, &dp_datapath_genl_family,
aaff4b55 1502 flags, cmd);
df2c07f4 1503 if (!ovs_header)
aaff4b55 1504 goto error;
064af421 1505
b063d9f0 1506 ovs_header->dp_ifindex = get_dpifindex(dp);
064af421 1507
850b6b3b 1508 err = nla_put_string(skb, OVS_DP_ATTR_NAME, ovs_dp_name(dp));
064af421 1509 if (err)
d6569377 1510 goto nla_put_failure;
064af421 1511
4fa72a95 1512 get_dp_stats(dp, &dp_stats, &dp_megaflow_stats);
91b37647
PS		 1513 if (nla_put_64bit(skb, OVS_DP_ATTR_STATS, sizeof(struct ovs_dp_stats),
1514 &dp_stats, OVS_DP_ATTR_PAD))
4fa72a95
AZ		 1515 goto nla_put_failure;
1516
91b37647
PS		 1517 if (nla_put_64bit(skb, OVS_DP_ATTR_MEGAFLOW_STATS,
1518 sizeof(struct ovs_dp_megaflow_stats),
1519 &dp_megaflow_stats, OVS_DP_ATTR_PAD))
c3cc8c03 1520 goto nla_put_failure;
d6569377 1521
c58cc9a4
TG		 1522 if (nla_put_u32(skb, OVS_DP_ATTR_USER_FEATURES, dp->user_features))
1523 goto nla_put_failure;
1524
23b48dc1
TG		 1525 genlmsg_end(skb, ovs_header);
1526 return 0;
d6569377
BP		 1527
1528nla_put_failure:
df2c07f4 1529 genlmsg_cancel(skb, ovs_header);
aaff4b55
BP		 1530error:
1531 return -EMSGSIZE;
d6569377
BP		 1532}
1533
40c08cda 1534static struct sk_buff *ovs_dp_cmd_alloc_info(void)
d6569377 1535{
40c08cda 1536 return genlmsg_new(ovs_dp_cmd_msg_size(), GFP_KERNEL);
aaff4b55 1537}
9dca7bd5 1538
aa917006 1539/* Called with rcu_read_lock or ovs_mutex. */
2a4999f3 1540static struct datapath *lookup_datapath(struct net *net,
f1f60b85 1541 const struct ovs_header *ovs_header,
6455100f 1542 struct nlattr *a[OVS_DP_ATTR_MAX + 1])
d6569377 1543{
254f2dc8
BP		 1544 struct datapath *dp;
1545
df2c07f4 1546 if (!a[OVS_DP_ATTR_NAME])
2a4999f3 1547 dp = get_dp(net, ovs_header->dp_ifindex);
254f2dc8 1548 else {
d6569377 1549 struct vport *vport;
d6569377 1550
2a4999f3 1551 vport = ovs_vport_locate(net, nla_data(a[OVS_DP_ATTR_NAME]));
df2c07f4 1552 dp = vport && vport->port_no == OVSP_LOCAL ? vport->dp : NULL;
d6569377 1553 }
254f2dc8 1554 return dp ? dp : ERR_PTR(-ENODEV);
d6569377
BP		 1555}
1556
94358dcf
TG		 1557static void ovs_dp_reset_user_features(struct sk_buff *skb, struct genl_info *info)
1558{
1559 struct datapath *dp;
1560
1561 dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
09350a3d 1562 if (IS_ERR(dp))
94358dcf
TG		 1563 return;
1564
1565 WARN(dp->user_features, "Dropping previously announced user features\n");
1566 dp->user_features = 0;
1567}
1568
f1f60b85 1569static void ovs_dp_change(struct datapath *dp, struct nlattr *a[])
c58cc9a4
TG		 1570{
1571 if (a[OVS_DP_ATTR_USER_FEATURES])
1572 dp->user_features = nla_get_u32(a[OVS_DP_ATTR_USER_FEATURES]);
1573}
1574
df2c07f4 1575static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)
d6569377 1576{
aaff4b55 1577 struct nlattr **a = info->attrs;
d6569377 1578 struct vport_parms parms;
aaff4b55 1579 struct sk_buff *reply;
d6569377
BP		 1580 struct datapath *dp;
1581 struct vport *vport;
2a4999f3 1582 struct ovs_net *ovs_net;
95b1d73a 1583 int err, i;
d6569377 1584
d6569377 1585 err = -EINVAL;
ea36840f 1586 if (!a[OVS_DP_ATTR_NAME] || !a[OVS_DP_ATTR_UPCALL_PID])
aaff4b55
BP		 1587 goto err;
1588
40c08cda 1589 reply = ovs_dp_cmd_alloc_info();
d81eef1b
JR		 1590 if (!reply)
1591 return -ENOMEM;
d6569377 1592
d6569377
BP		 1593 err = -ENOMEM;
1594 dp = kzalloc(sizeof(*dp), GFP_KERNEL);
1595 if (dp == NULL)
d81eef1b 1596 goto err_free_reply;
2a4999f3 1597
c0cddcec 1598 ovs_dp_set_net(dp, sock_net(skb->sk));
0ceaa66c 1599
d6569377 1600 /* Allocate table. */
994dc286
PS		 1601 err = ovs_flow_tbl_init(&dp->table);
1602 if (err)
d6569377
BP		 1603 goto err_free_dp;
1604
08fb1bbd 1605 dp->stats_percpu = netdev_alloc_pcpu_stats(struct dp_stats_percpu);
99769a40
JG		 1606 if (!dp->stats_percpu) {
1607 err = -ENOMEM;
1608 goto err_destroy_table;
1609 }
1610
8401bb6a
KC		 1611 dp->ports = kmalloc_array(DP_VPORT_HASH_BUCKETS,
1612 sizeof(struct hlist_head),
1613 GFP_KERNEL);
95b1d73a
PS		 1614 if (!dp->ports) {
1615 err = -ENOMEM;
1616 goto err_destroy_percpu;
1617 }
1618
1619 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++)
1620 INIT_HLIST_HEAD(&dp->ports[i]);
1621
1cb57039
AZ		 1622 err = ovs_meters_init(dp);
1623 if (err)
1624 goto err_destroy_ports_array;
1625
d6569377 1626 /* Set up our datapath device. */
df2c07f4
JP		 1627 parms.name = nla_data(a[OVS_DP_ATTR_NAME]);
1628 parms.type = OVS_VPORT_TYPE_INTERNAL;
d6569377
BP		 1629 parms.options = NULL;
1630 parms.dp = dp;
df2c07f4 1631 parms.port_no = OVSP_LOCAL;
beb1c69a 1632 parms.upcall_portids = a[OVS_DP_ATTR_UPCALL_PID];
b063d9f0 1633
c58cc9a4
TG		 1634 ovs_dp_change(dp, a);
1635
d81eef1b
JR		 1636 /* So far only local changes have been made, now need the lock. */
1637 ovs_lock();
1638
d6569377
BP		 1639 vport = new_vport(&parms);
1640 if (IS_ERR(vport)) {
1641 err = PTR_ERR(vport);
1642 if (err == -EBUSY)
1643 err = -EEXIST;
1644
94358dcf
TG		 1645 if (err == -EEXIST) {
1646 /* An outdated user space instance that does not understand
1647 * the concept of user_features has attempted to create a new
1648 * datapath and is likely to reuse it. Drop all user features.
1649 */
1650 if (info->genlhdr->version < OVS_DP_VER_FEATURES)
1651 ovs_dp_reset_user_features(skb, info);
1652 }
1653
1cb57039 1654 goto err_destroy_meters;
d6569377 1655 }
d6569377 1656
d81eef1b
JR		 1657 err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1658 info->snd_seq, 0, OVS_DP_CMD_NEW);
1659 BUG_ON(err < 0);
aaff4b55 1660
2a4999f3 1661 ovs_net = net_generic(ovs_dp_get_net(dp), ovs_net_id);
fb93e9aa 1662 list_add_tail_rcu(&dp->list_node, &ovs_net->dps);
a0fb56c1 1663
cd2a59e9 1664 ovs_unlock();
d6569377 1665
cb25142c 1666 ovs_notify(&dp_datapath_genl_family, &ovs_dp_datapath_multicast_group, reply, info);
d6569377
BP		 1667 return 0;
1668
1cb57039 1669err_destroy_meters:
d81eef1b 1670 ovs_unlock();
1cb57039
AZ		 1671 ovs_meters_exit(dp);
1672err_destroy_ports_array:
95b1d73a 1673 kfree(dp->ports);
99769a40
JG		 1674err_destroy_percpu:
1675 free_percpu(dp->stats_percpu);
d6569377 1676err_destroy_table:
e379e4d1 1677 ovs_flow_tbl_destroy(&dp->table);
d6569377 1678err_free_dp:
d6569377 1679 kfree(dp);
d81eef1b
JR		 1680err_free_reply:
1681 kfree_skb(reply);
d6569377 1682err:
064af421
BP		 1683 return err;
1684}
1685
cd2a59e9 1686/* Called with ovs_mutex. */
2a4999f3 1687static void __dp_destroy(struct datapath *dp)
44e05eca 1688{
95b1d73a 1689 int i;
44e05eca 1690
95b1d73a
PS		 1691 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
1692 struct vport *vport;
f8dfbcb7 1693 struct hlist_node *n;
95b1d73a 1694
f8dfbcb7 1695 hlist_for_each_entry_safe(vport, n, &dp->ports[i], dp_hash_node)
95b1d73a
PS		 1696 if (vport->port_no != OVSP_LOCAL)
1697 ovs_dp_detach_port(vport);
1698 }
ed099e92 1699
fb93e9aa 1700 list_del_rcu(&dp->list_node);
ed099e92 1701
cd2a59e9 1702 /* OVSP_LOCAL is datapath internal port. We need to make sure that
d103f479
AZ		 1703 * all ports in datapath are destroyed first before freeing datapath.
1704 */
cd2a59e9 1705 ovs_dp_detach_port(ovs_vport_ovsl(dp, OVSP_LOCAL));
99620d2c 1706
d103f479 1707 /* RCU destroy the flow table */
ed099e92 1708 call_rcu(&dp->rcu, destroy_dp_rcu);
2a4999f3
PS		 1709}
1710
1711static int ovs_dp_cmd_del(struct sk_buff *skb, struct genl_info *info)
1712{
1713 struct sk_buff *reply;
1714 struct datapath *dp;
1715 int err;
1716
40c08cda 1717 reply = ovs_dp_cmd_alloc_info();
d81eef1b
JR		 1718 if (!reply)
1719 return -ENOMEM;
1720
cd2a59e9 1721 ovs_lock();
2a4999f3
PS		 1722 dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1723 err = PTR_ERR(dp);
1724 if (IS_ERR(dp))
d81eef1b 1725 goto err_unlock_free;
2a4999f3 1726
d81eef1b
JR		 1727 err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1728 info->snd_seq, 0, OVS_DP_CMD_DEL);
1729 BUG_ON(err < 0);
2a4999f3
PS		 1730
1731 __dp_destroy(dp);
d81eef1b 1732 ovs_unlock();
7d16c847 1733
cb25142c 1734 ovs_notify(&dp_datapath_genl_family, &ovs_dp_datapath_multicast_group, reply, info);
99620d2c 1735 return 0;
d81eef1b
JR		 1736
1737err_unlock_free:
cd2a59e9 1738 ovs_unlock();
d81eef1b 1739 kfree_skb(reply);
cd2a59e9 1740 return err;
44e05eca
BP		 1741}
1742
df2c07f4 1743static int ovs_dp_cmd_set(struct sk_buff *skb, struct genl_info *info)
064af421 1744{
aaff4b55 1745 struct sk_buff *reply;
d6569377 1746 struct datapath *dp;
d6569377 1747 int err;
064af421 1748
40c08cda 1749 reply = ovs_dp_cmd_alloc_info();
d81eef1b
JR		 1750 if (!reply)
1751 return -ENOMEM;
1752
cd2a59e9 1753 ovs_lock();
2a4999f3 1754 dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
cd2a59e9 1755 err = PTR_ERR(dp);
d6569377 1756 if (IS_ERR(dp))
d81eef1b 1757 goto err_unlock_free;
38c6ecbc 1758
c58cc9a4
TG		 1759 ovs_dp_change(dp, info->attrs);
1760
d81eef1b 1761 err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
229486c2 1762 info->snd_seq, 0, OVS_DP_CMD_GET);
d81eef1b 1763 BUG_ON(err < 0);
a0fb56c1 1764
cd2a59e9 1765 ovs_unlock();
7d16c847 1766
cb25142c 1767 ovs_notify(&dp_datapath_genl_family, &ovs_dp_datapath_multicast_group, reply, info);
aaff4b55 1768 return 0;
d81eef1b
JR		 1769
1770err_unlock_free:
cd2a59e9 1771 ovs_unlock();
d81eef1b 1772 kfree_skb(reply);
cd2a59e9 1773 return err;
064af421
BP		 1774}
1775
df2c07f4 1776static int ovs_dp_cmd_get(struct sk_buff *skb, struct genl_info *info)
1dcf111b 1777{
aaff4b55 1778 struct sk_buff *reply;
d6569377 1779 struct datapath *dp;
d6569377 1780 int err;
1dcf111b 1781
40c08cda 1782 reply = ovs_dp_cmd_alloc_info();
d81eef1b
JR		 1783 if (!reply)
1784 return -ENOMEM;
1785
d637497c 1786 ovs_lock();
2a4999f3 1787 dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
cd2a59e9
PS		 1788 if (IS_ERR(dp)) {
1789 err = PTR_ERR(dp);
d81eef1b 1790 goto err_unlock_free;
cd2a59e9 1791 }
d81eef1b 1792 err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
229486c2 1793 info->snd_seq, 0, OVS_DP_CMD_GET);
d81eef1b 1794 BUG_ON(err < 0);
d637497c 1795 ovs_unlock();
aaff4b55
BP		 1796
1797 return genlmsg_reply(reply, info);
cd2a59e9 1798
d81eef1b 1799err_unlock_free:
d637497c 1800 ovs_unlock();
d81eef1b 1801 kfree_skb(reply);
cd2a59e9 1802 return err;
1dcf111b
JP		 1803}
1804
df2c07f4 1805static int ovs_dp_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
a7786963 1806{
2a4999f3 1807 struct ovs_net *ovs_net = net_generic(sock_net(skb->sk), ovs_net_id);
254f2dc8
BP		 1808 struct datapath *dp;
1809 int skip = cb->args[0];
1810 int i = 0;
a7786963 1811
d637497c
PS		 1812 ovs_lock();
1813 list_for_each_entry(dp, &ovs_net->dps, list_node) {
a2bab2f0 1814 if (i >= skip &&
28aea917 1815 ovs_dp_cmd_fill_info(dp, skb, NETLINK_CB(cb->skb).portid,
aaff4b55 1816 cb->nlh->nlmsg_seq, NLM_F_MULTI,
229486c2 1817 OVS_DP_CMD_GET) < 0)
aaff4b55 1818 break;
254f2dc8 1819 i++;
a7786963 1820 }
d637497c 1821 ovs_unlock();
aaff4b55 1822
254f2dc8
BP		 1823 cb->args[0] = i;
1824
aaff4b55 1825 return skb->len;
c19e6535
BP		 1826}
1827
cb25142c
PS		 1828static const struct nla_policy datapath_policy[OVS_DP_ATTR_MAX + 1] = {
1829 [OVS_DP_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
1830 [OVS_DP_ATTR_UPCALL_PID] = { .type = NLA_U32 },
1831 [OVS_DP_ATTR_USER_FEATURES] = { .type = NLA_U32 },
1832};
1833
18fd3a52 1834static struct genl_ops dp_datapath_genl_ops[] = {
df2c07f4 1835 { .cmd = OVS_DP_CMD_NEW,
a6a8674d 1836 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
aaff4b55 1837 .policy = datapath_policy,
df2c07f4 1838 .doit = ovs_dp_cmd_new
aaff4b55 1839 },
df2c07f4 1840 { .cmd = OVS_DP_CMD_DEL,
a6a8674d 1841 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
aaff4b55 1842 .policy = datapath_policy,
df2c07f4 1843 .doit = ovs_dp_cmd_del
aaff4b55 1844 },
df2c07f4 1845 { .cmd = OVS_DP_CMD_GET,
aaff4b55
BP		 1846 .flags = 0, /* OK for unprivileged users. */
1847 .policy = datapath_policy,
df2c07f4
JP		 1848 .doit = ovs_dp_cmd_get,
1849 .dumpit = ovs_dp_cmd_dump
aaff4b55 1850 },
df2c07f4 1851 { .cmd = OVS_DP_CMD_SET,
a6a8674d 1852 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
aaff4b55 1853 .policy = datapath_policy,
df2c07f4 1854 .doit = ovs_dp_cmd_set,
aaff4b55
BP		 1855 },
1856};
1857
ba63fe26 1858static struct genl_family dp_datapath_genl_family __ro_after_init = {
df2c07f4 1859 .hdrsize = sizeof(struct ovs_header),
cb25142c
PS		 1860 .name = OVS_DATAPATH_FAMILY,
1861 .version = OVS_DATAPATH_VERSION,
1862 .maxattr = OVS_DP_ATTR_MAX,
b3dcb73c 1863 .netnsok = true,
cb25142c
PS		 1864 .parallel_ops = true,
1865 .ops = dp_datapath_genl_ops,
1866 .n_ops = ARRAY_SIZE(dp_datapath_genl_ops),
1867 .mcgrps = &ovs_dp_datapath_multicast_group,
1868 .n_mcgrps = 1,
ba63fe26 1869 .module = THIS_MODULE,
f0fef760
BP		 1870};
1871
cd2a59e9 1872/* Called with ovs_mutex or RCU read lock. */
df2c07f4 1873static int ovs_vport_cmd_fill_info(struct vport *vport, struct sk_buff *skb,
b147f2e9
JB		 1874 struct net *net, u32 portid, u32 seq,
1875 u32 flags, u8 cmd)
064af421 1876{
df2c07f4 1877 struct ovs_header *ovs_header;
e926dfe3 1878 struct ovs_vport_stats vport_stats;
c19e6535
BP		 1879 int err;
1880
28aea917 1881 ovs_header = genlmsg_put(skb, portid, seq, &dp_vport_genl_family,
f0fef760 1882 flags, cmd);
df2c07f4 1883 if (!ovs_header)
f0fef760 1884 return -EMSGSIZE;
c19e6535 1885
99769a40 1886 ovs_header->dp_ifindex = get_dpifindex(vport->dp);
c19e6535 1887
c3cc8c03
DM		 1888 if (nla_put_u32(skb, OVS_VPORT_ATTR_PORT_NO, vport->port_no) ||
1889 nla_put_u32(skb, OVS_VPORT_ATTR_TYPE, vport->ops->type) ||
e23775f2 1890 nla_put_string(skb, OVS_VPORT_ATTR_NAME,
b147f2e9
JB		 1891 ovs_vport_name(vport)) ||
1892 nla_put_u32(skb, OVS_VPORT_ATTR_IFINDEX, vport->dev->ifindex))
c3cc8c03 1893 goto nla_put_failure;
c19e6535 1894
b147f2e9
JB		 1895#ifdef HAVE_PEERNET2ID_ALLOC
1896 if (!net_eq(net, dev_net(vport->dev))) {
1897 int id = peernet2id_alloc(net, dev_net(vport->dev));
1898
1899 if (nla_put_s32(skb, OVS_VPORT_ATTR_NETNSID, id))
1900 goto nla_put_failure;
1901 }
1902
1903#endif
850b6b3b 1904 ovs_vport_get_stats(vport, &vport_stats);
91b37647
PS		 1905 if (nla_put_64bit(skb, OVS_VPORT_ATTR_STATS,
1906 sizeof(struct ovs_vport_stats), &vport_stats,
1907 OVS_VPORT_ATTR_PAD))
c3cc8c03 1908 goto nla_put_failure;
c19e6535 1909
beb1c69a
AW		 1910 if (ovs_vport_get_upcall_portids(vport, skb))
1911 goto nla_put_failure;
1912
850b6b3b 1913 err = ovs_vport_get_options(vport, skb);
f0fef760
BP		 1914 if (err == -EMSGSIZE)
1915 goto error;
c19e6535 1916
23b48dc1
TG		 1917 genlmsg_end(skb, ovs_header);
1918 return 0;
c19e6535
BP		 1919
1920nla_put_failure:
1921 err = -EMSGSIZE;
f0fef760 1922error:
df2c07f4 1923 genlmsg_cancel(skb, ovs_header);
f0fef760 1924 return err;
064af421
BP		 1925}
1926
d81eef1b
JR		 1927static struct sk_buff *ovs_vport_cmd_alloc_info(void)
1928{
1929 return nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1930}
1931
1932/* Called with ovs_mutex, only via ovs_dp_notify_wq(). */
b147f2e9
JB		 1933struct sk_buff *ovs_vport_cmd_build_info(struct vport *vport, struct net *net,
1934 u32 portid, u32 seq, u8 cmd)
064af421 1935{
c19e6535 1936 struct sk_buff *skb;
f0fef760 1937 int retval;
c19e6535 1938
f0fef760 1939 skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_ATOMIC);
c19e6535
BP		 1940 if (!skb)
1941 return ERR_PTR(-ENOMEM);
1942
b147f2e9 1943 retval = ovs_vport_cmd_fill_info(vport, skb, net, portid, seq, 0, cmd);
c25ea534
JG		 1944 BUG_ON(retval < 0);
1945
c19e6535 1946 return skb;
f0fef760 1947}
c19e6535 1948
cd2a59e9 1949/* Called with ovs_mutex or RCU read lock. */
2a4999f3 1950static struct vport *lookup_vport(struct net *net,
f1f60b85 1951 const struct ovs_header *ovs_header,
df2c07f4 1952 struct nlattr *a[OVS_VPORT_ATTR_MAX + 1])
c19e6535
BP		 1953{
1954 struct datapath *dp;
1955 struct vport *vport;
1956
b147f2e9
JB		 1957 if (a[OVS_VPORT_ATTR_IFINDEX])
1958 return ERR_PTR(-EOPNOTSUPP);
df2c07f4 1959 if (a[OVS_VPORT_ATTR_NAME]) {
2a4999f3 1960 vport = ovs_vport_locate(net, nla_data(a[OVS_VPORT_ATTR_NAME]));
ed099e92 1961 if (!vport)
c19e6535 1962 return ERR_PTR(-ENODEV);
24ce832d
BP		 1963 if (ovs_header->dp_ifindex &&
1964 ovs_header->dp_ifindex != get_dpifindex(vport->dp))
1965 return ERR_PTR(-ENODEV);
c19e6535 1966 return vport;
df2c07f4
JP		 1967 } else if (a[OVS_VPORT_ATTR_PORT_NO]) {
1968 u32 port_no = nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]);
c19e6535
BP		 1969
1970 if (port_no >= DP_MAX_PORTS)
f0fef760 1971 return ERR_PTR(-EFBIG);
c19e6535 1972
2a4999f3 1973 dp = get_dp(net, ovs_header->dp_ifindex);
c19e6535
BP		 1974 if (!dp)
1975 return ERR_PTR(-ENODEV);
f2459fe7 1976
cd2a59e9 1977 vport = ovs_vport_ovsl_rcu(dp, port_no);
ed099e92 1978 if (!vport)
17535c57 1979 return ERR_PTR(-ENODEV);
c19e6535
BP		 1980 return vport;
1981 } else
1982 return ERR_PTR(-EINVAL);
b147f2e9 1983
064af421
BP		 1984}
1985
8ce37339
PS		 1986/* Called with ovs_mutex */
1987static void update_headroom(struct datapath *dp)
1988{
1989 unsigned dev_headroom, max_headroom = 0;
1990 struct net_device *dev;
1991 struct vport *vport;
1992 int i;
1993
1994 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
1995 hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node) {
1996 dev = vport->dev;
1997 dev_headroom = netdev_get_fwd_headroom(dev);
1998 if (dev_headroom > max_headroom)
1999 max_headroom = dev_headroom;
2000 }
2001 }
2002
2003 dp->max_headroom = max_headroom;
2004 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++)
2005 hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node)
2006 netdev_set_rx_headroom(vport->dev, max_headroom);
2007}
2008
df2c07f4 2009static int ovs_vport_cmd_new(struct sk_buff *skb, struct genl_info *info)
c19e6535 2010{
f0fef760 2011 struct nlattr **a = info->attrs;
df2c07f4 2012 struct ovs_header *ovs_header = info->userhdr;
c19e6535 2013 struct vport_parms parms;
ed099e92 2014 struct sk_buff *reply;
c19e6535 2015 struct vport *vport;
c19e6535 2016 struct datapath *dp;
b0ec0f27 2017 u32 port_no;
c19e6535 2018 int err;
b0ec0f27 2019
ea36840f
BP		 2020 if (!a[OVS_VPORT_ATTR_NAME] || !a[OVS_VPORT_ATTR_TYPE] ||
2021 !a[OVS_VPORT_ATTR_UPCALL_PID])
d81eef1b 2022 return -EINVAL;
b147f2e9
JB		 2023 if (a[OVS_VPORT_ATTR_IFINDEX])
2024 return -EOPNOTSUPP;
d81eef1b
JR		 2025
2026 port_no = a[OVS_VPORT_ATTR_PORT_NO]
2027 ? nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]) : 0;
2028 if (port_no >= DP_MAX_PORTS)
2029 return -EFBIG;
2030
2031 reply = ovs_vport_cmd_alloc_info();
2032 if (!reply)
2033 return -ENOMEM;
f0fef760 2034
cd2a59e9 2035 ovs_lock();
5a38795f 2036restart:
2a4999f3 2037 dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
c19e6535
BP		 2038 err = -ENODEV;
2039 if (!dp)
d81eef1b 2040 goto exit_unlock_free;
c19e6535 2041
d81eef1b 2042 if (port_no) {
cd2a59e9 2043 vport = ovs_vport_ovsl(dp, port_no);
c19e6535
BP		 2044 err = -EBUSY;
2045 if (vport)
d81eef1b 2046 goto exit_unlock_free;
c19e6535
BP		 2047 } else {
2048 for (port_no = 1; ; port_no++) {
2049 if (port_no >= DP_MAX_PORTS) {
2050 err = -EFBIG;
d81eef1b 2051 goto exit_unlock_free;
c19e6535 2052 }
cd2a59e9 2053 vport = ovs_vport_ovsl(dp, port_no);
c19e6535
BP		 2054 if (!vport)
2055 break;
51d4d598 2056 }
064af421 2057 }
b0ec0f27 2058
df2c07f4
JP		 2059 parms.name = nla_data(a[OVS_VPORT_ATTR_NAME]);
2060 parms.type = nla_get_u32(a[OVS_VPORT_ATTR_TYPE]);
2061 parms.options = a[OVS_VPORT_ATTR_OPTIONS];
c19e6535
BP		 2062 parms.dp = dp;
2063 parms.port_no = port_no;
beb1c69a 2064 parms.upcall_portids = a[OVS_VPORT_ATTR_UPCALL_PID];
c19e6535
BP		 2065
2066 vport = new_vport(&parms);
2067 err = PTR_ERR(vport);
5a38795f
TG		 2068 if (IS_ERR(vport)) {
2069 if (err == -EAGAIN)
2070 goto restart;
d81eef1b 2071 goto exit_unlock_free;
5a38795f 2072 }
c19e6535 2073
b147f2e9
JB		 2074 err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2075 info->snd_portid, info->snd_seq, 0,
2076 OVS_VPORT_CMD_NEW);
d81eef1b 2077 BUG_ON(err < 0);
8ce37339
PS		 2078
2079 if (netdev_get_fwd_headroom(vport->dev) > dp->max_headroom)
2080 update_headroom(dp);
2081 else
2082 netdev_set_rx_headroom(vport->dev, dp->max_headroom);
2083
d81eef1b 2084 ovs_unlock();
e297c6b7 2085
cb25142c 2086 ovs_notify(&dp_vport_genl_family, &ovs_dp_vport_multicast_group, reply, info);
d81eef1b 2087 return 0;
c19e6535 2088
d81eef1b 2089exit_unlock_free:
cd2a59e9 2090 ovs_unlock();
d81eef1b 2091 kfree_skb(reply);
c19e6535 2092 return err;
44e05eca
BP		 2093}
2094
df2c07f4 2095static int ovs_vport_cmd_set(struct sk_buff *skb, struct genl_info *info)
44e05eca 2096{
f0fef760
BP		 2097 struct nlattr **a = info->attrs;
2098 struct sk_buff *reply;
c19e6535 2099 struct vport *vport;
c19e6535 2100 int err;
44e05eca 2101
d81eef1b
JR		 2102 reply = ovs_vport_cmd_alloc_info();
2103 if (!reply)
2104 return -ENOMEM;
2105
cd2a59e9 2106 ovs_lock();
2a4999f3 2107 vport = lookup_vport(sock_net(skb->sk), info->userhdr, a);
c19e6535
BP		 2108 err = PTR_ERR(vport);
2109 if (IS_ERR(vport))
d81eef1b 2110 goto exit_unlock_free;
44e05eca 2111
6455100f 2112 if (a[OVS_VPORT_ATTR_TYPE] &&
17ec1d04 2113 nla_get_u32(a[OVS_VPORT_ATTR_TYPE]) != vport->ops->type) {
4879d4c7 2114 err = -EINVAL;
d81eef1b 2115 goto exit_unlock_free;
c25ea534
JG		 2116 }
2117
17ec1d04 2118 if (a[OVS_VPORT_ATTR_OPTIONS]) {
850b6b3b 2119 err = ovs_vport_set_options(vport, a[OVS_VPORT_ATTR_OPTIONS]);
17ec1d04 2120 if (err)
d81eef1b 2121 goto exit_unlock_free;
17ec1d04 2122 }
1fc7083d 2123
beb1c69a 2124 if (a[OVS_VPORT_ATTR_UPCALL_PID]) {
7d16c847
PS		 2125 struct nlattr *ids = a[OVS_VPORT_ATTR_UPCALL_PID];
2126
2127 err = ovs_vport_set_upcall_portids(vport, ids);
beb1c69a
AW		 2128 if (err)
2129 goto exit_unlock_free;
2130 }
c19e6535 2131
b147f2e9
JB		 2132 err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2133 info->snd_portid, info->snd_seq, 0,
229486c2 2134 OVS_VPORT_CMD_SET);
c25ea534 2135 BUG_ON(err < 0);
cd2a59e9 2136 ovs_unlock();
d81eef1b 2137
cb25142c 2138 ovs_notify(&dp_vport_genl_family, &ovs_dp_vport_multicast_group, reply, info);
c25ea534
JG		 2139 return 0;
2140
d81eef1b 2141exit_unlock_free:
cd2a59e9 2142 ovs_unlock();
d81eef1b 2143 kfree_skb(reply);
c19e6535 2144 return err;
064af421
BP		 2145}
2146
df2c07f4 2147static int ovs_vport_cmd_del(struct sk_buff *skb, struct genl_info *info)
7c40efc9 2148{
8ce37339 2149 bool must_update_headroom = false;
f0fef760
BP		 2150 struct nlattr **a = info->attrs;
2151 struct sk_buff *reply;
8ce37339 2152 struct datapath *dp;
c19e6535 2153 struct vport *vport;
c19e6535
BP		 2154 int err;
2155
d81eef1b
JR		 2156 reply = ovs_vport_cmd_alloc_info();
2157 if (!reply)
2158 return -ENOMEM;
2159
cd2a59e9 2160 ovs_lock();
2a4999f3 2161 vport = lookup_vport(sock_net(skb->sk), info->userhdr, a);
c19e6535 2162 err = PTR_ERR(vport);
f0fef760 2163 if (IS_ERR(vport))
d81eef1b 2164 goto exit_unlock_free;
c19e6535 2165
df2c07f4 2166 if (vport->port_no == OVSP_LOCAL) {
f0fef760 2167 err = -EINVAL;
d81eef1b 2168 goto exit_unlock_free;
f0fef760
BP		 2169 }
2170
b147f2e9
JB		 2171 err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2172 info->snd_portid, info->snd_seq, 0,
2173 OVS_VPORT_CMD_DEL);
d81eef1b 2174 BUG_ON(err < 0);
8ce37339
PS		 2175
2176 /* the vport deletion may trigger dp headroom update */
2177 dp = vport->dp;
2178 if (netdev_get_fwd_headroom(vport->dev) == dp->max_headroom)
2179 must_update_headroom = true;
2180 netdev_reset_rx_headroom(vport->dev);
850b6b3b 2181 ovs_dp_detach_port(vport);
8ce37339
PS		 2182
2183 if (must_update_headroom)
2184 update_headroom(dp);
2185
d81eef1b 2186 ovs_unlock();
f0fef760 2187
cb25142c 2188 ovs_notify(&dp_vport_genl_family, &ovs_dp_vport_multicast_group, reply, info);
d81eef1b 2189 return 0;
f0fef760 2190
d81eef1b 2191exit_unlock_free:
cd2a59e9 2192 ovs_unlock();
d81eef1b 2193 kfree_skb(reply);
c19e6535 2194 return err;
7c40efc9
BP		 2195}
2196
df2c07f4 2197static int ovs_vport_cmd_get(struct sk_buff *skb, struct genl_info *info)
7c40efc9 2198{
f0fef760 2199 struct nlattr **a = info->attrs;
df2c07f4 2200 struct ovs_header *ovs_header = info->userhdr;
ed099e92 2201 struct sk_buff *reply;
c19e6535 2202 struct vport *vport;
c19e6535
BP		 2203 int err;
2204
d81eef1b
JR		 2205 reply = ovs_vport_cmd_alloc_info();
2206 if (!reply)
2207 return -ENOMEM;
2208
ed099e92 2209 rcu_read_lock();
2a4999f3 2210 vport = lookup_vport(sock_net(skb->sk), ovs_header, a);
c19e6535
BP		 2211 err = PTR_ERR(vport);
2212 if (IS_ERR(vport))
d81eef1b 2213 goto exit_unlock_free;
b147f2e9
JB		 2214 err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2215 info->snd_portid, info->snd_seq, 0,
229486c2 2216 OVS_VPORT_CMD_GET);
d81eef1b 2217 BUG_ON(err < 0);
df2fa9b5
JG		 2218 rcu_read_unlock();
2219
2220 return genlmsg_reply(reply, info);
ed099e92 2221
d81eef1b 2222exit_unlock_free:
ed099e92 2223 rcu_read_unlock();
d81eef1b 2224 kfree_skb(reply);
c19e6535
BP		 2225 return err;
2226}
2227
df2c07f4 2228static int ovs_vport_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
c19e6535 2229{
df2c07f4 2230 struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
c19e6535 2231 struct datapath *dp;
95b1d73a
PS		 2232 int bucket = cb->args[0], skip = cb->args[1];
2233 int i, j = 0;
c19e6535 2234
03fc2881 2235 rcu_read_lock();
01ac0970 2236 dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
03fc2881
JR		 2237 if (!dp) {
2238 rcu_read_unlock();
f0fef760 2239 return -ENODEV;
03fc2881 2240 }
95b1d73a 2241 for (i = bucket; i < DP_VPORT_HASH_BUCKETS; i++) {
ed099e92 2242 struct vport *vport;
95b1d73a
PS		 2243
2244 j = 0;
f8dfbcb7 2245 hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node) {
95b1d73a
PS		 2246 if (j >= skip &&
2247 ovs_vport_cmd_fill_info(vport, skb,
b147f2e9 2248 sock_net(skb->sk),
28aea917 2249 NETLINK_CB(cb->skb).portid,
95b1d73a
PS		 2250 cb->nlh->nlmsg_seq,
2251 NLM_F_MULTI,
229486c2 2252 OVS_VPORT_CMD_GET) < 0)
95b1d73a
PS		 2253 goto out;
2254
2255 j++;
2256 }
2257 skip = 0;
c19e6535 2258 }
95b1d73a 2259out:
ed099e92 2260 rcu_read_unlock();
c19e6535 2261
95b1d73a
PS		 2262 cb->args[0] = i;
2263 cb->args[1] = j;
f0fef760 2264
95b1d73a 2265 return skb->len;
7c40efc9
BP		 2266}
2267
cb25142c
PS		 2268static const struct nla_policy vport_policy[OVS_VPORT_ATTR_MAX + 1] = {
2269 [OVS_VPORT_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
2270 [OVS_VPORT_ATTR_STATS] = { .len = sizeof(struct ovs_vport_stats) },
2271 [OVS_VPORT_ATTR_PORT_NO] = { .type = NLA_U32 },
2272 [OVS_VPORT_ATTR_TYPE] = { .type = NLA_U32 },
8fc8986d 2273 [OVS_VPORT_ATTR_UPCALL_PID] = { .type = NLA_UNSPEC },
cb25142c 2274 [OVS_VPORT_ATTR_OPTIONS] = { .type = NLA_NESTED },
b147f2e9
JB		 2275 [OVS_VPORT_ATTR_IFINDEX] = { .type = NLA_U32 },
2276 [OVS_VPORT_ATTR_NETNSID] = { .type = NLA_S32 },
cb25142c
PS		 2277};
2278
18fd3a52 2279static struct genl_ops dp_vport_genl_ops[] = {
df2c07f4 2280 { .cmd = OVS_VPORT_CMD_NEW,
a6a8674d 2281 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
f0fef760 2282 .policy = vport_policy,
df2c07f4 2283 .doit = ovs_vport_cmd_new
f0fef760 2284 },
df2c07f4 2285 { .cmd = OVS_VPORT_CMD_DEL,
a6a8674d 2286 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
f0fef760 2287 .policy = vport_policy,
df2c07f4 2288 .doit = ovs_vport_cmd_del
f0fef760 2289 },
df2c07f4 2290 { .cmd = OVS_VPORT_CMD_GET,
f0fef760
BP		 2291 .flags = 0, /* OK for unprivileged users. */
2292 .policy = vport_policy,
df2c07f4
JP		 2293 .doit = ovs_vport_cmd_get,
2294 .dumpit = ovs_vport_cmd_dump
f0fef760 2295 },
df2c07f4 2296 { .cmd = OVS_VPORT_CMD_SET,
a6a8674d 2297 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
f0fef760 2298 .policy = vport_policy,
df2c07f4 2299 .doit = ovs_vport_cmd_set,
f0fef760
BP		 2300 },
2301};
2302
ba63fe26 2303struct genl_family dp_vport_genl_family __ro_after_init = {
cb25142c
PS		 2304 .hdrsize = sizeof(struct ovs_header),
2305 .name = OVS_VPORT_FAMILY,
2306 .version = OVS_VPORT_VERSION,
2307 .maxattr = OVS_VPORT_ATTR_MAX,
2308 .netnsok = true,
2309 .parallel_ops = true,
2310 .ops = dp_vport_genl_ops,
2311 .n_ops = ARRAY_SIZE(dp_vport_genl_ops),
2312 .mcgrps = &ovs_dp_vport_multicast_group,
2313 .n_mcgrps = 1,
ba63fe26 2314 .module = THIS_MODULE,
982b8810 2315};
ed099e92 2316
18fd3a52 2317static struct genl_family *dp_genl_families[] = {
cb25142c
PS		 2318 &dp_datapath_genl_family,
2319 &dp_vport_genl_family,
2320 &dp_flow_genl_family,
2321 &dp_packet_genl_family,
1cb57039 2322 &dp_meter_genl_family,
cb2a5486
YHW		 2323#if IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT)
2324 &dp_ct_limit_genl_family,
2325#endif
982b8810 2326};
ed099e92 2327
982b8810
BP		 2328static void dp_unregister_genl(int n_families)
2329{
2330 int i;
ed099e92 2331
b867ca75 2332 for (i = 0; i < n_families; i++)
cb25142c 2333 genl_unregister_family(dp_genl_families[i]);
ed099e92
BP		 2334}
2335
ba63fe26 2336static int __init dp_register_genl(void)
064af421 2337{
982b8810
BP		 2338 int err;
2339 int i;
064af421 2340
982b8810 2341 for (i = 0; i < ARRAY_SIZE(dp_genl_families); i++) {
064af421 2342
cb25142c 2343 err = genl_register_family(dp_genl_families[i]);
982b8810
BP		 2344 if (err)
2345 goto error;
982b8810 2346 }
9cc8b4e4 2347
982b8810 2348 return 0;
064af421
BP		 2349
2350error:
cb25142c 2351 dp_unregister_genl(i);
982b8810 2352 return err;
064af421
BP		 2353}
2354
2a4999f3
PS		 2355static int __net_init ovs_init_net(struct net *net)
2356{
2357 struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
2358
2359 INIT_LIST_HEAD(&ovs_net->dps);
cd2a59e9 2360 INIT_WORK(&ovs_net->dp_notify_work, ovs_dp_notify_wq);
7f4a5d68 2361 ovs_netns_frags_init(net);
2362 ovs_netns_frags6_init(net);
cb2a5486 2363 return ovs_ct_init(net);
2a4999f3
PS		 2364}
2365
cabd5516
PS		 2366static void __net_exit list_vports_from_net(struct net *net, struct net *dnet,
2367 struct list_head *head)
2a4999f3
PS		 2368{
2369 struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
cabd5516
PS		 2370 struct datapath *dp;
2371
2372 list_for_each_entry(dp, &ovs_net->dps, list_node) {
2373 int i;
2374
2375 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
2376 struct vport *vport;
2377
2378 hlist_for_each_entry(vport, &dp->ports[i], dp_hash_node) {
cabd5516
PS		 2379
2380 if (vport->ops->type != OVS_VPORT_TYPE_INTERNAL)
2381 continue;
2382
e23775f2 2383 if (dev_net(vport->dev) == dnet)
cabd5516
PS		 2384 list_add(&vport->detach_list, head);
2385 }
2386 }
2387 }
2388}
2389
2390static void __net_exit ovs_exit_net(struct net *dnet)
2391{
2392 struct datapath *dp, *dp_next;
2393 struct ovs_net *ovs_net = net_generic(dnet, ovs_net_id);
2394 struct vport *vport, *vport_next;
2395 struct net *net;
2396 LIST_HEAD(head);
2a4999f3 2397
7f4a5d68 2398 ovs_netns_frags6_exit(dnet);
2399 ovs_netns_frags_exit(dnet);
038e34ab 2400 ovs_ct_exit(dnet);
cd2a59e9
PS		 2401 ovs_lock();
2402 list_for_each_entry_safe(dp, dp_next, &ovs_net->dps, list_node)
2403 __dp_destroy(dp);
cabd5516 2404
3d10a0c8
YHW		 2405#ifdef HAVE_NET_RWSEM
2406 down_read(&net_rwsem);
2407#else
cabd5516 2408 rtnl_lock();
3d10a0c8 2409#endif
cabd5516
PS		 2410 for_each_net(net)
2411 list_vports_from_net(net, dnet, &head);
3d10a0c8
YHW		 2412#ifdef HAVE_NET_RWSEM
2413 up_read(&net_rwsem);
2414#else
cabd5516 2415 rtnl_unlock();
3d10a0c8 2416#endif
cabd5516
PS		 2417
2418 /* Detach all vports from given namespace. */
2419 list_for_each_entry_safe(vport, vport_next, &head, detach_list) {
2420 list_del(&vport->detach_list);
2421 ovs_dp_detach_port(vport);
2422 }
2423
cd2a59e9
PS		 2424 ovs_unlock();
2425
2426 cancel_work_sync(&ovs_net->dp_notify_work);
2a4999f3
PS		 2427}
2428
2429static struct pernet_operations ovs_net_ops = {
2430 .init = ovs_init_net,
2431 .exit = ovs_exit_net,
2432 .id = &ovs_net_id,
2433 .size = sizeof(struct ovs_net),
2434};
2435
22d24ebf
BP		 2436static int __init dp_init(void)
2437{
2438 int err;
2439
f3d85db3 2440 BUILD_BUG_ON(sizeof(struct ovs_skb_cb) > FIELD_SIZEOF(struct sk_buff, cb));
22d24ebf 2441
26bfaeaa 2442 pr_info("Open vSwitch switching datapath %s\n", VERSION);
064af421 2443
907c26a8 2444 ovs_nsh_init();
595e069a
JS		 2445 err = action_fifos_init();
2446 if (err)
7f4a5d68 2447 goto error;
595e069a 2448
5282e284 2449 err = ovs_internal_dev_rtnl_link_register();
2c8c4fb7
AZ		 2450 if (err)
2451 goto error_action_fifos_exit;
2452
5282e284
TG		 2453 err = ovs_flow_init();
2454 if (err)
2455 goto error_unreg_rtnl_link;
2456
850b6b3b 2457 err = ovs_vport_init();
064af421
BP		 2458 if (err)
2459 goto error_flow_exit;
2460
2a4999f3 2461 err = register_pernet_device(&ovs_net_ops);
f2459fe7
JG		 2462 if (err)
2463 goto error_vport_exit;
2464
7f4a5d68 2465 err = compat_init();
2a4999f3
PS		 2466 if (err)
2467 goto error_netns_exit;
2468
7f4a5d68 2469 err = register_netdevice_notifier(&ovs_dp_device_notifier);
2470 if (err)
2471 goto error_compat_exit;
2472
5a38795f
TG		 2473 err = ovs_netdev_init();
2474 if (err)
2475 goto error_unreg_notifier;
2476
982b8810
BP		 2477 err = dp_register_genl();
2478 if (err < 0)
5a38795f 2479 goto error_unreg_netdev;
982b8810 2480
064af421
BP		 2481 return 0;
2482
5a38795f
TG		 2483error_unreg_netdev:
2484 ovs_netdev_exit();
064af421 2485error_unreg_notifier:
850b6b3b 2486 unregister_netdevice_notifier(&ovs_dp_device_notifier);
7f4a5d68 2487error_compat_exit:
2488 compat_exit();
2a4999f3
PS		 2489error_netns_exit:
2490 unregister_pernet_device(&ovs_net_ops);
f2459fe7 2491error_vport_exit:
850b6b3b 2492 ovs_vport_exit();
064af421 2493error_flow_exit:
850b6b3b 2494 ovs_flow_exit();
5282e284
TG		 2495error_unreg_rtnl_link:
2496 ovs_internal_dev_rtnl_link_unregister();
2c8c4fb7
AZ		 2497error_action_fifos_exit:
2498 action_fifos_exit();
064af421 2499error:
907c26a8 2500 ovs_nsh_cleanup();
064af421
BP		 2501 return err;
2502}
2503
2504static void dp_cleanup(void)
2505{
982b8810 2506 dp_unregister_genl(ARRAY_SIZE(dp_genl_families));
5a38795f 2507 ovs_netdev_exit();
850b6b3b 2508 unregister_netdevice_notifier(&ovs_dp_device_notifier);
7f4a5d68 2509 compat_exit();
2a4999f3
PS		 2510 unregister_pernet_device(&ovs_net_ops);
2511 rcu_barrier();
850b6b3b
JG		 2512 ovs_vport_exit();
2513 ovs_flow_exit();
5282e284 2514 ovs_internal_dev_rtnl_link_unregister();
2c8c4fb7 2515 action_fifos_exit();
907c26a8 2516 ovs_nsh_cleanup();
064af421
BP		 2517}
2518
2519module_init(dp_init);
2520module_exit(dp_cleanup);
2521
2522MODULE_DESCRIPTION("Open vSwitch switching datapath");
2523MODULE_LICENSE("GPL");
3d0666d2 2524MODULE_VERSION(VERSION);
75e2077e
TLSC		 2525MODULE_ALIAS_GENL_FAMILY(OVS_DATAPATH_FAMILY);
2526MODULE_ALIAS_GENL_FAMILY(OVS_VPORT_FAMILY);
2527MODULE_ALIAS_GENL_FAMILY(OVS_FLOW_FAMILY);
2528MODULE_ALIAS_GENL_FAMILY(OVS_PACKET_FAMILY);
1cb57039 2529MODULE_ALIAS_GENL_FAMILY(OVS_METER_FAMILY);
cb2a5486 2530MODULE_ALIAS_GENL_FAMILY(OVS_CT_LIMIT_FAMILY);
openvswitch mirror