]> git.proxmox.com Git - mirror_ovs.git/blame - datapath/datapath.c
projects / mirror_ovs.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
datapath: Remove non-standard nla_put_be{16,32,64} functions.
[mirror_ovs.git] / datapath / datapath.c
CommitLineData
064af421 1/*
f632c8fc 2 * Copyright (c) 2007, 2008, 2009, 2010, 2011 Nicira Networks.
a14bc59f
BP		 3 * Distributed under the terms of the GNU GPL version 2.
4 *
5 * Significant portions of this file may be copied from parts of the Linux
6 * kernel, by Linus Torvalds and others.
064af421
BP		 7 */
8
9/* Functions for managing the dp interface/device. */
10
dfffaef1
JP		 11#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12
064af421
BP		 13#include <linux/init.h>
14#include <linux/module.h>
15#include <linux/fs.h>
16#include <linux/if_arp.h>
064af421
BP		 17#include <linux/if_vlan.h>
18#include <linux/in.h>
19#include <linux/ip.h>
20#include <linux/delay.h>
21#include <linux/time.h>
22#include <linux/etherdevice.h>
23#include <linux/kernel.h>
24#include <linux/kthread.h>
064af421
BP		 25#include <linux/mutex.h>
26#include <linux/percpu.h>
27#include <linux/rcupdate.h>
28#include <linux/tcp.h>
29#include <linux/udp.h>
30#include <linux/version.h>
31#include <linux/ethtool.h>
064af421
BP		 32#include <linux/wait.h>
33#include <asm/system.h>
34#include <asm/div64.h>
35#include <asm/bug.h>
656a0e37 36#include <linux/highmem.h>
064af421
BP		 37#include <linux/netfilter_bridge.h>
38#include <linux/netfilter_ipv4.h>
39#include <linux/inetdevice.h>
40#include <linux/list.h>
41#include <linux/rculist.h>
064af421 42#include <linux/dmi.h>
3c5f6de3 43#include <net/inet_ecn.h>
36956a7d 44#include <net/genetlink.h>
3fbd517a 45#include <linux/compat.h>
064af421
BP		 46
47#include "openvswitch/datapath-protocol.h"
dd8d6b8c 48#include "checksum.h"
064af421
BP		 49#include "datapath.h"
50#include "actions.h"
064af421 51#include "flow.h"
7eaa9830 52#include "loop_counter.h"
3fbd517a 53#include "odp-compat.h"
8d5ebd83 54#include "table.h"
f2459fe7 55#include "vport-internal_dev.h"
064af421 56
064af421
BP		 57int (*dp_ioctl_hook)(struct net_device *dev, struct ifreq *rq, int cmd);
58EXPORT_SYMBOL(dp_ioctl_hook);
59
064af421 60/* Datapaths. Protected on the read side by rcu_read_lock, on the write side
0d3b8a34 61 * by dp_mutex.
064af421
BP		 62 *
63 * dp_mutex nests inside the RTNL lock: if you need both you must take the RTNL
64 * lock first.
65 *
e779d8d9 66 * It is safe to access the datapath and vport structures with just
064af421
BP		 67 * dp_mutex.
68 */
d6569377 69static struct datapath __rcu *dps[256];
064af421
BP		 70static DEFINE_MUTEX(dp_mutex);
71
c19e6535 72static struct vport *new_vport(const struct vport_parms *);
064af421
BP		 73
74/* Must be called with rcu_read_lock or dp_mutex. */
75struct datapath *get_dp(int dp_idx)
76{
d6569377 77 if (dp_idx < 0 || dp_idx >= ARRAY_SIZE(dps))
064af421 78 return NULL;
eb3ccf11
JG		 79 return rcu_dereference_check(dps[dp_idx], rcu_read_lock_held() ||
80 lockdep_is_held(&dp_mutex));
064af421
BP		 81}
82EXPORT_SYMBOL_GPL(get_dp);
83
35f7605b 84static struct datapath *get_dp_locked(int dp_idx)
064af421
BP		 85{
86 struct datapath *dp;
87
88 mutex_lock(&dp_mutex);
89 dp = get_dp(dp_idx);
90 if (dp)
91 mutex_lock(&dp->mutex);
92 mutex_unlock(&dp_mutex);
93 return dp;
94}
95
027f9007 96static struct tbl *get_table_protected(struct datapath *dp)
9abaf6b3 97{
1452b28c
JG		 98 return rcu_dereference_protected(dp->table,
99 lockdep_is_held(&dp->mutex));
100}
101
027f9007 102static struct vport *get_vport_protected(struct datapath *dp, u16 port_no)
1452b28c
JG		 103{
104 return rcu_dereference_protected(dp->ports[port_no],
105 lockdep_is_held(&dp->mutex));
9abaf6b3
JG		 106}
107
f2459fe7
JG		 108/* Must be called with rcu_read_lock or RTNL lock. */
109const char *dp_name(const struct datapath *dp)
110{
ad919711 111 return vport_get_name(rcu_dereference_rtnl(dp->ports[ODPP_LOCAL]));
f2459fe7
JG		 112}
113
064af421
BP		 114static inline size_t br_nlmsg_size(void)
115{
116 return NLMSG_ALIGN(sizeof(struct ifinfomsg))
117 + nla_total_size(IFNAMSIZ) /* IFLA_IFNAME */
118 + nla_total_size(MAX_ADDR_LEN) /* IFLA_ADDRESS */
119 + nla_total_size(4) /* IFLA_MASTER */
120 + nla_total_size(4) /* IFLA_MTU */
121 + nla_total_size(4) /* IFLA_LINK */
122 + nla_total_size(1); /* IFLA_OPERSTATE */
123}
124
125static int dp_fill_ifinfo(struct sk_buff *skb,
e779d8d9 126 const struct vport *port,
064af421
BP		 127 int event, unsigned int flags)
128{
027f9007 129 struct datapath *dp = port->dp;
e779d8d9
BP		 130 int ifindex = vport_get_ifindex(port);
131 int iflink = vport_get_iflink(port);
064af421
BP		 132 struct ifinfomsg *hdr;
133 struct nlmsghdr *nlh;
134
f2459fe7
JG		 135 if (ifindex < 0)
136 return ifindex;
137
138 if (iflink < 0)
139 return iflink;
140
064af421
BP		 141 nlh = nlmsg_put(skb, 0, 0, event, sizeof(*hdr), flags);
142 if (nlh == NULL)
143 return -EMSGSIZE;
144
145 hdr = nlmsg_data(nlh);
146 hdr->ifi_family = AF_BRIDGE;
147 hdr->__ifi_pad = 0;
f2459fe7
JG		 148 hdr->ifi_type = ARPHRD_ETHER;
149 hdr->ifi_index = ifindex;
e779d8d9 150 hdr->ifi_flags = vport_get_flags(port);
064af421
BP		 151 hdr->ifi_change = 0;
152
e779d8d9 153 NLA_PUT_STRING(skb, IFLA_IFNAME, vport_get_name(port));
ad919711 154 NLA_PUT_U32(skb, IFLA_MASTER,
1452b28c 155 vport_get_ifindex(get_vport_protected(dp, ODPP_LOCAL)));
e779d8d9 156 NLA_PUT_U32(skb, IFLA_MTU, vport_get_mtu(port));
064af421
BP		 157#ifdef IFLA_OPERSTATE
158 NLA_PUT_U8(skb, IFLA_OPERSTATE,
e779d8d9
BP		 159 vport_is_running(port)
160 ? vport_get_operstate(port)
f2459fe7 161 : IF_OPER_DOWN);
064af421
BP		 162#endif
163
e779d8d9 164 NLA_PUT(skb, IFLA_ADDRESS, ETH_ALEN, vport_get_addr(port));
064af421 165
f2459fe7
JG		 166 if (ifindex != iflink)
167 NLA_PUT_U32(skb, IFLA_LINK,iflink);
064af421
BP		 168
169 return nlmsg_end(skb, nlh);
170
171nla_put_failure:
172 nlmsg_cancel(skb, nlh);
173 return -EMSGSIZE;
174}
175
e779d8d9 176static void dp_ifinfo_notify(int event, struct vport *port)
064af421 177{
064af421
BP		 178 struct sk_buff *skb;
179 int err = -ENOBUFS;
180
181 skb = nlmsg_new(br_nlmsg_size(), GFP_KERNEL);
182 if (skb == NULL)
183 goto errout;
184
185 err = dp_fill_ifinfo(skb, port, event, 0);
186 if (err < 0) {
187 /* -EMSGSIZE implies BUG in br_nlmsg_size() */
188 WARN_ON(err == -EMSGSIZE);
189 kfree_skb(skb);
190 goto errout;
191 }
f2459fe7 192 rtnl_notify(skb, &init_net, 0, RTNLGRP_LINK, NULL, GFP_KERNEL);
cfe7c1f5 193 return;
064af421
BP		 194errout:
195 if (err < 0)
f2459fe7 196 rtnl_set_sk_err(&init_net, RTNLGRP_LINK, err);
064af421
BP		 197}
198
58c342f6
BP		 199static void release_dp(struct kobject *kobj)
200{
201 struct datapath *dp = container_of(kobj, struct datapath, ifobj);
202 kfree(dp);
203}
204
35f7605b 205static struct kobj_type dp_ktype = {
58c342f6
BP		 206 .release = release_dp
207};
208
46c6a11d
JG		 209static void destroy_dp_rcu(struct rcu_head *rcu)
210{
211 struct datapath *dp = container_of(rcu, struct datapath, rcu);
212 int i;
213
214 for (i = 0; i < DP_N_QUEUES; i++)
215 skb_queue_purge(&dp->queues[i]);
216
217 tbl_destroy((struct tbl __force *)dp->table, flow_free_tbl);
218 free_percpu(dp->stats_percpu);
219 kobject_put(&dp->ifobj);
220}
221
d6569377
BP		 222/* Caller must hold RTNL, dp_mutex, and dp->mutex. */
223static void destroy_dp(struct datapath *dp)
064af421 224{
e779d8d9 225 struct vport *p, *n;
064af421 226
6fba0d0b
BP		 227 list_for_each_entry_safe (p, n, &dp->port_list, node)
228 if (p->port_no != ODPP_LOCAL)
c3827f61 229 dp_detach_port(p);
6fba0d0b 230
2ba9026e 231 dp_sysfs_del_dp(dp);
064af421 232 rcu_assign_pointer(dps[dp->dp_idx], NULL);
1452b28c 233 dp_detach_port(get_vport_protected(dp, ODPP_LOCAL));
8f843b6f 234
b0fb95ac 235 mutex_unlock(&dp->mutex);
46c6a11d 236 call_rcu(&dp->rcu, destroy_dp_rcu);
064af421 237 module_put(THIS_MODULE);
064af421
BP		 238}
239
f072ebdd 240/* Called with RTNL lock and dp->mutex. */
c19e6535 241static struct vport *new_vport(const struct vport_parms *parms)
064af421 242{
f2459fe7 243 struct vport *vport;
f2459fe7 244
c3827f61 245 vport_lock();
c19e6535
BP		 246 vport = vport_add(parms);
247 if (!IS_ERR(vport)) {
248 struct datapath *dp = parms->dp;
064af421 249
c19e6535
BP		 250 rcu_assign_pointer(dp->ports[parms->port_no], vport);
251 list_add_rcu(&vport->node, &dp->port_list);
064af421 252
c19e6535
BP		 253 dp_ifinfo_notify(RTM_NEWLINK, vport);
254 }
255 vport_unlock();
064af421 256
c19e6535 257 return vport;
064af421
BP		 258}
259
e779d8d9 260int dp_detach_port(struct vport *p)
064af421 261{
f2459fe7
JG		 262 int err;
263
064af421
BP		 264 ASSERT_RTNL();
265
2e7dd8ec 266 if (p->port_no != ODPP_LOCAL)
0515ceb3 267 dp_sysfs_del_if(p);
064af421
BP		 268 dp_ifinfo_notify(RTM_DELLINK, p);
269
064af421 270 /* First drop references to device. */
064af421
BP		 271 list_del_rcu(&p->node);
272 rcu_assign_pointer(p->dp->ports[p->port_no], NULL);
f2459fe7 273
7237e4f4 274 /* Then destroy it. */
c3827f61 275 vport_lock();
7237e4f4 276 err = vport_del(p);
c3827f61 277 vport_unlock();
f2459fe7 278
7237e4f4 279 return err;
064af421
BP		 280}
281
8819fac7 282/* Must be called with rcu_read_lock. */
e779d8d9 283void dp_process_received_packet(struct vport *p, struct sk_buff *skb)
064af421
BP		 284{
285 struct datapath *dp = p->dp;
286 struct dp_stats_percpu *stats;
8819fac7 287 int stats_counter_off;
55574bb0
BP		 288 struct sw_flow_actions *acts;
289 struct loop_counter *loop;
4c1ad233 290 int error;
064af421 291
e779d8d9 292 OVS_CB(skb)->vport = p;
a063b0df 293
3976f6d5 294 if (!OVS_CB(skb)->flow) {
36956a7d 295 struct sw_flow_key key;
3976f6d5 296 struct tbl_node *flow_node;
b7a31ec1 297 bool is_frag;
4c1ad233 298
3976f6d5 299 /* Extract flow from 'skb' into 'key'. */
b7a31ec1 300 error = flow_extract(skb, p ? p->port_no : ODPP_NONE, &key, &is_frag);
3976f6d5
JG		 301 if (unlikely(error)) {
302 kfree_skb(skb);
303 return;
304 }
064af421 305
b7a31ec1 306 if (is_frag && dp->drop_frags) {
3976f6d5
JG		 307 kfree_skb(skb);
308 stats_counter_off = offsetof(struct dp_stats_percpu, n_frags);
309 goto out;
310 }
311
312 /* Look up flow. */
313 flow_node = tbl_lookup(rcu_dereference(dp->table), &key,
314 flow_hash(&key), flow_cmp);
315 if (unlikely(!flow_node)) {
856081f6
BP		 316 struct dp_upcall_info upcall;
317
318 upcall.type = _ODPL_MISS_NR;
319 upcall.key = &key;
320 upcall.userdata = 0;
321 upcall.sample_pool = 0;
322 upcall.actions = NULL;
323 upcall.actions_len = 0;
324 dp_upcall(dp, skb, &upcall);
3976f6d5
JG		 325 stats_counter_off = offsetof(struct dp_stats_percpu, n_missed);
326 goto out;
327 }
328
329 OVS_CB(skb)->flow = flow_cast(flow_node);
55574bb0
BP		 330 }
331
f267de8a 332 stats_counter_off = offsetof(struct dp_stats_percpu, n_hit);
3976f6d5 333 flow_used(OVS_CB(skb)->flow, skb);
55574bb0 334
3976f6d5 335 acts = rcu_dereference(OVS_CB(skb)->flow->sf_acts);
55574bb0
BP		 336
337 /* Check whether we've looped too much. */
7eaa9830
JG		 338 loop = loop_get_counter();
339 if (unlikely(++loop->count > MAX_LOOPS))
55574bb0
BP		 340 loop->looping = true;
341 if (unlikely(loop->looping)) {
7eaa9830 342 loop_suppress(dp, acts);
f267de8a 343 kfree_skb(skb);
55574bb0 344 goto out_loop;
064af421 345 }
8819fac7 346
55574bb0 347 /* Execute actions. */
3976f6d5 348 execute_actions(dp, skb, &OVS_CB(skb)->flow->key, acts->actions,
cdee00fd 349 acts->actions_len);
55574bb0
BP		 350
351 /* Check whether sub-actions looped too much. */
352 if (unlikely(loop->looping))
7eaa9830 353 loop_suppress(dp, acts);
55574bb0
BP		 354
355out_loop:
356 /* Decrement loop counter. */
357 if (!--loop->count)
358 loop->looping = false;
7eaa9830 359 loop_put_counter();
55574bb0 360
8819fac7 361out:
55574bb0 362 /* Update datapath statistics. */
8819fac7
JG		 363 local_bh_disable();
364 stats = per_cpu_ptr(dp->stats_percpu, smp_processor_id());
38c6ecbc
JG		 365
366 write_seqcount_begin(&stats->seqlock);
8819fac7 367 (*(u64 *)((u8 *)stats + stats_counter_off))++;
38c6ecbc
JG		 368 write_seqcount_end(&stats->seqlock);
369
8819fac7 370 local_bh_enable();
064af421
BP		 371}
372
856081f6
BP		 373static void copy_and_csum_skb(struct sk_buff *skb, void *to)
374{
375 u16 csum_start, csum_offset;
376 __wsum csum;
377
378 get_skb_csum_pointers(skb, &csum_start, &csum_offset);
379 csum_start -= skb_headroom(skb);
380 BUG_ON(csum_start >= skb_headlen(skb));
381
382 skb_copy_bits(skb, 0, to, csum_start);
383
384 csum = skb_copy_and_csum_bits(skb, csum_start, to + csum_start,
385 skb->len - csum_start, 0);
386 *(__sum16 *)(to + csum_start + csum_offset) = csum_fold(csum);
387}
388
cb5087ca
BP		 389/* Append each packet in 'skb' list to 'queue'. There will be only one packet
390 * unless we broke up a GSO packet. */
856081f6
BP		 391static int queue_control_packets(struct datapath *dp, struct sk_buff *skb,
392 const struct dp_upcall_info *upcall_info)
cb5087ca
BP		 393{
394 struct sk_buff *nskb;
395 int port_no;
396 int err;
397
e779d8d9
BP		 398 if (OVS_CB(skb)->vport)
399 port_no = OVS_CB(skb)->vport->port_no;
f2459fe7
JG		 400 else
401 port_no = ODPP_LOCAL;
cb5087ca
BP		 402
403 do {
856081f6
BP		 404 struct odp_packet *upcall;
405 struct sk_buff *user_skb; /* to be queued to userspace */
406 struct nlattr *nla;
407 unsigned int len;
cb5087ca
BP		 408
409 nskb = skb->next;
410 skb->next = NULL;
411
856081f6
BP		 412 len = sizeof(struct odp_packet);
413 len += nla_total_size(4); /* ODP_PACKET_ATTR_TYPE. */
414 len += nla_total_size(skb->len);
415 len += nla_total_size(FLOW_BUFSIZE);
416 if (upcall_info->userdata)
417 len += nla_total_size(8);
418 if (upcall_info->sample_pool)
419 len += nla_total_size(4);
420 if (upcall_info->actions_len)
421 len += nla_total_size(upcall_info->actions_len);
422
423 user_skb = alloc_skb(len, GFP_ATOMIC);
424 if (!user_skb)
cb5087ca
BP		 425 goto err_kfree_skbs;
426
856081f6
BP		 427 upcall = (struct odp_packet *)__skb_put(user_skb, sizeof(*upcall));
428 upcall->dp_idx = dp->dp_idx;
429
430 nla_put_u32(user_skb, ODP_PACKET_ATTR_TYPE, upcall_info->type);
431
432 nla = nla_nest_start(user_skb, ODP_PACKET_ATTR_KEY);
433 flow_to_nlattrs(upcall_info->key, user_skb);
434 nla_nest_end(user_skb, nla);
cb5087ca 435
856081f6
BP		 436 if (upcall_info->userdata)
437 nla_put_u64(user_skb, ODP_PACKET_ATTR_USERDATA, upcall_info->userdata);
438 if (upcall_info->sample_pool)
439 nla_put_u32(user_skb, ODP_PACKET_ATTR_SAMPLE_POOL, upcall_info->sample_pool);
440 if (upcall_info->actions_len) {
441 const struct nlattr *actions = upcall_info->actions;
442 u32 actions_len = upcall_info->actions_len;
443
444 nla = nla_nest_start(user_skb, ODP_PACKET_ATTR_ACTIONS);
445 memcpy(__skb_put(user_skb, actions_len), actions, actions_len);
446 nla_nest_end(user_skb, nla);
447 }
448
449 nla = __nla_reserve(user_skb, ODP_PACKET_ATTR_PACKET, skb->len);
450 if (skb->ip_summed == CHECKSUM_PARTIAL)
451 copy_and_csum_skb(skb, nla_data(nla));
452 else
453 skb_copy_bits(skb, 0, nla_data(nla), skb->len);
454
455 upcall->len = user_skb->len;
456 skb_queue_tail(&dp->queues[upcall_info->type], user_skb);
457
458 kfree_skb(skb);
cb5087ca
BP		 459 skb = nskb;
460 } while (skb);
461 return 0;
462
463err_kfree_skbs:
464 kfree_skb(skb);
465 while ((skb = nskb) != NULL) {
466 nskb = skb->next;
467 kfree_skb(skb);
468 }
469 return err;
470}
471
856081f6 472int dp_upcall(struct datapath *dp, struct sk_buff *skb, const struct dp_upcall_info *upcall_info)
064af421
BP		 473{
474 struct dp_stats_percpu *stats;
475 struct sk_buff_head *queue;
064af421
BP		 476 int err;
477
478 WARN_ON_ONCE(skb_shared(skb));
856081f6
BP		 479 BUG_ON(upcall_info->type >= DP_N_QUEUES);
480
481 queue = &dp->queues[upcall_info->type];
064af421
BP		 482 err = -ENOBUFS;
483 if (skb_queue_len(queue) >= DP_MAX_QUEUE_LEN)
484 goto err_kfree_skb;
485
a6057323
JG		 486 forward_ip_summed(skb);
487
a2377e44
JG		 488 err = vswitch_skb_checksum_setup(skb);
489 if (err)
490 goto err_kfree_skb;
491
064af421
BP		 492 /* Break apart GSO packets into their component pieces. Otherwise
493 * userspace may try to stuff a 64kB packet into a 1500-byte MTU. */
494 if (skb_is_gso(skb)) {
9cc8b4e4 495 struct sk_buff *nskb = skb_gso_segment(skb, NETIF_F_SG | NETIF_F_HW_CSUM);
2d7ce2ee
JG		 496
497 kfree_skb(skb);
498 skb = nskb;
40796b34 499 if (IS_ERR(skb)) {
2d7ce2ee
JG		 500 err = PTR_ERR(skb);
501 goto err;
064af421
BP		 502 }
503 }
504
856081f6 505 err = queue_control_packets(dp, skb, upcall_info);
064af421 506 wake_up_interruptible(&dp->waitqueue);
cb5087ca 507 return err;
064af421
BP		 508
509err_kfree_skb:
510 kfree_skb(skb);
511err:
1c075d0a
JG		 512 local_bh_disable();
513 stats = per_cpu_ptr(dp->stats_percpu, smp_processor_id());
38c6ecbc
JG		 514
515 write_seqcount_begin(&stats->seqlock);
064af421 516 stats->n_lost++;
38c6ecbc
JG		 517 write_seqcount_end(&stats->seqlock);
518
1c075d0a 519 local_bh_enable();
064af421
BP		 520
521 return err;
522}
523
9c52546b 524static int flush_flows(int dp_idx)
064af421 525{
9c52546b 526 struct tbl *old_table;
8d5ebd83 527 struct tbl *new_table;
9c52546b
BP		 528 struct datapath *dp;
529 int err;
530
531 dp = get_dp_locked(dp_idx);
532 err = -ENODEV;
533 if (!dp)
534 goto exit;
8d5ebd83 535
9c52546b 536 old_table = get_table_protected(dp);
c6fadeb1 537 new_table = tbl_create(TBL_MIN_BUCKETS);
9c52546b 538 err = -ENOMEM;
8d5ebd83 539 if (!new_table)
9c52546b 540 goto exit_unlock;
8d5ebd83
JG		 541
542 rcu_assign_pointer(dp->table, new_table);
543
544 tbl_deferred_destroy(old_table, flow_free_tbl);
545
9c52546b
BP		 546 err = 0;
547
548exit_unlock:
549 mutex_unlock(&dp->mutex);
550exit:
551 return err;
064af421
BP		 552}
553
cdee00fd 554static int validate_actions(const struct nlattr *actions, u32 actions_len)
064af421 555{
23cad98c
BP		 556 const struct nlattr *a;
557 int rem;
558
559 nla_for_each_attr(a, actions, actions_len, rem) {
560 static const u32 action_lens[ODPAT_MAX + 1] = {
561 [ODPAT_OUTPUT] = 4,
562 [ODPAT_CONTROLLER] = 8,
563 [ODPAT_SET_DL_TCI] = 2,
564 [ODPAT_STRIP_VLAN] = 0,
565 [ODPAT_SET_DL_SRC] = ETH_ALEN,
566 [ODPAT_SET_DL_DST] = ETH_ALEN,
567 [ODPAT_SET_NW_SRC] = 4,
568 [ODPAT_SET_NW_DST] = 4,
569 [ODPAT_SET_NW_TOS] = 1,
570 [ODPAT_SET_TP_SRC] = 2,
571 [ODPAT_SET_TP_DST] = 2,
572 [ODPAT_SET_TUNNEL] = 8,
573 [ODPAT_SET_PRIORITY] = 4,
574 [ODPAT_POP_PRIORITY] = 0,
575 [ODPAT_DROP_SPOOFED_ARP] = 0,
576 };
577 int type = nla_type(a);
578
579 if (type > ODPAT_MAX || nla_len(a) != action_lens[type])
580 return -EINVAL;
581
582 switch (type) {
cdee00fd
BP		 583 case ODPAT_UNSPEC:
584 return -EINVAL;
064af421 585
23cad98c
BP		 586 case ODPAT_CONTROLLER:
587 case ODPAT_STRIP_VLAN:
588 case ODPAT_SET_DL_SRC:
589 case ODPAT_SET_DL_DST:
590 case ODPAT_SET_NW_SRC:
591 case ODPAT_SET_NW_DST:
592 case ODPAT_SET_TP_SRC:
593 case ODPAT_SET_TP_DST:
594 case ODPAT_SET_TUNNEL:
595 case ODPAT_SET_PRIORITY:
596 case ODPAT_POP_PRIORITY:
597 case ODPAT_DROP_SPOOFED_ARP:
598 /* No validation needed. */
599 break;
600
601 case ODPAT_OUTPUT:
602 if (nla_get_u32(a) >= DP_MAX_PORTS)
603 return -EINVAL;
3b1fc5f3 604 break;
cdee00fd 605
23cad98c 606 case ODPAT_SET_DL_TCI:
cdee00fd 607 if (nla_get_be16(a) & htons(VLAN_CFI_MASK))
064af421 608 return -EINVAL;
23cad98c 609 break;
064af421 610
23cad98c
BP		 611 case ODPAT_SET_NW_TOS:
612 if (nla_get_u8(a) & INET_ECN_MASK)
613 return -EINVAL;
614 break;
064af421 615
23cad98c
BP		 616 default:
617 return -EOPNOTSUPP;
618 }
619 }
3c5f6de3 620
23cad98c
BP		 621 if (rem > 0)
622 return -EINVAL;
064af421 623
23cad98c 624 return 0;
064af421
BP		 625}
626
d6569377
BP		 627struct dp_flowcmd {
628 u32 nlmsg_flags;
629 u32 dp_idx;
630 u32 total_len;
631 struct sw_flow_key key;
632 const struct nlattr *actions;
633 u32 actions_len;
634 bool clear;
635 u64 state;
636};
637
638static struct sw_flow_actions *get_actions(const struct dp_flowcmd *flowcmd)
064af421
BP		 639{
640 struct sw_flow_actions *actions;
064af421 641
d6569377
BP		 642 actions = flow_actions_alloc(flowcmd->actions_len);
643 if (!IS_ERR(actions) && flowcmd->actions_len)
644 memcpy(actions->actions, flowcmd->actions, flowcmd->actions_len);
064af421 645 return actions;
064af421
BP		 646}
647
648static void clear_stats(struct sw_flow *flow)
649{
6bfafa55 650 flow->used = 0;
064af421 651 flow->tcp_flags = 0;
064af421
BP		 652 flow->packet_count = 0;
653 flow->byte_count = 0;
654}
655
8d5ebd83
JG		 656static int expand_table(struct datapath *dp)
657{
9abaf6b3 658 struct tbl *old_table = get_table_protected(dp);
8d5ebd83
JG		 659 struct tbl *new_table;
660
661 new_table = tbl_expand(old_table);
662 if (IS_ERR(new_table))
663 return PTR_ERR(new_table);
664
665 rcu_assign_pointer(dp->table, new_table);
666 tbl_deferred_destroy(old_table, NULL);
667
d6569377 668 return 0;
8d5ebd83
JG		 669}
670
d6569377 671static int do_execute(struct datapath *dp, const struct odp_execute *execute)
064af421 672{
36956a7d 673 struct sw_flow_key key;
d6569377
BP		 674 struct sk_buff *skb;
675 struct sw_flow_actions *actions;
676 struct ethhdr *eth;
677 bool is_frag;
678 int err;
064af421 679
d6569377
BP		 680 err = -EINVAL;
681 if (execute->length < ETH_HLEN || execute->length > 65535)
682 goto error;
064af421 683
d6569377
BP		 684 actions = flow_actions_alloc(execute->actions_len);
685 if (IS_ERR(actions)) {
686 err = PTR_ERR(actions);
687 goto error;
688 }
064af421 689
d6569377
BP		 690 err = -EFAULT;
691 if (copy_from_user(actions->actions,
692 (struct nlattr __user __force *)execute->actions, execute->actions_len))
693 goto error_free_actions;
064af421 694
d6569377
BP		 695 err = validate_actions(actions->actions, execute->actions_len);
696 if (err)
697 goto error_free_actions;
8d5ebd83 698
d6569377
BP		 699 err = -ENOMEM;
700 skb = alloc_skb(execute->length, GFP_KERNEL);
701 if (!skb)
702 goto error_free_actions;
064af421 703
d6569377
BP		 704 err = -EFAULT;
705 if (copy_from_user(skb_put(skb, execute->length),
706 (const void __user __force *)execute->data,
707 execute->length))
708 goto error_free_skb;
8d5ebd83 709
d6569377
BP		 710 skb_reset_mac_header(skb);
711 eth = eth_hdr(skb);
064af421 712
d6569377
BP		 713 /* Normally, setting the skb 'protocol' field would be handled by a
714 * call to eth_type_trans(), but it assumes there's a sending
715 * device, which we may not have. */
716 if (ntohs(eth->h_proto) >= 1536)
717 skb->protocol = eth->h_proto;
718 else
719 skb->protocol = htons(ETH_P_802_2);
d3c54451 720
d6569377
BP		 721 err = flow_extract(skb, -1, &key, &is_frag);
722 if (err)
723 goto error_free_skb;
064af421 724
d6569377
BP		 725 rcu_read_lock();
726 err = execute_actions(dp, skb, &key, actions->actions, actions->actions_len);
727 rcu_read_unlock();
064af421 728
d6569377
BP		 729 kfree(actions);
730 return err;
064af421 731
d6569377
BP		 732error_free_skb:
733 kfree_skb(skb);
734error_free_actions:
735 kfree(actions);
064af421 736error:
d6569377 737 return err;
064af421
BP		 738}
739
d6569377 740static int execute_packet(const struct odp_execute __user *executep)
44e05eca 741{
d6569377 742 struct odp_execute execute;
9c52546b 743 struct datapath *dp;
44e05eca
BP		 744 int error;
745
d6569377 746 if (copy_from_user(&execute, executep, sizeof(execute)))
44e05eca
BP		 747 return -EFAULT;
748
d6569377 749 dp = get_dp_locked(execute.dp_idx);
9c52546b
BP		 750 if (!dp)
751 return -ENODEV;
d6569377 752 error = do_execute(dp, &execute);
9c52546b 753 mutex_unlock(&dp->mutex);
44e05eca 754
9c52546b 755 return error;
44e05eca
BP		 756}
757
d6569377 758static void get_dp_stats(struct datapath *dp, struct odp_stats *stats)
064af421 759{
d6569377 760 int i;
064af421 761
d6569377
BP		 762 stats->n_frags = stats->n_hit = stats->n_missed = stats->n_lost = 0;
763 for_each_possible_cpu(i) {
764 const struct dp_stats_percpu *percpu_stats;
765 struct dp_stats_percpu local_stats;
766 unsigned seqcount;
44e05eca 767
d6569377 768 percpu_stats = per_cpu_ptr(dp->stats_percpu, i);
064af421 769
d6569377
BP		 770 do {
771 seqcount = read_seqcount_begin(&percpu_stats->seqlock);
772 local_stats = *percpu_stats;
773 } while (read_seqcount_retry(&percpu_stats->seqlock, seqcount));
064af421 774
d6569377
BP		 775 stats->n_frags += local_stats.n_frags;
776 stats->n_hit += local_stats.n_hit;
777 stats->n_missed += local_stats.n_missed;
778 stats->n_lost += local_stats.n_lost;
779 }
780}
064af421 781
d6569377
BP		 782/* MTU of the dp pseudo-device: ETH_DATA_LEN or the minimum of the ports */
783int dp_min_mtu(const struct datapath *dp)
784{
785 struct vport *p;
786 int mtu = 0;
787
788 ASSERT_RTNL();
789
790 list_for_each_entry_rcu (p, &dp->port_list, node) {
791 int dev_mtu;
792
793 /* Skip any internal ports, since that's what we're trying to
794 * set. */
795 if (is_internal_vport(p))
796 continue;
797
798 dev_mtu = vport_get_mtu(p);
799 if (!mtu || dev_mtu < mtu)
800 mtu = dev_mtu;
801 }
802
803 return mtu ? mtu : ETH_DATA_LEN;
064af421
BP		 804}
805
d6569377
BP		 806/* Sets the MTU of all datapath devices to the minimum of the ports. Must
807 * be called with RTNL lock. */
808void set_internal_devs_mtu(const struct datapath *dp)
064af421 809{
d6569377
BP		 810 struct vport *p;
811 int mtu;
064af421 812
d6569377
BP		 813 ASSERT_RTNL();
814
815 mtu = dp_min_mtu(dp);
44e05eca 816
d6569377
BP		 817 list_for_each_entry_rcu (p, &dp->port_list, node) {
818 if (is_internal_vport(p))
819 vport_set_mtu(p, mtu);
820 }
064af421
BP		 821}
822
d6569377 823static int get_listen_mask(const struct file *f)
064af421 824{
d6569377
BP		 825 return (long)f->private_data;
826}
827
828static void set_listen_mask(struct file *f, int listen_mask)
829{
830 f->private_data = (void*)(long)listen_mask;
831}
064af421 832
d6569377
BP		 833static const struct nla_policy flow_policy[ODP_FLOW_ATTR_MAX + 1] = {
834 [ODP_FLOW_ATTR_KEY] = { .type = NLA_NESTED },
835 [ODP_FLOW_ATTR_ACTIONS] = { .type = NLA_NESTED },
836 [ODP_FLOW_ATTR_CLEAR] = { .type = NLA_FLAG },
837 [ODP_FLOW_ATTR_STATE] = { .type = NLA_U64 },
838};
36956a7d 839
d6569377
BP		 840static int copy_flow_to_user(struct odp_flow __user *dst, struct datapath *dp,
841 struct sw_flow *flow, u32 total_len, u64 state)
842{
843 const struct sw_flow_actions *sf_acts;
844 struct odp_flow_stats stats;
845 struct odp_flow *odp_flow;
846 struct sk_buff *skb;
847 struct nlattr *nla;
848 unsigned long used;
849 u8 tcp_flags;
850 int err;
064af421 851
d6569377
BP		 852 sf_acts = rcu_dereference_protected(flow->sf_acts,
853 lockdep_is_held(&dp->mutex));
064af421 854
d6569377
BP		 855 skb = alloc_skb(128 + FLOW_BUFSIZE + sf_acts->actions_len, GFP_KERNEL);
856 err = -ENOMEM;
857 if (!skb)
858 goto exit;
859
860 rcu_read_lock();
861 odp_flow = (struct odp_flow*)__skb_put(skb, sizeof(struct odp_flow));
862 odp_flow->dp_idx = dp->dp_idx;
863 odp_flow->total_len = total_len;
864
865 nla = nla_nest_start(skb, ODP_FLOW_ATTR_KEY);
866 if (!nla)
867 goto nla_put_failure;
868 err = flow_to_nlattrs(&flow->key, skb);
869 if (err)
870 goto exit_unlock;
871 nla_nest_end(skb, nla);
872
873 nla = nla_nest_start(skb, ODP_FLOW_ATTR_ACTIONS);
874 if (!nla || skb_tailroom(skb) < sf_acts->actions_len)
875 goto nla_put_failure;
876 memcpy(__skb_put(skb, sf_acts->actions_len), sf_acts->actions, sf_acts->actions_len);
877 nla_nest_end(skb, nla);
878
879 spin_lock_bh(&flow->lock);
880 used = flow->used;
881 stats.n_packets = flow->packet_count;
882 stats.n_bytes = flow->byte_count;
883 tcp_flags = flow->tcp_flags;
884 spin_unlock_bh(&flow->lock);
885
886 if (used)
887 NLA_PUT_MSECS(skb, ODP_FLOW_ATTR_USED, used);
888
889 if (stats.n_packets)
890 NLA_PUT(skb, ODP_FLOW_ATTR_STATS, sizeof(struct odp_flow_stats), &stats);
891
892 if (tcp_flags)
893 NLA_PUT_U8(skb, ODP_FLOW_ATTR_TCP_FLAGS, tcp_flags);
894
895 if (state)
896 NLA_PUT_U64(skb, ODP_FLOW_ATTR_STATE, state);
897
898 if (skb->len > total_len)
899 goto nla_put_failure;
900
901 odp_flow->len = skb->len;
902 err = copy_to_user(dst, skb->data, skb->len) ? -EFAULT : 0;
903 goto exit_unlock;
904
905nla_put_failure:
906 err = -EMSGSIZE;
907exit_unlock:
908 rcu_read_unlock();
909 kfree_skb(skb);
910exit:
911 return err;
44e05eca
BP		 912}
913
d6569377
BP		 914static struct sk_buff *copy_flow_from_user(struct odp_flow __user *uodp_flow,
915 struct dp_flowcmd *flowcmd)
44e05eca 916{
d6569377
BP		 917 struct nlattr *a[ODP_FLOW_ATTR_MAX + 1];
918 struct odp_flow *odp_flow;
919 struct sk_buff *skb;
920 u32 len;
921 int err;
44e05eca 922
d6569377
BP		 923 if (get_user(len, &uodp_flow->len))
924 return ERR_PTR(-EFAULT);
925 if (len < sizeof(struct odp_flow))
926 return ERR_PTR(-EINVAL);
44e05eca 927
d6569377
BP		 928 skb = alloc_skb(len, GFP_KERNEL);
929 if (!skb)
930 return ERR_PTR(-ENOMEM);
9c52546b 931
d6569377
BP		 932 err = -EFAULT;
933 if (copy_from_user(__skb_put(skb, len), uodp_flow, len))
934 goto error_free_skb;
935
936 odp_flow = (struct odp_flow *)skb->data;
937 err = -EINVAL;
938 if (odp_flow->len != len)
939 goto error_free_skb;
940
941 flowcmd->nlmsg_flags = odp_flow->nlmsg_flags;
942 flowcmd->dp_idx = odp_flow->dp_idx;
943 flowcmd->total_len = odp_flow->total_len;
944
945 err = nla_parse(a, ODP_FLOW_ATTR_MAX,
946 (struct nlattr *)(skb->data + sizeof(struct odp_flow)),
947 skb->len - sizeof(struct odp_flow), flow_policy);
948 if (err)
949 goto error_free_skb;
950
951 /* ODP_FLOW_ATTR_KEY. */
952 if (a[ODP_FLOW_ATTR_KEY]) {
953 err = flow_from_nlattrs(&flowcmd->key, a[ODP_FLOW_ATTR_KEY]);
954 if (err)
955 goto error_free_skb;
956 } else
957 memset(&flowcmd->key, 0, sizeof(struct sw_flow_key));
958
959 /* ODP_FLOW_ATTR_ACTIONS. */
960 if (a[ODP_FLOW_ATTR_ACTIONS]) {
961 flowcmd->actions = nla_data(a[ODP_FLOW_ATTR_ACTIONS]);
962 flowcmd->actions_len = nla_len(a[ODP_FLOW_ATTR_ACTIONS]);
963 err = validate_actions(flowcmd->actions, flowcmd->actions_len);
964 if (err)
965 goto error_free_skb;
966 } else {
967 flowcmd->actions = NULL;
968 flowcmd->actions_len = 0;
9c52546b 969 }
8d5ebd83 970
d6569377
BP		 971 flowcmd->clear = a[ODP_FLOW_ATTR_CLEAR] != NULL;
972
973 flowcmd->state = a[ODP_FLOW_ATTR_STATE] ? nla_get_u64(a[ODP_FLOW_ATTR_STATE]) : 0;
974
975 return skb;
976
977error_free_skb:
978 kfree_skb(skb);
979 return ERR_PTR(err);
064af421
BP		 980}
981
d6569377 982static int new_flow(unsigned int cmd, struct odp_flow __user *uodp_flow)
064af421 983{
bc4a05c6 984 struct tbl_node *flow_node;
d6569377
BP		 985 struct dp_flowcmd flowcmd;
986 struct sw_flow *flow;
987 struct sk_buff *skb;
9c52546b 988 struct datapath *dp;
d6569377
BP		 989 struct tbl *table;
990 u32 hash;
bc4a05c6 991 int error;
064af421 992
d6569377
BP		 993 skb = copy_flow_from_user(uodp_flow, &flowcmd);
994 error = PTR_ERR(skb);
995 if (IS_ERR(skb))
996 goto exit;
064af421 997
d6569377
BP		 998 dp = get_dp_locked(flowcmd.dp_idx);
999 error = -ENODEV;
9c52546b 1000 if (!dp)
d6569377 1001 goto error_kfree_skb;
704a1e09 1002
d6569377
BP		 1003 hash = flow_hash(&flowcmd.key);
1004 table = get_table_protected(dp);
1005 flow_node = tbl_lookup(table, &flowcmd.key, hash, flow_cmp);
1006 if (!flow_node) {
1007 struct sw_flow_actions *acts;
1008
1009 /* Bail out if we're not allowed to create a new flow. */
1010 error = -ENOENT;
1011 if (cmd == ODP_FLOW_SET)
1012 goto error_unlock_dp;
1013
1014 /* Expand table, if necessary, to make room. */
1015 if (tbl_count(table) >= tbl_n_buckets(table)) {
1016 error = expand_table(dp);
1017 if (error)
1018 goto error_unlock_dp;
1019 table = get_table_protected(dp);
1020 }
1021
1022 /* Allocate flow. */
1023 flow = flow_alloc();
1024 if (IS_ERR(flow)) {
1025 error = PTR_ERR(flow);
1026 goto error_unlock_dp;
1027 }
1028 flow->key = flowcmd.key;
1029 clear_stats(flow);
1030
1031 /* Obtain actions. */
1032 acts = get_actions(&flowcmd);
1033 error = PTR_ERR(acts);
1034 if (IS_ERR(acts))
1035 goto error_free_flow;
1036 rcu_assign_pointer(flow->sf_acts, acts);
1037
1038 error = copy_flow_to_user(uodp_flow, dp, flow, flowcmd.total_len, 0);
1039 if (error)
1040 goto error_free_flow;
1041
1042 /* Put flow in bucket. */
1043 error = tbl_insert(table, &flow->tbl_node, hash);
1044 if (error)
1045 goto error_free_flow;
1046 } else {
1047 /* We found a matching flow. */
1048 struct sw_flow_actions *old_acts;
1049
1050 /* Bail out if we're not allowed to modify an existing flow.
1051 * We accept NLM_F_CREATE in place of the intended NLM_F_EXCL
1052 * because Generic Netlink treats the latter as a dump
1053 * request. We also accept NLM_F_EXCL in case that bug ever
1054 * gets fixed.
1055 */
1056 error = -EEXIST;
1057 if (flowcmd.nlmsg_flags & (NLM_F_CREATE | NLM_F_EXCL))
1058 goto error_kfree_skb;
1059
1060 /* Update actions. */
1061 flow = flow_cast(flow_node);
1062 old_acts = rcu_dereference_protected(flow->sf_acts,
1063 lockdep_is_held(&dp->mutex));
1064 if (flowcmd.actions &&
1065 (old_acts->actions_len != flowcmd.actions_len ||
1066 memcmp(old_acts->actions, flowcmd.actions,
1067 flowcmd.actions_len))) {
1068 struct sw_flow_actions *new_acts;
1069
1070 new_acts = get_actions(&flowcmd);
1071 error = PTR_ERR(new_acts);
1072 if (IS_ERR(new_acts))
1073 goto error_kfree_skb;
1074
1075 rcu_assign_pointer(flow->sf_acts, new_acts);
1076 flow_deferred_free_acts(old_acts);
1077 }
1078
1079 error = copy_flow_to_user(uodp_flow, dp, flow, flowcmd.total_len, 0);
1080 if (error)
1081 goto error_kfree_skb;
1082
1083 /* Clear stats. */
1084 if (flowcmd.clear) {
1085 spin_lock_bh(&flow->lock);
1086 clear_stats(flow);
1087 spin_unlock_bh(&flow->lock);
1088 }
9c52546b 1089 }
d6569377 1090 kfree_skb(skb);
9c52546b 1091 mutex_unlock(&dp->mutex);
d6569377 1092 return 0;
704a1e09 1093
d6569377
BP		 1094error_free_flow:
1095 flow_put(flow);
1096error_unlock_dp:
1097 mutex_unlock(&dp->mutex);
1098error_kfree_skb:
1099 kfree_skb(skb);
1100exit:
9c52546b 1101 return error;
704a1e09
BP		 1102}
1103
d6569377 1104static int get_or_del_flow(unsigned int cmd, struct odp_flow __user *uodp_flow)
704a1e09 1105{
d6569377
BP		 1106 struct tbl_node *flow_node;
1107 struct dp_flowcmd flowcmd;
704a1e09 1108 struct sw_flow *flow;
d6569377 1109 struct sk_buff *skb;
9c52546b
BP		 1110 struct datapath *dp;
1111 struct tbl *table;
9c52546b 1112 int err;
704a1e09 1113
d6569377
BP		 1114 skb = copy_flow_from_user(uodp_flow, &flowcmd);
1115 err = PTR_ERR(skb);
1116 if (IS_ERR(skb))
9c52546b 1117 goto exit;
704a1e09 1118
d6569377 1119 dp = get_dp_locked(flowcmd.dp_idx);
9c52546b
BP		 1120 err = -ENODEV;
1121 if (!dp)
d6569377 1122 goto exit_kfree_skb;
704a1e09 1123
9c52546b 1124 table = get_table_protected(dp);
d6569377
BP		 1125 flow_node = tbl_lookup(table, &flowcmd.key, flow_hash(&flowcmd.key), flow_cmp);
1126 err = -ENOENT;
1127 if (!flow_node)
1128 goto exit_unlock_dp;
1129
1130 if (cmd == ODP_FLOW_DEL) {
1131 err = tbl_remove(table, flow_node);
1132 if (err)
1133 goto exit_unlock_dp;
9c52546b 1134 }
704a1e09 1135
d6569377
BP		 1136 flow = flow_cast(flow_node);
1137 err = copy_flow_to_user(uodp_flow, dp, flow, flowcmd.total_len, 0);
1138 if (!err && cmd == ODP_FLOW_DEL)
1139 flow_deferred_free(flow);
36956a7d 1140
d6569377
BP		 1141exit_unlock_dp:
1142 mutex_unlock(&dp->mutex);
1143exit_kfree_skb:
1144 kfree_skb(skb);
1145exit:
1146 return err;
1147}
9c52546b 1148
d6569377
BP		 1149static int dump_flow(struct odp_flow __user *uodp_flow)
1150{
1151 struct tbl_node *flow_node;
1152 struct dp_flowcmd flowcmd;
1153 struct sw_flow *flow;
1154 struct sk_buff *skb;
1155 struct datapath *dp;
1156 u32 bucket, obj;
1157 int err;
36956a7d 1158
d6569377
BP		 1159 skb = copy_flow_from_user(uodp_flow, &flowcmd);
1160 err = PTR_ERR(skb);
1161 if (IS_ERR(skb))
1162 goto exit;
1163
1164 dp = get_dp_locked(flowcmd.dp_idx);
1165 err = -ENODEV;
1166 if (!dp)
1167 goto exit_free;
1168
1169 bucket = flowcmd.state >> 32;
1170 obj = flowcmd.state;
1171 flow_node = tbl_next(dp->table, &bucket, &obj);
1172 err = -ENODEV;
1173 if (!flow_node)
1174 goto exit_unlock_dp;
1175
1176 flow = flow_cast(flow_node);
1177 err = copy_flow_to_user(uodp_flow, dp, flow, flowcmd.total_len,
1178 ((u64)bucket << 32) | obj);
1179
1180exit_unlock_dp:
9c52546b 1181 mutex_unlock(&dp->mutex);
d6569377
BP		 1182exit_free:
1183 kfree_skb(skb);
9c52546b
BP		 1184exit:
1185 return err;
704a1e09
BP		 1186}
1187
d6569377
BP		 1188static const struct nla_policy datapath_policy[ODP_DP_ATTR_MAX + 1] = {
1189 [ODP_DP_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
1190 [ODP_DP_ATTR_IPV4_FRAGS] = { .type = NLA_U32 },
1191 [ODP_DP_ATTR_SAMPLING] = { .type = NLA_U32 },
1192};
1193
1194static int copy_datapath_to_user(void __user *dst, struct datapath *dp, uint32_t total_len)
064af421 1195{
d6569377 1196 struct odp_datapath *odp_datapath;
064af421 1197 struct sk_buff *skb;
d6569377 1198 struct nlattr *nla;
064af421
BP		 1199 int err;
1200
d6569377
BP		 1201 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1202 err = -ENOMEM;
1203 if (!skb)
1204 goto exit;
064af421 1205
d6569377
BP		 1206 odp_datapath = (struct odp_datapath*)__skb_put(skb, sizeof(struct odp_datapath));
1207 odp_datapath->dp_idx = dp->dp_idx;
1208 odp_datapath->total_len = total_len;
064af421 1209
d6569377
BP		 1210 rcu_read_lock();
1211 err = nla_put_string(skb, ODP_DP_ATTR_NAME, dp_name(dp));
1212 rcu_read_unlock();
064af421 1213 if (err)
d6569377 1214 goto nla_put_failure;
064af421 1215
d6569377
BP		 1216 nla = nla_reserve(skb, ODP_DP_ATTR_STATS, sizeof(struct odp_stats));
1217 if (!nla)
1218 goto nla_put_failure;
1219 get_dp_stats(dp, nla_data(nla));
1220
1221 NLA_PUT_U32(skb, ODP_DP_ATTR_IPV4_FRAGS,
1222 dp->drop_frags ? ODP_DP_FRAG_DROP : ODP_DP_FRAG_ZERO);
1223
1224 if (dp->sflow_probability)
1225 NLA_PUT_U32(skb, ODP_DP_ATTR_SAMPLING, dp->sflow_probability);
1226
1227 if (skb->len > total_len)
1228 goto nla_put_failure;
1229
1230 odp_datapath->len = skb->len;
1231 err = copy_to_user(dst, skb->data, skb->len) ? -EFAULT : 0;
1232 goto exit_free_skb;
1233
1234nla_put_failure:
1235 err = -EMSGSIZE;
1236exit_free_skb:
1237 kfree_skb(skb);
1238exit:
1239 return err;
1240}
1241
1242static struct sk_buff *copy_datapath_from_user(struct odp_datapath __user *uodp_datapath, struct nlattr *a[ODP_DP_ATTR_MAX + 1])
1243{
1244 struct odp_datapath *odp_datapath;
1245 struct sk_buff *skb;
1246 u32 len;
1247 int err;
1248
1249 if (get_user(len, &uodp_datapath->len))
1250 return ERR_PTR(-EFAULT);
1251 if (len < sizeof(struct odp_datapath))
1252 return ERR_PTR(-EINVAL);
1253
1254 skb = alloc_skb(len, GFP_KERNEL);
064af421 1255 if (!skb)
d6569377 1256 return ERR_PTR(-ENOMEM);
659586ef 1257
064af421 1258 err = -EFAULT;
d6569377 1259 if (copy_from_user(__skb_put(skb, len), uodp_datapath, len))
064af421
BP		 1260 goto error_free_skb;
1261
d6569377
BP		 1262 odp_datapath = (struct odp_datapath *)skb->data;
1263 err = -EINVAL;
1264 if (odp_datapath->len != len)
1265 goto error_free_skb;
a393b897 1266
d6569377
BP		 1267 err = nla_parse(a, ODP_DP_ATTR_MAX,
1268 (struct nlattr *)(skb->data + sizeof(struct odp_datapath)),
1269 skb->len - sizeof(struct odp_datapath), datapath_policy);
4c1ad233
BP		 1270 if (err)
1271 goto error_free_skb;
9dca7bd5 1272
d6569377
BP		 1273 if (a[ODP_DP_ATTR_IPV4_FRAGS]) {
1274 u32 frags = nla_get_u32(a[ODP_DP_ATTR_IPV4_FRAGS]);
9dca7bd5 1275
d6569377
BP		 1276 err = -EINVAL;
1277 if (frags != ODP_DP_FRAG_ZERO && frags != ODP_DP_FRAG_DROP)
1278 goto error_free_skb;
1279 }
1280
1281 err = VERIFY_NUL_STRING(a[ODP_DP_ATTR_NAME], IFNAMSIZ - 1);
1282 if (err)
1283 goto error_free_skb;
1284
1285 return skb;
064af421
BP		 1286
1287error_free_skb:
1288 kfree_skb(skb);
d6569377
BP		 1289 return ERR_PTR(err);
1290}
1291
1292/* Called with dp_mutex and optionally with RTNL lock also.
1293 * Holds the returned datapath's mutex on return.
1294 */
1295static struct datapath *lookup_datapath(struct odp_datapath *odp_datapath, struct nlattr *a[ODP_DP_ATTR_MAX + 1])
1296{
1297 WARN_ON_ONCE(!mutex_is_locked(&dp_mutex));
1298
1299 if (!a[ODP_DP_ATTR_NAME]) {
1300 struct datapath *dp;
1301
1302 dp = get_dp(odp_datapath->dp_idx);
1303 if (!dp)
1304 return ERR_PTR(-ENODEV);
1305 mutex_lock(&dp->mutex);
1306 return dp;
1307 } else {
1308 struct datapath *dp;
1309 struct vport *vport;
1310 int dp_idx;
1311
1312 vport_lock();
1313 vport = vport_locate(nla_data(a[ODP_DP_ATTR_NAME]));
1314 dp_idx = vport && vport->port_no == ODPP_LOCAL ? vport->dp->dp_idx : -1;
1315 vport_unlock();
1316
1317 if (dp_idx < 0)
1318 return ERR_PTR(-ENODEV);
1319
1320 dp = get_dp(dp_idx);
1321 mutex_lock(&dp->mutex);
1322 return dp;
1323 }
1324}
1325
1326static void change_datapath(struct datapath *dp, struct nlattr *a[ODP_DP_ATTR_MAX + 1])
1327{
1328 if (a[ODP_DP_ATTR_IPV4_FRAGS])
1329 dp->drop_frags = nla_get_u32(a[ODP_DP_ATTR_IPV4_FRAGS]) == ODP_DP_FRAG_DROP;
1330 if (a[ODP_DP_ATTR_SAMPLING])
1331 dp->sflow_probability = nla_get_u32(a[ODP_DP_ATTR_SAMPLING]);
1332}
1333
1334static int new_datapath(struct odp_datapath __user *uodp_datapath)
1335{
1336 struct nlattr *a[ODP_DP_ATTR_MAX + 1];
1337 struct odp_datapath *odp_datapath;
1338 struct vport_parms parms;
1339 struct sk_buff *skb;
1340 struct datapath *dp;
1341 struct vport *vport;
1342 int dp_idx;
1343 int err;
1344 int i;
1345
1346 skb = copy_datapath_from_user(uodp_datapath, a);
1347 err = PTR_ERR(skb);
1348 if (IS_ERR(skb))
1349 goto err;
1350 odp_datapath = (struct odp_datapath *)skb->data;
1351
1352 err = -EINVAL;
1353 if (!a[ODP_DP_ATTR_NAME])
1354 goto err_free_skb;
1355
1356 rtnl_lock();
1357 mutex_lock(&dp_mutex);
1358 err = -ENODEV;
1359 if (!try_module_get(THIS_MODULE))
1360 goto err_unlock_dp_mutex;
1361
1362 dp_idx = odp_datapath->dp_idx;
1363 if (dp_idx < 0) {
1364 err = -EFBIG;
1365 for (dp_idx = 0; dp_idx < ARRAY_SIZE(dps); dp_idx++) {
1366 if (get_dp(dp_idx))
1367 continue;
1368 err = 0;
1369 break;
1370 }
1371 } else if (dp_idx < ARRAY_SIZE(dps))
1372 err = get_dp(dp_idx) ? -EBUSY : 0;
1373 else
1374 err = -EINVAL;
1375 if (err)
1376 goto err_put_module;
1377
1378 err = -ENOMEM;
1379 dp = kzalloc(sizeof(*dp), GFP_KERNEL);
1380 if (dp == NULL)
1381 goto err_put_module;
1382 INIT_LIST_HEAD(&dp->port_list);
1383 mutex_init(&dp->mutex);
1384 mutex_lock(&dp->mutex);
1385 dp->dp_idx = dp_idx;
1386 for (i = 0; i < DP_N_QUEUES; i++)
1387 skb_queue_head_init(&dp->queues[i]);
1388 init_waitqueue_head(&dp->waitqueue);
1389
1390 /* Initialize kobject for bridge. This will be added as
1391 * /sys/class/net/<devname>/brif later, if sysfs is enabled. */
1392 dp->ifobj.kset = NULL;
1393 kobject_init(&dp->ifobj, &dp_ktype);
1394
1395 /* Allocate table. */
1396 err = -ENOMEM;
1397 rcu_assign_pointer(dp->table, tbl_create(TBL_MIN_BUCKETS));
1398 if (!dp->table)
1399 goto err_free_dp;
1400
1401 /* Set up our datapath device. */
1402 parms.name = nla_data(a[ODP_DP_ATTR_NAME]);
1403 parms.type = ODP_VPORT_TYPE_INTERNAL;
1404 parms.options = NULL;
1405 parms.dp = dp;
1406 parms.port_no = ODPP_LOCAL;
1407 vport = new_vport(&parms);
1408 if (IS_ERR(vport)) {
1409 err = PTR_ERR(vport);
1410 if (err == -EBUSY)
1411 err = -EEXIST;
1412
1413 goto err_destroy_table;
1414 }
1415
1416 dp->drop_frags = 0;
1417 dp->stats_percpu = alloc_percpu(struct dp_stats_percpu);
1418 if (!dp->stats_percpu) {
1419 err = -ENOMEM;
1420 goto err_destroy_local_port;
1421 }
1422
1423 change_datapath(dp, a);
1424
1425 rcu_assign_pointer(dps[dp_idx], dp);
1426 dp_sysfs_add_dp(dp);
1427
1428 mutex_unlock(&dp->mutex);
1429 mutex_unlock(&dp_mutex);
1430 rtnl_unlock();
1431
1432 return 0;
1433
1434err_destroy_local_port:
1435 dp_detach_port(get_vport_protected(dp, ODPP_LOCAL));
1436err_destroy_table:
1437 tbl_destroy(get_table_protected(dp), NULL);
1438err_free_dp:
1439 mutex_unlock(&dp->mutex);
1440 kfree(dp);
1441err_put_module:
1442 module_put(THIS_MODULE);
1443err_unlock_dp_mutex:
1444 mutex_unlock(&dp_mutex);
1445 rtnl_unlock();
1446err_free_skb:
1447 kfree_skb(skb);
1448err:
064af421
BP		 1449 return err;
1450}
1451
d6569377 1452static int del_datapath(struct odp_datapath __user *uodp_datapath)
44e05eca 1453{
d6569377 1454 struct nlattr *a[ODP_DP_ATTR_MAX + 1];
9c52546b 1455 struct datapath *dp;
d6569377
BP		 1456 struct sk_buff *skb;
1457 int err;
44e05eca 1458
d6569377
BP		 1459 skb = copy_datapath_from_user(uodp_datapath, a);
1460 err = PTR_ERR(skb);
1461 if (IS_ERR(skb))
1462 goto exit;
44e05eca 1463
d6569377
BP		 1464 rtnl_lock();
1465 mutex_lock(&dp_mutex);
1466 dp = lookup_datapath((struct odp_datapath *)skb->data, a);
1467 err = PTR_ERR(dp);
1468 if (IS_ERR(dp))
1469 goto exit_free;
9c52546b 1470
d6569377
BP		 1471 destroy_dp(dp);
1472 err = 0;
1473
1474exit_free:
1475 kfree_skb(skb);
1476 mutex_unlock(&dp_mutex);
1477 rtnl_unlock();
1478exit:
1479 return err;
44e05eca
BP		 1480}
1481
d6569377 1482static int set_datapath(struct odp_datapath __user *uodp_datapath)
064af421 1483{
d6569377
BP		 1484 struct nlattr *a[ODP_DP_ATTR_MAX + 1];
1485 struct datapath *dp;
1486 struct sk_buff *skb;
1487 int err;
064af421 1488
d6569377
BP		 1489 skb = copy_datapath_from_user(uodp_datapath, a);
1490 err = PTR_ERR(skb);
1491 if (IS_ERR(skb))
1492 goto exit;
38c6ecbc 1493
d6569377
BP		 1494 mutex_lock(&dp_mutex);
1495 dp = lookup_datapath((struct odp_datapath *)skb->data, a);
1496 err = PTR_ERR(dp);
1497 if (IS_ERR(dp))
1498 goto exit_free;
38c6ecbc 1499
d6569377
BP		 1500 change_datapath(dp, a);
1501 mutex_unlock(&dp->mutex);
1502 err = 0;
38c6ecbc 1503
d6569377
BP		 1504exit_free:
1505 kfree_skb(skb);
1506 mutex_unlock(&dp_mutex);
1507exit:
1508 return err;
064af421
BP		 1509}
1510
d6569377 1511static int get_datapath(struct odp_datapath __user *uodp_datapath)
1dcf111b 1512{
d6569377
BP		 1513 struct nlattr *a[ODP_DP_ATTR_MAX + 1];
1514 struct odp_datapath *odp_datapath;
1515 struct datapath *dp;
1516 struct sk_buff *skb;
1517 int err;
1dcf111b 1518
d6569377
BP		 1519 skb = copy_datapath_from_user(uodp_datapath, a);
1520 err = PTR_ERR(skb);
1521 if (IS_ERR(skb))
1522 goto exit;
1523 odp_datapath = (struct odp_datapath *)skb->data;
1dcf111b 1524
d6569377
BP		 1525 mutex_lock(&dp_mutex);
1526 dp = lookup_datapath(odp_datapath, a);
1527 mutex_unlock(&dp_mutex);
1dcf111b 1528
d6569377
BP		 1529 err = PTR_ERR(dp);
1530 if (IS_ERR(dp))
1531 goto exit_free;
1dcf111b 1532
d6569377
BP		 1533 err = copy_datapath_to_user(uodp_datapath, dp, odp_datapath->total_len);
1534 mutex_unlock(&dp->mutex);
1535exit_free:
1536 kfree_skb(skb);
1537exit:
1538 return err;
1dcf111b
JP		 1539}
1540
d6569377 1541static int dump_datapath(struct odp_datapath __user *uodp_datapath)
a7786963 1542{
d6569377
BP		 1543 struct nlattr *a[ODP_DP_ATTR_MAX + 1];
1544 struct odp_datapath *odp_datapath;
1545 struct sk_buff *skb;
1546 u32 dp_idx;
1547 int err;
a7786963 1548
d6569377
BP		 1549 skb = copy_datapath_from_user(uodp_datapath, a);
1550 err = PTR_ERR(skb);
1551 if (IS_ERR(skb))
1552 goto exit;
1553 odp_datapath = (struct odp_datapath *)skb->data;
a7786963 1554
d6569377
BP		 1555 mutex_lock(&dp_mutex);
1556 for (dp_idx = odp_datapath->dp_idx; dp_idx < ARRAY_SIZE(dps); dp_idx++) {
1557 struct datapath *dp = get_dp(dp_idx);
1558 if (!dp)
1559 continue;
a7786963 1560
d6569377
BP		 1561 mutex_lock(&dp->mutex);
1562 mutex_unlock(&dp_mutex);
1563 err = copy_datapath_to_user(uodp_datapath, dp, odp_datapath->total_len);
1564 mutex_unlock(&dp->mutex);
1565 goto exit_free;
a7786963 1566 }
d6569377
BP		 1567 mutex_unlock(&dp_mutex);
1568 err = -ENODEV;
c19e6535 1569
d6569377
BP		 1570exit_free:
1571 kfree_skb(skb);
1572exit:
1573 return err;
c19e6535
BP		 1574}
1575
1576static const struct nla_policy vport_policy[ODP_VPORT_ATTR_MAX + 1] = {
1577 [ODP_VPORT_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
1578 [ODP_VPORT_ATTR_PORT_NO] = { .type = NLA_U32 },
1579 [ODP_VPORT_ATTR_TYPE] = { .type = NLA_U32 },
1580 [ODP_VPORT_ATTR_STATS] = { .len = sizeof(struct rtnl_link_stats64) },
1581 [ODP_VPORT_ATTR_ADDRESS] = { .len = ETH_ALEN },
1582 [ODP_VPORT_ATTR_MTU] = { .type = NLA_U32 },
1583 [ODP_VPORT_ATTR_OPTIONS] = { .type = NLA_NESTED },
1584};
1585
1586static int copy_vport_to_user(void __user *dst, struct vport *vport, uint32_t total_len)
064af421 1587{
c19e6535
BP		 1588 struct odp_vport *odp_vport;
1589 struct sk_buff *skb;
1590 struct nlattr *nla;
1591 int ifindex, iflink;
1592 int err;
1593
1594 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1595 err = -ENOMEM;
1596 if (!skb)
1597 goto exit;
1598
f2459fe7 1599 rcu_read_lock();
c19e6535
BP		 1600 odp_vport = (struct odp_vport*)__skb_put(skb, sizeof(struct odp_vport));
1601 odp_vport->dp_idx = vport->dp->dp_idx;
1602 odp_vport->total_len = total_len;
1603
1604 NLA_PUT_U32(skb, ODP_VPORT_ATTR_PORT_NO, vport->port_no);
1605 NLA_PUT_U32(skb, ODP_VPORT_ATTR_TYPE, vport_get_type(vport));
1606 NLA_PUT_STRING(skb, ODP_VPORT_ATTR_NAME, vport_get_name(vport));
1607
1608 nla = nla_reserve(skb, ODP_VPORT_ATTR_STATS, sizeof(struct rtnl_link_stats64));
1609 if (!nla)
1610 goto nla_put_failure;
1611 if (vport_get_stats(vport, nla_data(nla)))
1612 __skb_trim(skb, skb->len - nla->nla_len);
1613
1614 NLA_PUT(skb, ODP_VPORT_ATTR_ADDRESS, ETH_ALEN, vport_get_addr(vport));
1615
1616 NLA_PUT_U32(skb, ODP_VPORT_ATTR_MTU, vport_get_mtu(vport));
1617
1618 err = vport_get_options(vport, skb);
1619
1620 ifindex = vport_get_ifindex(vport);
1621 if (ifindex > 0)
1622 NLA_PUT_U32(skb, ODP_VPORT_ATTR_IFINDEX, ifindex);
1623
1624 iflink = vport_get_iflink(vport);
1625 if (iflink > 0)
1626 NLA_PUT_U32(skb, ODP_VPORT_ATTR_IFLINK, iflink);
1627
1628 err = -EMSGSIZE;
1629 if (skb->len > total_len)
1630 goto exit_unlock;
1631
1632 odp_vport->len = skb->len;
1633 err = copy_to_user(dst, skb->data, skb->len) ? -EFAULT : 0;
1634 goto exit_unlock;
1635
1636nla_put_failure:
1637 err = -EMSGSIZE;
1638exit_unlock:
f2459fe7 1639 rcu_read_unlock();
c19e6535
BP		 1640 kfree_skb(skb);
1641exit:
1642 return err;
064af421
BP		 1643}
1644
c19e6535
BP		 1645static struct sk_buff *copy_vport_from_user(struct odp_vport __user *uodp_vport,
1646 struct nlattr *a[ODP_VPORT_ATTR_MAX + 1])
064af421 1647{
c19e6535
BP		 1648 struct odp_vport *odp_vport;
1649 struct sk_buff *skb;
1650 u32 len;
1651 int err;
064af421 1652
c19e6535
BP		 1653 if (get_user(len, &uodp_vport->len))
1654 return ERR_PTR(-EFAULT);
1655 if (len < sizeof(struct odp_vport))
1656 return ERR_PTR(-EINVAL);
1657
1658 skb = alloc_skb(len, GFP_KERNEL);
1659 if (!skb)
1660 return ERR_PTR(-ENOMEM);
1661
1662 err = -EFAULT;
1663 if (copy_from_user(__skb_put(skb, len), uodp_vport, len))
1664 goto error_free_skb;
f2459fe7 1665
c19e6535
BP		 1666 odp_vport = (struct odp_vport *)skb->data;
1667 err = -EINVAL;
1668 if (odp_vport->len != len)
1669 goto error_free_skb;
51d4d598 1670
c19e6535
BP		 1671 err = nla_parse(a, ODP_VPORT_ATTR_MAX, (struct nlattr *)(skb->data + sizeof(struct odp_vport)),
1672 skb->len - sizeof(struct odp_vport), vport_policy);
1673 if (err)
1674 goto error_free_skb;
064af421 1675
c19e6535
BP		 1676 err = VERIFY_NUL_STRING(a[ODP_VPORT_ATTR_NAME], IFNAMSIZ - 1);
1677 if (err)
1678 goto error_free_skb;
f2459fe7 1679
c19e6535
BP		 1680 return skb;
1681
1682error_free_skb:
1683 kfree_skb(skb);
1684 return ERR_PTR(err);
1685}
51d4d598 1686
c19e6535
BP		 1687
1688/* Called without any locks (or with RTNL lock).
1689 * Returns holding vport->dp->mutex.
1690 */
1691static struct vport *lookup_vport(struct odp_vport *odp_vport,
1692 struct nlattr *a[ODP_VPORT_ATTR_MAX + 1])
1693{
1694 struct datapath *dp;
1695 struct vport *vport;
1696
1697 if (a[ODP_VPORT_ATTR_NAME]) {
1698 int dp_idx, port_no;
1699
1700 retry:
1701 vport_lock();
1702 vport = vport_locate(nla_data(a[ODP_VPORT_ATTR_NAME]));
1703 if (!vport) {
1704 vport_unlock();
1705 return ERR_PTR(-ENODEV);
1706 }
1707 dp_idx = vport->dp->dp_idx;
1708 port_no = vport->port_no;
1709 vport_unlock();
7e71ab66 1710
51d4d598
BP		 1711 dp = get_dp_locked(dp_idx);
1712 if (!dp)
c19e6535 1713 goto retry;
51d4d598 1714
c19e6535
BP		 1715 vport = get_vport_protected(dp, port_no);
1716 if (!vport ||
1717 strcmp(vport_get_name(vport), nla_data(a[ODP_VPORT_ATTR_NAME]))) {
1718 mutex_unlock(&dp->mutex);
1719 goto retry;
1720 }
51d4d598 1721
c19e6535
BP		 1722 return vport;
1723 } else if (a[ODP_VPORT_ATTR_PORT_NO]) {
1724 u32 port_no = nla_get_u32(a[ODP_VPORT_ATTR_PORT_NO]);
1725
1726 if (port_no >= DP_MAX_PORTS)
1727 return ERR_PTR(-EINVAL);
1728
1729 dp = get_dp_locked(odp_vport->dp_idx);
1730 if (!dp)
1731 return ERR_PTR(-ENODEV);
f2459fe7 1732
c19e6535
BP		 1733 vport = get_vport_protected(dp, port_no);
1734 if (!vport) {
1735 mutex_unlock(&dp->mutex);
1736 return ERR_PTR(-ENOENT);
1737 }
1738 return vport;
1739 } else
1740 return ERR_PTR(-EINVAL);
064af421
BP		 1741}
1742
c19e6535 1743static int change_vport(struct vport *vport, struct nlattr *a[ODP_VPORT_ATTR_MAX + 1])
064af421 1744{
c19e6535
BP		 1745 int err = 0;
1746 if (a[ODP_VPORT_ATTR_STATS])
1747 err = vport_set_stats(vport, nla_data(a[ODP_VPORT_ATTR_STATS]));
1748 if (!err && a[ODP_VPORT_ATTR_ADDRESS])
1749 err = vport_set_addr(vport, nla_data(a[ODP_VPORT_ATTR_ADDRESS]));
1750 if (!err && a[ODP_VPORT_ATTR_MTU])
1751 err = vport_set_mtu(vport, nla_get_u32(a[ODP_VPORT_ATTR_MTU]));
1752 return err;
1753}
1754
1755static int attach_vport(struct odp_vport __user *uodp_vport)
1756{
1757 struct nlattr *a[ODP_VPORT_ATTR_MAX + 1];
1758 struct odp_vport *odp_vport;
1759 struct vport_parms parms;
1760 struct vport *vport;
1761 struct sk_buff *skb;
1762 struct datapath *dp;
b0ec0f27 1763 u32 port_no;
c19e6535 1764 int err;
b0ec0f27 1765
c19e6535
BP		 1766 skb = copy_vport_from_user(uodp_vport, a);
1767 err = PTR_ERR(skb);
1768 if (IS_ERR(skb))
1769 goto exit;
1770 odp_vport = (struct odp_vport *)skb->data;
1771
1772 err = -EINVAL;
1773 if (!a[ODP_VPORT_ATTR_NAME] || !a[ODP_VPORT_ATTR_TYPE])
1774 goto exit_kfree_skb;
51d4d598 1775
c19e6535
BP		 1776 rtnl_lock();
1777
1778 dp = get_dp_locked(odp_vport->dp_idx);
1779 err = -ENODEV;
1780 if (!dp)
1781 goto exit_unlock_rtnl;
1782
1783 if (a[ODP_VPORT_ATTR_PORT_NO]) {
1784 port_no = nla_get_u32(a[ODP_VPORT_ATTR_PORT_NO]);
1785
1786 err = -EFBIG;
1787 if (port_no >= DP_MAX_PORTS)
1788 goto exit_unlock_dp;
1789
1790 vport = get_vport_protected(dp, port_no);
1791 err = -EBUSY;
1792 if (vport)
1793 goto exit_unlock_dp;
1794 } else {
1795 for (port_no = 1; ; port_no++) {
1796 if (port_no >= DP_MAX_PORTS) {
1797 err = -EFBIG;
1798 goto exit_unlock_dp;
1799 }
1800 vport = get_vport_protected(dp, port_no);
1801 if (!vport)
1802 break;
51d4d598 1803 }
064af421 1804 }
b0ec0f27 1805
c19e6535
BP		 1806 parms.name = nla_data(a[ODP_VPORT_ATTR_NAME]);
1807 parms.type = nla_get_u32(a[ODP_VPORT_ATTR_TYPE]);
1808 parms.options = a[ODP_VPORT_ATTR_OPTIONS];
1809 parms.dp = dp;
1810 parms.port_no = port_no;
1811
1812 vport = new_vport(&parms);
1813 err = PTR_ERR(vport);
1814 if (IS_ERR(vport))
1815 goto exit_unlock_dp;
1816
1817 set_internal_devs_mtu(dp);
1818 dp_sysfs_add_if(vport);
1819
1820 err = change_vport(vport, a);
1821 if (err) {
1822 dp_detach_port(vport);
1823 goto exit_unlock_dp;
1824 }
1825
1826 err = copy_vport_to_user(uodp_vport, vport, odp_vport->total_len);
1827
1828exit_unlock_dp:
1829 mutex_unlock(&dp->mutex);
1830exit_unlock_rtnl:
1831 rtnl_unlock();
1832exit_kfree_skb:
1833 kfree_skb(skb);
1834exit:
1835 return err;
44e05eca
BP		 1836}
1837
c19e6535 1838static int set_vport(unsigned int cmd, struct odp_vport __user *uodp_vport)
44e05eca 1839{
c19e6535
BP		 1840 struct nlattr *a[ODP_VPORT_ATTR_MAX + 1];
1841 struct vport *vport;
1842 struct sk_buff *skb;
1843 int err;
44e05eca 1844
c19e6535
BP		 1845 skb = copy_vport_from_user(uodp_vport, a);
1846 err = PTR_ERR(skb);
1847 if (IS_ERR(skb))
1848 goto exit;
1849
1850 rtnl_lock();
1851 vport = lookup_vport((struct odp_vport *)skb->data, a);
1852 err = PTR_ERR(vport);
1853 if (IS_ERR(vport))
1854 goto exit_free;
44e05eca 1855
c19e6535
BP		 1856 err = 0;
1857 if (a[ODP_VPORT_ATTR_OPTIONS])
1858 err = vport_set_options(vport, a[ODP_VPORT_ATTR_OPTIONS]);
1859 if (!err)
1860 err = change_vport(vport, a);
1861
1862 mutex_unlock(&vport->dp->mutex);
1863exit_free:
1864 kfree_skb(skb);
1865 rtnl_unlock();
1866exit:
1867 return err;
064af421
BP		 1868}
1869
c19e6535 1870static int del_vport(unsigned int cmd, struct odp_vport __user *uodp_vport)
7c40efc9 1871{
c19e6535
BP		 1872 struct nlattr *a[ODP_VPORT_ATTR_MAX + 1];
1873 struct datapath *dp;
1874 struct vport *vport;
1875 struct sk_buff *skb;
1876 int err;
1877
1878 skb = copy_vport_from_user(uodp_vport, a);
1879 err = PTR_ERR(skb);
1880 if (IS_ERR(skb))
1881 goto exit;
1882
1883 rtnl_lock();
1884 vport = lookup_vport((struct odp_vport *)skb->data, a);
1885 err = PTR_ERR(vport);
1886 if (IS_ERR(vport))
1887 goto exit_free;
1888 dp = vport->dp;
1889
1890 err = -EINVAL;
1891 if (vport->port_no == ODPP_LOCAL)
1892 goto exit_free;
1893
1894 err = dp_detach_port(vport);
1895 mutex_unlock(&dp->mutex);
1896exit_free:
1897 kfree_skb(skb);
1898 rtnl_unlock();
1899exit:
1900 return err;
7c40efc9
BP		 1901}
1902
c19e6535 1903static int get_vport(struct odp_vport __user *uodp_vport)
7c40efc9 1904{
c19e6535
BP		 1905 struct nlattr *a[ODP_VPORT_ATTR_MAX + 1];
1906 struct odp_vport *odp_vport;
1907 struct vport *vport;
1908 struct sk_buff *skb;
1909 int err;
1910
1911 skb = copy_vport_from_user(uodp_vport, a);
1912 err = PTR_ERR(skb);
1913 if (IS_ERR(skb))
1914 goto exit;
1915 odp_vport = (struct odp_vport *)skb->data;
1916
1917 vport = lookup_vport(odp_vport, a);
1918 err = PTR_ERR(vport);
1919 if (IS_ERR(vport))
1920 goto exit_free;
1921
1922 err = copy_vport_to_user(uodp_vport, vport, odp_vport->total_len);
1923 mutex_unlock(&vport->dp->mutex);
1924exit_free:
1925 kfree_skb(skb);
1926exit:
1927 return err;
1928}
1929
1930static int dump_vport(struct odp_vport __user *uodp_vport)
1931{
1932 struct nlattr *a[ODP_VPORT_ATTR_MAX + 1];
1933 struct odp_vport *odp_vport;
1934 struct sk_buff *skb;
1935 struct datapath *dp;
1936 u32 port_no;
1937 int err;
1938
1939 skb = copy_vport_from_user(uodp_vport, a);
1940 err = PTR_ERR(skb);
1941 if (IS_ERR(skb))
1942 goto exit;
1943 odp_vport = (struct odp_vport *)skb->data;
1944
1945 dp = get_dp_locked(odp_vport->dp_idx);
1946 err = -ENODEV;
1947 if (!dp)
1948 goto exit_free;
1949
1950 port_no = 0;
1951 if (a[ODP_VPORT_ATTR_PORT_NO])
1952 port_no = nla_get_u32(a[ODP_VPORT_ATTR_PORT_NO]);
1953 for (; port_no < DP_MAX_PORTS; port_no++) {
1954 struct vport *vport = get_vport_protected(dp, port_no);
1955 if (vport) {
1956 err = copy_vport_to_user(uodp_vport, vport, odp_vport->total_len);
1957 goto exit_unlock_dp;
1958 }
1959 }
1960 err = -ENODEV;
1961
1962exit_unlock_dp:
1963 mutex_unlock(&dp->mutex);
1964exit_free:
1965 kfree_skb(skb);
1966exit:
1967 return err;
7c40efc9
BP		 1968}
1969
064af421
BP		 1970static long openvswitch_ioctl(struct file *f, unsigned int cmd,
1971 unsigned long argp)
1972{
1973 int dp_idx = iminor(f->f_dentry->d_inode);
1974 struct datapath *dp;
d6569377 1975 int listeners;
064af421
BP		 1976 int err;
1977
1978 /* Handle commands with special locking requirements up front. */
1979 switch (cmd) {
d6569377
BP		 1980 case ODP_DP_NEW:
1981 err = new_datapath((struct odp_datapath __user *)argp);
e86c8696 1982 goto exit;
064af421 1983
d6569377
BP		 1984 case ODP_DP_GET:
1985 err = get_datapath((struct odp_datapath __user *)argp);
1986 goto exit;
1987
1988 case ODP_DP_DEL:
1989 err = del_datapath((struct odp_datapath __user *)argp);
1990 goto exit;
1991
1992 case ODP_DP_SET:
1993 err = set_datapath((struct odp_datapath __user *)argp);
1994 goto exit;
1995
1996 case ODP_DP_DUMP:
1997 err = dump_datapath((struct odp_datapath __user *)argp);
e86c8696 1998 goto exit;
064af421 1999
c19e6535
BP		 2000 case ODP_VPORT_NEW:
2001 err = attach_vport((struct odp_vport __user *)argp);
e86c8696 2002 goto exit;
064af421 2003
c19e6535
BP		 2004 case ODP_VPORT_GET:
2005 err = get_vport((struct odp_vport __user *)argp);
f2459fe7
JG		 2006 goto exit;
2007
c19e6535
BP		 2008 case ODP_VPORT_DEL:
2009 err = del_vport(cmd, (struct odp_vport __user *)argp);
780e6207
JG		 2010 goto exit;
2011
c19e6535
BP		 2012 case ODP_VPORT_SET:
2013 err = set_vport(cmd, (struct odp_vport __user *)argp);
f2459fe7
JG		 2014 goto exit;
2015
c19e6535
BP		 2016 case ODP_VPORT_DUMP:
2017 err = dump_vport((struct odp_vport __user *)argp);
e86c8696 2018 goto exit;
9c52546b
BP		 2019
2020 case ODP_FLOW_FLUSH:
2021 err = flush_flows(argp);
2022 goto exit;
2023
d6569377
BP		 2024 case ODP_FLOW_NEW:
2025 case ODP_FLOW_SET:
2026 err = new_flow(cmd, (struct odp_flow __user *)argp);
9c52546b
BP		 2027 goto exit;
2028
2029 case ODP_FLOW_GET:
d6569377
BP		 2030 case ODP_FLOW_DEL:
2031 err = get_or_del_flow(cmd, (struct odp_flow __user *)argp);
9c52546b
BP		 2032 goto exit;
2033
2034 case ODP_FLOW_DUMP:
d6569377 2035 err = dump_flow((struct odp_flow __user *)argp);
9c52546b
BP		 2036 goto exit;
2037
2038 case ODP_EXECUTE:
2039 err = execute_packet((struct odp_execute __user *)argp);
2040 goto exit;
064af421
BP		 2041 }
2042
2043 dp = get_dp_locked(dp_idx);
e86c8696 2044 err = -ENODEV;
064af421 2045 if (!dp)
e86c8696 2046 goto exit;
064af421
BP		 2047
2048 switch (cmd) {
064af421 2049 case ODP_GET_LISTEN_MASK:
7c40efc9 2050 err = put_user(get_listen_mask(f), (int __user *)argp);
064af421
BP		 2051 break;
2052
2053 case ODP_SET_LISTEN_MASK:
2054 err = get_user(listeners, (int __user *)argp);
2055 if (err)
2056 break;
2057 err = -EINVAL;
2058 if (listeners & ~ODPL_ALL)
2059 break;
2060 err = 0;
7c40efc9 2061 set_listen_mask(f, listeners);
064af421
BP		 2062 break;
2063
064af421
BP		 2064 default:
2065 err = -ENOIOCTLCMD;
2066 break;
2067 }
2068 mutex_unlock(&dp->mutex);
e86c8696 2069exit:
064af421
BP		 2070 return err;
2071}
2072
2073static int dp_has_packet_of_interest(struct datapath *dp, int listeners)
2074{
2075 int i;
2076 for (i = 0; i < DP_N_QUEUES; i++) {
2077 if (listeners & (1 << i) && !skb_queue_empty(&dp->queues[i]))
2078 return 1;
2079 }
2080 return 0;
2081}
2082
3fbd517a 2083#ifdef CONFIG_COMPAT
9c52546b 2084static int compat_execute(const struct compat_odp_execute __user *uexecute)
3fbd517a
BP		 2085{
2086 struct odp_execute execute;
2087 compat_uptr_t actions;
2088 compat_uptr_t data;
9c52546b
BP		 2089 struct datapath *dp;
2090 int error;
3fbd517a
BP		 2091
2092 if (!access_ok(VERIFY_READ, uexecute, sizeof(struct compat_odp_execute)) ||
9c52546b 2093 __get_user(execute.dp_idx, &uexecute->dp_idx) ||
3fbd517a 2094 __get_user(actions, &uexecute->actions) ||
cdee00fd 2095 __get_user(execute.actions_len, &uexecute->actions_len) ||
3fbd517a
BP		 2096 __get_user(data, &uexecute->data) ||
2097 __get_user(execute.length, &uexecute->length))
2098 return -EFAULT;
2099
1b29ebe5
JG		 2100 execute.actions = (struct nlattr __force *)compat_ptr(actions);
2101 execute.data = (const void __force *)compat_ptr(data);
3fbd517a 2102
9c52546b
BP		 2103 dp = get_dp_locked(execute.dp_idx);
2104 if (!dp)
2105 return -ENODEV;
2106 error = do_execute(dp, &execute);
2107 mutex_unlock(&dp->mutex);
2108
2109 return error;
3fbd517a
BP		 2110}
2111
2112static long openvswitch_compat_ioctl(struct file *f, unsigned int cmd, unsigned long argp)
2113{
3fbd517a 2114 switch (cmd) {
3fbd517a
BP		 2115 case ODP_FLOW_FLUSH:
2116 /* Ioctls that don't need any translation at all. */
2117 return openvswitch_ioctl(f, cmd, argp);
2118
d6569377
BP		 2119 case ODP_DP_NEW:
2120 case ODP_DP_GET:
2121 case ODP_DP_DEL:
2122 case ODP_DP_SET:
2123 case ODP_DP_DUMP:
c19e6535
BP		 2124 case ODP_VPORT_NEW:
2125 case ODP_VPORT_DEL:
2126 case ODP_VPORT_GET:
2127 case ODP_VPORT_SET:
2128 case ODP_VPORT_DUMP:
d6569377
BP		 2129 case ODP_FLOW_NEW:
2130 case ODP_FLOW_DEL:
2131 case ODP_FLOW_GET:
2132 case ODP_FLOW_SET:
2133 case ODP_FLOW_DUMP:
3fbd517a
BP		 2134 case ODP_SET_LISTEN_MASK:
2135 case ODP_GET_LISTEN_MASK:
3fbd517a
BP		 2136 /* Ioctls that just need their pointer argument extended. */
2137 return openvswitch_ioctl(f, cmd, (unsigned long)compat_ptr(argp));
3fbd517a 2138
3fbd517a 2139 case ODP_EXECUTE32:
9c52546b 2140 return compat_execute(compat_ptr(argp));
3fbd517a
BP		 2141
2142 default:
9c52546b 2143 return -ENOIOCTLCMD;
3fbd517a 2144 }
3fbd517a
BP		 2145}
2146#endif
2147
33b38b63
JG		 2148static ssize_t openvswitch_read(struct file *f, char __user *buf,
2149 size_t nbytes, loff_t *ppos)
064af421 2150{
7c40efc9 2151 int listeners = get_listen_mask(f);
064af421 2152 int dp_idx = iminor(f->f_dentry->d_inode);
e22d4953 2153 struct datapath *dp = get_dp_locked(dp_idx);
064af421 2154 struct sk_buff *skb;
856081f6 2155 struct iovec iov;
064af421
BP		 2156 int retval;
2157
2158 if (!dp)
2159 return -ENODEV;
2160
2161 if (nbytes == 0 || !listeners)
2162 return 0;
2163
2164 for (;;) {
2165 int i;
2166
2167 for (i = 0; i < DP_N_QUEUES; i++) {
2168 if (listeners & (1 << i)) {
2169 skb = skb_dequeue(&dp->queues[i]);
2170 if (skb)
2171 goto success;
2172 }
2173 }
2174
2175 if (f->f_flags & O_NONBLOCK) {
2176 retval = -EAGAIN;
2177 goto error;
2178 }
2179
2180 wait_event_interruptible(dp->waitqueue,
2181 dp_has_packet_of_interest(dp,
2182 listeners));
2183
2184 if (signal_pending(current)) {
2185 retval = -ERESTARTSYS;
2186 goto error;
2187 }
2188 }
2189success:
e22d4953
JG		 2190 mutex_unlock(&dp->mutex);
2191
856081f6
BP		 2192 iov.iov_base = buf;
2193 iov.iov_len = min_t(size_t, skb->len, nbytes);
2194 retval = skb_copy_datagram_iovec(skb, 0, &iov, iov.iov_len);
064af421 2195 if (!retval)
856081f6 2196 retval = skb->len;
9cc8b4e4 2197
064af421 2198 kfree_skb(skb);
e22d4953 2199 return retval;
064af421
BP		 2200
2201error:
e22d4953 2202 mutex_unlock(&dp->mutex);
064af421
BP		 2203 return retval;
2204}
2205
2206static unsigned int openvswitch_poll(struct file *file, poll_table *wait)
2207{
2208 int dp_idx = iminor(file->f_dentry->d_inode);
e22d4953 2209 struct datapath *dp = get_dp_locked(dp_idx);
064af421
BP		 2210 unsigned int mask;
2211
2212 if (dp) {
2213 mask = 0;
2214 poll_wait(file, &dp->waitqueue, wait);
7c40efc9 2215 if (dp_has_packet_of_interest(dp, get_listen_mask(file)))
064af421 2216 mask |= POLLIN | POLLRDNORM;
e22d4953 2217 mutex_unlock(&dp->mutex);
064af421
BP		 2218 } else {
2219 mask = POLLIN | POLLRDNORM | POLLHUP;
2220 }
2221 return mask;
2222}
2223
33b38b63 2224static struct file_operations openvswitch_fops = {
609af740 2225 .owner = THIS_MODULE,
064af421
BP		 2226 .read = openvswitch_read,
2227 .poll = openvswitch_poll,
2228 .unlocked_ioctl = openvswitch_ioctl,
3fbd517a
BP		 2229#ifdef CONFIG_COMPAT
2230 .compat_ioctl = openvswitch_compat_ioctl,
2231#endif
064af421
BP		 2232};
2233
2234static int major;
22d24ebf 2235
22d24ebf
BP		 2236static int __init dp_init(void)
2237{
f2459fe7 2238 struct sk_buff *dummy_skb;
22d24ebf
BP		 2239 int err;
2240
f2459fe7 2241 BUILD_BUG_ON(sizeof(struct ovs_skb_cb) > sizeof(dummy_skb->cb));
22d24ebf 2242
f2459fe7 2243 printk("Open vSwitch %s, built "__DATE__" "__TIME__"\n", VERSION BUILDNR);
064af421
BP		 2244
2245 err = flow_init();
2246 if (err)
2247 goto error;
2248
f2459fe7 2249 err = vport_init();
064af421
BP		 2250 if (err)
2251 goto error_flow_exit;
2252
f2459fe7
JG		 2253 err = register_netdevice_notifier(&dp_device_notifier);
2254 if (err)
2255 goto error_vport_exit;
2256
064af421
BP		 2257 major = register_chrdev(0, "openvswitch", &openvswitch_fops);
2258 if (err < 0)
2259 goto error_unreg_notifier;
2260
064af421
BP		 2261 return 0;
2262
2263error_unreg_notifier:
2264 unregister_netdevice_notifier(&dp_device_notifier);
f2459fe7
JG		 2265error_vport_exit:
2266 vport_exit();
064af421
BP		 2267error_flow_exit:
2268 flow_exit();
2269error:
2270 return err;
2271}
2272
2273static void dp_cleanup(void)
2274{
2275 rcu_barrier();
2276 unregister_chrdev(major, "openvswitch");
2277 unregister_netdevice_notifier(&dp_device_notifier);
f2459fe7 2278 vport_exit();
064af421 2279 flow_exit();
064af421
BP		 2280}
2281
2282module_init(dp_init);
2283module_exit(dp_cleanup);
2284
2285MODULE_DESCRIPTION("Open vSwitch switching datapath");
2286MODULE_LICENSE("GPL");
openvswitch mirror