[mirror_ubuntu-jammy-kernel.git] / net / xdp / xsk.c

// SPDX-License-Identifier: GPL-2.0
/* XDP sockets
 *
 * AF_XDP sockets allows a channel between XDP programs and userspace
 * applications.
 * Copyright(c) 2018 Intel Corporation.
 *
 * Author(s): Björn Töpel <bjorn.topel@intel.com>
 *	      Magnus Karlsson <magnus.karlsson@intel.com>
 */

#define pr_fmt(fmt) "AF_XDP: %s: " fmt, __func__

#include <linux/if_xdp.h>
#include <linux/init.h>
#include <linux/sched/mm.h>
#include <linux/sched/signal.h>
#include <linux/sched/task.h>
#include <linux/socket.h>
#include <linux/file.h>
#include <linux/uaccess.h>
#include <linux/net.h>
#include <linux/netdevice.h>
#include <linux/rculist.h>
#include <net/xdp_sock.h>
#include <net/xdp.h>

#include "xsk_queue.h"
#include "xdp_umem.h"

#define TX_BATCH_SIZE 16

static struct xdp_sock *xdp_sk(struct sock *sk)
{
	return (struct xdp_sock *)sk;
}

bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs)
{
	return READ_ONCE(xs->rx) &&  READ_ONCE(xs->umem) &&
		READ_ONCE(xs->umem->fq);
}

u64 *xsk_umem_peek_addr(struct xdp_umem *umem, u64 *addr)
{
	return xskq_peek_addr(umem->fq, addr);
}
EXPORT_SYMBOL(xsk_umem_peek_addr);

void xsk_umem_discard_addr(struct xdp_umem *umem)
{
	xskq_discard_addr(umem->fq);
}
EXPORT_SYMBOL(xsk_umem_discard_addr);

static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
{
	void *to_buf, *from_buf;
	u32 metalen;
	u64 addr;
	int err;

	if (!xskq_peek_addr(xs->umem->fq, &addr) ||
	    len > xs->umem->chunk_size_nohr - XDP_PACKET_HEADROOM) {
		xs->rx_dropped++;
		return -ENOSPC;
	}

	addr += xs->umem->headroom;

	if (unlikely(xdp_data_meta_unsupported(xdp))) {
		from_buf = xdp->data;
		metalen = 0;
	} else {
		from_buf = xdp->data_meta;
		metalen = xdp->data - xdp->data_meta;
	}

	to_buf = xdp_umem_get_data(xs->umem, addr);
	memcpy(to_buf, from_buf, len + metalen);
	addr += metalen;
	err = xskq_produce_batch_desc(xs->rx, addr, len);
	if (!err) {
		xskq_discard_addr(xs->umem->fq);
		xdp_return_buff(xdp);
		return 0;
	}

	xs->rx_dropped++;
	return err;
}

static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
{
	int err = xskq_produce_batch_desc(xs->rx, (u64)xdp->handle, len);

	if (err)
		xs->rx_dropped++;

	return err;
}

int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
{
	u32 len;

	if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index)
		return -EINVAL;

	len = xdp->data_end - xdp->data;

	return (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY) ?
		__xsk_rcv_zc(xs, xdp, len) : __xsk_rcv(xs, xdp, len);
}

void xsk_flush(struct xdp_sock *xs)
{
	xskq_produce_flush_desc(xs->rx);
	xs->sk.sk_data_ready(&xs->sk);
}

int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
{
	u32 metalen = xdp->data - xdp->data_meta;
	u32 len = xdp->data_end - xdp->data;
	void *buffer;
	u64 addr;
	int err;

	if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index)
		return -EINVAL;

	if (!xskq_peek_addr(xs->umem->fq, &addr) ||
	    len > xs->umem->chunk_size_nohr - XDP_PACKET_HEADROOM) {
		xs->rx_dropped++;
		return -ENOSPC;
	}

	addr += xs->umem->headroom;

	buffer = xdp_umem_get_data(xs->umem, addr);
	memcpy(buffer, xdp->data_meta, len + metalen);
	addr += metalen;
	err = xskq_produce_batch_desc(xs->rx, addr, len);
	if (!err) {
		xskq_discard_addr(xs->umem->fq);
		xsk_flush(xs);
		return 0;
	}

	xs->rx_dropped++;
	return err;
}

void xsk_umem_complete_tx(struct xdp_umem *umem, u32 nb_entries)
{
	xskq_produce_flush_addr_n(umem->cq, nb_entries);
}
EXPORT_SYMBOL(xsk_umem_complete_tx);

void xsk_umem_consume_tx_done(struct xdp_umem *umem)
{
	struct xdp_sock *xs;

	rcu_read_lock();
	list_for_each_entry_rcu(xs, &umem->xsk_list, list) {
		xs->sk.sk_write_space(&xs->sk);
	}
	rcu_read_unlock();
}
EXPORT_SYMBOL(xsk_umem_consume_tx_done);

bool xsk_umem_consume_tx(struct xdp_umem *umem, dma_addr_t *dma, u32 *len)
{
	struct xdp_desc desc;
	struct xdp_sock *xs;

	rcu_read_lock();
	list_for_each_entry_rcu(xs, &umem->xsk_list, list) {
		if (!xskq_peek_desc(xs->tx, &desc))
			continue;

		if (xskq_produce_addr_lazy(umem->cq, desc.addr))
			goto out;

		*dma = xdp_umem_get_dma(umem, desc.addr);
		*len = desc.len;

		xskq_discard_desc(xs->tx);
		rcu_read_unlock();
		return true;
	}

out:
	rcu_read_unlock();
	return false;
}
EXPORT_SYMBOL(xsk_umem_consume_tx);

static int xsk_zc_xmit(struct sock *sk)
{
	struct xdp_sock *xs = xdp_sk(sk);
	struct net_device *dev = xs->dev;

	return dev->netdev_ops->ndo_xsk_async_xmit(dev, xs->queue_id);
}

static void xsk_destruct_skb(struct sk_buff *skb)
{
	u64 addr = (u64)(long)skb_shinfo(skb)->destructor_arg;
	struct xdp_sock *xs = xdp_sk(skb->sk);
	unsigned long flags;

	spin_lock_irqsave(&xs->tx_completion_lock, flags);
	WARN_ON_ONCE(xskq_produce_addr(xs->umem->cq, addr));
	spin_unlock_irqrestore(&xs->tx_completion_lock, flags);

	sock_wfree(skb);
}

static int xsk_generic_xmit(struct sock *sk, struct msghdr *m,
			    size_t total_len)
{
	u32 max_batch = TX_BATCH_SIZE;
	struct xdp_sock *xs = xdp_sk(sk);
	bool sent_frame = false;
	struct xdp_desc desc;
	struct sk_buff *skb;
	int err = 0;

	mutex_lock(&xs->mutex);

	while (xskq_peek_desc(xs->tx, &desc)) {
		char *buffer;
		u64 addr;
		u32 len;

		if (max_batch-- == 0) {
			err = -EAGAIN;
			goto out;
		}

		if (xskq_reserve_addr(xs->umem->cq))
			goto out;

		if (xs->queue_id >= xs->dev->real_num_tx_queues)
			goto out;

		len = desc.len;
		skb = sock_alloc_send_skb(sk, len, 1, &err);
		if (unlikely(!skb)) {
			err = -EAGAIN;
			goto out;
		}

		skb_put(skb, len);
		addr = desc.addr;
		buffer = xdp_umem_get_data(xs->umem, addr);
		err = skb_store_bits(skb, 0, buffer, len);
		if (unlikely(err)) {
			kfree_skb(skb);
			goto out;
		}

		skb->dev = xs->dev;
		skb->priority = sk->sk_priority;
		skb->mark = sk->sk_mark;
		skb_shinfo(skb)->destructor_arg = (void *)(long)addr;
		skb->destructor = xsk_destruct_skb;

		err = dev_direct_xmit(skb, xs->queue_id);
		xskq_discard_desc(xs->tx);
		/* Ignore NET_XMIT_CN as packet might have been sent */
		if (err == NET_XMIT_DROP || err == NETDEV_TX_BUSY) {
			/* SKB completed but not sent */
			err = -EBUSY;
			goto out;
		}

		sent_frame = true;
	}

out:
	if (sent_frame)
		sk->sk_write_space(sk);

	mutex_unlock(&xs->mutex);
	return err;
}

static int xsk_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
{
	bool need_wait = !(m->msg_flags & MSG_DONTWAIT);
	struct sock *sk = sock->sk;
	struct xdp_sock *xs = xdp_sk(sk);

	if (unlikely(!xs->dev))
		return -ENXIO;
	if (unlikely(!(xs->dev->flags & IFF_UP)))
		return -ENETDOWN;
	if (unlikely(!xs->tx))
		return -ENOBUFS;
	if (need_wait)
		return -EOPNOTSUPP;

	return (xs->zc) ? xsk_zc_xmit(sk) : xsk_generic_xmit(sk, m, total_len);
}

static unsigned int xsk_poll(struct file *file, struct socket *sock,
			     struct poll_table_struct *wait)
{
	unsigned int mask = datagram_poll(file, sock, wait);
	struct sock *sk = sock->sk;
	struct xdp_sock *xs = xdp_sk(sk);

	if (xs->rx && !xskq_empty_desc(xs->rx))
		mask |= POLLIN | POLLRDNORM;
	if (xs->tx && !xskq_full_desc(xs->tx))
		mask |= POLLOUT | POLLWRNORM;

	return mask;
}

static int xsk_init_queue(u32 entries, struct xsk_queue **queue,
			  bool umem_queue)
{
	struct xsk_queue *q;

	if (entries == 0 || *queue || !is_power_of_2(entries))
		return -EINVAL;

	q = xskq_create(entries, umem_queue);
	if (!q)
		return -ENOMEM;

	/* Make sure queue is ready before it can be seen by others */
	smp_wmb();
	*queue = q;
	return 0;
}

static int xsk_release(struct socket *sock)
{
	struct sock *sk = sock->sk;
	struct xdp_sock *xs = xdp_sk(sk);
	struct net *net;

	if (!sk)
		return 0;

	net = sock_net(sk);

	local_bh_disable();
	sock_prot_inuse_add(net, sk->sk_prot, -1);
	local_bh_enable();

	if (xs->dev) {
		struct net_device *dev = xs->dev;

		/* Wait for driver to stop using the xdp socket. */
		xdp_del_sk_umem(xs->umem, xs);
		xs->dev = NULL;
		synchronize_net();
		dev_put(dev);
	}

	xskq_destroy(xs->rx);
	xskq_destroy(xs->tx);
	xdp_put_umem(xs->umem);

	sock_orphan(sk);
	sock->sk = NULL;

	sk_refcnt_debug_release(sk);
	sock_put(sk);

	return 0;
}

static struct socket *xsk_lookup_xsk_from_fd(int fd)
{
	struct socket *sock;
	int err;

	sock = sockfd_lookup(fd, &err);
	if (!sock)
		return ERR_PTR(-ENOTSOCK);

	if (sock->sk->sk_family != PF_XDP) {
		sockfd_put(sock);
		return ERR_PTR(-ENOPROTOOPT);
	}

	return sock;
}

static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len)
{
	struct sockaddr_xdp *sxdp = (struct sockaddr_xdp *)addr;
	struct sock *sk = sock->sk;
	struct xdp_sock *xs = xdp_sk(sk);
	struct net_device *dev;
	u32 flags, qid;
	int err = 0;

	if (addr_len < sizeof(struct sockaddr_xdp))
		return -EINVAL;
	if (sxdp->sxdp_family != AF_XDP)
		return -EINVAL;

	mutex_lock(&xs->mutex);
	if (xs->dev) {
		err = -EBUSY;
		goto out_release;
	}

	dev = dev_get_by_index(sock_net(sk), sxdp->sxdp_ifindex);
	if (!dev) {
		err = -ENODEV;
		goto out_release;
	}

	if (!xs->rx && !xs->tx) {
		err = -EINVAL;
		goto out_unlock;
	}

	qid = sxdp->sxdp_queue_id;
	flags = sxdp->sxdp_flags;

	if (flags & XDP_SHARED_UMEM) {
		struct xdp_sock *umem_xs;
		struct socket *sock;

		if ((flags & XDP_COPY) || (flags & XDP_ZEROCOPY)) {
			/* Cannot specify flags for shared sockets. */
			err = -EINVAL;
			goto out_unlock;
		}

		if (xs->umem) {
			/* We have already our own. */
			err = -EINVAL;
			goto out_unlock;
		}

		sock = xsk_lookup_xsk_from_fd(sxdp->sxdp_shared_umem_fd);
		if (IS_ERR(sock)) {
			err = PTR_ERR(sock);
			goto out_unlock;
		}

		umem_xs = xdp_sk(sock->sk);
		if (!umem_xs->umem) {
			/* No umem to inherit. */
			err = -EBADF;
			sockfd_put(sock);
			goto out_unlock;
		} else if (umem_xs->dev != dev || umem_xs->queue_id != qid) {
			err = -EINVAL;
			sockfd_put(sock);
			goto out_unlock;
		}

		xdp_get_umem(umem_xs->umem);
		xs->umem = umem_xs->umem;
		sockfd_put(sock);
	} else if (!xs->umem || !xdp_umem_validate_queues(xs->umem)) {
		err = -EINVAL;
		goto out_unlock;
	} else {
		/* This xsk has its own umem. */
		xskq_set_umem(xs->umem->fq, xs->umem->size,
			      xs->umem->chunk_mask);
		xskq_set_umem(xs->umem->cq, xs->umem->size,
			      xs->umem->chunk_mask);

		err = xdp_umem_assign_dev(xs->umem, dev, qid, flags);
		if (err)
			goto out_unlock;
	}

	xs->dev = dev;
	xs->zc = xs->umem->zc;
	xs->queue_id = qid;
	xskq_set_umem(xs->rx, xs->umem->size, xs->umem->chunk_mask);
	xskq_set_umem(xs->tx, xs->umem->size, xs->umem->chunk_mask);
	xdp_add_sk_umem(xs->umem, xs);

out_unlock:
	if (err)
		dev_put(dev);
out_release:
	mutex_unlock(&xs->mutex);
	return err;
}

static int xsk_setsockopt(struct socket *sock, int level, int optname,
			  char __user *optval, unsigned int optlen)
{
	struct sock *sk = sock->sk;
	struct xdp_sock *xs = xdp_sk(sk);
	int err;

	if (level != SOL_XDP)
		return -ENOPROTOOPT;

	switch (optname) {
	case XDP_RX_RING:
	case XDP_TX_RING:
	{
		struct xsk_queue **q;
		int entries;

		if (optlen < sizeof(entries))
			return -EINVAL;
		if (copy_from_user(&entries, optval, sizeof(entries)))
			return -EFAULT;

		mutex_lock(&xs->mutex);
		q = (optname == XDP_TX_RING) ? &xs->tx : &xs->rx;
		err = xsk_init_queue(entries, q, false);
		mutex_unlock(&xs->mutex);
		return err;
	}
	case XDP_UMEM_REG:
	{
		struct xdp_umem_reg mr;
		struct xdp_umem *umem;

		if (copy_from_user(&mr, optval, sizeof(mr)))
			return -EFAULT;

		mutex_lock(&xs->mutex);
		if (xs->umem) {
			mutex_unlock(&xs->mutex);
			return -EBUSY;
		}

		umem = xdp_umem_create(&mr);
		if (IS_ERR(umem)) {
			mutex_unlock(&xs->mutex);
			return PTR_ERR(umem);
		}

		/* Make sure umem is ready before it can be seen by others */
		smp_wmb();
		xs->umem = umem;
		mutex_unlock(&xs->mutex);
		return 0;
	}
	case XDP_UMEM_FILL_RING:
	case XDP_UMEM_COMPLETION_RING:
	{
		struct xsk_queue **q;
		int entries;

		if (copy_from_user(&entries, optval, sizeof(entries)))
			return -EFAULT;

		mutex_lock(&xs->mutex);
		if (!xs->umem) {
			mutex_unlock(&xs->mutex);
			return -EINVAL;
		}

		q = (optname == XDP_UMEM_FILL_RING) ? &xs->umem->fq :
			&xs->umem->cq;
		err = xsk_init_queue(entries, q, true);
		mutex_unlock(&xs->mutex);
		return err;
	}
	default:
		break;
	}

	return -ENOPROTOOPT;
}

static int xsk_getsockopt(struct socket *sock, int level, int optname,
			  char __user *optval, int __user *optlen)
{
	struct sock *sk = sock->sk;
	struct xdp_sock *xs = xdp_sk(sk);
	int len;

	if (level != SOL_XDP)
		return -ENOPROTOOPT;

	if (get_user(len, optlen))
		return -EFAULT;
	if (len < 0)
		return -EINVAL;

	switch (optname) {
	case XDP_STATISTICS:
	{
		struct xdp_statistics stats;

		if (len < sizeof(stats))
			return -EINVAL;

		mutex_lock(&xs->mutex);
		stats.rx_dropped = xs->rx_dropped;
		stats.rx_invalid_descs = xskq_nb_invalid_descs(xs->rx);
		stats.tx_invalid_descs = xskq_nb_invalid_descs(xs->tx);
		mutex_unlock(&xs->mutex);

		if (copy_to_user(optval, &stats, sizeof(stats)))
			return -EFAULT;
		if (put_user(sizeof(stats), optlen))
			return -EFAULT;

		return 0;
	}
	case XDP_MMAP_OFFSETS:
	{
		struct xdp_mmap_offsets off;

		if (len < sizeof(off))
			return -EINVAL;

		off.rx.producer = offsetof(struct xdp_rxtx_ring, ptrs.producer);
		off.rx.consumer = offsetof(struct xdp_rxtx_ring, ptrs.consumer);
		off.rx.desc	= offsetof(struct xdp_rxtx_ring, desc);
		off.tx.producer = offsetof(struct xdp_rxtx_ring, ptrs.producer);
		off.tx.consumer = offsetof(struct xdp_rxtx_ring, ptrs.consumer);
		off.tx.desc	= offsetof(struct xdp_rxtx_ring, desc);

		off.fr.producer = offsetof(struct xdp_umem_ring, ptrs.producer);
		off.fr.consumer = offsetof(struct xdp_umem_ring, ptrs.consumer);
		off.fr.desc	= offsetof(struct xdp_umem_ring, desc);
		off.cr.producer = offsetof(struct xdp_umem_ring, ptrs.producer);
		off.cr.consumer = offsetof(struct xdp_umem_ring, ptrs.consumer);
		off.cr.desc	= offsetof(struct xdp_umem_ring, desc);

		len = sizeof(off);
		if (copy_to_user(optval, &off, len))
			return -EFAULT;
		if (put_user(len, optlen))
			return -EFAULT;

		return 0;
	}
	default:
		break;
	}

	return -EOPNOTSUPP;
}

static int xsk_mmap(struct file *file, struct socket *sock,
		    struct vm_area_struct *vma)
{
	loff_t offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT;
	unsigned long size = vma->vm_end - vma->vm_start;
	struct xdp_sock *xs = xdp_sk(sock->sk);
	struct xsk_queue *q = NULL;
	struct xdp_umem *umem;
	unsigned long pfn;
	struct page *qpg;

	if (offset == XDP_PGOFF_RX_RING) {
		q = READ_ONCE(xs->rx);
	} else if (offset == XDP_PGOFF_TX_RING) {
		q = READ_ONCE(xs->tx);
	} else {
		umem = READ_ONCE(xs->umem);
		if (!umem)
			return -EINVAL;

		if (offset == XDP_UMEM_PGOFF_FILL_RING)
			q = READ_ONCE(umem->fq);
		else if (offset == XDP_UMEM_PGOFF_COMPLETION_RING)
			q = READ_ONCE(umem->cq);
	}

	if (!q)
		return -EINVAL;

	qpg = virt_to_head_page(q->ring);
	if (size > (PAGE_SIZE << compound_order(qpg)))
		return -EINVAL;

	pfn = virt_to_phys(q->ring) >> PAGE_SHIFT;
	return remap_pfn_range(vma, vma->vm_start, pfn,
			       size, vma->vm_page_prot);
}

static struct proto xsk_proto = {
	.name =		"XDP",
	.owner =	THIS_MODULE,
	.obj_size =	sizeof(struct xdp_sock),
};

static const struct proto_ops xsk_proto_ops = {
	.family		= PF_XDP,
	.owner		= THIS_MODULE,
	.release	= xsk_release,
	.bind		= xsk_bind,
	.connect	= sock_no_connect,
	.socketpair	= sock_no_socketpair,
	.accept		= sock_no_accept,
	.getname	= sock_no_getname,
	.poll		= xsk_poll,
	.ioctl		= sock_no_ioctl,
	.listen		= sock_no_listen,
	.shutdown	= sock_no_shutdown,
	.setsockopt	= xsk_setsockopt,
	.getsockopt	= xsk_getsockopt,
	.sendmsg	= xsk_sendmsg,
	.recvmsg	= sock_no_recvmsg,
	.mmap		= xsk_mmap,
	.sendpage	= sock_no_sendpage,
};

static int xsk_create(struct net *net, struct socket *sock, int protocol,
		      int kern)
{
	struct sock *sk;
	struct xdp_sock *xs;

	if (!ns_capable(net->user_ns, CAP_NET_RAW))
		return -EPERM;
	if (sock->type != SOCK_RAW)
		return -ESOCKTNOSUPPORT;

	if (protocol)
		return -EPROTONOSUPPORT;

	sock->state = SS_UNCONNECTED;

	sk = sk_alloc(net, PF_XDP, GFP_KERNEL, &xsk_proto, kern);
	if (!sk)
		return -ENOBUFS;

	sock->ops = &xsk_proto_ops;

	sock_init_data(sock, sk);

	sk->sk_family = PF_XDP;

	sock_set_flag(sk, SOCK_RCU_FREE);

	xs = xdp_sk(sk);
	mutex_init(&xs->mutex);
	spin_lock_init(&xs->tx_completion_lock);

	local_bh_disable();
	sock_prot_inuse_add(net, &xsk_proto, 1);
	local_bh_enable();

	return 0;
}

static const struct net_proto_family xsk_family_ops = {
	.family = PF_XDP,
	.create = xsk_create,
	.owner	= THIS_MODULE,
};

static int __init xsk_init(void)
{
	int err;

	err = proto_register(&xsk_proto, 0 /* no slab */);
	if (err)
		goto out;

	err = sock_register(&xsk_family_ops);
	if (err)
		goto out_proto;

	return 0;

out_proto:
	proto_unregister(&xsk_proto);
out:
	return err;
}

fs_initcall(xsk_init);
Commit	Line	Data
c0c77d8f BT	1	// SPDX-License-Identifier: GPL-2.0
	2	/* XDP sockets
	3	*
	4	* AF_XDP sockets allows a channel between XDP programs and userspace
	5	* applications.
	6	* Copyright(c) 2018 Intel Corporation.
	7	*
c0c77d8f BT	8	* Author(s): Björn Töpel <bjorn.topel@intel.com>
	9	* Magnus Karlsson <magnus.karlsson@intel.com>
	10	*/
	11
	12	#define pr_fmt(fmt) "AF_XDP: %s: " fmt, __func__
	13
	14	#include <linux/if_xdp.h>
	15	#include <linux/init.h>
	16	#include <linux/sched/mm.h>
	17	#include <linux/sched/signal.h>
	18	#include <linux/sched/task.h>
	19	#include <linux/socket.h>
	20	#include <linux/file.h>
	21	#include <linux/uaccess.h>
	22	#include <linux/net.h>
	23	#include <linux/netdevice.h>
ac98d8aa	24	#include <linux/rculist.h>
c0c77d8f	25	#include <net/xdp_sock.h>
b9b6b68e	26	#include <net/xdp.h>
c0c77d8f	27
423f3832	28	#include "xsk_queue.h"
c0c77d8f BT	29	#include "xdp_umem.h"
c0c77d8f BT	30
35fcde7f MK	31	#define TX_BATCH_SIZE 16
35fcde7f MK	32
c0c77d8f BT	33	static struct xdp_sock xdp_sk(struct sock sk)
	34	{
	35	return (struct xdp_sock *)sk;
	36	}
	37
fbfc504a BT	38	bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs)
fbfc504a BT	39	{
173d3adb BT	40	return READ_ONCE(xs->rx) && READ_ONCE(xs->umem) &&
173d3adb BT	41	READ_ONCE(xs->umem->fq);
fbfc504a BT	42	}
fbfc504a BT	43
173d3adb BT	44	u64 xsk_umem_peek_addr(struct xdp_umem umem, u64 *addr)
	45	{
	46	return xskq_peek_addr(umem->fq, addr);
	47	}
	48	EXPORT_SYMBOL(xsk_umem_peek_addr);
	49
	50	void xsk_umem_discard_addr(struct xdp_umem *umem)
	51	{
	52	xskq_discard_addr(umem->fq);
	53	}
	54	EXPORT_SYMBOL(xsk_umem_discard_addr);
	55
	56	static int __xsk_rcv(struct xdp_sock xs, struct xdp_buff xdp, u32 len)
c497176c	57	{
18baed26 BT	58	void to_buf, from_buf;
18baed26 BT	59	u32 metalen;
bbff2f32	60	u64 addr;
4e64c835	61	int err;
c497176c	62
bbff2f32	63	if (!xskq_peek_addr(xs->umem->fq, &addr) \|\|
18baed26	64	len > xs->umem->chunk_size_nohr - XDP_PACKET_HEADROOM) {
a509a955	65	xs->rx_dropped++;
c497176c	66	return -ENOSPC;
a509a955	67	}
c497176c	68
bbff2f32 BT	69	addr += xs->umem->headroom;
bbff2f32 BT	70
18baed26 BT	71	if (unlikely(xdp_data_meta_unsupported(xdp))) {
	72	from_buf = xdp->data;
	73	metalen = 0;
	74	} else {
	75	from_buf = xdp->data_meta;
	76	metalen = xdp->data - xdp->data_meta;
	77	}
	78
	79	to_buf = xdp_umem_get_data(xs->umem, addr);
	80	memcpy(to_buf, from_buf, len + metalen);
	81	addr += metalen;
bbff2f32	82	err = xskq_produce_batch_desc(xs->rx, addr, len);
173d3adb	83	if (!err) {
bbff2f32	84	xskq_discard_addr(xs->umem->fq);
173d3adb BT	85	xdp_return_buff(xdp);
	86	return 0;
	87	}
c497176c	88
173d3adb	89	xs->rx_dropped++;
c497176c BT	90	return err;
	91	}
	92
173d3adb	93	static int __xsk_rcv_zc(struct xdp_sock xs, struct xdp_buff xdp, u32 len)
c497176c	94	{
173d3adb	95	int err = xskq_produce_batch_desc(xs->rx, (u64)xdp->handle, len);
c497176c	96
2d55d614	97	if (err)
173d3adb	98	xs->rx_dropped++;
c497176c BT	99
	100	return err;
	101	}
	102
173d3adb BT	103	int xsk_rcv(struct xdp_sock xs, struct xdp_buff xdp)
	104	{
	105	u32 len;
	106
	107	if (xs->dev != xdp->rxq->dev \|\| xs->queue_id != xdp->rxq->queue_index)
	108	return -EINVAL;
	109
	110	len = xdp->data_end - xdp->data;
	111
	112	return (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY) ?
	113	__xsk_rcv_zc(xs, xdp, len) : __xsk_rcv(xs, xdp, len);
	114	}
	115
c497176c BT	116	void xsk_flush(struct xdp_sock *xs)
	117	{
	118	xskq_produce_flush_desc(xs->rx);
	119	xs->sk.sk_data_ready(&xs->sk);
	120	}
	121
	122	int xsk_generic_rcv(struct xdp_sock xs, struct xdp_buff xdp)
	123	{
18baed26	124	u32 metalen = xdp->data - xdp->data_meta;
173d3adb BT	125	u32 len = xdp->data_end - xdp->data;
	126	void *buffer;
	127	u64 addr;
c497176c BT	128	int err;
c497176c BT	129
5d902372 BT	130	if (xs->dev != xdp->rxq->dev \|\| xs->queue_id != xdp->rxq->queue_index)
	131	return -EINVAL;
	132
173d3adb	133	if (!xskq_peek_addr(xs->umem->fq, &addr) \|\|
18baed26	134	len > xs->umem->chunk_size_nohr - XDP_PACKET_HEADROOM) {
173d3adb BT	135	xs->rx_dropped++;
	136	return -ENOSPC;
	137	}
	138
	139	addr += xs->umem->headroom;
	140
	141	buffer = xdp_umem_get_data(xs->umem, addr);
18baed26 BT	142	memcpy(buffer, xdp->data_meta, len + metalen);
18baed26 BT	143	addr += metalen;
173d3adb BT	144	err = xskq_produce_batch_desc(xs->rx, addr, len);
	145	if (!err) {
	146	xskq_discard_addr(xs->umem->fq);
c497176c	147	xsk_flush(xs);
173d3adb BT	148	return 0;
173d3adb BT	149	}
c497176c	150
173d3adb	151	xs->rx_dropped++;
c497176c BT	152	return err;
	153	}
	154
ac98d8aa MK	155	void xsk_umem_complete_tx(struct xdp_umem *umem, u32 nb_entries)
	156	{
	157	xskq_produce_flush_addr_n(umem->cq, nb_entries);
	158	}
	159	EXPORT_SYMBOL(xsk_umem_complete_tx);
	160
	161	void xsk_umem_consume_tx_done(struct xdp_umem *umem)
	162	{
	163	struct xdp_sock *xs;
	164
	165	rcu_read_lock();
	166	list_for_each_entry_rcu(xs, &umem->xsk_list, list) {
	167	xs->sk.sk_write_space(&xs->sk);
	168	}
	169	rcu_read_unlock();
	170	}
	171	EXPORT_SYMBOL(xsk_umem_consume_tx_done);
	172
	173	bool xsk_umem_consume_tx(struct xdp_umem umem, dma_addr_t dma, u32 *len)
	174	{
	175	struct xdp_desc desc;
	176	struct xdp_sock *xs;
	177
	178	rcu_read_lock();
	179	list_for_each_entry_rcu(xs, &umem->xsk_list, list) {
	180	if (!xskq_peek_desc(xs->tx, &desc))
	181	continue;
	182
	183	if (xskq_produce_addr_lazy(umem->cq, desc.addr))
	184	goto out;
	185
	186	*dma = xdp_umem_get_dma(umem, desc.addr);
	187	*len = desc.len;
	188
	189	xskq_discard_desc(xs->tx);
	190	rcu_read_unlock();
	191	return true;
	192	}
	193
	194	out:
	195	rcu_read_unlock();
	196	return false;
	197	}
	198	EXPORT_SYMBOL(xsk_umem_consume_tx);
	199
	200	static int xsk_zc_xmit(struct sock *sk)
	201	{
	202	struct xdp_sock *xs = xdp_sk(sk);
	203	struct net_device *dev = xs->dev;
	204
	205	return dev->netdev_ops->ndo_xsk_async_xmit(dev, xs->queue_id);
	206	}
	207
35fcde7f MK	208	static void xsk_destruct_skb(struct sk_buff *skb)
35fcde7f MK	209	{
bbff2f32	210	u64 addr = (u64)(long)skb_shinfo(skb)->destructor_arg;
35fcde7f	211	struct xdp_sock *xs = xdp_sk(skb->sk);
a9744f7c	212	unsigned long flags;
35fcde7f	213
a9744f7c	214	spin_lock_irqsave(&xs->tx_completion_lock, flags);
bbff2f32	215	WARN_ON_ONCE(xskq_produce_addr(xs->umem->cq, addr));
a9744f7c	216	spin_unlock_irqrestore(&xs->tx_completion_lock, flags);
35fcde7f MK	217
	218	sock_wfree(skb);
	219	}
	220
	221	static int xsk_generic_xmit(struct sock sk, struct msghdr m,
	222	size_t total_len)
	223	{
35fcde7f MK	224	u32 max_batch = TX_BATCH_SIZE;
	225	struct xdp_sock *xs = xdp_sk(sk);
	226	bool sent_frame = false;
	227	struct xdp_desc desc;
	228	struct sk_buff *skb;
	229	int err = 0;
	230
35fcde7f MK	231	mutex_lock(&xs->mutex);
	232
	233	while (xskq_peek_desc(xs->tx, &desc)) {
	234	char *buffer;
bbff2f32 BT	235	u64 addr;
bbff2f32 BT	236	u32 len;
35fcde7f MK	237
	238	if (max_batch-- == 0) {
	239	err = -EAGAIN;
	240	goto out;
	241	}
	242
9684f5e7	243	if (xskq_reserve_addr(xs->umem->cq))
35fcde7f	244	goto out;
35fcde7f	245
509d7648	246	if (xs->queue_id >= xs->dev->real_num_tx_queues)
2e59dd5e	247	goto out;
2e59dd5e	248
09210c4b	249	len = desc.len;
ac98d8aa	250	skb = sock_alloc_send_skb(sk, len, 1, &err);
35fcde7f MK	251	if (unlikely(!skb)) {
	252	err = -EAGAIN;
	253	goto out;
	254	}
	255
	256	skb_put(skb, len);
bbff2f32 BT	257	addr = desc.addr;
bbff2f32 BT	258	buffer = xdp_umem_get_data(xs->umem, addr);
35fcde7f MK	259	err = skb_store_bits(skb, 0, buffer, len);
	260	if (unlikely(err)) {
	261	kfree_skb(skb);
	262	goto out;
	263	}
	264
	265	skb->dev = xs->dev;
	266	skb->priority = sk->sk_priority;
	267	skb->mark = sk->sk_mark;
bbff2f32	268	skb_shinfo(skb)->destructor_arg = (void *)(long)addr;
35fcde7f MK	269	skb->destructor = xsk_destruct_skb;
	270
	271	err = dev_direct_xmit(skb, xs->queue_id);
fe588685	272	xskq_discard_desc(xs->tx);
35fcde7f MK	273	/* Ignore NET_XMIT_CN as packet might have been sent */
35fcde7f MK	274	if (err == NET_XMIT_DROP \|\| err == NETDEV_TX_BUSY) {
fe588685 MK	275	/* SKB completed but not sent */
fe588685 MK	276	err = -EBUSY;
35fcde7f MK	277	goto out;
	278	}
	279
	280	sent_frame = true;
35fcde7f MK	281	}
	282
	283	out:
	284	if (sent_frame)
	285	sk->sk_write_space(sk);
	286
	287	mutex_unlock(&xs->mutex);
	288	return err;
	289	}
	290
	291	static int xsk_sendmsg(struct socket sock, struct msghdr m, size_t total_len)
	292	{
ac98d8aa	293	bool need_wait = !(m->msg_flags & MSG_DONTWAIT);
35fcde7f MK	294	struct sock *sk = sock->sk;
	295	struct xdp_sock *xs = xdp_sk(sk);
	296
	297	if (unlikely(!xs->dev))
	298	return -ENXIO;
	299	if (unlikely(!(xs->dev->flags & IFF_UP)))
	300	return -ENETDOWN;
6efb4436 MK	301	if (unlikely(!xs->tx))
6efb4436 MK	302	return -ENOBUFS;
ac98d8aa MK	303	if (need_wait)
ac98d8aa MK	304	return -EOPNOTSUPP;
35fcde7f	305
ac98d8aa	306	return (xs->zc) ? xsk_zc_xmit(sk) : xsk_generic_xmit(sk, m, total_len);
35fcde7f MK	307	}
35fcde7f MK	308
a11e1d43 LT	309	static unsigned int xsk_poll(struct file file, struct socket sock,
a11e1d43 LT	310	struct poll_table_struct *wait)
c497176c	311	{
a11e1d43	312	unsigned int mask = datagram_poll(file, sock, wait);
c497176c BT	313	struct sock *sk = sock->sk;
	314	struct xdp_sock *xs = xdp_sk(sk);
	315
	316	if (xs->rx && !xskq_empty_desc(xs->rx))
	317	mask \|= POLLIN \| POLLRDNORM;
35fcde7f MK	318	if (xs->tx && !xskq_full_desc(xs->tx))
35fcde7f MK	319	mask \|= POLLOUT \| POLLWRNORM;
c497176c BT	320
	321	return mask;
	322	}
	323
b9b6b68e BT	324	static int xsk_init_queue(u32 entries, struct xsk_queue **queue,
b9b6b68e BT	325	bool umem_queue)
423f3832 MK	326	{
	327	struct xsk_queue *q;
	328
	329	if (entries == 0 \|\| *queue \|\| !is_power_of_2(entries))
	330	return -EINVAL;
	331
b9b6b68e	332	q = xskq_create(entries, umem_queue);
423f3832 MK	333	if (!q)
	334	return -ENOMEM;
	335
37b07693 BT	336	/* Make sure queue is ready before it can be seen by others */
37b07693 BT	337	smp_wmb();
423f3832 MK	338	*queue = q;
	339	return 0;
	340	}
	341
c0c77d8f BT	342	static int xsk_release(struct socket *sock)
	343	{
	344	struct sock *sk = sock->sk;
965a9909	345	struct xdp_sock *xs = xdp_sk(sk);
c0c77d8f BT	346	struct net *net;
	347
	348	if (!sk)
	349	return 0;
	350
	351	net = sock_net(sk);
	352
	353	local_bh_disable();
	354	sock_prot_inuse_add(net, sk->sk_prot, -1);
	355	local_bh_enable();
	356
965a9909	357	if (xs->dev) {
541d7fdd BT	358	struct net_device *dev = xs->dev;
541d7fdd BT	359
959b71db	360	/* Wait for driver to stop using the xdp socket. */
541d7fdd	361	xdp_del_sk_umem(xs->umem, xs);
965a9909	362	xs->dev = NULL;
541d7fdd BT	363	synchronize_net();
541d7fdd BT	364	dev_put(dev);
965a9909 MK	365	}
965a9909 MK	366
541d7fdd BT	367	xskq_destroy(xs->rx);
541d7fdd BT	368	xskq_destroy(xs->tx);
e2ce3674	369	xdp_put_umem(xs->umem);
541d7fdd	370
c0c77d8f BT	371	sock_orphan(sk);
	372	sock->sk = NULL;
	373
	374	sk_refcnt_debug_release(sk);
	375	sock_put(sk);
	376
	377	return 0;
	378	}
	379
965a9909 MK	380	static struct socket *xsk_lookup_xsk_from_fd(int fd)
	381	{
	382	struct socket *sock;
	383	int err;
	384
	385	sock = sockfd_lookup(fd, &err);
	386	if (!sock)
	387	return ERR_PTR(-ENOTSOCK);
	388
	389	if (sock->sk->sk_family != PF_XDP) {
	390	sockfd_put(sock);
	391	return ERR_PTR(-ENOPROTOOPT);
	392	}
	393
	394	return sock;
	395	}
	396
	397	static int xsk_bind(struct socket sock, struct sockaddr addr, int addr_len)
	398	{
	399	struct sockaddr_xdp sxdp = (struct sockaddr_xdp )addr;
	400	struct sock *sk = sock->sk;
965a9909	401	struct xdp_sock *xs = xdp_sk(sk);
959b71db	402	struct net_device *dev;
173d3adb	403	u32 flags, qid;
965a9909 MK	404	int err = 0;
	405
	406	if (addr_len < sizeof(struct sockaddr_xdp))
	407	return -EINVAL;
	408	if (sxdp->sxdp_family != AF_XDP)
	409	return -EINVAL;
	410
	411	mutex_lock(&xs->mutex);
959b71db BT	412	if (xs->dev) {
	413	err = -EBUSY;
	414	goto out_release;
	415	}
	416
965a9909 MK	417	dev = dev_get_by_index(sock_net(sk), sxdp->sxdp_ifindex);
	418	if (!dev) {
	419	err = -ENODEV;
	420	goto out_release;
	421	}
	422
f6145903	423	if (!xs->rx && !xs->tx) {
965a9909 MK	424	err = -EINVAL;
	425	goto out_unlock;
	426	}
	427
173d3adb	428	qid = sxdp->sxdp_queue_id;
173d3adb BT	429	flags = sxdp->sxdp_flags;
	430
	431	if (flags & XDP_SHARED_UMEM) {
965a9909 MK	432	struct xdp_sock *umem_xs;
	433	struct socket *sock;
	434
173d3adb BT	435	if ((flags & XDP_COPY) \|\| (flags & XDP_ZEROCOPY)) {
	436	/* Cannot specify flags for shared sockets. */
	437	err = -EINVAL;
	438	goto out_unlock;
	439	}
	440
965a9909 MK	441	if (xs->umem) {
	442	/* We have already our own. */
	443	err = -EINVAL;
	444	goto out_unlock;
	445	}
	446
	447	sock = xsk_lookup_xsk_from_fd(sxdp->sxdp_shared_umem_fd);
	448	if (IS_ERR(sock)) {
	449	err = PTR_ERR(sock);
	450	goto out_unlock;
	451	}
	452
	453	umem_xs = xdp_sk(sock->sk);
	454	if (!umem_xs->umem) {
	455	/* No umem to inherit. */
	456	err = -EBADF;
	457	sockfd_put(sock);
	458	goto out_unlock;
173d3adb	459	} else if (umem_xs->dev != dev \|\| umem_xs->queue_id != qid) {
965a9909 MK	460	err = -EINVAL;
	461	sockfd_put(sock);
	462	goto out_unlock;
	463	}
	464
	465	xdp_get_umem(umem_xs->umem);
965a9909 MK	466	xs->umem = umem_xs->umem;
	467	sockfd_put(sock);
	468	} else if (!xs->umem \|\| !xdp_umem_validate_queues(xs->umem)) {
	469	err = -EINVAL;
	470	goto out_unlock;
c497176c BT	471	} else {
c497176c BT	472	/* This xsk has its own umem. */
93ee30f3 MK	473	xskq_set_umem(xs->umem->fq, xs->umem->size,
	474	xs->umem->chunk_mask);
	475	xskq_set_umem(xs->umem->cq, xs->umem->size,
	476	xs->umem->chunk_mask);
173d3adb BT	477
	478	err = xdp_umem_assign_dev(xs->umem, dev, qid, flags);
	479	if (err)
	480	goto out_unlock;
965a9909 MK	481	}
965a9909 MK	482
965a9909	483	xs->dev = dev;
ac98d8aa MK	484	xs->zc = xs->umem->zc;
ac98d8aa MK	485	xs->queue_id = qid;
93ee30f3 MK	486	xskq_set_umem(xs->rx, xs->umem->size, xs->umem->chunk_mask);
93ee30f3 MK	487	xskq_set_umem(xs->tx, xs->umem->size, xs->umem->chunk_mask);
ac98d8aa	488	xdp_add_sk_umem(xs->umem, xs);
965a9909 MK	489
	490	out_unlock:
	491	if (err)
	492	dev_put(dev);
	493	out_release:
	494	mutex_unlock(&xs->mutex);
	495	return err;
	496	}
	497
c0c77d8f BT	498	static int xsk_setsockopt(struct socket *sock, int level, int optname,
	499	char __user *optval, unsigned int optlen)
	500	{
	501	struct sock *sk = sock->sk;
	502	struct xdp_sock *xs = xdp_sk(sk);
	503	int err;
	504
	505	if (level != SOL_XDP)
	506	return -ENOPROTOOPT;
	507
	508	switch (optname) {
b9b6b68e	509	case XDP_RX_RING:
f6145903	510	case XDP_TX_RING:
b9b6b68e BT	511	{
	512	struct xsk_queue **q;
	513	int entries;
	514
	515	if (optlen < sizeof(entries))
	516	return -EINVAL;
	517	if (copy_from_user(&entries, optval, sizeof(entries)))
	518	return -EFAULT;
	519
	520	mutex_lock(&xs->mutex);
f6145903	521	q = (optname == XDP_TX_RING) ? &xs->tx : &xs->rx;
b9b6b68e BT	522	err = xsk_init_queue(entries, q, false);
	523	mutex_unlock(&xs->mutex);
	524	return err;
	525	}
c0c77d8f BT	526	case XDP_UMEM_REG:
	527	{
	528	struct xdp_umem_reg mr;
	529	struct xdp_umem *umem;
	530
c0c77d8f BT	531	if (copy_from_user(&mr, optval, sizeof(mr)))
	532	return -EFAULT;
	533
	534	mutex_lock(&xs->mutex);
a49049ea BT	535	if (xs->umem) {
	536	mutex_unlock(&xs->mutex);
	537	return -EBUSY;
	538	}
c0c77d8f	539
a49049ea BT	540	umem = xdp_umem_create(&mr);
a49049ea BT	541	if (IS_ERR(umem)) {
c0c77d8f	542	mutex_unlock(&xs->mutex);
a49049ea	543	return PTR_ERR(umem);
c0c77d8f BT	544	}
	545
	546	/* Make sure umem is ready before it can be seen by others */
	547	smp_wmb();
c0c77d8f BT	548	xs->umem = umem;
	549	mutex_unlock(&xs->mutex);
	550	return 0;
	551	}
423f3832	552	case XDP_UMEM_FILL_RING:
fe230832	553	case XDP_UMEM_COMPLETION_RING:
423f3832 MK	554	{
	555	struct xsk_queue **q;
	556	int entries;
	557
423f3832 MK	558	if (copy_from_user(&entries, optval, sizeof(entries)))
	559	return -EFAULT;
	560
	561	mutex_lock(&xs->mutex);
a49049ea BT	562	if (!xs->umem) {
	563	mutex_unlock(&xs->mutex);
	564	return -EINVAL;
	565	}
	566
fe230832 MK	567	q = (optname == XDP_UMEM_FILL_RING) ? &xs->umem->fq :
fe230832 MK	568	&xs->umem->cq;
b9b6b68e	569	err = xsk_init_queue(entries, q, true);
423f3832 MK	570	mutex_unlock(&xs->mutex);
	571	return err;
	572	}
c0c77d8f BT	573	default:
	574	break;
	575	}
	576
	577	return -ENOPROTOOPT;
	578	}
	579
af75d9e0 MK	580	static int xsk_getsockopt(struct socket *sock, int level, int optname,
	581	char __user optval, int __user optlen)
	582	{
	583	struct sock *sk = sock->sk;
	584	struct xdp_sock *xs = xdp_sk(sk);
	585	int len;
	586
	587	if (level != SOL_XDP)
	588	return -ENOPROTOOPT;
	589
	590	if (get_user(len, optlen))
	591	return -EFAULT;
	592	if (len < 0)
	593	return -EINVAL;
	594
	595	switch (optname) {
	596	case XDP_STATISTICS:
	597	{
	598	struct xdp_statistics stats;
	599
	600	if (len < sizeof(stats))
	601	return -EINVAL;
	602
	603	mutex_lock(&xs->mutex);
	604	stats.rx_dropped = xs->rx_dropped;
	605	stats.rx_invalid_descs = xskq_nb_invalid_descs(xs->rx);
	606	stats.tx_invalid_descs = xskq_nb_invalid_descs(xs->tx);
	607	mutex_unlock(&xs->mutex);
	608
	609	if (copy_to_user(optval, &stats, sizeof(stats)))
	610	return -EFAULT;
	611	if (put_user(sizeof(stats), optlen))
	612	return -EFAULT;
	613
	614	return 0;
	615	}
b3a9e0be BT	616	case XDP_MMAP_OFFSETS:
	617	{
	618	struct xdp_mmap_offsets off;
	619
	620	if (len < sizeof(off))
	621	return -EINVAL;
	622
	623	off.rx.producer = offsetof(struct xdp_rxtx_ring, ptrs.producer);
	624	off.rx.consumer = offsetof(struct xdp_rxtx_ring, ptrs.consumer);
	625	off.rx.desc = offsetof(struct xdp_rxtx_ring, desc);
	626	off.tx.producer = offsetof(struct xdp_rxtx_ring, ptrs.producer);
	627	off.tx.consumer = offsetof(struct xdp_rxtx_ring, ptrs.consumer);
	628	off.tx.desc = offsetof(struct xdp_rxtx_ring, desc);
	629
	630	off.fr.producer = offsetof(struct xdp_umem_ring, ptrs.producer);
	631	off.fr.consumer = offsetof(struct xdp_umem_ring, ptrs.consumer);
	632	off.fr.desc = offsetof(struct xdp_umem_ring, desc);
	633	off.cr.producer = offsetof(struct xdp_umem_ring, ptrs.producer);
	634	off.cr.consumer = offsetof(struct xdp_umem_ring, ptrs.consumer);
	635	off.cr.desc = offsetof(struct xdp_umem_ring, desc);
	636
	637	len = sizeof(off);
	638	if (copy_to_user(optval, &off, len))
	639	return -EFAULT;
	640	if (put_user(len, optlen))
	641	return -EFAULT;
	642
	643	return 0;
	644	}
af75d9e0 MK	645	default:
	646	break;
	647	}
	648
	649	return -EOPNOTSUPP;
	650	}
	651
423f3832 MK	652	static int xsk_mmap(struct file file, struct socket sock,
	653	struct vm_area_struct *vma)
	654	{
a5a16e43	655	loff_t offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT;
423f3832 MK	656	unsigned long size = vma->vm_end - vma->vm_start;
	657	struct xdp_sock *xs = xdp_sk(sock->sk);
	658	struct xsk_queue *q = NULL;
37b07693	659	struct xdp_umem *umem;
423f3832 MK	660	unsigned long pfn;
	661	struct page *qpg;
	662
b9b6b68e	663	if (offset == XDP_PGOFF_RX_RING) {
37b07693	664	q = READ_ONCE(xs->rx);
f6145903	665	} else if (offset == XDP_PGOFF_TX_RING) {
37b07693	666	q = READ_ONCE(xs->tx);
b9b6b68e	667	} else {
37b07693 BT	668	umem = READ_ONCE(xs->umem);
37b07693 BT	669	if (!umem)
b9b6b68e	670	return -EINVAL;
423f3832	671
b9b6b68e	672	if (offset == XDP_UMEM_PGOFF_FILL_RING)
37b07693	673	q = READ_ONCE(umem->fq);
fe230832	674	else if (offset == XDP_UMEM_PGOFF_COMPLETION_RING)
37b07693	675	q = READ_ONCE(umem->cq);
b9b6b68e	676	}
423f3832 MK	677
	678	if (!q)
	679	return -EINVAL;
	680
	681	qpg = virt_to_head_page(q->ring);
	682	if (size > (PAGE_SIZE << compound_order(qpg)))
	683	return -EINVAL;
	684
	685	pfn = virt_to_phys(q->ring) >> PAGE_SHIFT;
	686	return remap_pfn_range(vma, vma->vm_start, pfn,
	687	size, vma->vm_page_prot);
	688	}
	689
c0c77d8f BT	690	static struct proto xsk_proto = {
	691	.name = "XDP",
	692	.owner = THIS_MODULE,
	693	.obj_size = sizeof(struct xdp_sock),
	694	};
	695
	696	static const struct proto_ops xsk_proto_ops = {
c2f4374b BT	697	.family = PF_XDP,
	698	.owner = THIS_MODULE,
	699	.release = xsk_release,
	700	.bind = xsk_bind,
	701	.connect = sock_no_connect,
	702	.socketpair = sock_no_socketpair,
	703	.accept = sock_no_accept,
	704	.getname = sock_no_getname,
a11e1d43	705	.poll = xsk_poll,
c2f4374b BT	706	.ioctl = sock_no_ioctl,
	707	.listen = sock_no_listen,
	708	.shutdown = sock_no_shutdown,
	709	.setsockopt = xsk_setsockopt,
	710	.getsockopt = xsk_getsockopt,
	711	.sendmsg = xsk_sendmsg,
	712	.recvmsg = sock_no_recvmsg,
	713	.mmap = xsk_mmap,
	714	.sendpage = sock_no_sendpage,
c0c77d8f BT	715	};
c0c77d8f BT	716
c0c77d8f BT	717	static int xsk_create(struct net net, struct socket sock, int protocol,
	718	int kern)
	719	{
	720	struct sock *sk;
	721	struct xdp_sock *xs;
	722
	723	if (!ns_capable(net->user_ns, CAP_NET_RAW))
	724	return -EPERM;
	725	if (sock->type != SOCK_RAW)
	726	return -ESOCKTNOSUPPORT;
	727
	728	if (protocol)
	729	return -EPROTONOSUPPORT;
	730
	731	sock->state = SS_UNCONNECTED;
	732
	733	sk = sk_alloc(net, PF_XDP, GFP_KERNEL, &xsk_proto, kern);
	734	if (!sk)
	735	return -ENOBUFS;
	736
	737	sock->ops = &xsk_proto_ops;
	738
	739	sock_init_data(sock, sk);
	740
	741	sk->sk_family = PF_XDP;
	742
cee27167 BT	743	sock_set_flag(sk, SOCK_RCU_FREE);
cee27167 BT	744
c0c77d8f BT	745	xs = xdp_sk(sk);
c0c77d8f BT	746	mutex_init(&xs->mutex);
a9744f7c	747	spin_lock_init(&xs->tx_completion_lock);
c0c77d8f BT	748
	749	local_bh_disable();
	750	sock_prot_inuse_add(net, &xsk_proto, 1);
	751	local_bh_enable();
	752
	753	return 0;
	754	}
	755
	756	static const struct net_proto_family xsk_family_ops = {
	757	.family = PF_XDP,
	758	.create = xsk_create,
	759	.owner = THIS_MODULE,
	760	};
	761
	762	static int __init xsk_init(void)
	763	{
	764	int err;
	765
	766	err = proto_register(&xsk_proto, 0 /* no slab */);
	767	if (err)
	768	goto out;
	769
	770	err = sock_register(&xsk_family_ops);
	771	if (err)
	772	goto out_proto;
	773
	774	return 0;
	775
	776	out_proto:
	777	proto_unregister(&xsk_proto);
	778	out:
	779	return err;
	780	}
	781
	782	fs_initcall(xsk_init);