[ceph.git] / ceph / src / spdk / dpdk / drivers / net / tap / tap_bpf_program.c

/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
 * Copyright 2017 Mellanox Technologies, Ltd
 */

#include <stdint.h>
#include <stdbool.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <asm/types.h>
#include <linux/in.h>
#include <linux/if.h>
#include <linux/if_ether.h>
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/if_tunnel.h>
#include <linux/filter.h>
#include <linux/bpf.h>

#include "tap_rss.h"

/** Create IPv4 address */
#define IPv4(a, b, c, d) ((__u32)(((a) & 0xff) << 24) | \
		(((b) & 0xff) << 16) | \
		(((c) & 0xff) << 8)  | \
		((d) & 0xff))

#define PORT(a, b) ((__u16)(((a) & 0xff) << 8) | \
		((b) & 0xff))

/*
 * The queue number is offset by a unique QUEUE_OFFSET, to distinguish
 * packets that have gone through this rule (skb->cb[1] != 0) from others.
 */
#define QUEUE_OFFSET		0x7cafe800
#define PIN_GLOBAL_NS		2

#define KEY_IDX			0
#define BPF_MAP_ID_KEY	1

struct vlan_hdr {
	__be16 proto;
	__be16 tci;
};

struct bpf_elf_map __attribute__((section("maps"), used))
map_keys = {
	.type           =       BPF_MAP_TYPE_HASH,
	.id             =       BPF_MAP_ID_KEY,
	.size_key       =       sizeof(__u32),
	.size_value     =       sizeof(struct rss_key),
	.max_elem       =       256,
	.pinning        =       PIN_GLOBAL_NS,
};

__section("cls_q") int
match_q(struct __sk_buff *skb)
{
	__u32 queue = skb->cb[1];
	volatile __u32 q = 0xdeadbeef;
	__u32 match_queue = QUEUE_OFFSET + q;

	/* printt("match_q$i() queue = %d\n", queue); */

	if (queue != match_queue)
		return TC_ACT_OK;

	/* queue match */
	skb->cb[1] = 0;
	return TC_ACT_UNSPEC;
}


struct ipv4_l3_l4_tuple {
	__u32    src_addr;
	__u32    dst_addr;
	__u16    dport;
	__u16    sport;
} __attribute__((packed));

struct ipv6_l3_l4_tuple {
	__u8        src_addr[16];
	__u8        dst_addr[16];
	__u16       dport;
	__u16       sport;
} __attribute__((packed));

static const __u8 def_rss_key[TAP_RSS_HASH_KEY_SIZE] = {
	0xd1, 0x81, 0xc6, 0x2c,
	0xf7, 0xf4, 0xdb, 0x5b,
	0x19, 0x83, 0xa2, 0xfc,
	0x94, 0x3e, 0x1a, 0xdb,
	0xd9, 0x38, 0x9e, 0x6b,
	0xd1, 0x03, 0x9c, 0x2c,
	0xa7, 0x44, 0x99, 0xad,
	0x59, 0x3d, 0x56, 0xd9,
	0xf3, 0x25, 0x3c, 0x06,
	0x2a, 0xdc, 0x1f, 0xfc,
};

static __u32  __attribute__((always_inline))
rte_softrss_be(const __u32 *input_tuple, const uint8_t *rss_key,
		__u8 input_len)
{
	__u32 i, j, hash = 0;
#pragma unroll
	for (j = 0; j < input_len; j++) {
#pragma unroll
		for (i = 0; i < 32; i++) {
			if (input_tuple[j] & (1U << (31 - i))) {
				hash ^= ((const __u32 *)def_rss_key)[j] << i |
				(__u32)((uint64_t)
				(((const __u32 *)def_rss_key)[j + 1])
					>> (32 - i));
			}
		}
	}
	return hash;
}

static int __attribute__((always_inline))
rss_l3_l4(struct __sk_buff *skb)
{
	void *data_end = (void *)(long)skb->data_end;
	void *data = (void *)(long)skb->data;
	__u16 proto = (__u16)skb->protocol;
	__u32 key_idx = 0xdeadbeef;
	__u32 hash;
	struct rss_key *rsskey;
	__u64 off = ETH_HLEN;
	int j;
	__u8 *key = 0;
	__u32 len;
	__u32 queue = 0;

	rsskey = map_lookup_elem(&map_keys, &key_idx);
	if (!rsskey) {
		printt("hash(): rss key is not configured\n");
		return TC_ACT_OK;
	}
	key = (__u8 *)rsskey->key;

	/* Get correct proto for 802.1ad */
	if (skb->vlan_present && skb->vlan_proto == htons(ETH_P_8021AD)) {
		if (data + ETH_ALEN * 2 + sizeof(struct vlan_hdr) +
		    sizeof(proto) > data_end)
			return TC_ACT_OK;
		proto = *(__u16 *)(data + ETH_ALEN * 2 +
				   sizeof(struct vlan_hdr));
		off += sizeof(struct vlan_hdr);
	}

	if (proto == htons(ETH_P_IP)) {
		if (data + off + sizeof(struct iphdr) + sizeof(__u32)
			> data_end)
			return TC_ACT_OK;

		__u8 *src_dst_addr = data + off + offsetof(struct iphdr, saddr);
		__u8 *src_dst_port = data + off + sizeof(struct iphdr);
		struct ipv4_l3_l4_tuple v4_tuple = {
			.src_addr = IPv4(*(src_dst_addr + 0),
					*(src_dst_addr + 1),
					*(src_dst_addr + 2),
					*(src_dst_addr + 3)),
			.dst_addr = IPv4(*(src_dst_addr + 4),
					*(src_dst_addr + 5),
					*(src_dst_addr + 6),
					*(src_dst_addr + 7)),
			.sport = PORT(*(src_dst_port + 0),
					*(src_dst_port + 1)),
			.dport = PORT(*(src_dst_port + 2),
					*(src_dst_port + 3)),
		};
		__u8 input_len = sizeof(v4_tuple) / sizeof(__u32);
		if (rsskey->hash_fields & (1 << HASH_FIELD_IPV4_L3))
			input_len--;
		hash = rte_softrss_be((__u32 *)&v4_tuple, key, 3);
	} else if (proto == htons(ETH_P_IPV6)) {
		if (data + off + sizeof(struct ipv6hdr) +
					sizeof(__u32) > data_end)
			return TC_ACT_OK;
		__u8 *src_dst_addr = data + off +
					offsetof(struct ipv6hdr, saddr);
		__u8 *src_dst_port = data + off +
					sizeof(struct ipv6hdr);
		struct ipv6_l3_l4_tuple v6_tuple;
		for (j = 0; j < 4; j++)
			*((uint32_t *)&v6_tuple.src_addr + j) =
				__builtin_bswap32(*((uint32_t *)
						src_dst_addr + j));
		for (j = 0; j < 4; j++)
			*((uint32_t *)&v6_tuple.dst_addr + j) =
				__builtin_bswap32(*((uint32_t *)
						src_dst_addr + 4 + j));
		v6_tuple.sport = PORT(*(src_dst_port + 0),
			      *(src_dst_port + 1));
		v6_tuple.dport = PORT(*(src_dst_port + 2),
			      *(src_dst_port + 3));

		__u8 input_len = sizeof(v6_tuple) / sizeof(__u32);
		if (rsskey->hash_fields & (1 << HASH_FIELD_IPV6_L3))
			input_len--;
		hash = rte_softrss_be((__u32 *)&v6_tuple, key, 9);
	} else {
		return TC_ACT_PIPE;
	}

	queue = rsskey->queues[(hash % rsskey->nb_queues) &
				       (TAP_MAX_QUEUES - 1)];
	skb->cb[1] = QUEUE_OFFSET + queue;
	/* printt(">>>>> rss_l3_l4 hash=0x%x queue=%u\n", hash, queue); */

	return TC_ACT_RECLASSIFY;
}

#define RSS(L)						\
	__section(#L) int				\
		L ## _hash(struct __sk_buff *skb)	\
	{						\
		return rss_ ## L (skb);			\
	}

RSS(l3_l4)

BPF_LICENSE("Dual BSD/GPL");
Commit	Line	Data
11fdf7f2 TL	1	/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
	2	* Copyright 2017 Mellanox Technologies, Ltd
	3	*/
	4
	5	#include <stdint.h>
	6	#include <stdbool.h>
	7	#include <sys/types.h>
	8	#include <sys/socket.h>
	9	#include <asm/types.h>
	10	#include <linux/in.h>
	11	#include <linux/if.h>
	12	#include <linux/if_ether.h>
	13	#include <linux/ip.h>
	14	#include <linux/ipv6.h>
	15	#include <linux/if_tunnel.h>
	16	#include <linux/filter.h>
	17	#include <linux/bpf.h>
	18
	19	#include "tap_rss.h"
	20
	21	/** Create IPv4 address */
	22	#define IPv4(a, b, c, d) ((__u32)(((a) & 0xff) << 24) \| \
	23	(((b) & 0xff) << 16) \| \
	24	(((c) & 0xff) << 8) \| \
	25	((d) & 0xff))
	26
	27	#define PORT(a, b) ((__u16)(((a) & 0xff) << 8) \| \
	28	((b) & 0xff))
	29
	30	/*
	31	* The queue number is offset by a unique QUEUE_OFFSET, to distinguish
	32	* packets that have gone through this rule (skb->cb[1] != 0) from others.
	33	*/
	34	#define QUEUE_OFFSET 0x7cafe800
	35	#define PIN_GLOBAL_NS 2
	36
	37	#define KEY_IDX 0
	38	#define BPF_MAP_ID_KEY 1
	39
	40	struct vlan_hdr {
	41	__be16 proto;
	42	__be16 tci;
	43	};
	44
	45	struct bpf_elf_map __attribute__((section("maps"), used))
	46	map_keys = {
	47	.type = BPF_MAP_TYPE_HASH,
	48	.id = BPF_MAP_ID_KEY,
	49	.size_key = sizeof(__u32),
	50	.size_value = sizeof(struct rss_key),
	51	.max_elem = 256,
	52	.pinning = PIN_GLOBAL_NS,
	53	};
	54
	55	__section("cls_q") int
	56	match_q(struct __sk_buff *skb)
	57	{
	58	__u32 queue = skb->cb[1];
	59	volatile __u32 q = 0xdeadbeef;
	60	__u32 match_queue = QUEUE_OFFSET + q;
	61
	62	/* printt("match_q$i() queue = %d\n", queue); */
	63
	64	if (queue != match_queue)
65	return TC_ACT_OK;
66
67	/* queue match */
68	skb->cb[1] = 0;
69	return TC_ACT_UNSPEC;
70	}
71
72
73	struct ipv4_l3_l4_tuple {
74	__u32 src_addr;
75	__u32 dst_addr;
76	__u16 dport;
77	__u16 sport;
78	} __attribute__((packed));
79
80	struct ipv6_l3_l4_tuple {
81	__u8 src_addr[16];
82	__u8 dst_addr[16];
83	__u16 dport;
84	__u16 sport;
85	} __attribute__((packed));
86
87	static const __u8 def_rss_key[TAP_RSS_HASH_KEY_SIZE] = {
88	0xd1, 0x81, 0xc6, 0x2c,
89	0xf7, 0xf4, 0xdb, 0x5b,
90	0x19, 0x83, 0xa2, 0xfc,
91	0x94, 0x3e, 0x1a, 0xdb,
92	0xd9, 0x38, 0x9e, 0x6b,
93	0xd1, 0x03, 0x9c, 0x2c,
94	0xa7, 0x44, 0x99, 0xad,
95	0x59, 0x3d, 0x56, 0xd9,
96	0xf3, 0x25, 0x3c, 0x06,
97	0x2a, 0xdc, 0x1f, 0xfc,
98	};
99
100	static __u32 __attribute__((always_inline))
101	rte_softrss_be(const __u32 input_tuple, const uint8_t rss_key,
102	__u8 input_len)
103	{
104	__u32 i, j, hash = 0;
105	#pragma unroll
106	for (j = 0; j < input_len; j++) {
107	#pragma unroll
108	for (i = 0; i < 32; i++) {
9f95a23c	109	if (input_tuple[j] & (1U << (31 - i))) {
11fdf7f2 TL	110	hash ^= ((const __u32 *)def_rss_key)[j] << i \|
	111	(__u32)((uint64_t)
	112	(((const __u32 *)def_rss_key)[j + 1])
	113	>> (32 - i));
	114	}
	115	}
	116	}
	117	return hash;
	118	}
	119
	120	static int __attribute__((always_inline))
	121	rss_l3_l4(struct __sk_buff *skb)
	122	{
	123	void data_end = (void )(long)skb->data_end;
	124	void data = (void )(long)skb->data;
	125	__u16 proto = (__u16)skb->protocol;
	126	__u32 key_idx = 0xdeadbeef;
	127	__u32 hash;
	128	struct rss_key *rsskey;
	129	__u64 off = ETH_HLEN;
	130	int j;
	131	__u8 *key = 0;
	132	__u32 len;
	133	__u32 queue = 0;
	134
	135	rsskey = map_lookup_elem(&map_keys, &key_idx);
	136	if (!rsskey) {
	137	printt("hash(): rss key is not configured\n");
	138	return TC_ACT_OK;
	139	}
	140	key = (__u8 *)rsskey->key;
	141
	142	/* Get correct proto for 802.1ad */
	143	if (skb->vlan_present && skb->vlan_proto == htons(ETH_P_8021AD)) {
	144	if (data + ETH_ALEN * 2 + sizeof(struct vlan_hdr) +
	145	sizeof(proto) > data_end)
	146	return TC_ACT_OK;
	147	proto = (__u16 )(data + ETH_ALEN * 2 +
	148	sizeof(struct vlan_hdr));
	149	off += sizeof(struct vlan_hdr);
	150	}
	151
	152	if (proto == htons(ETH_P_IP)) {
	153	if (data + off + sizeof(struct iphdr) + sizeof(__u32)
	154	> data_end)
	155	return TC_ACT_OK;
	156
	157	__u8 *src_dst_addr = data + off + offsetof(struct iphdr, saddr);
	158	__u8 *src_dst_port = data + off + sizeof(struct iphdr);
	159	struct ipv4_l3_l4_tuple v4_tuple = {
	160	.src_addr = IPv4(*(src_dst_addr + 0),
	161	*(src_dst_addr + 1),
	162	*(src_dst_addr + 2),
	163	*(src_dst_addr + 3)),
	164	.dst_addr = IPv4(*(src_dst_addr + 4),
	165	*(src_dst_addr + 5),
	166	*(src_dst_addr + 6),
	167	*(src_dst_addr + 7)),
	168	.sport = PORT(*(src_dst_port + 0),
	169	*(src_dst_port + 1)),
	170	.dport = PORT(*(src_dst_port + 2),
	171	*(src_dst_port + 3)),
	172	};
	173	__u8 input_len = sizeof(v4_tuple) / sizeof(__u32);
174	if (rsskey->hash_fields & (1 << HASH_FIELD_IPV4_L3))
175	input_len--;
176	hash = rte_softrss_be((__u32 *)&v4_tuple, key, 3);
177	} else if (proto == htons(ETH_P_IPV6)) {
178	if (data + off + sizeof(struct ipv6hdr) +
179	sizeof(__u32) > data_end)
180	return TC_ACT_OK;
181	__u8 *src_dst_addr = data + off +
182	offsetof(struct ipv6hdr, saddr);
183	__u8 *src_dst_port = data + off +
184	sizeof(struct ipv6hdr);
185	struct ipv6_l3_l4_tuple v6_tuple;
186	for (j = 0; j < 4; j++)
187	((uint32_t )&v6_tuple.src_addr + j) =
188	__builtin_bswap32(((uint32_t )
189	src_dst_addr + j));
190	for (j = 0; j < 4; j++)
191	((uint32_t )&v6_tuple.dst_addr + j) =
192	__builtin_bswap32(((uint32_t )
193	src_dst_addr + 4 + j));
194	v6_tuple.sport = PORT(*(src_dst_port + 0),
195	*(src_dst_port + 1));
196	v6_tuple.dport = PORT(*(src_dst_port + 2),
197	*(src_dst_port + 3));
198
199	__u8 input_len = sizeof(v6_tuple) / sizeof(__u32);
200	if (rsskey->hash_fields & (1 << HASH_FIELD_IPV6_L3))
201	input_len--;
202	hash = rte_softrss_be((__u32 *)&v6_tuple, key, 9);
203	} else {
204	return TC_ACT_PIPE;
205	}
206
207	queue = rsskey->queues[(hash % rsskey->nb_queues) &
208	(TAP_MAX_QUEUES - 1)];
209	skb->cb[1] = QUEUE_OFFSET + queue;
210	/* printt(">>>>> rss_l3_l4 hash=0x%x queue=%u\n", hash, queue); */
211
212	return TC_ACT_RECLASSIFY;
213	}
214
215	#define RSS(L) \
216	__section(#L) int \
217	L ## _hash(struct __sk_buff *skb) \
218	{ \
219	return rss_ ## L (skb); \
220	}
221
222	RSS(l3_l4)
223
224	BPF_LICENSE("Dual BSD/GPL");