[ceph.git] / ceph / src / seastar / dpdk / lib / librte_hash / rte_thash.h

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2015-2019 Vladimir Medvedkin <medvedkinv@gmail.com>
 */

#ifndef _RTE_THASH_H
#define _RTE_THASH_H

/**
 * @file
 *
 * toeplitz hash functions.
 */

#ifdef __cplusplus
extern "C" {
#endif

/**
 * Software implementation of the Toeplitz hash function used by RSS.
 * Can be used either for packet distribution on single queue NIC
 * or for simulating of RSS computation on specific NIC (for example
 * after GRE header decapsulating)
 */

#include <stdint.h>
#include <rte_byteorder.h>
#include <rte_config.h>
#include <rte_ip.h>
#include <rte_common.h>

#if defined(RTE_ARCH_X86) || defined(RTE_MACHINE_CPUFLAG_NEON)
#include <rte_vect.h>
#endif

#ifdef RTE_ARCH_X86
/* Byte swap mask used for converting IPv6 address
 * 4-byte chunks to CPU byte order
 */
static const __m128i rte_thash_ipv6_bswap_mask = {
		0x0405060700010203ULL, 0x0C0D0E0F08090A0BULL};
#endif

/**
 * length in dwords of input tuple to
 * calculate hash of ipv4 header only
 */
#define RTE_THASH_V4_L3_LEN	((sizeof(struct rte_ipv4_tuple) -	\
			sizeof(((struct rte_ipv4_tuple *)0)->sctp_tag)) / 4)

/**
 * length in dwords of input tuple to
 * calculate hash of ipv4 header +
 * transport header
 */
#define RTE_THASH_V4_L4_LEN	 ((sizeof(struct rte_ipv4_tuple)) / 4)

/**
 * length in dwords of input tuple to
 * calculate hash of ipv6 header only
 */
#define RTE_THASH_V6_L3_LEN	((sizeof(struct rte_ipv6_tuple) -       \
			sizeof(((struct rte_ipv6_tuple *)0)->sctp_tag)) / 4)

/**
 * length in dwords of input tuple to
 * calculate hash of ipv6 header +
 * transport header
 */
#define RTE_THASH_V6_L4_LEN	((sizeof(struct rte_ipv6_tuple)) / 4)

/**
 * IPv4 tuple
 * addresses and ports/sctp_tag have to be CPU byte order
 */
struct rte_ipv4_tuple {
	uint32_t	src_addr;
	uint32_t	dst_addr;
	RTE_STD_C11
	union {
		struct {
			uint16_t dport;
			uint16_t sport;
		};
		uint32_t        sctp_tag;
	};
};

/**
 * IPv6 tuple
 * Addresses have to be filled by rte_thash_load_v6_addr()
 * ports/sctp_tag have to be CPU byte order
 */
struct rte_ipv6_tuple {
	uint8_t		src_addr[16];
	uint8_t		dst_addr[16];
	RTE_STD_C11
	union {
		struct {
			uint16_t dport;
			uint16_t sport;
		};
		uint32_t        sctp_tag;
	};
};

union rte_thash_tuple {
	struct rte_ipv4_tuple	v4;
	struct rte_ipv6_tuple	v6;
#ifdef RTE_ARCH_X86
} __attribute__((aligned(XMM_SIZE)));
#else
};
#endif

/**
 * Prepare special converted key to use with rte_softrss_be()
 * @param orig
 *   pointer to original RSS key
 * @param targ
 *   pointer to target RSS key
 * @param len
 *   RSS key length
 */
static inline void
rte_convert_rss_key(const uint32_t *orig, uint32_t *targ, int len)
{
	int i;

	for (i = 0; i < (len >> 2); i++)
		targ[i] = rte_be_to_cpu_32(orig[i]);
}

/**
 * Prepare and load IPv6 addresses (src and dst)
 * into target tuple
 * @param orig
 *   Pointer to ipv6 header of the original packet
 * @param targ
 *   Pointer to rte_ipv6_tuple structure
 */
static inline void
rte_thash_load_v6_addrs(const struct ipv6_hdr *orig, union rte_thash_tuple *targ)
{
#ifdef RTE_ARCH_X86
	__m128i ipv6 = _mm_loadu_si128((const __m128i *)orig->src_addr);
	*(__m128i *)targ->v6.src_addr =
			_mm_shuffle_epi8(ipv6, rte_thash_ipv6_bswap_mask);
	ipv6 = _mm_loadu_si128((const __m128i *)orig->dst_addr);
	*(__m128i *)targ->v6.dst_addr =
			_mm_shuffle_epi8(ipv6, rte_thash_ipv6_bswap_mask);
#elif defined(RTE_MACHINE_CPUFLAG_NEON)
	uint8x16_t ipv6 = vld1q_u8((uint8_t const *)orig->src_addr);
	vst1q_u8((uint8_t *)targ->v6.src_addr, vrev32q_u8(ipv6));
	ipv6 = vld1q_u8((uint8_t const *)orig->dst_addr);
	vst1q_u8((uint8_t *)targ->v6.dst_addr, vrev32q_u8(ipv6));
#else
	int i;
	for (i = 0; i < 4; i++) {
		*((uint32_t *)targ->v6.src_addr + i) =
			rte_be_to_cpu_32(*((const uint32_t *)orig->src_addr + i));
		*((uint32_t *)targ->v6.dst_addr + i) =
			rte_be_to_cpu_32(*((const uint32_t *)orig->dst_addr + i));
	}
#endif
}

/**
 * Generic implementation. Can be used with original rss_key
 * @param input_tuple
 *   Pointer to input tuple
 * @param input_len
 *   Length of input_tuple in 4-bytes chunks
 * @param rss_key
 *   Pointer to RSS hash key.
 * @return
 *   Calculated hash value.
 */
static inline uint32_t
rte_softrss(uint32_t *input_tuple, uint32_t input_len,
		const uint8_t *rss_key)
{
	uint32_t i, j, map, ret = 0;

	for (j = 0; j < input_len; j++) {
		for (map = input_tuple[j]; map;	map &= (map - 1)) {
			i = rte_bsf32(map);
			ret ^= rte_cpu_to_be_32(((const uint32_t *)rss_key)[j]) << (31 - i) |
					(uint32_t)((uint64_t)(rte_cpu_to_be_32(((const uint32_t *)rss_key)[j + 1])) >>
					(i + 1));
		}
	}
	return ret;
}

/**
 * Optimized implementation.
 * If you want the calculated hash value matches NIC RSS value
 * you have to use special converted key with rte_convert_rss_key() fn.
 * @param input_tuple
 *   Pointer to input tuple
 * @param input_len
 *   Length of input_tuple in 4-bytes chunks
 * @param *rss_key
 *   Pointer to RSS hash key.
 * @return
 *   Calculated hash value.
 */
static inline uint32_t
rte_softrss_be(uint32_t *input_tuple, uint32_t input_len,
		const uint8_t *rss_key)
{
	uint32_t i, j, map, ret = 0;

	for (j = 0; j < input_len; j++) {
		for (map = input_tuple[j]; map;	map &= (map - 1)) {
			i = rte_bsf32(map);
			ret ^= ((const uint32_t *)rss_key)[j] << (31 - i) |
				(uint32_t)((uint64_t)(((const uint32_t *)rss_key)[j + 1]) >> (i + 1));
		}
	}
	return ret;
}

#ifdef __cplusplus
}
#endif

#endif /* _RTE_THASH_H */
Commit	Line	Data
9f95a23c TL	1	/* SPDX-License-Identifier: BSD-3-Clause
9f95a23c TL	2	* Copyright(c) 2015-2019 Vladimir Medvedkin <medvedkinv@gmail.com>
7c673cae FG	3	*/
	4
	5	#ifndef _RTE_THASH_H
	6	#define _RTE_THASH_H
	7
	8	/**
	9	* @file
	10	*
	11	* toeplitz hash functions.
	12	*/
	13
	14	#ifdef __cplusplus
	15	extern "C" {
	16	#endif
	17
	18	/**
	19	* Software implementation of the Toeplitz hash function used by RSS.
	20	* Can be used either for packet distribution on single queue NIC
	21	* or for simulating of RSS computation on specific NIC (for example
	22	* after GRE header decapsulating)
	23	*/
	24
	25	#include <stdint.h>
	26	#include <rte_byteorder.h>
9f95a23c	27	#include <rte_config.h>
7c673cae FG	28	#include <rte_ip.h>
	29	#include <rte_common.h>
	30
9f95a23c	31	#if defined(RTE_ARCH_X86) \|\| defined(RTE_MACHINE_CPUFLAG_NEON)
7c673cae FG	32	#include <rte_vect.h>
	33	#endif
	34
9f95a23c	35	#ifdef RTE_ARCH_X86
7c673cae FG	36	/* Byte swap mask used for converting IPv6 address
	37	* 4-byte chunks to CPU byte order
	38	*/
	39	static const __m128i rte_thash_ipv6_bswap_mask = {
	40	0x0405060700010203ULL, 0x0C0D0E0F08090A0BULL};
	41	#endif
	42
	43	/**
	44	* length in dwords of input tuple to
	45	* calculate hash of ipv4 header only
	46	*/
	47	#define RTE_THASH_V4_L3_LEN ((sizeof(struct rte_ipv4_tuple) - \
	48	sizeof(((struct rte_ipv4_tuple *)0)->sctp_tag)) / 4)
	49
	50	/**
	51	* length in dwords of input tuple to
	52	* calculate hash of ipv4 header +
	53	* transport header
	54	*/
	55	#define RTE_THASH_V4_L4_LEN ((sizeof(struct rte_ipv4_tuple)) / 4)
	56
	57	/**
	58	* length in dwords of input tuple to
	59	* calculate hash of ipv6 header only
	60	*/
	61	#define RTE_THASH_V6_L3_LEN ((sizeof(struct rte_ipv6_tuple) - \
	62	sizeof(((struct rte_ipv6_tuple *)0)->sctp_tag)) / 4)
	63
	64	/**
	65	* length in dwords of input tuple to
	66	* calculate hash of ipv6 header +
	67	* transport header
	68	*/
	69	#define RTE_THASH_V6_L4_LEN ((sizeof(struct rte_ipv6_tuple)) / 4)
	70
	71	/**
	72	* IPv4 tuple
	73	* addresses and ports/sctp_tag have to be CPU byte order
	74	*/
	75	struct rte_ipv4_tuple {
	76	uint32_t src_addr;
	77	uint32_t dst_addr;
	78	RTE_STD_C11
	79	union {
	80	struct {
	81	uint16_t dport;
	82	uint16_t sport;
	83	};
	84	uint32_t sctp_tag;
	85	};
	86	};
	87
	88	/**
	89	* IPv6 tuple
	90	* Addresses have to be filled by rte_thash_load_v6_addr()
	91	* ports/sctp_tag have to be CPU byte order
	92	*/
	93	struct rte_ipv6_tuple {
	94	uint8_t src_addr[16];
	95	uint8_t dst_addr[16];
	96	RTE_STD_C11
	97	union {
	98	struct {
	99	uint16_t dport;
100	uint16_t sport;
101	};
102	uint32_t sctp_tag;
103	};
104	};
105
106	union rte_thash_tuple {
107	struct rte_ipv4_tuple v4;
108	struct rte_ipv6_tuple v6;
9f95a23c	109	#ifdef RTE_ARCH_X86
7c673cae FG	110	} __attribute__((aligned(XMM_SIZE)));
	111	#else
	112	};
	113	#endif
	114
	115	/**
	116	* Prepare special converted key to use with rte_softrss_be()
	117	* @param orig
	118	* pointer to original RSS key
	119	* @param targ
	120	* pointer to target RSS key
	121	* @param len
	122	* RSS key length
	123	*/
	124	static inline void
	125	rte_convert_rss_key(const uint32_t orig, uint32_t targ, int len)
	126	{
	127	int i;
	128
	129	for (i = 0; i < (len >> 2); i++)
	130	targ[i] = rte_be_to_cpu_32(orig[i]);
	131	}
	132
	133	/**
	134	* Prepare and load IPv6 addresses (src and dst)
	135	* into target tuple
	136	* @param orig
	137	* Pointer to ipv6 header of the original packet
	138	* @param targ
	139	* Pointer to rte_ipv6_tuple structure
	140	*/
	141	static inline void
	142	rte_thash_load_v6_addrs(const struct ipv6_hdr orig, union rte_thash_tuple targ)
	143	{
9f95a23c	144	#ifdef RTE_ARCH_X86
7c673cae FG	145	__m128i ipv6 = _mm_loadu_si128((const __m128i *)orig->src_addr);
	146	(__m128i )targ->v6.src_addr =
	147	_mm_shuffle_epi8(ipv6, rte_thash_ipv6_bswap_mask);
	148	ipv6 = _mm_loadu_si128((const __m128i *)orig->dst_addr);
	149	(__m128i )targ->v6.dst_addr =
	150	_mm_shuffle_epi8(ipv6, rte_thash_ipv6_bswap_mask);
9f95a23c TL	151	#elif defined(RTE_MACHINE_CPUFLAG_NEON)
	152	uint8x16_t ipv6 = vld1q_u8((uint8_t const *)orig->src_addr);
	153	vst1q_u8((uint8_t *)targ->v6.src_addr, vrev32q_u8(ipv6));
	154	ipv6 = vld1q_u8((uint8_t const *)orig->dst_addr);
	155	vst1q_u8((uint8_t *)targ->v6.dst_addr, vrev32q_u8(ipv6));
7c673cae FG	156	#else
	157	int i;
	158	for (i = 0; i < 4; i++) {
	159	((uint32_t )targ->v6.src_addr + i) =
	160	rte_be_to_cpu_32(((const uint32_t )orig->src_addr + i));
	161	((uint32_t )targ->v6.dst_addr + i) =
	162	rte_be_to_cpu_32(((const uint32_t )orig->dst_addr + i));
	163	}
	164	#endif
	165	}
	166
	167	/**
	168	* Generic implementation. Can be used with original rss_key
	169	* @param input_tuple
	170	* Pointer to input tuple
	171	* @param input_len
	172	* Length of input_tuple in 4-bytes chunks
	173	* @param rss_key
	174	* Pointer to RSS hash key.
	175	* @return
	176	* Calculated hash value.
	177	*/
	178	static inline uint32_t
	179	rte_softrss(uint32_t *input_tuple, uint32_t input_len,
	180	const uint8_t *rss_key)
	181	{
9f95a23c	182	uint32_t i, j, map, ret = 0;
7c673cae FG	183
7c673cae FG	184	for (j = 0; j < input_len; j++) {
9f95a23c TL	185	for (map = input_tuple[j]; map; map &= (map - 1)) {
	186	i = rte_bsf32(map);
	187	ret ^= rte_cpu_to_be_32(((const uint32_t *)rss_key)[j]) << (31 - i) \|
7c673cae	188	(uint32_t)((uint64_t)(rte_cpu_to_be_32(((const uint32_t *)rss_key)[j + 1])) >>
9f95a23c	189	(i + 1));
7c673cae FG	190	}
	191	}
	192	return ret;
	193	}
	194
	195	/**
	196	* Optimized implementation.
	197	* If you want the calculated hash value matches NIC RSS value
	198	* you have to use special converted key with rte_convert_rss_key() fn.
	199	* @param input_tuple
	200	* Pointer to input tuple
	201	* @param input_len
	202	* Length of input_tuple in 4-bytes chunks
	203	* @param *rss_key
	204	* Pointer to RSS hash key.
	205	* @return
	206	* Calculated hash value.
	207	*/
	208	static inline uint32_t
	209	rte_softrss_be(uint32_t *input_tuple, uint32_t input_len,
	210	const uint8_t *rss_key)
	211	{
9f95a23c	212	uint32_t i, j, map, ret = 0;
7c673cae FG	213
7c673cae FG	214	for (j = 0; j < input_len; j++) {
9f95a23c TL	215	for (map = input_tuple[j]; map; map &= (map - 1)) {
	216	i = rte_bsf32(map);
	217	ret ^= ((const uint32_t *)rss_key)[j] << (31 - i) \|
	218	(uint32_t)((uint64_t)(((const uint32_t *)rss_key)[j + 1]) >> (i + 1));
7c673cae FG	219	}
	220	}
	221	return ret;
	222	}
	223
	224	#ifdef __cplusplus
	225	}
	226	#endif
	227
	228	#endif /* _RTE_THASH_H */