[ceph.git] / ceph / src / spdk / dpdk / lib / librte_table / rte_table_lpm_ipv6.c

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2010-2014 Intel Corporation
 */

#include <string.h>
#include <stdio.h>

#include <rte_common.h>
#include <rte_mbuf.h>
#include <rte_memory.h>
#include <rte_malloc.h>
#include <rte_byteorder.h>
#include <rte_log.h>
#include <rte_lpm6.h>

#include "rte_table_lpm_ipv6.h"

#define RTE_TABLE_LPM_MAX_NEXT_HOPS                        256

#ifdef RTE_TABLE_STATS_COLLECT

#define RTE_TABLE_LPM_IPV6_STATS_PKTS_IN_ADD(table, val) \
	table->stats.n_pkts_in += val
#define RTE_TABLE_LPM_IPV6_STATS_PKTS_LOOKUP_MISS(table, val) \
	table->stats.n_pkts_lookup_miss += val

#else

#define RTE_TABLE_LPM_IPV6_STATS_PKTS_IN_ADD(table, val)
#define RTE_TABLE_LPM_IPV6_STATS_PKTS_LOOKUP_MISS(table, val)

#endif

struct rte_table_lpm_ipv6 {
	struct rte_table_stats stats;

	/* Input parameters */
	uint32_t entry_size;
	uint32_t entry_unique_size;
	uint32_t n_rules;
	uint32_t offset;

	/* Handle to low-level LPM table */
	struct rte_lpm6 *lpm;

	/* Next Hop Table (NHT) */
	uint32_t nht_users[RTE_TABLE_LPM_MAX_NEXT_HOPS];
	uint8_t nht[0] __rte_cache_aligned;
};

static void *
rte_table_lpm_ipv6_create(void *params, int socket_id, uint32_t entry_size)
{
	struct rte_table_lpm_ipv6_params *p =
		params;
	struct rte_table_lpm_ipv6 *lpm;
	struct rte_lpm6_config lpm6_config;
	uint32_t total_size, nht_size;

	/* Check input parameters */
	if (p == NULL) {
		RTE_LOG(ERR, TABLE, "%s: NULL input parameters\n", __func__);
		return NULL;
	}
	if (p->n_rules == 0) {
		RTE_LOG(ERR, TABLE, "%s: Invalid n_rules\n", __func__);
		return NULL;
	}
	if (p->number_tbl8s == 0) {
		RTE_LOG(ERR, TABLE, "%s: Invalid n_rules\n", __func__);
		return NULL;
	}
	if (p->entry_unique_size == 0) {
		RTE_LOG(ERR, TABLE, "%s: Invalid entry_unique_size\n",
			__func__);
		return NULL;
	}
	if (p->entry_unique_size > entry_size) {
		RTE_LOG(ERR, TABLE, "%s: Invalid entry_unique_size\n",
			__func__);
		return NULL;
	}
	if (p->name == NULL) {
		RTE_LOG(ERR, TABLE, "%s: Table name is NULL\n",
			__func__);
		return NULL;
	}
	entry_size = RTE_ALIGN(entry_size, sizeof(uint64_t));

	/* Memory allocation */
	nht_size = RTE_TABLE_LPM_MAX_NEXT_HOPS * entry_size;
	total_size = sizeof(struct rte_table_lpm_ipv6) + nht_size;
	lpm = rte_zmalloc_socket("TABLE", total_size, RTE_CACHE_LINE_SIZE,
		socket_id);
	if (lpm == NULL) {
		RTE_LOG(ERR, TABLE,
			"%s: Cannot allocate %u bytes for LPM IPv6 table\n",
			__func__, total_size);
		return NULL;
	}

	/* LPM low-level table creation */
	lpm6_config.max_rules = p->n_rules;
	lpm6_config.number_tbl8s = p->number_tbl8s;
	lpm6_config.flags = 0;
	lpm->lpm = rte_lpm6_create(p->name, socket_id, &lpm6_config);
	if (lpm->lpm == NULL) {
		rte_free(lpm);
		RTE_LOG(ERR, TABLE,
			"Unable to create low-level LPM IPv6 table\n");
		return NULL;
	}

	/* Memory initialization */
	lpm->entry_size = entry_size;
	lpm->entry_unique_size = p->entry_unique_size;
	lpm->n_rules = p->n_rules;
	lpm->offset = p->offset;

	return lpm;
}

static int
rte_table_lpm_ipv6_free(void *table)
{
	struct rte_table_lpm_ipv6 *lpm = table;

	/* Check input parameters */
	if (lpm == NULL) {
		RTE_LOG(ERR, TABLE, "%s: table parameter is NULL\n", __func__);
		return -EINVAL;
	}

	/* Free previously allocated resources */
	rte_lpm6_free(lpm->lpm);
	rte_free(lpm);

	return 0;
}

static int
nht_find_free(struct rte_table_lpm_ipv6 *lpm, uint32_t *pos)
{
	uint32_t i;

	for (i = 0; i < RTE_TABLE_LPM_MAX_NEXT_HOPS; i++) {
		if (lpm->nht_users[i] == 0) {
			*pos = i;
			return 1;
		}
	}

	return 0;
}

static int
nht_find_existing(struct rte_table_lpm_ipv6 *lpm, void *entry, uint32_t *pos)
{
	uint32_t i;

	for (i = 0; i < RTE_TABLE_LPM_MAX_NEXT_HOPS; i++) {
		uint8_t *nht_entry = &lpm->nht[i * lpm->entry_size];

		if ((lpm->nht_users[i] > 0) && (memcmp(nht_entry, entry,
			lpm->entry_unique_size) == 0)) {
			*pos = i;
			return 1;
		}
	}

	return 0;
}

static int
rte_table_lpm_ipv6_entry_add(
	void *table,
	void *key,
	void *entry,
	int *key_found,
	void **entry_ptr)
{
	struct rte_table_lpm_ipv6 *lpm = table;
	struct rte_table_lpm_ipv6_key *ip_prefix =
		key;
	uint32_t nht_pos = 0, nht_pos0 = 0, nht_pos0_valid = 0;
	int status;

	/* Check input parameters */
	if (lpm == NULL) {
		RTE_LOG(ERR, TABLE, "%s: table parameter is NULL\n", __func__);
		return -EINVAL;
	}
	if (ip_prefix == NULL) {
		RTE_LOG(ERR, TABLE, "%s: ip_prefix parameter is NULL\n",
			__func__);
		return -EINVAL;
	}
	if (entry == NULL) {
		RTE_LOG(ERR, TABLE, "%s: entry parameter is NULL\n", __func__);
		return -EINVAL;
	}

	if ((ip_prefix->depth == 0) || (ip_prefix->depth > 128)) {
		RTE_LOG(ERR, TABLE, "%s: invalid depth (%d)\n", __func__,
			ip_prefix->depth);
		return -EINVAL;
	}

	/* Check if rule is already present in the table */
	status = rte_lpm6_is_rule_present(lpm->lpm, ip_prefix->ip,
		ip_prefix->depth, &nht_pos0);
	nht_pos0_valid = status > 0;

	/* Find existing or free NHT entry */
	if (nht_find_existing(lpm, entry, &nht_pos) == 0) {
		uint8_t *nht_entry;

		if (nht_find_free(lpm, &nht_pos) == 0) {
			RTE_LOG(ERR, TABLE, "%s: NHT full\n", __func__);
			return -1;
		}

		nht_entry = &lpm->nht[nht_pos * lpm->entry_size];
		memcpy(nht_entry, entry, lpm->entry_size);
	}

	/* Add rule to low level LPM table */
	if (rte_lpm6_add(lpm->lpm, ip_prefix->ip, ip_prefix->depth,
		nht_pos) < 0) {
		RTE_LOG(ERR, TABLE, "%s: LPM IPv6 rule add failed\n", __func__);
		return -1;
	}

	/* Commit NHT changes */
	lpm->nht_users[nht_pos]++;
	lpm->nht_users[nht_pos0] -= nht_pos0_valid;

	*key_found = nht_pos0_valid;
	*entry_ptr = (void *) &lpm->nht[nht_pos * lpm->entry_size];
	return 0;
}

static int
rte_table_lpm_ipv6_entry_delete(
	void *table,
	void *key,
	int *key_found,
	void *entry)
{
	struct rte_table_lpm_ipv6 *lpm = table;
	struct rte_table_lpm_ipv6_key *ip_prefix =
		key;
	uint32_t nht_pos;
	int status;

	/* Check input parameters */
	if (lpm == NULL) {
		RTE_LOG(ERR, TABLE, "%s: table parameter is NULL\n", __func__);
		return -EINVAL;
	}
	if (ip_prefix == NULL) {
		RTE_LOG(ERR, TABLE, "%s: ip_prefix parameter is NULL\n",
			__func__);
		return -EINVAL;
	}
	if ((ip_prefix->depth == 0) || (ip_prefix->depth > 128)) {
		RTE_LOG(ERR, TABLE, "%s: invalid depth (%d)\n", __func__,
			ip_prefix->depth);
		return -EINVAL;
	}

	/* Return if rule is not present in the table */
	status = rte_lpm6_is_rule_present(lpm->lpm, ip_prefix->ip,
		ip_prefix->depth, &nht_pos);
	if (status < 0) {
		RTE_LOG(ERR, TABLE, "%s: LPM IPv6 algorithmic error\n",
			__func__);
		return -1;
	}
	if (status == 0) {
		*key_found = 0;
		return 0;
	}

	/* Delete rule from the low-level LPM table */
	status = rte_lpm6_delete(lpm->lpm, ip_prefix->ip, ip_prefix->depth);
	if (status) {
		RTE_LOG(ERR, TABLE, "%s: LPM IPv6 rule delete failed\n",
			__func__);
		return -1;
	}

	/* Commit NHT changes */
	lpm->nht_users[nht_pos]--;

	*key_found = 1;
	if (entry)
		memcpy(entry, &lpm->nht[nht_pos * lpm->entry_size],
			lpm->entry_size);

	return 0;
}

static int
rte_table_lpm_ipv6_lookup(
	void *table,
	struct rte_mbuf **pkts,
	uint64_t pkts_mask,
	uint64_t *lookup_hit_mask,
	void **entries)
{
	struct rte_table_lpm_ipv6 *lpm = (struct rte_table_lpm_ipv6 *) table;
	uint64_t pkts_out_mask = 0;
	uint32_t i;

	__rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask);
	RTE_TABLE_LPM_IPV6_STATS_PKTS_IN_ADD(lpm, n_pkts_in);

	pkts_out_mask = 0;
	for (i = 0; i < (uint32_t)(RTE_PORT_IN_BURST_SIZE_MAX -
		__builtin_clzll(pkts_mask)); i++) {
		uint64_t pkt_mask = 1LLU << i;

		if (pkt_mask & pkts_mask) {
			struct rte_mbuf *pkt = pkts[i];
			uint8_t *ip = RTE_MBUF_METADATA_UINT8_PTR(pkt,
				lpm->offset);
			int status;
			uint32_t nht_pos;

			status = rte_lpm6_lookup(lpm->lpm, ip, &nht_pos);
			if (status == 0) {
				pkts_out_mask |= pkt_mask;
				entries[i] = (void *) &lpm->nht[nht_pos *
					lpm->entry_size];
			}
		}
	}

	*lookup_hit_mask = pkts_out_mask;
	RTE_TABLE_LPM_IPV6_STATS_PKTS_LOOKUP_MISS(lpm, n_pkts_in - __builtin_popcountll(pkts_out_mask));
	return 0;
}

static int
rte_table_lpm_ipv6_stats_read(void *table, struct rte_table_stats *stats, int clear)
{
	struct rte_table_lpm_ipv6 *t = table;

	if (stats != NULL)
		memcpy(stats, &t->stats, sizeof(t->stats));

	if (clear)
		memset(&t->stats, 0, sizeof(t->stats));

	return 0;
}

struct rte_table_ops rte_table_lpm_ipv6_ops = {
	.f_create = rte_table_lpm_ipv6_create,
	.f_free = rte_table_lpm_ipv6_free,
	.f_add = rte_table_lpm_ipv6_entry_add,
	.f_delete = rte_table_lpm_ipv6_entry_delete,
	.f_add_bulk = NULL,
	.f_delete_bulk = NULL,
	.f_lookup = rte_table_lpm_ipv6_lookup,
	.f_stats = rte_table_lpm_ipv6_stats_read,
};
Commit	Line	Data
11fdf7f2 TL	1	/* SPDX-License-Identifier: BSD-3-Clause
11fdf7f2 TL	2	* Copyright(c) 2010-2014 Intel Corporation
7c673cae FG	3	*/
	4
	5	#include <string.h>
	6	#include <stdio.h>
	7
	8	#include <rte_common.h>
	9	#include <rte_mbuf.h>
	10	#include <rte_memory.h>
	11	#include <rte_malloc.h>
	12	#include <rte_byteorder.h>
	13	#include <rte_log.h>
	14	#include <rte_lpm6.h>
	15
	16	#include "rte_table_lpm_ipv6.h"
	17
	18	#define RTE_TABLE_LPM_MAX_NEXT_HOPS 256
	19
	20	#ifdef RTE_TABLE_STATS_COLLECT
	21
	22	#define RTE_TABLE_LPM_IPV6_STATS_PKTS_IN_ADD(table, val) \
	23	table->stats.n_pkts_in += val
	24	#define RTE_TABLE_LPM_IPV6_STATS_PKTS_LOOKUP_MISS(table, val) \
	25	table->stats.n_pkts_lookup_miss += val
	26
	27	#else
	28
	29	#define RTE_TABLE_LPM_IPV6_STATS_PKTS_IN_ADD(table, val)
	30	#define RTE_TABLE_LPM_IPV6_STATS_PKTS_LOOKUP_MISS(table, val)
	31
	32	#endif
	33
	34	struct rte_table_lpm_ipv6 {
	35	struct rte_table_stats stats;
	36
	37	/* Input parameters */
	38	uint32_t entry_size;
	39	uint32_t entry_unique_size;
	40	uint32_t n_rules;
	41	uint32_t offset;
	42
	43	/* Handle to low-level LPM table */
	44	struct rte_lpm6 *lpm;
	45
	46	/* Next Hop Table (NHT) */
	47	uint32_t nht_users[RTE_TABLE_LPM_MAX_NEXT_HOPS];
	48	uint8_t nht[0] __rte_cache_aligned;
	49	};
	50
	51	static void *
	52	rte_table_lpm_ipv6_create(void *params, int socket_id, uint32_t entry_size)
	53	{
	54	struct rte_table_lpm_ipv6_params *p =
11fdf7f2	55	params;
7c673cae FG	56	struct rte_table_lpm_ipv6 *lpm;
	57	struct rte_lpm6_config lpm6_config;
	58	uint32_t total_size, nht_size;
	59
	60	/* Check input parameters */
	61	if (p == NULL) {
	62	RTE_LOG(ERR, TABLE, "%s: NULL input parameters\n", __func__);
	63	return NULL;
	64	}
	65	if (p->n_rules == 0) {
	66	RTE_LOG(ERR, TABLE, "%s: Invalid n_rules\n", __func__);
	67	return NULL;
	68	}
	69	if (p->number_tbl8s == 0) {
	70	RTE_LOG(ERR, TABLE, "%s: Invalid n_rules\n", __func__);
	71	return NULL;
	72	}
	73	if (p->entry_unique_size == 0) {
	74	RTE_LOG(ERR, TABLE, "%s: Invalid entry_unique_size\n",
	75	__func__);
	76	return NULL;
	77	}
	78	if (p->entry_unique_size > entry_size) {
	79	RTE_LOG(ERR, TABLE, "%s: Invalid entry_unique_size\n",
	80	__func__);
	81	return NULL;
	82	}
	83	if (p->name == NULL) {
	84	RTE_LOG(ERR, TABLE, "%s: Table name is NULL\n",
	85	__func__);
	86	return NULL;
	87	}
	88	entry_size = RTE_ALIGN(entry_size, sizeof(uint64_t));
	89
	90	/* Memory allocation */
	91	nht_size = RTE_TABLE_LPM_MAX_NEXT_HOPS * entry_size;
	92	total_size = sizeof(struct rte_table_lpm_ipv6) + nht_size;
	93	lpm = rte_zmalloc_socket("TABLE", total_size, RTE_CACHE_LINE_SIZE,
	94	socket_id);
	95	if (lpm == NULL) {
	96	RTE_LOG(ERR, TABLE,
	97	"%s: Cannot allocate %u bytes for LPM IPv6 table\n",
	98	__func__, total_size);
	99	return NULL;
	100	}
	101
	102	/* LPM low-level table creation */
	103	lpm6_config.max_rules = p->n_rules;
	104	lpm6_config.number_tbl8s = p->number_tbl8s;
	105	lpm6_config.flags = 0;
	106	lpm->lpm = rte_lpm6_create(p->name, socket_id, &lpm6_config);
	107	if (lpm->lpm == NULL) {
	108	rte_free(lpm);
	109	RTE_LOG(ERR, TABLE,
	110	"Unable to create low-level LPM IPv6 table\n");
	111	return NULL;
	112	}
	113
	114	/* Memory initialization */
	115	lpm->entry_size = entry_size;
	116	lpm->entry_unique_size = p->entry_unique_size;
	117	lpm->n_rules = p->n_rules;
	118	lpm->offset = p->offset;
	119
120	return lpm;
121	}
122
123	static int
124	rte_table_lpm_ipv6_free(void *table)
125	{
11fdf7f2	126	struct rte_table_lpm_ipv6 *lpm = table;
7c673cae FG	127
	128	/* Check input parameters */
	129	if (lpm == NULL) {
	130	RTE_LOG(ERR, TABLE, "%s: table parameter is NULL\n", __func__);
	131	return -EINVAL;
	132	}
	133
	134	/* Free previously allocated resources */
	135	rte_lpm6_free(lpm->lpm);
	136	rte_free(lpm);
	137
	138	return 0;
	139	}
	140
	141	static int
	142	nht_find_free(struct rte_table_lpm_ipv6 lpm, uint32_t pos)
	143	{
	144	uint32_t i;
	145
	146	for (i = 0; i < RTE_TABLE_LPM_MAX_NEXT_HOPS; i++) {
	147	if (lpm->nht_users[i] == 0) {
	148	*pos = i;
	149	return 1;
	150	}
	151	}
	152
	153	return 0;
	154	}
	155
	156	static int
	157	nht_find_existing(struct rte_table_lpm_ipv6 lpm, void entry, uint32_t *pos)
	158	{
	159	uint32_t i;
	160
	161	for (i = 0; i < RTE_TABLE_LPM_MAX_NEXT_HOPS; i++) {
	162	uint8_t nht_entry = &lpm->nht[i lpm->entry_size];
	163
	164	if ((lpm->nht_users[i] > 0) && (memcmp(nht_entry, entry,
	165	lpm->entry_unique_size) == 0)) {
	166	*pos = i;
	167	return 1;
	168	}
	169	}
	170
	171	return 0;
	172	}
	173
	174	static int
	175	rte_table_lpm_ipv6_entry_add(
	176	void *table,
	177	void *key,
	178	void *entry,
	179	int *key_found,
	180	void **entry_ptr)
	181	{
11fdf7f2	182	struct rte_table_lpm_ipv6 *lpm = table;
7c673cae	183	struct rte_table_lpm_ipv6_key *ip_prefix =
11fdf7f2	184	key;
f67539c2	185	uint32_t nht_pos = 0, nht_pos0 = 0, nht_pos0_valid = 0;
7c673cae	186	int status;
7c673cae FG	187
	188	/* Check input parameters */
	189	if (lpm == NULL) {
	190	RTE_LOG(ERR, TABLE, "%s: table parameter is NULL\n", __func__);
	191	return -EINVAL;
	192	}
	193	if (ip_prefix == NULL) {
	194	RTE_LOG(ERR, TABLE, "%s: ip_prefix parameter is NULL\n",
	195	__func__);
	196	return -EINVAL;
	197	}
	198	if (entry == NULL) {
	199	RTE_LOG(ERR, TABLE, "%s: entry parameter is NULL\n", __func__);
	200	return -EINVAL;
	201	}
	202
	203	if ((ip_prefix->depth == 0) \|\| (ip_prefix->depth > 128)) {
	204	RTE_LOG(ERR, TABLE, "%s: invalid depth (%d)\n", __func__,
	205	ip_prefix->depth);
	206	return -EINVAL;
	207	}
	208
	209	/* Check if rule is already present in the table */
	210	status = rte_lpm6_is_rule_present(lpm->lpm, ip_prefix->ip,
	211	ip_prefix->depth, &nht_pos0);
	212	nht_pos0_valid = status > 0;
	213
	214	/* Find existing or free NHT entry */
	215	if (nht_find_existing(lpm, entry, &nht_pos) == 0) {
	216	uint8_t *nht_entry;
	217
	218	if (nht_find_free(lpm, &nht_pos) == 0) {
	219	RTE_LOG(ERR, TABLE, "%s: NHT full\n", __func__);
	220	return -1;
	221	}
	222
	223	nht_entry = &lpm->nht[nht_pos * lpm->entry_size];
	224	memcpy(nht_entry, entry, lpm->entry_size);
	225	}
	226
	227	/* Add rule to low level LPM table */
	228	if (rte_lpm6_add(lpm->lpm, ip_prefix->ip, ip_prefix->depth,
11fdf7f2	229	nht_pos) < 0) {
7c673cae FG	230	RTE_LOG(ERR, TABLE, "%s: LPM IPv6 rule add failed\n", __func__);
	231	return -1;
	232	}
	233
	234	/* Commit NHT changes */
	235	lpm->nht_users[nht_pos]++;
	236	lpm->nht_users[nht_pos0] -= nht_pos0_valid;
	237
	238	*key_found = nht_pos0_valid;
	239	entry_ptr = (void ) &lpm->nht[nht_pos * lpm->entry_size];
	240	return 0;
	241	}
	242
	243	static int
	244	rte_table_lpm_ipv6_entry_delete(
	245	void *table,
	246	void *key,
	247	int *key_found,
	248	void *entry)
	249	{
11fdf7f2	250	struct rte_table_lpm_ipv6 *lpm = table;
7c673cae	251	struct rte_table_lpm_ipv6_key *ip_prefix =
11fdf7f2 TL	252	key;
11fdf7f2 TL	253	uint32_t nht_pos;
7c673cae FG	254	int status;
	255
	256	/* Check input parameters */
	257	if (lpm == NULL) {
	258	RTE_LOG(ERR, TABLE, "%s: table parameter is NULL\n", __func__);
	259	return -EINVAL;
	260	}
	261	if (ip_prefix == NULL) {
	262	RTE_LOG(ERR, TABLE, "%s: ip_prefix parameter is NULL\n",
	263	__func__);
	264	return -EINVAL;
	265	}
	266	if ((ip_prefix->depth == 0) \|\| (ip_prefix->depth > 128)) {
	267	RTE_LOG(ERR, TABLE, "%s: invalid depth (%d)\n", __func__,
	268	ip_prefix->depth);
	269	return -EINVAL;
	270	}
	271
	272	/* Return if rule is not present in the table */
	273	status = rte_lpm6_is_rule_present(lpm->lpm, ip_prefix->ip,
	274	ip_prefix->depth, &nht_pos);
	275	if (status < 0) {
	276	RTE_LOG(ERR, TABLE, "%s: LPM IPv6 algorithmic error\n",
	277	__func__);
	278	return -1;
	279	}
	280	if (status == 0) {
	281	*key_found = 0;
	282	return 0;
	283	}
	284
	285	/* Delete rule from the low-level LPM table */
	286	status = rte_lpm6_delete(lpm->lpm, ip_prefix->ip, ip_prefix->depth);
	287	if (status) {
	288	RTE_LOG(ERR, TABLE, "%s: LPM IPv6 rule delete failed\n",
	289	__func__);
	290	return -1;
	291	}
	292
	293	/* Commit NHT changes */
	294	lpm->nht_users[nht_pos]--;
	295
	296	*key_found = 1;
	297	if (entry)
	298	memcpy(entry, &lpm->nht[nht_pos * lpm->entry_size],
	299	lpm->entry_size);
	300
	301	return 0;
	302	}
	303
	304	static int
	305	rte_table_lpm_ipv6_lookup(
	306	void *table,
	307	struct rte_mbuf **pkts,
	308	uint64_t pkts_mask,
	309	uint64_t *lookup_hit_mask,
	310	void **entries)
	311	{
	312	struct rte_table_lpm_ipv6 lpm = (struct rte_table_lpm_ipv6 ) table;
	313	uint64_t pkts_out_mask = 0;
	314	uint32_t i;
	315
	316	__rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask);
	317	RTE_TABLE_LPM_IPV6_STATS_PKTS_IN_ADD(lpm, n_pkts_in);
318
319	pkts_out_mask = 0;
320	for (i = 0; i < (uint32_t)(RTE_PORT_IN_BURST_SIZE_MAX -
321	__builtin_clzll(pkts_mask)); i++) {
322	uint64_t pkt_mask = 1LLU << i;
323
324	if (pkt_mask & pkts_mask) {
325	struct rte_mbuf *pkt = pkts[i];
326	uint8_t *ip = RTE_MBUF_METADATA_UINT8_PTR(pkt,
327	lpm->offset);
328	int status;
11fdf7f2	329	uint32_t nht_pos;
7c673cae FG	330
	331	status = rte_lpm6_lookup(lpm->lpm, ip, &nht_pos);
	332	if (status == 0) {
	333	pkts_out_mask \|= pkt_mask;
	334	entries[i] = (void ) &lpm->nht[nht_pos
	335	lpm->entry_size];
	336	}
	337	}
	338	}
	339
	340	*lookup_hit_mask = pkts_out_mask;
	341	RTE_TABLE_LPM_IPV6_STATS_PKTS_LOOKUP_MISS(lpm, n_pkts_in - __builtin_popcountll(pkts_out_mask));
	342	return 0;
	343	}
	344
	345	static int
	346	rte_table_lpm_ipv6_stats_read(void table, struct rte_table_stats stats, int clear)
	347	{
11fdf7f2	348	struct rte_table_lpm_ipv6 *t = table;
7c673cae FG	349
	350	if (stats != NULL)
	351	memcpy(stats, &t->stats, sizeof(t->stats));
	352
	353	if (clear)
	354	memset(&t->stats, 0, sizeof(t->stats));
	355
	356	return 0;
	357	}
	358
	359	struct rte_table_ops rte_table_lpm_ipv6_ops = {
	360	.f_create = rte_table_lpm_ipv6_create,
	361	.f_free = rte_table_lpm_ipv6_free,
	362	.f_add = rte_table_lpm_ipv6_entry_add,
	363	.f_delete = rte_table_lpm_ipv6_entry_delete,
	364	.f_add_bulk = NULL,
	365	.f_delete_bulk = NULL,
	366	.f_lookup = rte_table_lpm_ipv6_lookup,
	367	.f_stats = rte_table_lpm_ipv6_stats_read,
	368	};