[ceph.git] / ceph / src / spdk / dpdk / lib / librte_vhost / iotlb.c

/*-
 *   BSD LICENSE
 *
 *   Copyright (c) 2017 Red Hat, Inc.
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *     * Neither the name of Intel Corporation nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#ifdef RTE_LIBRTE_VHOST_NUMA
#include <numaif.h>
#endif

#include <rte_tailq.h>

#include "iotlb.h"
#include "vhost.h"

struct vhost_iotlb_entry {
	TAILQ_ENTRY(vhost_iotlb_entry) next;

	uint64_t iova;
	uint64_t uaddr;
	uint64_t size;
	uint8_t perm;
};

#define IOTLB_CACHE_SIZE 2048

static void
vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq);

static void
vhost_user_iotlb_pending_remove_all(struct vhost_virtqueue *vq)
{
	struct vhost_iotlb_entry *node, *temp_node;

	rte_rwlock_write_lock(&vq->iotlb_pending_lock);

	TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next, temp_node) {
		TAILQ_REMOVE(&vq->iotlb_pending_list, node, next);
		rte_mempool_put(vq->iotlb_pool, node);
	}

	rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
}

bool
vhost_user_iotlb_pending_miss(struct vhost_virtqueue *vq, uint64_t iova,
				uint8_t perm)
{
	struct vhost_iotlb_entry *node;
	bool found = false;

	rte_rwlock_read_lock(&vq->iotlb_pending_lock);

	TAILQ_FOREACH(node, &vq->iotlb_pending_list, next) {
		if ((node->iova == iova) && (node->perm == perm)) {
			found = true;
			break;
		}
	}

	rte_rwlock_read_unlock(&vq->iotlb_pending_lock);

	return found;
}

void
vhost_user_iotlb_pending_insert(struct vhost_virtqueue *vq,
				uint64_t iova, uint8_t perm)
{
	struct vhost_iotlb_entry *node;
	int ret;

	ret = rte_mempool_get(vq->iotlb_pool, (void **)&node);
	if (ret) {
		RTE_LOG(DEBUG, VHOST_CONFIG, "IOTLB pool empty, clear entries\n");
		if (!TAILQ_EMPTY(&vq->iotlb_pending_list))
			vhost_user_iotlb_pending_remove_all(vq);
		else
			vhost_user_iotlb_cache_random_evict(vq);
		ret = rte_mempool_get(vq->iotlb_pool, (void **)&node);
		if (ret) {
			RTE_LOG(ERR, VHOST_CONFIG, "IOTLB pool still empty, failure\n");
			return;
		}
	}

	node->iova = iova;
	node->perm = perm;

	rte_rwlock_write_lock(&vq->iotlb_pending_lock);

	TAILQ_INSERT_TAIL(&vq->iotlb_pending_list, node, next);

	rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
}

void
vhost_user_iotlb_pending_remove(struct vhost_virtqueue *vq,
				uint64_t iova, uint64_t size, uint8_t perm)
{
	struct vhost_iotlb_entry *node, *temp_node;

	rte_rwlock_write_lock(&vq->iotlb_pending_lock);

	TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next, temp_node) {
		if (node->iova < iova)
			continue;
		if (node->iova >= iova + size)
			continue;
		if ((node->perm & perm) != node->perm)
			continue;
		TAILQ_REMOVE(&vq->iotlb_pending_list, node, next);
		rte_mempool_put(vq->iotlb_pool, node);
	}

	rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
}

static void
vhost_user_iotlb_cache_remove_all(struct vhost_virtqueue *vq)
{
	struct vhost_iotlb_entry *node, *temp_node;

	rte_rwlock_write_lock(&vq->iotlb_lock);

	TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
		TAILQ_REMOVE(&vq->iotlb_list, node, next);
		rte_mempool_put(vq->iotlb_pool, node);
	}

	vq->iotlb_cache_nr = 0;

	rte_rwlock_write_unlock(&vq->iotlb_lock);
}

static void
vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq)
{
	struct vhost_iotlb_entry *node, *temp_node;
	int entry_idx;

	rte_rwlock_write_lock(&vq->iotlb_lock);

	entry_idx = rte_rand() % vq->iotlb_cache_nr;

	TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
		if (!entry_idx) {
			TAILQ_REMOVE(&vq->iotlb_list, node, next);
			rte_mempool_put(vq->iotlb_pool, node);
			vq->iotlb_cache_nr--;
			break;
		}
		entry_idx--;
	}

	rte_rwlock_write_unlock(&vq->iotlb_lock);
}

void
vhost_user_iotlb_cache_insert(struct vhost_virtqueue *vq, uint64_t iova,
				uint64_t uaddr, uint64_t size, uint8_t perm)
{
	struct vhost_iotlb_entry *node, *new_node;
	int ret;

	ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node);
	if (ret) {
		RTE_LOG(DEBUG, VHOST_CONFIG, "IOTLB pool empty, clear entries\n");
		if (!TAILQ_EMPTY(&vq->iotlb_list))
			vhost_user_iotlb_cache_random_evict(vq);
		else
			vhost_user_iotlb_pending_remove_all(vq);
		ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node);
		if (ret) {
			RTE_LOG(ERR, VHOST_CONFIG, "IOTLB pool still empty, failure\n");
			return;
		}
	}

	new_node->iova = iova;
	new_node->uaddr = uaddr;
	new_node->size = size;
	new_node->perm = perm;

	rte_rwlock_write_lock(&vq->iotlb_lock);

	TAILQ_FOREACH(node, &vq->iotlb_list, next) {
		/*
		 * Entries must be invalidated before being updated.
		 * So if iova already in list, assume identical.
		 */
		if (node->iova == new_node->iova) {
			rte_mempool_put(vq->iotlb_pool, new_node);
			goto unlock;
		} else if (node->iova > new_node->iova) {
			TAILQ_INSERT_BEFORE(node, new_node, next);
			vq->iotlb_cache_nr++;
			goto unlock;
		}
	}

	TAILQ_INSERT_TAIL(&vq->iotlb_list, new_node, next);
	vq->iotlb_cache_nr++;

unlock:
	vhost_user_iotlb_pending_remove(vq, iova, size, perm);

	rte_rwlock_write_unlock(&vq->iotlb_lock);

}

void
vhost_user_iotlb_cache_remove(struct vhost_virtqueue *vq,
					uint64_t iova, uint64_t size)
{
	struct vhost_iotlb_entry *node, *temp_node;

	if (unlikely(!size))
		return;

	rte_rwlock_write_lock(&vq->iotlb_lock);

	TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
		/* Sorted list */
		if (unlikely(iova + size < node->iova))
			break;

		if (iova < node->iova + node->size) {
			TAILQ_REMOVE(&vq->iotlb_list, node, next);
			rte_mempool_put(vq->iotlb_pool, node);
			vq->iotlb_cache_nr--;
		}
	}

	rte_rwlock_write_unlock(&vq->iotlb_lock);
}

uint64_t
vhost_user_iotlb_cache_find(struct vhost_virtqueue *vq, uint64_t iova,
						uint64_t *size, uint8_t perm)
{
	struct vhost_iotlb_entry *node;
	uint64_t offset, vva = 0, mapped = 0;

	if (unlikely(!*size))
		goto out;

	TAILQ_FOREACH(node, &vq->iotlb_list, next) {
		/* List sorted by iova */
		if (unlikely(iova < node->iova))
			break;

		if (iova >= node->iova + node->size)
			continue;

		if (unlikely((perm & node->perm) != perm)) {
			vva = 0;
			break;
		}

		offset = iova - node->iova;
		if (!vva)
			vva = node->uaddr + offset;

		mapped += node->size - offset;
		iova = node->iova + node->size;

		if (mapped >= *size)
			break;
	}

out:
	/* Only part of the requested chunk is mapped */
	if (unlikely(mapped < *size))
		*size = mapped;

	return vva;
}

void
vhost_user_iotlb_flush_all(struct vhost_virtqueue *vq)
{
	vhost_user_iotlb_cache_remove_all(vq);
	vhost_user_iotlb_pending_remove_all(vq);
}

int
vhost_user_iotlb_init(struct virtio_net *dev, int vq_index)
{
	char pool_name[RTE_MEMPOOL_NAMESIZE];
	struct vhost_virtqueue *vq = dev->virtqueue[vq_index];
	int socket = 0;

	if (vq->iotlb_pool) {
		/*
		 * The cache has already been initialized,
		 * just drop all cached and pending entries.
		 */
		vhost_user_iotlb_flush_all(vq);
	}

#ifdef RTE_LIBRTE_VHOST_NUMA
	if (get_mempolicy(&socket, NULL, 0, vq, MPOL_F_NODE | MPOL_F_ADDR) != 0)
		socket = 0;
#endif

	rte_rwlock_init(&vq->iotlb_lock);
	rte_rwlock_init(&vq->iotlb_pending_lock);

	TAILQ_INIT(&vq->iotlb_list);
	TAILQ_INIT(&vq->iotlb_pending_list);

	snprintf(pool_name, sizeof(pool_name), "iotlb_cache_%d_%d",
			dev->vid, vq_index);

	/* If already created, free it and recreate */
	vq->iotlb_pool = rte_mempool_lookup(pool_name);
	if (vq->iotlb_pool)
		rte_mempool_free(vq->iotlb_pool);

	vq->iotlb_pool = rte_mempool_create(pool_name,
			IOTLB_CACHE_SIZE, sizeof(struct vhost_iotlb_entry), 0,
			0, 0, NULL, NULL, NULL, socket,
			MEMPOOL_F_NO_CACHE_ALIGN |
			MEMPOOL_F_SP_PUT |
			MEMPOOL_F_SC_GET);
	if (!vq->iotlb_pool) {
		RTE_LOG(ERR, VHOST_CONFIG,
				"Failed to create IOTLB cache pool (%s)\n",
				pool_name);
		return -1;
	}

	vq->iotlb_cache_nr = 0;

	return 0;
}
Commit	Line	Data
11fdf7f2 TL	1	/*-
	2	* BSD LICENSE
	3	*
	4	* Copyright (c) 2017 Red Hat, Inc.
	5	*
	6	* Redistribution and use in source and binary forms, with or without
	7	* modification, are permitted provided that the following conditions
	8	* are met:
	9	*
	10	* * Redistributions of source code must retain the above copyright
	11	* notice, this list of conditions and the following disclaimer.
	12	* * Redistributions in binary form must reproduce the above copyright
	13	* notice, this list of conditions and the following disclaimer in
	14	* the documentation and/or other materials provided with the
	15	* distribution.
	16	* * Neither the name of Intel Corporation nor the names of its
	17	* contributors may be used to endorse or promote products derived
	18	* from this software without specific prior written permission.
	19	*
	20	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	21	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	22	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	23	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	24	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	25	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	26	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	27	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	28	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	29	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	30	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	31	*/
	32
	33	#ifdef RTE_LIBRTE_VHOST_NUMA
	34	#include <numaif.h>
	35	#endif
	36
	37	#include <rte_tailq.h>
	38
	39	#include "iotlb.h"
	40	#include "vhost.h"
	41
	42	struct vhost_iotlb_entry {
	43	TAILQ_ENTRY(vhost_iotlb_entry) next;
	44
	45	uint64_t iova;
	46	uint64_t uaddr;
	47	uint64_t size;
	48	uint8_t perm;
	49	};
	50
	51	#define IOTLB_CACHE_SIZE 2048
	52
	53	static void
	54	vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq);
	55
	56	static void
	57	vhost_user_iotlb_pending_remove_all(struct vhost_virtqueue *vq)
	58	{
	59	struct vhost_iotlb_entry node, temp_node;
	60
	61	rte_rwlock_write_lock(&vq->iotlb_pending_lock);
	62
	63	TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next, temp_node) {
	64	TAILQ_REMOVE(&vq->iotlb_pending_list, node, next);
65	rte_mempool_put(vq->iotlb_pool, node);
66	}
67
68	rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
69	}
70
71	bool
72	vhost_user_iotlb_pending_miss(struct vhost_virtqueue *vq, uint64_t iova,
73	uint8_t perm)
74	{
75	struct vhost_iotlb_entry *node;
76	bool found = false;
77
78	rte_rwlock_read_lock(&vq->iotlb_pending_lock);
79
80	TAILQ_FOREACH(node, &vq->iotlb_pending_list, next) {
81	if ((node->iova == iova) && (node->perm == perm)) {
82	found = true;
83	break;
84	}
85	}
86
87	rte_rwlock_read_unlock(&vq->iotlb_pending_lock);
88
89	return found;
90	}
91
92	void
93	vhost_user_iotlb_pending_insert(struct vhost_virtqueue *vq,
94	uint64_t iova, uint8_t perm)
95	{
96	struct vhost_iotlb_entry *node;
97	int ret;
98
99	ret = rte_mempool_get(vq->iotlb_pool, (void **)&node);
100	if (ret) {
101	RTE_LOG(DEBUG, VHOST_CONFIG, "IOTLB pool empty, clear entries\n");
102	if (!TAILQ_EMPTY(&vq->iotlb_pending_list))
103	vhost_user_iotlb_pending_remove_all(vq);
104	else
105	vhost_user_iotlb_cache_random_evict(vq);
106	ret = rte_mempool_get(vq->iotlb_pool, (void **)&node);
107	if (ret) {
108	RTE_LOG(ERR, VHOST_CONFIG, "IOTLB pool still empty, failure\n");
109	return;
110	}
111	}
112
113	node->iova = iova;
114	node->perm = perm;
115
116	rte_rwlock_write_lock(&vq->iotlb_pending_lock);
117
118	TAILQ_INSERT_TAIL(&vq->iotlb_pending_list, node, next);
119
120	rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
121	}
122
123	void
124	vhost_user_iotlb_pending_remove(struct vhost_virtqueue *vq,
125	uint64_t iova, uint64_t size, uint8_t perm)
126	{
127	struct vhost_iotlb_entry node, temp_node;
128
129	rte_rwlock_write_lock(&vq->iotlb_pending_lock);
130
131	TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next, temp_node) {
132	if (node->iova < iova)
133	continue;
134	if (node->iova >= iova + size)
135	continue;
136	if ((node->perm & perm) != node->perm)
137	continue;
138	TAILQ_REMOVE(&vq->iotlb_pending_list, node, next);
139	rte_mempool_put(vq->iotlb_pool, node);
140	}
141
142	rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
143	}
144
145	static void
146	vhost_user_iotlb_cache_remove_all(struct vhost_virtqueue *vq)
147	{
148	struct vhost_iotlb_entry node, temp_node;
149
150	rte_rwlock_write_lock(&vq->iotlb_lock);
151
152	TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
153	TAILQ_REMOVE(&vq->iotlb_list, node, next);
154	rte_mempool_put(vq->iotlb_pool, node);
155	}
156
157	vq->iotlb_cache_nr = 0;
158
159	rte_rwlock_write_unlock(&vq->iotlb_lock);
160	}
161
162	static void
163	vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq)
164	{
165	struct vhost_iotlb_entry node, temp_node;
166	int entry_idx;
167
168	rte_rwlock_write_lock(&vq->iotlb_lock);
169
170	entry_idx = rte_rand() % vq->iotlb_cache_nr;
171
172	TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
173	if (!entry_idx) {
174	TAILQ_REMOVE(&vq->iotlb_list, node, next);
175	rte_mempool_put(vq->iotlb_pool, node);
176	vq->iotlb_cache_nr--;
177	break;
178	}
179	entry_idx--;
180	}
181
182	rte_rwlock_write_unlock(&vq->iotlb_lock);
183	}
184
185	void
186	vhost_user_iotlb_cache_insert(struct vhost_virtqueue *vq, uint64_t iova,
187	uint64_t uaddr, uint64_t size, uint8_t perm)
188	{
189	struct vhost_iotlb_entry node, new_node;
190	int ret;
191
192	ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node);
193	if (ret) {
194	RTE_LOG(DEBUG, VHOST_CONFIG, "IOTLB pool empty, clear entries\n");
195	if (!TAILQ_EMPTY(&vq->iotlb_list))
196	vhost_user_iotlb_cache_random_evict(vq);
197	else
198	vhost_user_iotlb_pending_remove_all(vq);
199	ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node);
200	if (ret) {
201	RTE_LOG(ERR, VHOST_CONFIG, "IOTLB pool still empty, failure\n");
202	return;
203	}
204	}
205
206	new_node->iova = iova;
207	new_node->uaddr = uaddr;
208	new_node->size = size;
209	new_node->perm = perm;
210
211	rte_rwlock_write_lock(&vq->iotlb_lock);
212
213	TAILQ_FOREACH(node, &vq->iotlb_list, next) {
214	/*
215	* Entries must be invalidated before being updated.
216	* So if iova already in list, assume identical.
217	*/
218	if (node->iova == new_node->iova) {
219	rte_mempool_put(vq->iotlb_pool, new_node);
220	goto unlock;
221	} else if (node->iova > new_node->iova) {
222	TAILQ_INSERT_BEFORE(node, new_node, next);
223	vq->iotlb_cache_nr++;
224	goto unlock;
225	}
226	}
227
228	TAILQ_INSERT_TAIL(&vq->iotlb_list, new_node, next);
229	vq->iotlb_cache_nr++;
230
231	unlock:
232	vhost_user_iotlb_pending_remove(vq, iova, size, perm);
233
234	rte_rwlock_write_unlock(&vq->iotlb_lock);
235
236	}
237
238	void
239	vhost_user_iotlb_cache_remove(struct vhost_virtqueue *vq,
240	uint64_t iova, uint64_t size)
241	{
242	struct vhost_iotlb_entry node, temp_node;
243
244	if (unlikely(!size))
245	return;
246
247	rte_rwlock_write_lock(&vq->iotlb_lock);
248
249	TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
250	/* Sorted list */
251	if (unlikely(iova + size < node->iova))
252	break;
253
254	if (iova < node->iova + node->size) {
255	TAILQ_REMOVE(&vq->iotlb_list, node, next);
256	rte_mempool_put(vq->iotlb_pool, node);
257	vq->iotlb_cache_nr--;
258	}
259	}
260
261	rte_rwlock_write_unlock(&vq->iotlb_lock);
262	}
263
264	uint64_t
265	vhost_user_iotlb_cache_find(struct vhost_virtqueue *vq, uint64_t iova,
266	uint64_t *size, uint8_t perm)
267	{
268	struct vhost_iotlb_entry *node;
269	uint64_t offset, vva = 0, mapped = 0;
270
271	if (unlikely(!*size))
272	goto out;
273
274	TAILQ_FOREACH(node, &vq->iotlb_list, next) {
275	/* List sorted by iova */
276	if (unlikely(iova < node->iova))
277	break;
278
279	if (iova >= node->iova + node->size)
280	continue;
281
282	if (unlikely((perm & node->perm) != perm)) {
283	vva = 0;
284	break;
285	}
286
287	offset = iova - node->iova;
288	if (!vva)
289	vva = node->uaddr + offset;
290
291	mapped += node->size - offset;
292	iova = node->iova + node->size;
293
294	if (mapped >= *size)
295	break;
296	}
297
298	out:
299	/* Only part of the requested chunk is mapped */
300	if (unlikely(mapped < *size))
301	*size = mapped;
302
303	return vva;
304	}
305
306	void
307	vhost_user_iotlb_flush_all(struct vhost_virtqueue *vq)
308	{
309	vhost_user_iotlb_cache_remove_all(vq);
310	vhost_user_iotlb_pending_remove_all(vq);
311	}
312
313	int
314	vhost_user_iotlb_init(struct virtio_net *dev, int vq_index)
315	{
316	char pool_name[RTE_MEMPOOL_NAMESIZE];
317	struct vhost_virtqueue *vq = dev->virtqueue[vq_index];
318	int socket = 0;
319
320	if (vq->iotlb_pool) {
321	/*
322	* The cache has already been initialized,
323	* just drop all cached and pending entries.
324	*/
325	vhost_user_iotlb_flush_all(vq);
326	}
327
328	#ifdef RTE_LIBRTE_VHOST_NUMA
329	if (get_mempolicy(&socket, NULL, 0, vq, MPOL_F_NODE \| MPOL_F_ADDR) != 0)
330	socket = 0;
331	#endif
332
333	rte_rwlock_init(&vq->iotlb_lock);
334	rte_rwlock_init(&vq->iotlb_pending_lock);
335
336	TAILQ_INIT(&vq->iotlb_list);
337	TAILQ_INIT(&vq->iotlb_pending_list);
338
339	snprintf(pool_name, sizeof(pool_name), "iotlb_cache_%d_%d",
340	dev->vid, vq_index);
341
342	/* If already created, free it and recreate */
343	vq->iotlb_pool = rte_mempool_lookup(pool_name);
344	if (vq->iotlb_pool)
345	rte_mempool_free(vq->iotlb_pool);
346
347	vq->iotlb_pool = rte_mempool_create(pool_name,
348	IOTLB_CACHE_SIZE, sizeof(struct vhost_iotlb_entry), 0,
349	0, 0, NULL, NULL, NULL, socket,
350	MEMPOOL_F_NO_CACHE_ALIGN \|
351	MEMPOOL_F_SP_PUT \|
352	MEMPOOL_F_SC_GET);
353	if (!vq->iotlb_pool) {
354	RTE_LOG(ERR, VHOST_CONFIG,
355	"Failed to create IOTLB cache pool (%s)\n",
356	pool_name);
357	return -1;
358	}
359
360	vq->iotlb_cache_nr = 0;
361
362	return 0;
363	}