[ceph.git] / ceph / src / spdk / dpdk / lib / librte_vhost / iotlb.c

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright (c) 2017 Red Hat, Inc.
 */

#ifdef RTE_LIBRTE_VHOST_NUMA
#include <numaif.h>
#endif

#include <rte_tailq.h>

#include "iotlb.h"
#include "vhost.h"

struct vhost_iotlb_entry {
	TAILQ_ENTRY(vhost_iotlb_entry) next;

	uint64_t iova;
	uint64_t uaddr;
	uint64_t size;
	uint8_t perm;
};

#define IOTLB_CACHE_SIZE 2048

static void
vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq);

static void
vhost_user_iotlb_pending_remove_all(struct vhost_virtqueue *vq)
{
	struct vhost_iotlb_entry *node, *temp_node;

	rte_rwlock_write_lock(&vq->iotlb_pending_lock);

	TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next, temp_node) {
		TAILQ_REMOVE(&vq->iotlb_pending_list, node, next);
		rte_mempool_put(vq->iotlb_pool, node);
	}

	rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
}

bool
vhost_user_iotlb_pending_miss(struct vhost_virtqueue *vq, uint64_t iova,
				uint8_t perm)
{
	struct vhost_iotlb_entry *node;
	bool found = false;

	rte_rwlock_read_lock(&vq->iotlb_pending_lock);

	TAILQ_FOREACH(node, &vq->iotlb_pending_list, next) {
		if ((node->iova == iova) && (node->perm == perm)) {
			found = true;
			break;
		}
	}

	rte_rwlock_read_unlock(&vq->iotlb_pending_lock);

	return found;
}

void
vhost_user_iotlb_pending_insert(struct vhost_virtqueue *vq,
				uint64_t iova, uint8_t perm)
{
	struct vhost_iotlb_entry *node;
	int ret;

	ret = rte_mempool_get(vq->iotlb_pool, (void **)&node);
	if (ret) {
		VHOST_LOG_CONFIG(DEBUG, "IOTLB pool empty, clear entries\n");
		if (!TAILQ_EMPTY(&vq->iotlb_pending_list))
			vhost_user_iotlb_pending_remove_all(vq);
		else
			vhost_user_iotlb_cache_random_evict(vq);
		ret = rte_mempool_get(vq->iotlb_pool, (void **)&node);
		if (ret) {
			VHOST_LOG_CONFIG(ERR, "IOTLB pool still empty, failure\n");
			return;
		}
	}

	node->iova = iova;
	node->perm = perm;

	rte_rwlock_write_lock(&vq->iotlb_pending_lock);

	TAILQ_INSERT_TAIL(&vq->iotlb_pending_list, node, next);

	rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
}

void
vhost_user_iotlb_pending_remove(struct vhost_virtqueue *vq,
				uint64_t iova, uint64_t size, uint8_t perm)
{
	struct vhost_iotlb_entry *node, *temp_node;

	rte_rwlock_write_lock(&vq->iotlb_pending_lock);

	TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next, temp_node) {
		if (node->iova < iova)
			continue;
		if (node->iova >= iova + size)
			continue;
		if ((node->perm & perm) != node->perm)
			continue;
		TAILQ_REMOVE(&vq->iotlb_pending_list, node, next);
		rte_mempool_put(vq->iotlb_pool, node);
	}

	rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
}

static void
vhost_user_iotlb_cache_remove_all(struct vhost_virtqueue *vq)
{
	struct vhost_iotlb_entry *node, *temp_node;

	rte_rwlock_write_lock(&vq->iotlb_lock);

	TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
		TAILQ_REMOVE(&vq->iotlb_list, node, next);
		rte_mempool_put(vq->iotlb_pool, node);
	}

	vq->iotlb_cache_nr = 0;

	rte_rwlock_write_unlock(&vq->iotlb_lock);
}

static void
vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq)
{
	struct vhost_iotlb_entry *node, *temp_node;
	int entry_idx;

	rte_rwlock_write_lock(&vq->iotlb_lock);

	entry_idx = rte_rand() % vq->iotlb_cache_nr;

	TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
		if (!entry_idx) {
			TAILQ_REMOVE(&vq->iotlb_list, node, next);
			rte_mempool_put(vq->iotlb_pool, node);
			vq->iotlb_cache_nr--;
			break;
		}
		entry_idx--;
	}

	rte_rwlock_write_unlock(&vq->iotlb_lock);
}

void
vhost_user_iotlb_cache_insert(struct vhost_virtqueue *vq, uint64_t iova,
				uint64_t uaddr, uint64_t size, uint8_t perm)
{
	struct vhost_iotlb_entry *node, *new_node;
	int ret;

	ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node);
	if (ret) {
		VHOST_LOG_CONFIG(DEBUG, "IOTLB pool empty, clear entries\n");
		if (!TAILQ_EMPTY(&vq->iotlb_list))
			vhost_user_iotlb_cache_random_evict(vq);
		else
			vhost_user_iotlb_pending_remove_all(vq);
		ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node);
		if (ret) {
			VHOST_LOG_CONFIG(ERR, "IOTLB pool still empty, failure\n");
			return;
		}
	}

	new_node->iova = iova;
	new_node->uaddr = uaddr;
	new_node->size = size;
	new_node->perm = perm;

	rte_rwlock_write_lock(&vq->iotlb_lock);

	TAILQ_FOREACH(node, &vq->iotlb_list, next) {
		/*
		 * Entries must be invalidated before being updated.
		 * So if iova already in list, assume identical.
		 */
		if (node->iova == new_node->iova) {
			rte_mempool_put(vq->iotlb_pool, new_node);
			goto unlock;
		} else if (node->iova > new_node->iova) {
			TAILQ_INSERT_BEFORE(node, new_node, next);
			vq->iotlb_cache_nr++;
			goto unlock;
		}
	}

	TAILQ_INSERT_TAIL(&vq->iotlb_list, new_node, next);
	vq->iotlb_cache_nr++;

unlock:
	vhost_user_iotlb_pending_remove(vq, iova, size, perm);

	rte_rwlock_write_unlock(&vq->iotlb_lock);

}

void
vhost_user_iotlb_cache_remove(struct vhost_virtqueue *vq,
					uint64_t iova, uint64_t size)
{
	struct vhost_iotlb_entry *node, *temp_node;

	if (unlikely(!size))
		return;

	rte_rwlock_write_lock(&vq->iotlb_lock);

	TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
		/* Sorted list */
		if (unlikely(iova + size < node->iova))
			break;

		if (iova < node->iova + node->size) {
			TAILQ_REMOVE(&vq->iotlb_list, node, next);
			rte_mempool_put(vq->iotlb_pool, node);
			vq->iotlb_cache_nr--;
		}
	}

	rte_rwlock_write_unlock(&vq->iotlb_lock);
}

uint64_t
vhost_user_iotlb_cache_find(struct vhost_virtqueue *vq, uint64_t iova,
						uint64_t *size, uint8_t perm)
{
	struct vhost_iotlb_entry *node;
	uint64_t offset, vva = 0, mapped = 0;

	if (unlikely(!*size))
		goto out;

	TAILQ_FOREACH(node, &vq->iotlb_list, next) {
		/* List sorted by iova */
		if (unlikely(iova < node->iova))
			break;

		if (iova >= node->iova + node->size)
			continue;

		if (unlikely((perm & node->perm) != perm)) {
			vva = 0;
			break;
		}

		offset = iova - node->iova;
		if (!vva)
			vva = node->uaddr + offset;

		mapped += node->size - offset;
		iova = node->iova + node->size;

		if (mapped >= *size)
			break;
	}

out:
	/* Only part of the requested chunk is mapped */
	if (unlikely(mapped < *size))
		*size = mapped;

	return vva;
}

void
vhost_user_iotlb_flush_all(struct vhost_virtqueue *vq)
{
	vhost_user_iotlb_cache_remove_all(vq);
	vhost_user_iotlb_pending_remove_all(vq);
}

int
vhost_user_iotlb_init(struct virtio_net *dev, int vq_index)
{
	char pool_name[RTE_MEMPOOL_NAMESIZE];
	struct vhost_virtqueue *vq = dev->virtqueue[vq_index];
	int socket = 0;

	if (vq->iotlb_pool) {
		/*
		 * The cache has already been initialized,
		 * just drop all cached and pending entries.
		 */
		vhost_user_iotlb_flush_all(vq);
	}

#ifdef RTE_LIBRTE_VHOST_NUMA
	if (get_mempolicy(&socket, NULL, 0, vq, MPOL_F_NODE | MPOL_F_ADDR) != 0)
		socket = 0;
#endif

	rte_rwlock_init(&vq->iotlb_lock);
	rte_rwlock_init(&vq->iotlb_pending_lock);

	TAILQ_INIT(&vq->iotlb_list);
	TAILQ_INIT(&vq->iotlb_pending_list);

	snprintf(pool_name, sizeof(pool_name), "iotlb_%u_%d_%d",
			getpid(), dev->vid, vq_index);
	VHOST_LOG_CONFIG(DEBUG, "IOTLB cache name: %s\n", pool_name);

	/* If already created, free it and recreate */
	vq->iotlb_pool = rte_mempool_lookup(pool_name);
	if (vq->iotlb_pool)
		rte_mempool_free(vq->iotlb_pool);

	vq->iotlb_pool = rte_mempool_create(pool_name,
			IOTLB_CACHE_SIZE, sizeof(struct vhost_iotlb_entry), 0,
			0, 0, NULL, NULL, NULL, socket,
			MEMPOOL_F_NO_CACHE_ALIGN |
			MEMPOOL_F_SP_PUT |
			MEMPOOL_F_SC_GET);
	if (!vq->iotlb_pool) {
		VHOST_LOG_CONFIG(ERR,
				"Failed to create IOTLB cache pool (%s)\n",
				pool_name);
		return -1;
	}

	vq->iotlb_cache_nr = 0;

	return 0;
}
Commit	Line	Data
f67539c2 TL	1	/* SPDX-License-Identifier: BSD-3-Clause
f67539c2 TL	2	* Copyright (c) 2017 Red Hat, Inc.
11fdf7f2 TL	3	*/
	4
	5	#ifdef RTE_LIBRTE_VHOST_NUMA
	6	#include <numaif.h>
	7	#endif
	8
	9	#include <rte_tailq.h>
	10
	11	#include "iotlb.h"
	12	#include "vhost.h"
	13
	14	struct vhost_iotlb_entry {
	15	TAILQ_ENTRY(vhost_iotlb_entry) next;
	16
	17	uint64_t iova;
	18	uint64_t uaddr;
	19	uint64_t size;
	20	uint8_t perm;
	21	};
	22
	23	#define IOTLB_CACHE_SIZE 2048
	24
	25	static void
	26	vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq);
	27
	28	static void
	29	vhost_user_iotlb_pending_remove_all(struct vhost_virtqueue *vq)
	30	{
	31	struct vhost_iotlb_entry node, temp_node;
	32
	33	rte_rwlock_write_lock(&vq->iotlb_pending_lock);
	34
	35	TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next, temp_node) {
	36	TAILQ_REMOVE(&vq->iotlb_pending_list, node, next);
	37	rte_mempool_put(vq->iotlb_pool, node);
	38	}
	39
	40	rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
	41	}
	42
	43	bool
	44	vhost_user_iotlb_pending_miss(struct vhost_virtqueue *vq, uint64_t iova,
	45	uint8_t perm)
	46	{
	47	struct vhost_iotlb_entry *node;
	48	bool found = false;
	49
	50	rte_rwlock_read_lock(&vq->iotlb_pending_lock);
	51
	52	TAILQ_FOREACH(node, &vq->iotlb_pending_list, next) {
	53	if ((node->iova == iova) && (node->perm == perm)) {
	54	found = true;
	55	break;
	56	}
	57	}
	58
	59	rte_rwlock_read_unlock(&vq->iotlb_pending_lock);
	60
	61	return found;
	62	}
	63
	64	void
	65	vhost_user_iotlb_pending_insert(struct vhost_virtqueue *vq,
	66	uint64_t iova, uint8_t perm)
67	{
68	struct vhost_iotlb_entry *node;
69	int ret;
70
71	ret = rte_mempool_get(vq->iotlb_pool, (void **)&node);
72	if (ret) {
f67539c2	73	VHOST_LOG_CONFIG(DEBUG, "IOTLB pool empty, clear entries\n");
11fdf7f2 TL	74	if (!TAILQ_EMPTY(&vq->iotlb_pending_list))
	75	vhost_user_iotlb_pending_remove_all(vq);
	76	else
	77	vhost_user_iotlb_cache_random_evict(vq);
	78	ret = rte_mempool_get(vq->iotlb_pool, (void **)&node);
	79	if (ret) {
f67539c2	80	VHOST_LOG_CONFIG(ERR, "IOTLB pool still empty, failure\n");
11fdf7f2 TL	81	return;
	82	}
	83	}
	84
	85	node->iova = iova;
	86	node->perm = perm;
	87
	88	rte_rwlock_write_lock(&vq->iotlb_pending_lock);
	89
	90	TAILQ_INSERT_TAIL(&vq->iotlb_pending_list, node, next);
	91
	92	rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
	93	}
	94
	95	void
	96	vhost_user_iotlb_pending_remove(struct vhost_virtqueue *vq,
	97	uint64_t iova, uint64_t size, uint8_t perm)
	98	{
	99	struct vhost_iotlb_entry node, temp_node;
	100
	101	rte_rwlock_write_lock(&vq->iotlb_pending_lock);
	102
	103	TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next, temp_node) {
	104	if (node->iova < iova)
	105	continue;
	106	if (node->iova >= iova + size)
	107	continue;
	108	if ((node->perm & perm) != node->perm)
	109	continue;
	110	TAILQ_REMOVE(&vq->iotlb_pending_list, node, next);
	111	rte_mempool_put(vq->iotlb_pool, node);
	112	}
	113
	114	rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
	115	}
	116
	117	static void
	118	vhost_user_iotlb_cache_remove_all(struct vhost_virtqueue *vq)
	119	{
	120	struct vhost_iotlb_entry node, temp_node;
	121
	122	rte_rwlock_write_lock(&vq->iotlb_lock);
	123
	124	TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
	125	TAILQ_REMOVE(&vq->iotlb_list, node, next);
	126	rte_mempool_put(vq->iotlb_pool, node);
	127	}
	128
	129	vq->iotlb_cache_nr = 0;
	130
	131	rte_rwlock_write_unlock(&vq->iotlb_lock);
	132	}
	133
	134	static void
	135	vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq)
	136	{
	137	struct vhost_iotlb_entry node, temp_node;
	138	int entry_idx;
	139
	140	rte_rwlock_write_lock(&vq->iotlb_lock);
	141
	142	entry_idx = rte_rand() % vq->iotlb_cache_nr;
	143
	144	TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
145	if (!entry_idx) {
146	TAILQ_REMOVE(&vq->iotlb_list, node, next);
147	rte_mempool_put(vq->iotlb_pool, node);
148	vq->iotlb_cache_nr--;
149	break;
150	}
151	entry_idx--;
152	}
153
154	rte_rwlock_write_unlock(&vq->iotlb_lock);
155	}
156
157	void
158	vhost_user_iotlb_cache_insert(struct vhost_virtqueue *vq, uint64_t iova,
159	uint64_t uaddr, uint64_t size, uint8_t perm)
160	{
161	struct vhost_iotlb_entry node, new_node;
162	int ret;
163
164	ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node);
165	if (ret) {
f67539c2	166	VHOST_LOG_CONFIG(DEBUG, "IOTLB pool empty, clear entries\n");
11fdf7f2 TL	167	if (!TAILQ_EMPTY(&vq->iotlb_list))
	168	vhost_user_iotlb_cache_random_evict(vq);
	169	else
	170	vhost_user_iotlb_pending_remove_all(vq);
	171	ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node);
	172	if (ret) {
f67539c2	173	VHOST_LOG_CONFIG(ERR, "IOTLB pool still empty, failure\n");
11fdf7f2 TL	174	return;
	175	}
	176	}
	177
	178	new_node->iova = iova;
	179	new_node->uaddr = uaddr;
	180	new_node->size = size;
	181	new_node->perm = perm;
	182
	183	rte_rwlock_write_lock(&vq->iotlb_lock);
	184
	185	TAILQ_FOREACH(node, &vq->iotlb_list, next) {
	186	/*
	187	* Entries must be invalidated before being updated.
	188	* So if iova already in list, assume identical.
	189	*/
	190	if (node->iova == new_node->iova) {
	191	rte_mempool_put(vq->iotlb_pool, new_node);
	192	goto unlock;
	193	} else if (node->iova > new_node->iova) {
	194	TAILQ_INSERT_BEFORE(node, new_node, next);
	195	vq->iotlb_cache_nr++;
	196	goto unlock;
	197	}
	198	}
	199
	200	TAILQ_INSERT_TAIL(&vq->iotlb_list, new_node, next);
	201	vq->iotlb_cache_nr++;
	202
	203	unlock:
	204	vhost_user_iotlb_pending_remove(vq, iova, size, perm);
	205
	206	rte_rwlock_write_unlock(&vq->iotlb_lock);
	207
	208	}
	209
	210	void
	211	vhost_user_iotlb_cache_remove(struct vhost_virtqueue *vq,
	212	uint64_t iova, uint64_t size)
	213	{
	214	struct vhost_iotlb_entry node, temp_node;
	215
	216	if (unlikely(!size))
	217	return;
	218
	219	rte_rwlock_write_lock(&vq->iotlb_lock);
	220
	221	TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
	222	/* Sorted list */
	223	if (unlikely(iova + size < node->iova))
	224	break;
	225
	226	if (iova < node->iova + node->size) {
	227	TAILQ_REMOVE(&vq->iotlb_list, node, next);
	228	rte_mempool_put(vq->iotlb_pool, node);
	229	vq->iotlb_cache_nr--;
	230	}
	231	}
	232
	233	rte_rwlock_write_unlock(&vq->iotlb_lock);
	234	}
	235
	236	uint64_t
	237	vhost_user_iotlb_cache_find(struct vhost_virtqueue *vq, uint64_t iova,
238	uint64_t *size, uint8_t perm)
239	{
240	struct vhost_iotlb_entry *node;
241	uint64_t offset, vva = 0, mapped = 0;
242
243	if (unlikely(!*size))
244	goto out;
245
246	TAILQ_FOREACH(node, &vq->iotlb_list, next) {
247	/* List sorted by iova */
248	if (unlikely(iova < node->iova))
249	break;
250
251	if (iova >= node->iova + node->size)
252	continue;
253
254	if (unlikely((perm & node->perm) != perm)) {
255	vva = 0;
256	break;
257	}
258
259	offset = iova - node->iova;
260	if (!vva)
261	vva = node->uaddr + offset;
262
263	mapped += node->size - offset;
264	iova = node->iova + node->size;
265
266	if (mapped >= *size)
267	break;
268	}
269
270	out:
271	/* Only part of the requested chunk is mapped */
272	if (unlikely(mapped < *size))
273	*size = mapped;
274
275	return vva;
276	}
277
278	void
279	vhost_user_iotlb_flush_all(struct vhost_virtqueue *vq)
280	{
281	vhost_user_iotlb_cache_remove_all(vq);
282	vhost_user_iotlb_pending_remove_all(vq);
283	}
284
285	int
286	vhost_user_iotlb_init(struct virtio_net *dev, int vq_index)
287	{
288	char pool_name[RTE_MEMPOOL_NAMESIZE];
289	struct vhost_virtqueue *vq = dev->virtqueue[vq_index];
290	int socket = 0;
291
292	if (vq->iotlb_pool) {
293	/*
294	* The cache has already been initialized,
295	* just drop all cached and pending entries.
296	*/
297	vhost_user_iotlb_flush_all(vq);
298	}
299
300	#ifdef RTE_LIBRTE_VHOST_NUMA
301	if (get_mempolicy(&socket, NULL, 0, vq, MPOL_F_NODE \| MPOL_F_ADDR) != 0)
302	socket = 0;
303	#endif
304
305	rte_rwlock_init(&vq->iotlb_lock);
306	rte_rwlock_init(&vq->iotlb_pending_lock);
307
308	TAILQ_INIT(&vq->iotlb_list);
309	TAILQ_INIT(&vq->iotlb_pending_list);
310
f67539c2 TL	311	snprintf(pool_name, sizeof(pool_name), "iotlb_%u_%d_%d",
	312	getpid(), dev->vid, vq_index);
	313	VHOST_LOG_CONFIG(DEBUG, "IOTLB cache name: %s\n", pool_name);
11fdf7f2 TL	314
	315	/* If already created, free it and recreate */
	316	vq->iotlb_pool = rte_mempool_lookup(pool_name);
	317	if (vq->iotlb_pool)
	318	rte_mempool_free(vq->iotlb_pool);
	319
	320	vq->iotlb_pool = rte_mempool_create(pool_name,
	321	IOTLB_CACHE_SIZE, sizeof(struct vhost_iotlb_entry), 0,
	322	0, 0, NULL, NULL, NULL, socket,
	323	MEMPOOL_F_NO_CACHE_ALIGN \|
	324	MEMPOOL_F_SP_PUT \|
	325	MEMPOOL_F_SC_GET);
	326	if (!vq->iotlb_pool) {
f67539c2	327	VHOST_LOG_CONFIG(ERR,
11fdf7f2 TL	328	"Failed to create IOTLB cache pool (%s)\n",
	329	pool_name);
	330	return -1;
	331	}
	332
	333	vq->iotlb_cache_nr = 0;
	334
	335	return 0;
	336	}