[ceph.git] / ceph / src / spdk / examples / ioat / verify / verify.c

/*-
 *   BSD LICENSE
 *
 *   Copyright (c) Intel Corporation.
 *   All rights reserved.
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *     * Neither the name of Intel Corporation nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "spdk/stdinc.h"

#include "spdk/ioat.h"
#include "spdk/env.h"
#include "spdk/queue.h"
#include "spdk/string.h"
#include "spdk/util.h"

#define SRC_BUFFER_SIZE (512*1024)

enum ioat_task_type {
	IOAT_COPY_TYPE,
	IOAT_FILL_TYPE,
};

struct user_config {
	int queue_depth;
	int time_in_sec;
	char *core_mask;
};

struct ioat_device {
	struct spdk_ioat_chan *ioat;
	TAILQ_ENTRY(ioat_device) tailq;
};

static TAILQ_HEAD(, ioat_device) g_devices;
static struct ioat_device *g_next_device;

static struct user_config g_user_config;

struct thread_entry {
	struct spdk_ioat_chan *chan;
	uint64_t xfer_completed;
	uint64_t xfer_failed;
	uint64_t fill_completed;
	uint64_t fill_failed;
	uint64_t current_queue_depth;
	unsigned lcore_id;
	bool is_draining;
	bool init_failed;
	struct spdk_mempool *data_pool;
	struct spdk_mempool *task_pool;
};

struct ioat_task {
	enum ioat_task_type type;
	struct thread_entry *thread_entry;
	void *buffer;
	int len;
	uint64_t fill_pattern;
	void *src;
	void *dst;
};

static __thread unsigned int seed = 0;

static unsigned char *g_src;

static void submit_single_xfer(struct ioat_task *ioat_task);

static void
construct_user_config(struct user_config *self)
{
	self->queue_depth = 32;
	self->time_in_sec = 10;
	self->core_mask = "0x1";
}

static void
dump_user_config(struct user_config *self)
{
	printf("User configuration:\n");
	printf("Run time:       %u seconds\n", self->time_in_sec);
	printf("Core mask:      %s\n", self->core_mask);
	printf("Queue depth:    %u\n", self->queue_depth);
}

static void
ioat_exit(void)
{
	struct ioat_device *dev;

	while (!TAILQ_EMPTY(&g_devices)) {
		dev = TAILQ_FIRST(&g_devices);
		TAILQ_REMOVE(&g_devices, dev, tailq);
		if (dev->ioat) {
			spdk_ioat_detach(dev->ioat);
		}
		free(dev);
	}
}
static void prepare_ioat_task(struct thread_entry *thread_entry, struct ioat_task *ioat_task)
{
	int len;
	uintptr_t src_offset;
	uintptr_t dst_offset;
	uint64_t fill_pattern;

	if (ioat_task->type == IOAT_FILL_TYPE) {
		fill_pattern = rand_r(&seed);
		fill_pattern = fill_pattern << 32 | rand_r(&seed);

		/* Ensure that the length of memset block is 8 Bytes aligned.
		 * In case the buffer crosses hugepage boundary and must be split,
		 * we also need to ensure 8 byte address alignment. We do it
		 * unconditionally to keep things simple.
		 */
		len = 8 + ((rand_r(&seed) % (SRC_BUFFER_SIZE - 16)) & ~0x7);
		dst_offset = 8 + rand_r(&seed) % (SRC_BUFFER_SIZE - 8 - len);
		ioat_task->fill_pattern = fill_pattern;
		ioat_task->dst = (void *)(((uintptr_t)ioat_task->buffer + dst_offset) & ~0x7);
	} else {
		src_offset = rand_r(&seed) % SRC_BUFFER_SIZE;
		len = rand_r(&seed) % (SRC_BUFFER_SIZE - src_offset);
		dst_offset = rand_r(&seed) % (SRC_BUFFER_SIZE - len);

		memset(ioat_task->buffer, 0, SRC_BUFFER_SIZE);
		ioat_task->src = (void *)((uintptr_t)g_src + src_offset);
		ioat_task->dst = (void *)((uintptr_t)ioat_task->buffer + dst_offset);
	}
	ioat_task->len = len;
	ioat_task->thread_entry = thread_entry;
}

static void
ioat_done(void *cb_arg)
{
	char *value;
	int i, failed = 0;
	struct ioat_task *ioat_task = (struct ioat_task *)cb_arg;
	struct thread_entry *thread_entry = ioat_task->thread_entry;

	if (ioat_task->type == IOAT_FILL_TYPE) {
		value = ioat_task->dst;
		for (i = 0; i < ioat_task->len / 8; i++) {
			if (memcmp(value, &ioat_task->fill_pattern, 8) != 0) {
				thread_entry->fill_failed++;
				failed = 1;
				break;
			}
			value += 8;
		}
		if (!failed) {
			thread_entry->fill_completed++;
		}
	} else {
		if (memcmp(ioat_task->src, ioat_task->dst, ioat_task->len)) {
			thread_entry->xfer_failed++;
		} else {
			thread_entry->xfer_completed++;
		}
	}

	thread_entry->current_queue_depth--;
	if (thread_entry->is_draining) {
		spdk_mempool_put(thread_entry->data_pool, ioat_task->buffer);
		spdk_mempool_put(thread_entry->task_pool, ioat_task);
	} else {
		prepare_ioat_task(thread_entry, ioat_task);
		submit_single_xfer(ioat_task);
	}
}

static bool
probe_cb(void *cb_ctx, struct spdk_pci_device *pci_dev)
{
	printf(" Found matching device at %04x:%02x:%02x.%x "
	       "vendor:0x%04x device:0x%04x\n",
	       spdk_pci_device_get_domain(pci_dev),
	       spdk_pci_device_get_bus(pci_dev), spdk_pci_device_get_dev(pci_dev),
	       spdk_pci_device_get_func(pci_dev),
	       spdk_pci_device_get_vendor_id(pci_dev), spdk_pci_device_get_device_id(pci_dev));

	return true;
}

static void
attach_cb(void *cb_ctx, struct spdk_pci_device *pci_dev, struct spdk_ioat_chan *ioat)
{
	struct ioat_device *dev;

	dev = malloc(sizeof(*dev));
	if (dev == NULL) {
		printf("Failed to allocate device struct\n");
		return;
	}
	memset(dev, 0, sizeof(*dev));

	dev->ioat = ioat;
	TAILQ_INSERT_TAIL(&g_devices, dev, tailq);
}

static int
ioat_init(void)
{
	TAILQ_INIT(&g_devices);

	if (spdk_ioat_probe(NULL, probe_cb, attach_cb) != 0) {
		fprintf(stderr, "ioat_probe() failed\n");
		return 1;
	}

	return 0;
}

static void
usage(char *program_name)
{
	printf("%s options\n", program_name);
	printf("\t[-h help message]\n");
	printf("\t[-c core mask for distributing I/O submission/completion work]\n");
	printf("\t[-t time in seconds]\n");
	printf("\t[-q queue depth]\n");
}

static int
parse_args(int argc, char **argv)
{
	int op;

	construct_user_config(&g_user_config);
	while ((op = getopt(argc, argv, "c:ht:q:")) != -1) {
		switch (op) {
		case 't':
			g_user_config.time_in_sec = spdk_strtol(optarg, 10);
			break;
		case 'c':
			g_user_config.core_mask = optarg;
			break;
		case 'q':
			g_user_config.queue_depth = spdk_strtol(optarg, 10);
			break;
		case 'h':
			usage(argv[0]);
			exit(0);
		default:
			usage(argv[0]);
			return 1;
		}
	}
	if (g_user_config.time_in_sec <= 0 || !g_user_config.core_mask ||
	    g_user_config.queue_depth <= 0) {
		usage(argv[0]);
		return 1;
	}

	return 0;
}

static void
drain_xfers(struct thread_entry *thread_entry)
{
	while (thread_entry->current_queue_depth > 0) {
		spdk_ioat_process_events(thread_entry->chan);
	}
}

static void
submit_single_xfer(struct ioat_task *ioat_task)
{
	if (ioat_task->type == IOAT_FILL_TYPE)
		spdk_ioat_submit_fill(ioat_task->thread_entry->chan, ioat_task, ioat_done,
				      ioat_task->dst, ioat_task->fill_pattern, ioat_task->len);
	else
		spdk_ioat_submit_copy(ioat_task->thread_entry->chan, ioat_task, ioat_done,
				      ioat_task->dst, ioat_task->src, ioat_task->len);
	ioat_task->thread_entry->current_queue_depth++;
}

static void
submit_xfers(struct thread_entry *thread_entry, uint64_t queue_depth)
{
	while (queue_depth-- > 0) {
		struct ioat_task *ioat_task = NULL;
		ioat_task = spdk_mempool_get(thread_entry->task_pool);
		ioat_task->buffer = spdk_mempool_get(thread_entry->data_pool);

		ioat_task->type = IOAT_COPY_TYPE;
		if (spdk_ioat_get_dma_capabilities(thread_entry->chan) & SPDK_IOAT_ENGINE_FILL_SUPPORTED) {
			if (queue_depth % 2) {
				ioat_task->type = IOAT_FILL_TYPE;
			}
		}
		prepare_ioat_task(thread_entry, ioat_task);
		submit_single_xfer(ioat_task);
	}
}

static int
work_fn(void *arg)
{
	uint64_t tsc_end;
	char buf_pool_name[20], task_pool_name[20];
	struct thread_entry *t = (struct thread_entry *)arg;

	if (!t->chan) {
		return 0;
	}

	t->lcore_id = spdk_env_get_current_core();

	snprintf(buf_pool_name, sizeof(buf_pool_name), "buf_pool_%u", t->lcore_id);
	snprintf(task_pool_name, sizeof(task_pool_name), "task_pool_%u", t->lcore_id);
	t->data_pool = spdk_mempool_create(buf_pool_name, g_user_config.queue_depth, SRC_BUFFER_SIZE,
					   SPDK_MEMPOOL_DEFAULT_CACHE_SIZE,
					   SPDK_ENV_SOCKET_ID_ANY);
	t->task_pool = spdk_mempool_create(task_pool_name, g_user_config.queue_depth,
					   sizeof(struct ioat_task),
					   SPDK_MEMPOOL_DEFAULT_CACHE_SIZE,
					   SPDK_ENV_SOCKET_ID_ANY);
	if (!t->data_pool || !t->task_pool) {
		fprintf(stderr, "Could not allocate buffer pool.\n");
		t->init_failed = true;
		return 1;
	}

	tsc_end = spdk_get_ticks() + g_user_config.time_in_sec * spdk_get_ticks_hz();

	submit_xfers(t, g_user_config.queue_depth);
	while (spdk_get_ticks() < tsc_end) {
		spdk_ioat_process_events(t->chan);
	}

	t->is_draining = true;
	drain_xfers(t);

	return 0;
}

static int
init_src_buffer(void)
{
	int i;

	g_src = spdk_dma_zmalloc(SRC_BUFFER_SIZE, 512, NULL);
	if (g_src == NULL) {
		fprintf(stderr, "Allocate src buffer failed\n");
		return -1;
	}

	for (i = 0; i < SRC_BUFFER_SIZE / 4; i++) {
		memset((g_src + (4 * i)), i, 4);
	}

	return 0;
}

static int
init(void)
{
	struct spdk_env_opts opts;

	spdk_env_opts_init(&opts);
	opts.name = "verify";
	opts.core_mask = g_user_config.core_mask;
	if (spdk_env_init(&opts) < 0) {
		fprintf(stderr, "Unable to initialize SPDK env\n");
		return 1;
	}

	if (init_src_buffer() != 0) {
		fprintf(stderr, "Could not init src buffer\n");
		return 1;
	}
	if (ioat_init() != 0) {
		fprintf(stderr, "Could not init ioat\n");
		return 1;
	}

	return 0;
}

static int
dump_result(struct thread_entry *threads, uint32_t num_threads)
{
	uint32_t i;
	uint64_t total_completed = 0;
	uint64_t total_failed = 0;

	for (i = 0; i < num_threads; i++) {
		struct thread_entry *t = &threads[i];

		if (!t->chan) {
			continue;
		}

		if (t->init_failed) {
			total_failed++;
			continue;
		}

		total_completed += t->xfer_completed;
		total_completed += t->fill_completed;
		total_failed += t->xfer_failed;
		total_failed += t->fill_failed;
		if (total_completed || total_failed)
			printf("lcore = %d, copy success = %ld, copy failed = %ld, fill success = %ld, fill failed = %ld\n",
			       t->lcore_id, t->xfer_completed, t->xfer_failed, t->fill_completed, t->fill_failed);
	}
	return total_failed ? 1 : 0;
}

static struct spdk_ioat_chan *
get_next_chan(void)
{
	struct spdk_ioat_chan *chan;

	if (g_next_device == NULL) {
		fprintf(stderr, "Not enough ioat channels found. Check that ioat channels are bound\n");
		fprintf(stderr, "to uio_pci_generic or vfio-pci.  scripts/setup.sh can help with this.\n");
		return NULL;
	}

	chan = g_next_device->ioat;

	g_next_device = TAILQ_NEXT(g_next_device, tailq);

	return chan;
}

static uint32_t
get_max_core(void)
{
	uint32_t i;
	uint32_t max_core = 0;

	SPDK_ENV_FOREACH_CORE(i) {
		if (i > max_core) {
			max_core = i;
		}
	}

	return max_core;
}

int
main(int argc, char **argv)
{
	uint32_t i, current_core;
	struct thread_entry *threads;
	uint32_t num_threads;
	int rc;

	if (parse_args(argc, argv) != 0) {
		return 1;
	}

	if (init() != 0) {
		return 1;
	}

	dump_user_config(&g_user_config);

	g_next_device = TAILQ_FIRST(&g_devices);

	num_threads = get_max_core() + 1;
	threads = calloc(num_threads, sizeof(*threads));
	if (!threads) {
		fprintf(stderr, "Thread memory allocation failed\n");
		rc = 1;
		goto cleanup;
	}

	current_core = spdk_env_get_current_core();
	SPDK_ENV_FOREACH_CORE(i) {
		if (i != current_core) {
			threads[i].chan = get_next_chan();
			spdk_env_thread_launch_pinned(i, work_fn, &threads[i]);
		}
	}

	threads[current_core].chan = get_next_chan();
	work_fn(&threads[current_core]);

	spdk_env_thread_wait_all();
	rc = dump_result(threads, num_threads);

cleanup:
	spdk_dma_free(g_src);
	ioat_exit();
	free(threads);

	return rc;
}
Commit	Line	Data
7c673cae FG	1	/*-
	2	* BSD LICENSE
	3	*
	4	* Copyright (c) Intel Corporation.
	5	* All rights reserved.
	6	*
	7	* Redistribution and use in source and binary forms, with or without
	8	* modification, are permitted provided that the following conditions
	9	* are met:
	10	*
	11	* * Redistributions of source code must retain the above copyright
	12	* notice, this list of conditions and the following disclaimer.
	13	* * Redistributions in binary form must reproduce the above copyright
	14	* notice, this list of conditions and the following disclaimer in
	15	* the documentation and/or other materials provided with the
	16	* distribution.
	17	* * Neither the name of Intel Corporation nor the names of its
	18	* contributors may be used to endorse or promote products derived
	19	* from this software without specific prior written permission.
	20	*
	21	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	22	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	23	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	24	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	25	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	26	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	27	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	28	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	29	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	30	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	31	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	32	*/
	33
11fdf7f2	34	#include "spdk/stdinc.h"
7c673cae FG	35
	36	#include "spdk/ioat.h"
	37	#include "spdk/env.h"
	38	#include "spdk/queue.h"
	39	#include "spdk/string.h"
11fdf7f2	40	#include "spdk/util.h"
7c673cae FG	41
	42	#define SRC_BUFFER_SIZE (512*1024)
	43
	44	enum ioat_task_type {
	45	IOAT_COPY_TYPE,
	46	IOAT_FILL_TYPE,
	47	};
	48
	49	struct user_config {
	50	int queue_depth;
	51	int time_in_sec;
	52	char *core_mask;
	53	};
	54
	55	struct ioat_device {
	56	struct spdk_ioat_chan *ioat;
	57	TAILQ_ENTRY(ioat_device) tailq;
	58	};
	59
	60	static TAILQ_HEAD(, ioat_device) g_devices;
	61	static struct ioat_device *g_next_device;
	62
	63	static struct user_config g_user_config;
	64
	65	struct thread_entry {
	66	struct spdk_ioat_chan *chan;
	67	uint64_t xfer_completed;
	68	uint64_t xfer_failed;
	69	uint64_t fill_completed;
	70	uint64_t fill_failed;
	71	uint64_t current_queue_depth;
	72	unsigned lcore_id;
	73	bool is_draining;
11fdf7f2	74	bool init_failed;
7c673cae FG	75	struct spdk_mempool *data_pool;
	76	struct spdk_mempool *task_pool;
	77	};
	78
	79	struct ioat_task {
	80	enum ioat_task_type type;
	81	struct thread_entry *thread_entry;
	82	void *buffer;
	83	int len;
	84	uint64_t fill_pattern;
	85	void *src;
	86	void *dst;
	87	};
	88
	89	static __thread unsigned int seed = 0;
	90
	91	static unsigned char *g_src;
	92
	93	static void submit_single_xfer(struct ioat_task *ioat_task);
	94
	95	static void
	96	construct_user_config(struct user_config *self)
	97	{
	98	self->queue_depth = 32;
	99	self->time_in_sec = 10;
	100	self->core_mask = "0x1";
	101	}
	102
	103	static void
	104	dump_user_config(struct user_config *self)
	105	{
	106	printf("User configuration:\n");
	107	printf("Run time: %u seconds\n", self->time_in_sec);
	108	printf("Core mask: %s\n", self->core_mask);
	109	printf("Queue depth: %u\n", self->queue_depth);
	110	}
	111
	112	static void
	113	ioat_exit(void)
	114	{
	115	struct ioat_device *dev;
	116
	117	while (!TAILQ_EMPTY(&g_devices)) {
	118	dev = TAILQ_FIRST(&g_devices);
	119	TAILQ_REMOVE(&g_devices, dev, tailq);
	120	if (dev->ioat) {
	121	spdk_ioat_detach(dev->ioat);
	122	}
	123	free(dev);
	124	}
	125	}
	126	static void prepare_ioat_task(struct thread_entry thread_entry, struct ioat_task ioat_task)
	127	{
	128	int len;
11fdf7f2 TL	129	uintptr_t src_offset;
11fdf7f2 TL	130	uintptr_t dst_offset;
7c673cae FG	131	uint64_t fill_pattern;
	132
	133	if (ioat_task->type == IOAT_FILL_TYPE) {
	134	fill_pattern = rand_r(&seed);
	135	fill_pattern = fill_pattern << 32 \| rand_r(&seed);
	136
11fdf7f2 TL	137	/* Ensure that the length of memset block is 8 Bytes aligned.
	138	* In case the buffer crosses hugepage boundary and must be split,
	139	* we also need to ensure 8 byte address alignment. We do it
	140	* unconditionally to keep things simple.
	141	*/
	142	len = 8 + ((rand_r(&seed) % (SRC_BUFFER_SIZE - 16)) & ~0x7);
	143	dst_offset = 8 + rand_r(&seed) % (SRC_BUFFER_SIZE - 8 - len);
7c673cae	144	ioat_task->fill_pattern = fill_pattern;
11fdf7f2	145	ioat_task->dst = (void *)(((uintptr_t)ioat_task->buffer + dst_offset) & ~0x7);
7c673cae FG	146	} else {
	147	src_offset = rand_r(&seed) % SRC_BUFFER_SIZE;
	148	len = rand_r(&seed) % (SRC_BUFFER_SIZE - src_offset);
	149	dst_offset = rand_r(&seed) % (SRC_BUFFER_SIZE - len);
	150
	151	memset(ioat_task->buffer, 0, SRC_BUFFER_SIZE);
11fdf7f2 TL	152	ioat_task->src = (void *)((uintptr_t)g_src + src_offset);
11fdf7f2 TL	153	ioat_task->dst = (void *)((uintptr_t)ioat_task->buffer + dst_offset);
7c673cae FG	154	}
7c673cae FG	155	ioat_task->len = len;
7c673cae FG	156	ioat_task->thread_entry = thread_entry;
	157	}
	158
	159	static void
	160	ioat_done(void *cb_arg)
	161	{
	162	char *value;
	163	int i, failed = 0;
	164	struct ioat_task ioat_task = (struct ioat_task )cb_arg;
	165	struct thread_entry *thread_entry = ioat_task->thread_entry;
	166
	167	if (ioat_task->type == IOAT_FILL_TYPE) {
	168	value = ioat_task->dst;
	169	for (i = 0; i < ioat_task->len / 8; i++) {
	170	if (memcmp(value, &ioat_task->fill_pattern, 8) != 0) {
	171	thread_entry->fill_failed++;
	172	failed = 1;
	173	break;
	174	}
	175	value += 8;
	176	}
11fdf7f2	177	if (!failed) {
7c673cae	178	thread_entry->fill_completed++;
11fdf7f2	179	}
7c673cae FG	180	} else {
	181	if (memcmp(ioat_task->src, ioat_task->dst, ioat_task->len)) {
	182	thread_entry->xfer_failed++;
	183	} else {
	184	thread_entry->xfer_completed++;
	185	}
	186	}
	187
	188	thread_entry->current_queue_depth--;
	189	if (thread_entry->is_draining) {
	190	spdk_mempool_put(thread_entry->data_pool, ioat_task->buffer);
	191	spdk_mempool_put(thread_entry->task_pool, ioat_task);
	192	} else {
	193	prepare_ioat_task(thread_entry, ioat_task);
	194	submit_single_xfer(ioat_task);
	195	}
	196	}
	197
	198	static bool
	199	probe_cb(void cb_ctx, struct spdk_pci_device pci_dev)
	200	{
	201	printf(" Found matching device at %04x:%02x:%02x.%x "
	202	"vendor:0x%04x device:0x%04x\n",
	203	spdk_pci_device_get_domain(pci_dev),
	204	spdk_pci_device_get_bus(pci_dev), spdk_pci_device_get_dev(pci_dev),
	205	spdk_pci_device_get_func(pci_dev),
	206	spdk_pci_device_get_vendor_id(pci_dev), spdk_pci_device_get_device_id(pci_dev));
	207
	208	return true;
	209	}
	210
	211	static void
	212	attach_cb(void cb_ctx, struct spdk_pci_device pci_dev, struct spdk_ioat_chan *ioat)
	213	{
	214	struct ioat_device *dev;
	215
	216	dev = malloc(sizeof(*dev));
	217	if (dev == NULL) {
	218	printf("Failed to allocate device struct\n");
	219	return;
	220	}
	221	memset(dev, 0, sizeof(*dev));
	222
	223	dev->ioat = ioat;
	224	TAILQ_INSERT_TAIL(&g_devices, dev, tailq);
	225	}
	226
	227	static int
	228	ioat_init(void)
	229	{
	230	TAILQ_INIT(&g_devices);
	231
	232	if (spdk_ioat_probe(NULL, probe_cb, attach_cb) != 0) {
	233	fprintf(stderr, "ioat_probe() failed\n");
	234	return 1;
	235	}
	236
	237	return 0;
	238	}
	239
	240	static void
	241	usage(char *program_name)
	242	{
	243	printf("%s options\n", program_name);
244	printf("\t[-h help message]\n");
245	printf("\t[-c core mask for distributing I/O submission/completion work]\n");
246	printf("\t[-t time in seconds]\n");
247	printf("\t[-q queue depth]\n");
248	}
249
250	static int
251	parse_args(int argc, char **argv)
252	{
253	int op;
254
255	construct_user_config(&g_user_config);
256	while ((op = getopt(argc, argv, "c:ht:q:")) != -1) {
257	switch (op) {
258	case 't':
9f95a23c	259	g_user_config.time_in_sec = spdk_strtol(optarg, 10);
7c673cae FG	260	break;
	261	case 'c':
	262	g_user_config.core_mask = optarg;
	263	break;
	264	case 'q':
9f95a23c	265	g_user_config.queue_depth = spdk_strtol(optarg, 10);
7c673cae FG	266	break;
	267	case 'h':
	268	usage(argv[0]);
	269	exit(0);
	270	default:
	271	usage(argv[0]);
	272	return 1;
	273	}
	274	}
9f95a23c TL	275	if (g_user_config.time_in_sec <= 0 \|\| !g_user_config.core_mask \|\|
9f95a23c TL	276	g_user_config.queue_depth <= 0) {
7c673cae FG	277	usage(argv[0]);
	278	return 1;
	279	}
11fdf7f2	280
7c673cae FG	281	return 0;
	282	}
	283
	284	static void
	285	drain_xfers(struct thread_entry *thread_entry)
	286	{
	287	while (thread_entry->current_queue_depth > 0) {
	288	spdk_ioat_process_events(thread_entry->chan);
	289	}
	290	}
	291
	292	static void
	293	submit_single_xfer(struct ioat_task *ioat_task)
	294	{
	295	if (ioat_task->type == IOAT_FILL_TYPE)
	296	spdk_ioat_submit_fill(ioat_task->thread_entry->chan, ioat_task, ioat_done,
	297	ioat_task->dst, ioat_task->fill_pattern, ioat_task->len);
	298	else
	299	spdk_ioat_submit_copy(ioat_task->thread_entry->chan, ioat_task, ioat_done,
	300	ioat_task->dst, ioat_task->src, ioat_task->len);
	301	ioat_task->thread_entry->current_queue_depth++;
	302	}
	303
	304	static void
	305	submit_xfers(struct thread_entry *thread_entry, uint64_t queue_depth)
	306	{
	307	while (queue_depth-- > 0) {
	308	struct ioat_task *ioat_task = NULL;
	309	ioat_task = spdk_mempool_get(thread_entry->task_pool);
	310	ioat_task->buffer = spdk_mempool_get(thread_entry->data_pool);
	311
	312	ioat_task->type = IOAT_COPY_TYPE;
	313	if (spdk_ioat_get_dma_capabilities(thread_entry->chan) & SPDK_IOAT_ENGINE_FILL_SUPPORTED) {
11fdf7f2	314	if (queue_depth % 2) {
7c673cae	315	ioat_task->type = IOAT_FILL_TYPE;
11fdf7f2	316	}
7c673cae FG	317	}
	318	prepare_ioat_task(thread_entry, ioat_task);
	319	submit_single_xfer(ioat_task);
	320	}
	321	}
	322
	323	static int
	324	work_fn(void *arg)
	325	{
	326	uint64_t tsc_end;
	327	char buf_pool_name[20], task_pool_name[20];
	328	struct thread_entry t = (struct thread_entry )arg;
	329
	330	if (!t->chan) {
	331	return 0;
	332	}
	333
11fdf7f2	334	t->lcore_id = spdk_env_get_current_core();
7c673cae	335
11fdf7f2 TL	336	snprintf(buf_pool_name, sizeof(buf_pool_name), "buf_pool_%u", t->lcore_id);
	337	snprintf(task_pool_name, sizeof(task_pool_name), "task_pool_%u", t->lcore_id);
	338	t->data_pool = spdk_mempool_create(buf_pool_name, g_user_config.queue_depth, SRC_BUFFER_SIZE,
	339	SPDK_MEMPOOL_DEFAULT_CACHE_SIZE,
7c673cae FG	340	SPDK_ENV_SOCKET_ID_ANY);
7c673cae FG	341	t->task_pool = spdk_mempool_create(task_pool_name, g_user_config.queue_depth,
11fdf7f2 TL	342	sizeof(struct ioat_task),
	343	SPDK_MEMPOOL_DEFAULT_CACHE_SIZE,
	344	SPDK_ENV_SOCKET_ID_ANY);
7c673cae FG	345	if (!t->data_pool \|\| !t->task_pool) {
7c673cae FG	346	fprintf(stderr, "Could not allocate buffer pool.\n");
11fdf7f2	347	t->init_failed = true;
7c673cae FG	348	return 1;
	349	}
	350
	351	tsc_end = spdk_get_ticks() + g_user_config.time_in_sec * spdk_get_ticks_hz();
	352
	353	submit_xfers(t, g_user_config.queue_depth);
	354	while (spdk_get_ticks() < tsc_end) {
	355	spdk_ioat_process_events(t->chan);
	356	}
	357
	358	t->is_draining = true;
	359	drain_xfers(t);
	360
	361	return 0;
	362	}
	363
	364	static int
	365	init_src_buffer(void)
	366	{
	367	int i;
	368
11fdf7f2	369	g_src = spdk_dma_zmalloc(SRC_BUFFER_SIZE, 512, NULL);
7c673cae FG	370	if (g_src == NULL) {
	371	fprintf(stderr, "Allocate src buffer failed\n");
	372	return -1;
	373	}
	374
	375	for (i = 0; i < SRC_BUFFER_SIZE / 4; i++) {
	376	memset((g_src + (4 * i)), i, 4);
	377	}
	378
	379	return 0;
	380	}
	381
	382	static int
	383	init(void)
	384	{
	385	struct spdk_env_opts opts;
	386
	387	spdk_env_opts_init(&opts);
	388	opts.name = "verify";
	389	opts.core_mask = g_user_config.core_mask;
11fdf7f2 TL	390	if (spdk_env_init(&opts) < 0) {
	391	fprintf(stderr, "Unable to initialize SPDK env\n");
	392	return 1;
	393	}
7c673cae FG	394
	395	if (init_src_buffer() != 0) {
	396	fprintf(stderr, "Could not init src buffer\n");
	397	return 1;
	398	}
	399	if (ioat_init() != 0) {
	400	fprintf(stderr, "Could not init ioat\n");
	401	return 1;
	402	}
	403
	404	return 0;
	405	}
	406
	407	static int
11fdf7f2	408	dump_result(struct thread_entry *threads, uint32_t num_threads)
7c673cae	409	{
11fdf7f2	410	uint32_t i;
7c673cae FG	411	uint64_t total_completed = 0;
	412	uint64_t total_failed = 0;
	413
11fdf7f2	414	for (i = 0; i < num_threads; i++) {
7c673cae	415	struct thread_entry *t = &threads[i];
11fdf7f2 TL	416
	417	if (!t->chan) {
	418	continue;
	419	}
	420
	421	if (t->init_failed) {
	422	total_failed++;
	423	continue;
	424	}
	425
7c673cae FG	426	total_completed += t->xfer_completed;
	427	total_completed += t->fill_completed;
	428	total_failed += t->xfer_failed;
	429	total_failed += t->fill_failed;
11fdf7f2 TL	430	if (total_completed \|\| total_failed)
11fdf7f2 TL	431	printf("lcore = %d, copy success = %ld, copy failed = %ld, fill success = %ld, fill failed = %ld\n",
7c673cae FG	432	t->lcore_id, t->xfer_completed, t->xfer_failed, t->fill_completed, t->fill_failed);
	433	}
	434	return total_failed ? 1 : 0;
	435	}
	436
	437	static struct spdk_ioat_chan *
	438	get_next_chan(void)
	439	{
	440	struct spdk_ioat_chan *chan;
	441
	442	if (g_next_device == NULL) {
11fdf7f2 TL	443	fprintf(stderr, "Not enough ioat channels found. Check that ioat channels are bound\n");
11fdf7f2 TL	444	fprintf(stderr, "to uio_pci_generic or vfio-pci. scripts/setup.sh can help with this.\n");
7c673cae FG	445	return NULL;
	446	}
	447
	448	chan = g_next_device->ioat;
	449
	450	g_next_device = TAILQ_NEXT(g_next_device, tailq);
	451
	452	return chan;
	453	}
	454
11fdf7f2 TL	455	static uint32_t
	456	get_max_core(void)
	457	{
	458	uint32_t i;
	459	uint32_t max_core = 0;
	460
	461	SPDK_ENV_FOREACH_CORE(i) {
	462	if (i > max_core) {
	463	max_core = i;
	464	}
	465	}
	466
	467	return max_core;
	468	}
	469
7c673cae FG	470	int
	471	main(int argc, char **argv)
	472	{
	473	uint32_t i, current_core;
11fdf7f2 TL	474	struct thread_entry *threads;
11fdf7f2 TL	475	uint32_t num_threads;
7c673cae FG	476	int rc;
	477
	478	if (parse_args(argc, argv) != 0) {
	479	return 1;
	480	}
	481
	482	if (init() != 0) {
	483	return 1;
	484	}
	485
	486	dump_user_config(&g_user_config);
	487
	488	g_next_device = TAILQ_FIRST(&g_devices);
	489
11fdf7f2 TL	490	num_threads = get_max_core() + 1;
	491	threads = calloc(num_threads, sizeof(*threads));
	492	if (!threads) {
	493	fprintf(stderr, "Thread memory allocation failed\n");
7c673cae FG	494	rc = 1;
	495	goto cleanup;
	496	}
	497
11fdf7f2	498	current_core = spdk_env_get_current_core();
7c673cae FG	499	SPDK_ENV_FOREACH_CORE(i) {
7c673cae FG	500	if (i != current_core) {
11fdf7f2 TL	501	threads[i].chan = get_next_chan();
11fdf7f2 TL	502	spdk_env_thread_launch_pinned(i, work_fn, &threads[i]);
7c673cae FG	503	}
	504	}
	505
11fdf7f2 TL	506	threads[current_core].chan = get_next_chan();
	507	work_fn(&threads[current_core]);
	508
	509	spdk_env_thread_wait_all();
	510	rc = dump_result(threads, num_threads);
7c673cae FG	511
7c673cae FG	512	cleanup:
11fdf7f2	513	spdk_dma_free(g_src);
7c673cae	514	ioat_exit();
11fdf7f2	515	free(threads);
7c673cae FG	516
	517	return rc;
	518	}