[ceph.git] / ceph / src / spdk / ocf / src / utils / utils_io.c

/*
 * Copyright(c) 2012-2018 Intel Corporation
 * SPDX-License-Identifier: BSD-3-Clause-Clear
 */

#include "ocf/ocf.h"
#include "../ocf_priv.h"
#include "../ocf_cache_priv.h"
#include "../ocf_volume_priv.h"
#include "../ocf_request.h"
#include "utils_io.h"
#include "utils_cache_line.h"

struct ocf_submit_volume_context {
	env_atomic req_remaining;
	int error;
	ocf_submit_end_t cmpl;
	void *priv;
};

static void _ocf_volume_flush_end(struct ocf_io *io, int error)
{
	ocf_submit_end_t cmpl = io->priv1;

	cmpl(io->priv2, error);
	ocf_io_put(io);
}

void ocf_submit_volume_flush(ocf_volume_t volume,
		ocf_submit_end_t cmpl, void *priv)
{
	struct ocf_io *io;

	io = ocf_volume_new_io(volume, NULL, 0, 0, OCF_WRITE, 0, 0);
	if (!io)
		OCF_CMPL_RET(priv, -OCF_ERR_NO_MEM);

	ocf_io_set_cmpl(io, cmpl, priv, _ocf_volume_flush_end);

	ocf_volume_submit_flush(io);
}

static void ocf_submit_volume_end(struct ocf_io *io, int error)
{
	struct ocf_submit_volume_context *context = io->priv1;

	if (error)
		context->error = error;

	ocf_io_put(io);

	if (env_atomic_dec_return(&context->req_remaining))
		return;

	context->cmpl(context->priv, context->error);
	env_vfree(context);
}

void ocf_submit_volume_discard(ocf_volume_t volume, uint64_t addr,
		uint64_t length, ocf_submit_end_t cmpl, void *priv)
{
	struct ocf_submit_volume_context *context;
	uint64_t bytes;
	uint64_t sector_mask = (1 << ENV_SECTOR_SHIFT) - 1;
	uint64_t max_length = (uint32_t)~0 & ~sector_mask;
	struct ocf_io *io;

	context = env_vzalloc(sizeof(*context));
	if (!context)
		OCF_CMPL_RET(priv, -OCF_ERR_NO_MEM);

	env_atomic_set(&context->req_remaining, 1);
	context->cmpl = cmpl;
	context->priv = priv;

	while (length) {
		bytes = OCF_MIN(length, max_length);

		io = ocf_volume_new_io(volume, NULL, addr, bytes,
				OCF_WRITE, 0, 0);
		if (!io) {
			context->error = -OCF_ERR_NO_MEM;
			break;
		}

		env_atomic_inc(&context->req_remaining);

		ocf_io_set_cmpl(io, context, NULL, ocf_submit_volume_end);
		ocf_volume_submit_discard(io);

		addr += bytes;
		length -= bytes;
	}

	if (env_atomic_dec_return(&context->req_remaining))
		return;

	cmpl(priv, context->error);
	env_vfree(context);
}

void ocf_submit_write_zeros(ocf_volume_t volume, uint64_t addr,
		uint64_t length, ocf_submit_end_t cmpl, void *priv)
{
	struct ocf_submit_volume_context *context;
	uint32_t bytes;
	uint32_t max_length = ~((uint32_t)PAGE_SIZE - 1);
	struct ocf_io *io;

	context = env_vzalloc(sizeof(*context));
	if (!context)
		OCF_CMPL_RET(priv, -OCF_ERR_NO_MEM);

	env_atomic_set(&context->req_remaining, 1);
	context->cmpl = cmpl;
	context->priv = priv;

	while (length) {
		bytes = OCF_MIN(length, max_length);

		io = ocf_volume_new_io(volume, NULL, addr, bytes,
				OCF_WRITE, 0, 0);
		if (!io) {
			context->error = -OCF_ERR_NO_MEM;
			break;
		}

		env_atomic_inc(&context->req_remaining);

		ocf_io_set_cmpl(io, context, NULL, ocf_submit_volume_end);
		ocf_volume_submit_write_zeroes(io);

		addr += bytes;
		length -= bytes;
	}

	if (env_atomic_dec_return(&context->req_remaining))
		return;

	cmpl(priv, context->error);
	env_vfree(context);
}

struct ocf_submit_cache_page_context {
	ocf_cache_t cache;
	void *buffer;
	ocf_submit_end_t cmpl;
	void *priv;
};

static void ocf_submit_cache_page_end(struct ocf_io *io, int error)
{
	struct ocf_submit_cache_page_context *context = io->priv1;
	ctx_data_t *data = ocf_io_get_data(io);

	if (io->dir == OCF_READ) {
		ctx_data_rd_check(context->cache->owner, context->buffer,
				data, PAGE_SIZE);
	}

	context->cmpl(context->priv, error);
	ctx_data_free(context->cache->owner, data);
	env_vfree(context);
	ocf_io_put(io);
}

void ocf_submit_cache_page(ocf_cache_t cache, uint64_t addr, int dir,
		void *buffer, ocf_submit_end_t cmpl, void *priv)
{
	struct ocf_submit_cache_page_context *context;
	ctx_data_t *data;
	struct ocf_io *io;
	int result = 0;

	context = env_vmalloc(sizeof(*context));
	if (!context)
		OCF_CMPL_RET(priv, -OCF_ERR_NO_MEM);

	context->cache = cache;
	context->buffer = buffer;
	context->cmpl = cmpl;
	context->priv = priv;

	io = ocf_new_cache_io(cache, NULL, addr, PAGE_SIZE, dir, 0, 0);
	if (!io) {
		result = -OCF_ERR_NO_MEM;
		goto err_io;
	}

	data = ctx_data_alloc(cache->owner, 1);
	if (!data) {
		result = -OCF_ERR_NO_MEM;
		goto err_data;
	}

	if (dir == OCF_WRITE)
		ctx_data_wr_check(cache->owner, data, buffer, PAGE_SIZE);

	result = ocf_io_set_data(io, data, 0);
	if (result)
		goto err_set_data;

	ocf_io_set_cmpl(io, context, NULL, ocf_submit_cache_page_end);

	ocf_volume_submit_io(io);
	return;

err_set_data:
	ctx_data_free(cache->owner, data);
err_data:
	ocf_io_put(io);
err_io:
	env_vfree(context);
	cmpl(priv, result);
}

static void ocf_submit_volume_req_cmpl(struct ocf_io *io, int error)
{
	struct ocf_request *req = io->priv1;
	ocf_req_end_t callback = io->priv2;

	callback(req, error);

	ocf_io_put(io);
}

void ocf_submit_cache_reqs(struct ocf_cache *cache,
		struct ocf_request *req, int dir, uint64_t offset,
		uint64_t size, unsigned int reqs, ocf_req_end_t callback)
{
	uint64_t flags = req->ioi.io.flags;
	uint32_t io_class = req->ioi.io.io_class;
	uint64_t addr, bytes, total_bytes = 0;
	struct ocf_io *io;
	int err;
	uint32_t i;
	uint32_t first_cl = ocf_bytes_2_lines(cache, req->byte_position +
			offset) - ocf_bytes_2_lines(cache, req->byte_position);

	ENV_BUG_ON(req->byte_length < offset + size);
	ENV_BUG_ON(first_cl + reqs > req->core_line_count);

	if (reqs == 1) {
		addr = ocf_metadata_map_lg2phy(cache,
					req->map[first_cl].coll_idx);
		addr *= ocf_line_size(cache);
		addr += cache->device->metadata_offset;
		addr += ((req->byte_position + offset) % ocf_line_size(cache));
		bytes = size;

		io = ocf_new_cache_io(cache, req->io_queue,
				addr, bytes, dir, io_class, flags);
		if (!io) {
			callback(req, -OCF_ERR_NO_MEM);
			return;
		}

		ocf_io_set_cmpl(io, req, callback, ocf_submit_volume_req_cmpl);

		err = ocf_io_set_data(io, req->data, offset);
		if (err) {
			ocf_io_put(io);
			callback(req, err);
			return;
		}

		ocf_core_stats_cache_block_update(req->core, io_class,
				dir, bytes);

		ocf_volume_submit_io(io);
		return;
	}

	/* Issue requests to cache. */
	for (i = 0; i < reqs; i++) {
		addr  = ocf_metadata_map_lg2phy(cache,
				req->map[first_cl + i].coll_idx);
		addr *= ocf_line_size(cache);
		addr += cache->device->metadata_offset;
		bytes = ocf_line_size(cache);

		if (i == 0) {
			uint64_t seek = ((req->byte_position + offset) %
					ocf_line_size(cache));

			addr += seek;
			bytes -= seek;
		} else  if (i == (reqs - 1)) {
			uint64_t skip = (ocf_line_size(cache) -
				((req->byte_position + offset + size) %
				ocf_line_size(cache))) % ocf_line_size(cache);

			bytes -= skip;
		}

		bytes = OCF_MIN(bytes, size - total_bytes);
		ENV_BUG_ON(bytes == 0);

		io = ocf_new_cache_io(cache, req->io_queue,
				addr, bytes, dir, io_class, flags);
		if (!io) {
			/* Finish all IOs which left with ERROR */
			for (; i < reqs; i++)
				callback(req, -OCF_ERR_NO_MEM);
			return;
		}

		ocf_io_set_cmpl(io, req, callback, ocf_submit_volume_req_cmpl);

		err = ocf_io_set_data(io, req->data, offset + total_bytes);
		if (err) {
			ocf_io_put(io);
			/* Finish all IOs which left with ERROR */
			for (; i < reqs; i++)
				callback(req, err);
			return;
		}
		ocf_core_stats_cache_block_update(req->core, io_class,
				dir, bytes);
		ocf_volume_submit_io(io);
		total_bytes += bytes;
	}

	ENV_BUG_ON(total_bytes != size);
}

void ocf_submit_volume_req(ocf_volume_t volume, struct ocf_request *req,
		ocf_req_end_t callback)
{
	uint64_t flags = req->ioi.io.flags;
	uint32_t io_class = req->ioi.io.io_class;
	int dir = req->rw;
	struct ocf_io *io;
	int err;

	ocf_core_stats_core_block_update(req->core, io_class, dir,
			req->byte_length);

	io = ocf_volume_new_io(volume, req->io_queue, req->byte_position,
			req->byte_length, dir, io_class, flags);
	if (!io) {
		callback(req, -OCF_ERR_NO_MEM);
		return;
	}

	ocf_io_set_cmpl(io, req, callback, ocf_submit_volume_req_cmpl);
	err = ocf_io_set_data(io, req->data, 0);
	if (err) {
		ocf_io_put(io);
		callback(req, err);
		return;
	}
	ocf_volume_submit_io(io);
}
Commit	Line	Data
9f95a23c TL	1	/*
	2	* Copyright(c) 2012-2018 Intel Corporation
	3	* SPDX-License-Identifier: BSD-3-Clause-Clear
	4	*/
	5
	6	#include "ocf/ocf.h"
	7	#include "../ocf_priv.h"
	8	#include "../ocf_cache_priv.h"
	9	#include "../ocf_volume_priv.h"
	10	#include "../ocf_request.h"
	11	#include "utils_io.h"
	12	#include "utils_cache_line.h"
	13
	14	struct ocf_submit_volume_context {
	15	env_atomic req_remaining;
	16	int error;
	17	ocf_submit_end_t cmpl;
	18	void *priv;
	19	};
	20
	21	static void _ocf_volume_flush_end(struct ocf_io *io, int error)
	22	{
	23	ocf_submit_end_t cmpl = io->priv1;
	24
	25	cmpl(io->priv2, error);
	26	ocf_io_put(io);
	27	}
	28
	29	void ocf_submit_volume_flush(ocf_volume_t volume,
	30	ocf_submit_end_t cmpl, void *priv)
	31	{
	32	struct ocf_io *io;
	33
f67539c2 TL	34	io = ocf_volume_new_io(volume, NULL, 0, 0, OCF_WRITE, 0, 0);
	35	if (!io)
	36	OCF_CMPL_RET(priv, -OCF_ERR_NO_MEM);
9f95a23c	37
9f95a23c TL	38	ocf_io_set_cmpl(io, cmpl, priv, _ocf_volume_flush_end);
	39
	40	ocf_volume_submit_flush(io);
	41	}
	42
	43	static void ocf_submit_volume_end(struct ocf_io *io, int error)
	44	{
	45	struct ocf_submit_volume_context *context = io->priv1;
	46
	47	if (error)
	48	context->error = error;
	49
	50	ocf_io_put(io);
	51
	52	if (env_atomic_dec_return(&context->req_remaining))
	53	return;
	54
	55	context->cmpl(context->priv, context->error);
	56	env_vfree(context);
	57	}
	58
	59	void ocf_submit_volume_discard(ocf_volume_t volume, uint64_t addr,
	60	uint64_t length, ocf_submit_end_t cmpl, void *priv)
	61	{
	62	struct ocf_submit_volume_context *context;
	63	uint64_t bytes;
f67539c2 TL	64	uint64_t sector_mask = (1 << ENV_SECTOR_SHIFT) - 1;
f67539c2 TL	65	uint64_t max_length = (uint32_t)~0 & ~sector_mask;
9f95a23c TL	66	struct ocf_io *io;
	67
	68	context = env_vzalloc(sizeof(*context));
f67539c2 TL	69	if (!context)
f67539c2 TL	70	OCF_CMPL_RET(priv, -OCF_ERR_NO_MEM);
9f95a23c TL	71
	72	env_atomic_set(&context->req_remaining, 1);
	73	context->cmpl = cmpl;
	74	context->priv = priv;
	75
	76	while (length) {
f67539c2 TL	77	bytes = OCF_MIN(length, max_length);
	78
	79	io = ocf_volume_new_io(volume, NULL, addr, bytes,
	80	OCF_WRITE, 0, 0);
9f95a23c TL	81	if (!io) {
	82	context->error = -OCF_ERR_NO_MEM;
	83	break;
	84	}
	85
	86	env_atomic_inc(&context->req_remaining);
	87
9f95a23c TL	88	ocf_io_set_cmpl(io, context, NULL, ocf_submit_volume_end);
	89	ocf_volume_submit_discard(io);
	90
	91	addr += bytes;
	92	length -= bytes;
	93	}
	94
	95	if (env_atomic_dec_return(&context->req_remaining))
	96	return;
	97
	98	cmpl(priv, context->error);
	99	env_vfree(context);
	100	}
	101
	102	void ocf_submit_write_zeros(ocf_volume_t volume, uint64_t addr,
	103	uint64_t length, ocf_submit_end_t cmpl, void *priv)
	104	{
	105	struct ocf_submit_volume_context *context;
	106	uint32_t bytes;
	107	uint32_t max_length = ~((uint32_t)PAGE_SIZE - 1);
	108	struct ocf_io *io;
	109
	110	context = env_vzalloc(sizeof(*context));
f67539c2 TL	111	if (!context)
f67539c2 TL	112	OCF_CMPL_RET(priv, -OCF_ERR_NO_MEM);
9f95a23c TL	113
	114	env_atomic_set(&context->req_remaining, 1);
	115	context->cmpl = cmpl;
	116	context->priv = priv;
	117
	118	while (length) {
f67539c2 TL	119	bytes = OCF_MIN(length, max_length);
	120
	121	io = ocf_volume_new_io(volume, NULL, addr, bytes,
	122	OCF_WRITE, 0, 0);
9f95a23c TL	123	if (!io) {
	124	context->error = -OCF_ERR_NO_MEM;
	125	break;
	126	}
	127
	128	env_atomic_inc(&context->req_remaining);
	129
9f95a23c TL	130	ocf_io_set_cmpl(io, context, NULL, ocf_submit_volume_end);
	131	ocf_volume_submit_write_zeroes(io);
	132
	133	addr += bytes;
	134	length -= bytes;
	135	}
	136
	137	if (env_atomic_dec_return(&context->req_remaining))
	138	return;
	139
	140	cmpl(priv, context->error);
	141	env_vfree(context);
	142	}
	143
	144	struct ocf_submit_cache_page_context {
	145	ocf_cache_t cache;
	146	void *buffer;
	147	ocf_submit_end_t cmpl;
	148	void *priv;
	149	};
	150
	151	static void ocf_submit_cache_page_end(struct ocf_io *io, int error)
	152	{
	153	struct ocf_submit_cache_page_context *context = io->priv1;
	154	ctx_data_t *data = ocf_io_get_data(io);
	155
	156	if (io->dir == OCF_READ) {
	157	ctx_data_rd_check(context->cache->owner, context->buffer,
	158	data, PAGE_SIZE);
	159	}
	160
	161	context->cmpl(context->priv, error);
	162	ctx_data_free(context->cache->owner, data);
	163	env_vfree(context);
	164	ocf_io_put(io);
	165	}
	166
	167	void ocf_submit_cache_page(ocf_cache_t cache, uint64_t addr, int dir,
	168	void buffer, ocf_submit_end_t cmpl, void priv)
	169	{
	170	struct ocf_submit_cache_page_context *context;
	171	ctx_data_t *data;
	172	struct ocf_io *io;
	173	int result = 0;
	174
	175	context = env_vmalloc(sizeof(*context));
f67539c2 TL	176	if (!context)
f67539c2 TL	177	OCF_CMPL_RET(priv, -OCF_ERR_NO_MEM);
9f95a23c TL	178
	179	context->cache = cache;
	180	context->buffer = buffer;
	181	context->cmpl = cmpl;
	182	context->priv = priv;
	183
f67539c2	184	io = ocf_new_cache_io(cache, NULL, addr, PAGE_SIZE, dir, 0, 0);
9f95a23c TL	185	if (!io) {
	186	result = -OCF_ERR_NO_MEM;
	187	goto err_io;
	188	}
	189
	190	data = ctx_data_alloc(cache->owner, 1);
	191	if (!data) {
	192	result = -OCF_ERR_NO_MEM;
	193	goto err_data;
	194	}
	195
	196	if (dir == OCF_WRITE)
	197	ctx_data_wr_check(cache->owner, data, buffer, PAGE_SIZE);
	198
	199	result = ocf_io_set_data(io, data, 0);
	200	if (result)
	201	goto err_set_data;
	202
9f95a23c TL	203	ocf_io_set_cmpl(io, context, NULL, ocf_submit_cache_page_end);
	204
	205	ocf_volume_submit_io(io);
	206	return;
	207
	208	err_set_data:
	209	ctx_data_free(cache->owner, data);
	210	err_data:
	211	ocf_io_put(io);
	212	err_io:
	213	env_vfree(context);
	214	cmpl(priv, result);
	215	}
	216
	217	static void ocf_submit_volume_req_cmpl(struct ocf_io *io, int error)
	218	{
	219	struct ocf_request *req = io->priv1;
	220	ocf_req_end_t callback = io->priv2;
	221
	222	callback(req, error);
	223
	224	ocf_io_put(io);
	225	}
	226
	227	void ocf_submit_cache_reqs(struct ocf_cache *cache,
f67539c2 TL	228	struct ocf_request *req, int dir, uint64_t offset,
f67539c2 TL	229	uint64_t size, unsigned int reqs, ocf_req_end_t callback)
9f95a23c	230	{
f67539c2 TL	231	uint64_t flags = req->ioi.io.flags;
f67539c2 TL	232	uint32_t io_class = req->ioi.io.io_class;
9f95a23c TL	233	uint64_t addr, bytes, total_bytes = 0;
9f95a23c TL	234	struct ocf_io *io;
9f95a23c	235	int err;
f67539c2 TL	236	uint32_t i;
	237	uint32_t first_cl = ocf_bytes_2_lines(cache, req->byte_position +
	238	offset) - ocf_bytes_2_lines(cache, req->byte_position);
9f95a23c	239
f67539c2 TL	240	ENV_BUG_ON(req->byte_length < offset + size);
f67539c2 TL	241	ENV_BUG_ON(first_cl + reqs > req->core_line_count);
9f95a23c TL	242
9f95a23c TL	243	if (reqs == 1) {
9f95a23c	244	addr = ocf_metadata_map_lg2phy(cache,
f67539c2	245	req->map[first_cl].coll_idx);
9f95a23c TL	246	addr *= ocf_line_size(cache);
9f95a23c TL	247	addr += cache->device->metadata_offset;
f67539c2 TL	248	addr += ((req->byte_position + offset) % ocf_line_size(cache));
	249	bytes = size;
	250
	251	io = ocf_new_cache_io(cache, req->io_queue,
	252	addr, bytes, dir, io_class, flags);
	253	if (!io) {
	254	callback(req, -OCF_ERR_NO_MEM);
	255	return;
	256	}
9f95a23c	257
9f95a23c TL	258	ocf_io_set_cmpl(io, req, callback, ocf_submit_volume_req_cmpl);
9f95a23c TL	259
f67539c2	260	err = ocf_io_set_data(io, req->data, offset);
9f95a23c TL	261	if (err) {
	262	ocf_io_put(io);
	263	callback(req, err);
f67539c2	264	return;
9f95a23c TL	265	}
9f95a23c TL	266
f67539c2 TL	267	ocf_core_stats_cache_block_update(req->core, io_class,
f67539c2 TL	268	dir, bytes);
9f95a23c	269
f67539c2 TL	270	ocf_volume_submit_io(io);
f67539c2 TL	271	return;
9f95a23c TL	272	}
	273
	274	/* Issue requests to cache. */
	275	for (i = 0; i < reqs; i++) {
9f95a23c	276	addr = ocf_metadata_map_lg2phy(cache,
f67539c2	277	req->map[first_cl + i].coll_idx);
9f95a23c TL	278	addr *= ocf_line_size(cache);
	279	addr += cache->device->metadata_offset;
	280	bytes = ocf_line_size(cache);
	281
	282	if (i == 0) {
f67539c2	283	uint64_t seek = ((req->byte_position + offset) %
9f95a23c TL	284	ocf_line_size(cache));
	285
	286	addr += seek;
	287	bytes -= seek;
	288	} else if (i == (reqs - 1)) {
	289	uint64_t skip = (ocf_line_size(cache) -
f67539c2	290	((req->byte_position + offset + size) %
9f95a23c TL	291	ocf_line_size(cache))) % ocf_line_size(cache);
	292
	293	bytes -= skip;
	294	}
	295
f67539c2 TL	296	bytes = OCF_MIN(bytes, size - total_bytes);
	297	ENV_BUG_ON(bytes == 0);
	298
	299	io = ocf_new_cache_io(cache, req->io_queue,
	300	addr, bytes, dir, io_class, flags);
	301	if (!io) {
	302	/* Finish all IOs which left with ERROR */
	303	for (; i < reqs; i++)
	304	callback(req, -OCF_ERR_NO_MEM);
	305	return;
	306	}
	307
9f95a23c TL	308	ocf_io_set_cmpl(io, req, callback, ocf_submit_volume_req_cmpl);
9f95a23c TL	309
f67539c2	310	err = ocf_io_set_data(io, req->data, offset + total_bytes);
9f95a23c TL	311	if (err) {
	312	ocf_io_put(io);
	313	/* Finish all IOs which left with ERROR */
	314	for (; i < reqs; i++)
	315	callback(req, err);
f67539c2	316	return;
9f95a23c	317	}
f67539c2 TL	318	ocf_core_stats_cache_block_update(req->core, io_class,
f67539c2 TL	319	dir, bytes);
9f95a23c TL	320	ocf_volume_submit_io(io);
	321	total_bytes += bytes;
	322	}
	323
f67539c2	324	ENV_BUG_ON(total_bytes != size);
9f95a23c TL	325	}
	326
	327	void ocf_submit_volume_req(ocf_volume_t volume, struct ocf_request *req,
	328	ocf_req_end_t callback)
	329	{
f67539c2 TL	330	uint64_t flags = req->ioi.io.flags;
f67539c2 TL	331	uint32_t io_class = req->ioi.io.io_class;
9f95a23c TL	332	int dir = req->rw;
	333	struct ocf_io *io;
	334	int err;
	335
f67539c2 TL	336	ocf_core_stats_core_block_update(req->core, io_class, dir,
f67539c2 TL	337	req->byte_length);
9f95a23c	338
f67539c2 TL	339	io = ocf_volume_new_io(volume, req->io_queue, req->byte_position,
f67539c2 TL	340	req->byte_length, dir, io_class, flags);
9f95a23c	341	if (!io) {
f67539c2	342	callback(req, -OCF_ERR_NO_MEM);
9f95a23c TL	343	return;
	344	}
	345
9f95a23c TL	346	ocf_io_set_cmpl(io, req, callback, ocf_submit_volume_req_cmpl);
	347	err = ocf_io_set_data(io, req->data, 0);
	348	if (err) {
	349	ocf_io_put(io);
	350	callback(req, err);
	351	return;
	352	}
	353	ocf_volume_submit_io(io);
	354	}