[mirror_ubuntu-jammy-kernel.git] / crypto / async_tx / async_raid6_recov.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Asynchronous RAID-6 recovery calculations ASYNC_TX API.
 * Copyright(c) 2009 Intel Corporation
 *
 * based on raid6recov.c:
 *   Copyright 2002 H. Peter Anvin
 */
#include <linux/kernel.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/dma-mapping.h>
#include <linux/raid/pq.h>
#include <linux/async_tx.h>
#include <linux/dmaengine.h>

static struct dma_async_tx_descriptor *
async_sum_product(struct page *dest, struct page **srcs, unsigned char *coef,
		  size_t len, struct async_submit_ctl *submit)
{
	struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
						      &dest, 1, srcs, 2, len);
	struct dma_device *dma = chan ? chan->device : NULL;
	struct dmaengine_unmap_data *unmap = NULL;
	const u8 *amul, *bmul;
	u8 ax, bx;
	u8 *a, *b, *c;

	if (dma)
		unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOWAIT);

	if (unmap) {
		struct device *dev = dma->dev;
		dma_addr_t pq[2];
		struct dma_async_tx_descriptor *tx;
		enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;

		if (submit->flags & ASYNC_TX_FENCE)
			dma_flags |= DMA_PREP_FENCE;
		unmap->addr[0] = dma_map_page(dev, srcs[0], 0, len, DMA_TO_DEVICE);
		unmap->addr[1] = dma_map_page(dev, srcs[1], 0, len, DMA_TO_DEVICE);
		unmap->to_cnt = 2;

		unmap->addr[2] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
		unmap->bidi_cnt = 1;
		/* engine only looks at Q, but expects it to follow P */
		pq[1] = unmap->addr[2];

		unmap->len = len;
		tx = dma->device_prep_dma_pq(chan, pq, unmap->addr, 2, coef,
					     len, dma_flags);
		if (tx) {
			dma_set_unmap(tx, unmap);
			async_tx_submit(chan, tx, submit);
			dmaengine_unmap_put(unmap);
			return tx;
		}

		/* could not get a descriptor, unmap and fall through to
		 * the synchronous path
		 */
		dmaengine_unmap_put(unmap);
	}

	/* run the operation synchronously */
	async_tx_quiesce(&submit->depend_tx);
	amul = raid6_gfmul[coef[0]];
	bmul = raid6_gfmul[coef[1]];
	a = page_address(srcs[0]);
	b = page_address(srcs[1]);
	c = page_address(dest);

	while (len--) {
		ax    = amul[*a++];
		bx    = bmul[*b++];
		*c++ = ax ^ bx;
	}

	return NULL;
}

static struct dma_async_tx_descriptor *
async_mult(struct page *dest, struct page *src, u8 coef, size_t len,
	   struct async_submit_ctl *submit)
{
	struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
						      &dest, 1, &src, 1, len);
	struct dma_device *dma = chan ? chan->device : NULL;
	struct dmaengine_unmap_data *unmap = NULL;
	const u8 *qmul; /* Q multiplier table */
	u8 *d, *s;

	if (dma)
		unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOWAIT);

	if (unmap) {
		dma_addr_t dma_dest[2];
		struct device *dev = dma->dev;
		struct dma_async_tx_descriptor *tx;
		enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;

		if (submit->flags & ASYNC_TX_FENCE)
			dma_flags |= DMA_PREP_FENCE;
		unmap->addr[0] = dma_map_page(dev, src, 0, len, DMA_TO_DEVICE);
		unmap->to_cnt++;
		unmap->addr[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
		dma_dest[1] = unmap->addr[1];
		unmap->bidi_cnt++;
		unmap->len = len;

		/* this looks funny, but the engine looks for Q at
		 * dma_dest[1] and ignores dma_dest[0] as a dest
		 * due to DMA_PREP_PQ_DISABLE_P
		 */
		tx = dma->device_prep_dma_pq(chan, dma_dest, unmap->addr,
					     1, &coef, len, dma_flags);

		if (tx) {
			dma_set_unmap(tx, unmap);
			dmaengine_unmap_put(unmap);
			async_tx_submit(chan, tx, submit);
			return tx;
		}

		/* could not get a descriptor, unmap and fall through to
		 * the synchronous path
		 */
		dmaengine_unmap_put(unmap);
	}

	/* no channel available, or failed to allocate a descriptor, so
	 * perform the operation synchronously
	 */
	async_tx_quiesce(&submit->depend_tx);
	qmul  = raid6_gfmul[coef];
	d = page_address(dest);
	s = page_address(src);

	while (len--)
		*d++ = qmul[*s++];

	return NULL;
}

static struct dma_async_tx_descriptor *
__2data_recov_4(int disks, size_t bytes, int faila, int failb,
		struct page **blocks, struct async_submit_ctl *submit)
{
	struct dma_async_tx_descriptor *tx = NULL;
	struct page *p, *q, *a, *b;
	struct page *srcs[2];
	unsigned char coef[2];
	enum async_tx_flags flags = submit->flags;
	dma_async_tx_callback cb_fn = submit->cb_fn;
	void *cb_param = submit->cb_param;
	void *scribble = submit->scribble;

	p = blocks[disks-2];
	q = blocks[disks-1];

	a = blocks[faila];
	b = blocks[failb];

	/* in the 4 disk case P + Pxy == P and Q + Qxy == Q */
	/* Dx = A*(P+Pxy) + B*(Q+Qxy) */
	srcs[0] = p;
	srcs[1] = q;
	coef[0] = raid6_gfexi[failb-faila];
	coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
	tx = async_sum_product(b, srcs, coef, bytes, submit);

	/* Dy = P+Pxy+Dx */
	srcs[0] = p;
	srcs[1] = b;
	init_async_submit(submit, flags | ASYNC_TX_XOR_ZERO_DST, tx, cb_fn,
			  cb_param, scribble);
	tx = async_xor(a, srcs, 0, 2, bytes, submit);

	return tx;

}

static struct dma_async_tx_descriptor *
__2data_recov_5(int disks, size_t bytes, int faila, int failb,
		struct page **blocks, struct async_submit_ctl *submit)
{
	struct dma_async_tx_descriptor *tx = NULL;
	struct page *p, *q, *g, *dp, *dq;
	struct page *srcs[2];
	unsigned char coef[2];
	enum async_tx_flags flags = submit->flags;
	dma_async_tx_callback cb_fn = submit->cb_fn;
	void *cb_param = submit->cb_param;
	void *scribble = submit->scribble;
	int good_srcs, good, i;

	good_srcs = 0;
	good = -1;
	for (i = 0; i < disks-2; i++) {
		if (blocks[i] == NULL)
			continue;
		if (i == faila || i == failb)
			continue;
		good = i;
		good_srcs++;
	}
	BUG_ON(good_srcs > 1);

	p = blocks[disks-2];
	q = blocks[disks-1];
	g = blocks[good];

	/* Compute syndrome with zero for the missing data pages
	 * Use the dead data pages as temporary storage for delta p and
	 * delta q
	 */
	dp = blocks[faila];
	dq = blocks[failb];

	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
	tx = async_memcpy(dp, g, 0, 0, bytes, submit);
	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
	tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit);

	/* compute P + Pxy */
	srcs[0] = dp;
	srcs[1] = p;
	init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
			  NULL, NULL, scribble);
	tx = async_xor(dp, srcs, 0, 2, bytes, submit);

	/* compute Q + Qxy */
	srcs[0] = dq;
	srcs[1] = q;
	init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
			  NULL, NULL, scribble);
	tx = async_xor(dq, srcs, 0, 2, bytes, submit);

	/* Dx = A*(P+Pxy) + B*(Q+Qxy) */
	srcs[0] = dp;
	srcs[1] = dq;
	coef[0] = raid6_gfexi[failb-faila];
	coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
	tx = async_sum_product(dq, srcs, coef, bytes, submit);

	/* Dy = P+Pxy+Dx */
	srcs[0] = dp;
	srcs[1] = dq;
	init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
			  cb_param, scribble);
	tx = async_xor(dp, srcs, 0, 2, bytes, submit);

	return tx;
}

static struct dma_async_tx_descriptor *
__2data_recov_n(int disks, size_t bytes, int faila, int failb,
	      struct page **blocks, struct async_submit_ctl *submit)
{
	struct dma_async_tx_descriptor *tx = NULL;
	struct page *p, *q, *dp, *dq;
	struct page *srcs[2];
	unsigned char coef[2];
	enum async_tx_flags flags = submit->flags;
	dma_async_tx_callback cb_fn = submit->cb_fn;
	void *cb_param = submit->cb_param;
	void *scribble = submit->scribble;

	p = blocks[disks-2];
	q = blocks[disks-1];

	/* Compute syndrome with zero for the missing data pages
	 * Use the dead data pages as temporary storage for
	 * delta p and delta q
	 */
	dp = blocks[faila];
	blocks[faila] = NULL;
	blocks[disks-2] = dp;
	dq = blocks[failb];
	blocks[failb] = NULL;
	blocks[disks-1] = dq;

	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
	tx = async_gen_syndrome(blocks, 0, disks, bytes, submit);

	/* Restore pointer table */
	blocks[faila]   = dp;
	blocks[failb]   = dq;
	blocks[disks-2] = p;
	blocks[disks-1] = q;

	/* compute P + Pxy */
	srcs[0] = dp;
	srcs[1] = p;
	init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
			  NULL, NULL, scribble);
	tx = async_xor(dp, srcs, 0, 2, bytes, submit);

	/* compute Q + Qxy */
	srcs[0] = dq;
	srcs[1] = q;
	init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
			  NULL, NULL, scribble);
	tx = async_xor(dq, srcs, 0, 2, bytes, submit);

	/* Dx = A*(P+Pxy) + B*(Q+Qxy) */
	srcs[0] = dp;
	srcs[1] = dq;
	coef[0] = raid6_gfexi[failb-faila];
	coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
	tx = async_sum_product(dq, srcs, coef, bytes, submit);

	/* Dy = P+Pxy+Dx */
	srcs[0] = dp;
	srcs[1] = dq;
	init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
			  cb_param, scribble);
	tx = async_xor(dp, srcs, 0, 2, bytes, submit);

	return tx;
}

/**
 * async_raid6_2data_recov - asynchronously calculate two missing data blocks
 * @disks: number of disks in the RAID-6 array
 * @bytes: block size
 * @faila: first failed drive index
 * @failb: second failed drive index
 * @blocks: array of source pointers where the last two entries are p and q
 * @submit: submission/completion modifiers
 */
struct dma_async_tx_descriptor *
async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb,
			struct page **blocks, struct async_submit_ctl *submit)
{
	void *scribble = submit->scribble;
	int non_zero_srcs, i;

	BUG_ON(faila == failb);
	if (failb < faila)
		swap(faila, failb);

	pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);

	/* if a dma resource is not available or a scribble buffer is not
	 * available punt to the synchronous path.  In the 'dma not
	 * available' case be sure to use the scribble buffer to
	 * preserve the content of 'blocks' as the caller intended.
	 */
	if (!async_dma_find_channel(DMA_PQ) || !scribble) {
		void **ptrs = scribble ? scribble : (void **) blocks;

		async_tx_quiesce(&submit->depend_tx);
		for (i = 0; i < disks; i++)
			if (blocks[i] == NULL)
				ptrs[i] = (void *) raid6_empty_zero_page;
			else
				ptrs[i] = page_address(blocks[i]);

		raid6_2data_recov(disks, bytes, faila, failb, ptrs);

		async_tx_sync_epilog(submit);

		return NULL;
	}

	non_zero_srcs = 0;
	for (i = 0; i < disks-2 && non_zero_srcs < 4; i++)
		if (blocks[i])
			non_zero_srcs++;
	switch (non_zero_srcs) {
	case 0:
	case 1:
		/* There must be at least 2 sources - the failed devices. */
		BUG();

	case 2:
		/* dma devices do not uniformly understand a zero source pq
		 * operation (in contrast to the synchronous case), so
		 * explicitly handle the special case of a 4 disk array with
		 * both data disks missing.
		 */
		return __2data_recov_4(disks, bytes, faila, failb, blocks, submit);
	case 3:
		/* dma devices do not uniformly understand a single
		 * source pq operation (in contrast to the synchronous
		 * case), so explicitly handle the special case of a 5 disk
		 * array with 2 of 3 data disks missing.
		 */
		return __2data_recov_5(disks, bytes, faila, failb, blocks, submit);
	default:
		return __2data_recov_n(disks, bytes, faila, failb, blocks, submit);
	}
}
EXPORT_SYMBOL_GPL(async_raid6_2data_recov);

/**
 * async_raid6_datap_recov - asynchronously calculate a data and the 'p' block
 * @disks: number of disks in the RAID-6 array
 * @bytes: block size
 * @faila: failed drive index
 * @blocks: array of source pointers where the last two entries are p and q
 * @submit: submission/completion modifiers
 */
struct dma_async_tx_descriptor *
async_raid6_datap_recov(int disks, size_t bytes, int faila,
			struct page **blocks, struct async_submit_ctl *submit)
{
	struct dma_async_tx_descriptor *tx = NULL;
	struct page *p, *q, *dq;
	u8 coef;
	enum async_tx_flags flags = submit->flags;
	dma_async_tx_callback cb_fn = submit->cb_fn;
	void *cb_param = submit->cb_param;
	void *scribble = submit->scribble;
	int good_srcs, good, i;
	struct page *srcs[2];

	pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);

	/* if a dma resource is not available or a scribble buffer is not
	 * available punt to the synchronous path.  In the 'dma not
	 * available' case be sure to use the scribble buffer to
	 * preserve the content of 'blocks' as the caller intended.
	 */
	if (!async_dma_find_channel(DMA_PQ) || !scribble) {
		void **ptrs = scribble ? scribble : (void **) blocks;

		async_tx_quiesce(&submit->depend_tx);
		for (i = 0; i < disks; i++)
			if (blocks[i] == NULL)
				ptrs[i] = (void*)raid6_empty_zero_page;
			else
				ptrs[i] = page_address(blocks[i]);

		raid6_datap_recov(disks, bytes, faila, ptrs);

		async_tx_sync_epilog(submit);

		return NULL;
	}

	good_srcs = 0;
	good = -1;
	for (i = 0; i < disks-2; i++) {
		if (i == faila)
			continue;
		if (blocks[i]) {
			good = i;
			good_srcs++;
			if (good_srcs > 1)
				break;
		}
	}
	BUG_ON(good_srcs == 0);

	p = blocks[disks-2];
	q = blocks[disks-1];

	/* Compute syndrome with zero for the missing data page
	 * Use the dead data page as temporary storage for delta q
	 */
	dq = blocks[faila];
	blocks[faila] = NULL;
	blocks[disks-1] = dq;

	/* in the 4-disk case we only need to perform a single source
	 * multiplication with the one good data block.
	 */
	if (good_srcs == 1) {
		struct page *g = blocks[good];

		init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
				  scribble);
		tx = async_memcpy(p, g, 0, 0, bytes, submit);

		init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
				  scribble);
		tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit);
	} else {
		init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
				  scribble);
		tx = async_gen_syndrome(blocks, 0, disks, bytes, submit);
	}

	/* Restore pointer table */
	blocks[faila]   = dq;
	blocks[disks-1] = q;

	/* calculate g^{-faila} */
	coef = raid6_gfinv[raid6_gfexp[faila]];

	srcs[0] = dq;
	srcs[1] = q;
	init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
			  NULL, NULL, scribble);
	tx = async_xor(dq, srcs, 0, 2, bytes, submit);

	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
	tx = async_mult(dq, dq, coef, bytes, submit);

	srcs[0] = p;
	srcs[1] = dq;
	init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
			  cb_param, scribble);
	tx = async_xor(p, srcs, 0, 2, bytes, submit);

	return tx;
}
EXPORT_SYMBOL_GPL(async_raid6_datap_recov);

MODULE_AUTHOR("Dan Williams <dan.j.williams@intel.com>");
MODULE_DESCRIPTION("asynchronous RAID-6 recovery api");
MODULE_LICENSE("GPL");
Commit	Line	Data
fd534e9b	1	// SPDX-License-Identifier: GPL-2.0-or-later
0a82a623 DW	2	/*
	3	* Asynchronous RAID-6 recovery calculations ASYNC_TX API.
	4	* Copyright(c) 2009 Intel Corporation
	5	*
	6	* based on raid6recov.c:
	7	* Copyright 2002 H. Peter Anvin
0a82a623 DW	8	*/
	9	#include <linux/kernel.h>
	10	#include <linux/interrupt.h>
4bb33cc8	11	#include <linux/module.h>
0a82a623 DW	12	#include <linux/dma-mapping.h>
	13	#include <linux/raid/pq.h>
	14	#include <linux/async_tx.h>
3bbdd498	15	#include <linux/dmaengine.h>
0a82a623 DW	16
	17	static struct dma_async_tx_descriptor *
	18	async_sum_product(struct page dest, struct page srcs, unsigned char coef,
	19	size_t len, struct async_submit_ctl *submit)
	20	{
	21	struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
	22	&dest, 1, srcs, 2, len);
	23	struct dma_device *dma = chan ? chan->device : NULL;
3bbdd498	24	struct dmaengine_unmap_data *unmap = NULL;
0a82a623 DW	25	const u8 amul, bmul;
	26	u8 ax, bx;
	27	u8 a, b, *c;
	28
3bbdd498	29	if (dma)
b02bab6b	30	unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOWAIT);
3bbdd498 DW	31
3bbdd498 DW	32	if (unmap) {
0a82a623	33	struct device *dev = dma->dev;
3bbdd498	34	dma_addr_t pq[2];
0a82a623	35	struct dma_async_tx_descriptor *tx;
0776ae7b	36	enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
0a82a623	37
0403e382 DW	38	if (submit->flags & ASYNC_TX_FENCE)
0403e382 DW	39	dma_flags \|= DMA_PREP_FENCE;
3bbdd498 DW	40	unmap->addr[0] = dma_map_page(dev, srcs[0], 0, len, DMA_TO_DEVICE);
	41	unmap->addr[1] = dma_map_page(dev, srcs[1], 0, len, DMA_TO_DEVICE);
	42	unmap->to_cnt = 2;
	43
	44	unmap->addr[2] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
	45	unmap->bidi_cnt = 1;
	46	/* engine only looks at Q, but expects it to follow P */
	47	pq[1] = unmap->addr[2];
	48
	49	unmap->len = len;
	50	tx = dma->device_prep_dma_pq(chan, pq, unmap->addr, 2, coef,
0a82a623 DW	51	len, dma_flags);
0a82a623 DW	52	if (tx) {
3bbdd498	53	dma_set_unmap(tx, unmap);
0a82a623	54	async_tx_submit(chan, tx, submit);
3bbdd498	55	dmaengine_unmap_put(unmap);
0a82a623 DW	56	return tx;
0a82a623 DW	57	}
1f6672d4 DW	58
	59	/* could not get a descriptor, unmap and fall through to
	60	* the synchronous path
	61	*/
3bbdd498	62	dmaengine_unmap_put(unmap);
0a82a623 DW	63	}
	64
	65	/* run the operation synchronously */
	66	async_tx_quiesce(&submit->depend_tx);
	67	amul = raid6_gfmul[coef[0]];
	68	bmul = raid6_gfmul[coef[1]];
	69	a = page_address(srcs[0]);
	70	b = page_address(srcs[1]);
	71	c = page_address(dest);
	72
	73	while (len--) {
	74	ax = amul[*a++];
	75	bx = bmul[*b++];
	76	*c++ = ax ^ bx;
	77	}
	78
	79	return NULL;
	80	}
	81
	82	static struct dma_async_tx_descriptor *
	83	async_mult(struct page dest, struct page src, u8 coef, size_t len,
	84	struct async_submit_ctl *submit)
	85	{
	86	struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
	87	&dest, 1, &src, 1, len);
	88	struct dma_device *dma = chan ? chan->device : NULL;
3bbdd498	89	struct dmaengine_unmap_data *unmap = NULL;
0a82a623 DW	90	const u8 qmul; / Q multiplier table */
	91	u8 d, s;
	92
3bbdd498	93	if (dma)
b02bab6b	94	unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOWAIT);
3bbdd498 DW	95
3bbdd498 DW	96	if (unmap) {
0a82a623	97	dma_addr_t dma_dest[2];
0a82a623 DW	98	struct device *dev = dma->dev;
0a82a623 DW	99	struct dma_async_tx_descriptor *tx;
0776ae7b	100	enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
0a82a623	101
0403e382 DW	102	if (submit->flags & ASYNC_TX_FENCE)
0403e382 DW	103	dma_flags \|= DMA_PREP_FENCE;
3bbdd498 DW	104	unmap->addr[0] = dma_map_page(dev, src, 0, len, DMA_TO_DEVICE);
	105	unmap->to_cnt++;
	106	unmap->addr[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
	107	dma_dest[1] = unmap->addr[1];
	108	unmap->bidi_cnt++;
	109	unmap->len = len;
	110
	111	/* this looks funny, but the engine looks for Q at
	112	* dma_dest[1] and ignores dma_dest[0] as a dest
	113	* due to DMA_PREP_PQ_DISABLE_P
	114	*/
	115	tx = dma->device_prep_dma_pq(chan, dma_dest, unmap->addr,
	116	1, &coef, len, dma_flags);
	117
0a82a623	118	if (tx) {
3bbdd498 DW	119	dma_set_unmap(tx, unmap);
3bbdd498 DW	120	dmaengine_unmap_put(unmap);
0a82a623 DW	121	async_tx_submit(chan, tx, submit);
	122	return tx;
	123	}
1f6672d4 DW	124
	125	/* could not get a descriptor, unmap and fall through to
	126	* the synchronous path
	127	*/
3bbdd498	128	dmaengine_unmap_put(unmap);
0a82a623 DW	129	}
	130
	131	/* no channel available, or failed to allocate a descriptor, so
	132	* perform the operation synchronously
	133	*/
	134	async_tx_quiesce(&submit->depend_tx);
	135	qmul = raid6_gfmul[coef];
	136	d = page_address(dest);
	137	s = page_address(src);
	138
	139	while (len--)
	140	d++ = qmul[s++];
	141
	142	return NULL;
	143	}
	144
	145	static struct dma_async_tx_descriptor *
da17bf43 DW	146	__2data_recov_4(int disks, size_t bytes, int faila, int failb,
da17bf43 DW	147	struct page *blocks, struct async_submit_ctl submit)
0a82a623 DW	148	{
	149	struct dma_async_tx_descriptor *tx = NULL;
	150	struct page p, q, a, b;
	151	struct page *srcs[2];
	152	unsigned char coef[2];
	153	enum async_tx_flags flags = submit->flags;
	154	dma_async_tx_callback cb_fn = submit->cb_fn;
	155	void *cb_param = submit->cb_param;
	156	void *scribble = submit->scribble;
	157
da17bf43 DW	158	p = blocks[disks-2];
da17bf43 DW	159	q = blocks[disks-1];
0a82a623 DW	160
	161	a = blocks[faila];
	162	b = blocks[failb];
	163
	164	/* in the 4 disk case P + Pxy == P and Q + Qxy == Q */
	165	/* Dx = A(P+Pxy) + B(Q+Qxy) */
	166	srcs[0] = p;
	167	srcs[1] = q;
	168	coef[0] = raid6_gfexi[failb-faila];
	169	coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
0403e382	170	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
0a82a623 DW	171	tx = async_sum_product(b, srcs, coef, bytes, submit);
	172
	173	/* Dy = P+Pxy+Dx */
	174	srcs[0] = p;
	175	srcs[1] = b;
	176	init_async_submit(submit, flags \| ASYNC_TX_XOR_ZERO_DST, tx, cb_fn,
	177	cb_param, scribble);
	178	tx = async_xor(a, srcs, 0, 2, bytes, submit);
	179
	180	return tx;
	181
	182	}
	183
	184	static struct dma_async_tx_descriptor *
da17bf43 DW	185	__2data_recov_5(int disks, size_t bytes, int faila, int failb,
da17bf43 DW	186	struct page *blocks, struct async_submit_ctl submit)
0a82a623 DW	187	{
	188	struct dma_async_tx_descriptor *tx = NULL;
	189	struct page p, q, g, dp, *dq;
	190	struct page *srcs[2];
	191	unsigned char coef[2];
	192	enum async_tx_flags flags = submit->flags;
	193	dma_async_tx_callback cb_fn = submit->cb_fn;
	194	void *cb_param = submit->cb_param;
	195	void *scribble = submit->scribble;
da17bf43	196	int good_srcs, good, i;
0a82a623	197
da17bf43 DW	198	good_srcs = 0;
	199	good = -1;
	200	for (i = 0; i < disks-2; i++) {
	201	if (blocks[i] == NULL)
	202	continue;
0a82a623 DW	203	if (i == faila \|\| i == failb)
0a82a623 DW	204	continue;
da17bf43 DW	205	good = i;
da17bf43 DW	206	good_srcs++;
0a82a623	207	}
da17bf43	208	BUG_ON(good_srcs > 1);
0a82a623	209
da17bf43 DW	210	p = blocks[disks-2];
da17bf43 DW	211	q = blocks[disks-1];
0a82a623 DW	212	g = blocks[good];
	213
	214	/* Compute syndrome with zero for the missing data pages
	215	* Use the dead data pages as temporary storage for delta p and
	216	* delta q
	217	*/
	218	dp = blocks[faila];
	219	dq = blocks[failb];
	220
0403e382	221	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
0a82a623	222	tx = async_memcpy(dp, g, 0, 0, bytes, submit);
0403e382	223	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
0a82a623 DW	224	tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit);
	225
	226	/* compute P + Pxy */
	227	srcs[0] = dp;
	228	srcs[1] = p;
0403e382 DW	229	init_async_submit(submit, ASYNC_TX_FENCE\|ASYNC_TX_XOR_DROP_DST, tx,
0403e382 DW	230	NULL, NULL, scribble);
0a82a623 DW	231	tx = async_xor(dp, srcs, 0, 2, bytes, submit);
	232
	233	/* compute Q + Qxy */
	234	srcs[0] = dq;
	235	srcs[1] = q;
0403e382 DW	236	init_async_submit(submit, ASYNC_TX_FENCE\|ASYNC_TX_XOR_DROP_DST, tx,
0403e382 DW	237	NULL, NULL, scribble);
0a82a623 DW	238	tx = async_xor(dq, srcs, 0, 2, bytes, submit);
	239
	240	/* Dx = A(P+Pxy) + B(Q+Qxy) */
	241	srcs[0] = dp;
	242	srcs[1] = dq;
	243	coef[0] = raid6_gfexi[failb-faila];
	244	coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
0403e382	245	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
0a82a623 DW	246	tx = async_sum_product(dq, srcs, coef, bytes, submit);
	247
	248	/* Dy = P+Pxy+Dx */
	249	srcs[0] = dp;
	250	srcs[1] = dq;
	251	init_async_submit(submit, flags \| ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
	252	cb_param, scribble);
	253	tx = async_xor(dp, srcs, 0, 2, bytes, submit);
	254
	255	return tx;
	256	}
	257
	258	static struct dma_async_tx_descriptor *
	259	__2data_recov_n(int disks, size_t bytes, int faila, int failb,
	260	struct page *blocks, struct async_submit_ctl submit)
	261	{
	262	struct dma_async_tx_descriptor *tx = NULL;
	263	struct page p, q, dp, dq;
	264	struct page *srcs[2];
	265	unsigned char coef[2];
	266	enum async_tx_flags flags = submit->flags;
	267	dma_async_tx_callback cb_fn = submit->cb_fn;
	268	void *cb_param = submit->cb_param;
	269	void *scribble = submit->scribble;
	270
	271	p = blocks[disks-2];
	272	q = blocks[disks-1];
	273
	274	/* Compute syndrome with zero for the missing data pages
	275	* Use the dead data pages as temporary storage for
	276	* delta p and delta q
	277	*/
	278	dp = blocks[faila];
5dd33c9a	279	blocks[faila] = NULL;
0a82a623 DW	280	blocks[disks-2] = dp;
0a82a623 DW	281	dq = blocks[failb];
5dd33c9a	282	blocks[failb] = NULL;
0a82a623 DW	283	blocks[disks-1] = dq;
0a82a623 DW	284
0403e382	285	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
0a82a623 DW	286	tx = async_gen_syndrome(blocks, 0, disks, bytes, submit);
	287
	288	/* Restore pointer table */
	289	blocks[faila] = dp;
	290	blocks[failb] = dq;
	291	blocks[disks-2] = p;
	292	blocks[disks-1] = q;
	293
	294	/* compute P + Pxy */
	295	srcs[0] = dp;
	296	srcs[1] = p;
0403e382 DW	297	init_async_submit(submit, ASYNC_TX_FENCE\|ASYNC_TX_XOR_DROP_DST, tx,
0403e382 DW	298	NULL, NULL, scribble);
0a82a623 DW	299	tx = async_xor(dp, srcs, 0, 2, bytes, submit);
	300
	301	/* compute Q + Qxy */
	302	srcs[0] = dq;
	303	srcs[1] = q;
0403e382 DW	304	init_async_submit(submit, ASYNC_TX_FENCE\|ASYNC_TX_XOR_DROP_DST, tx,
0403e382 DW	305	NULL, NULL, scribble);
0a82a623 DW	306	tx = async_xor(dq, srcs, 0, 2, bytes, submit);
	307
	308	/* Dx = A(P+Pxy) + B(Q+Qxy) */
	309	srcs[0] = dp;
	310	srcs[1] = dq;
	311	coef[0] = raid6_gfexi[failb-faila];
	312	coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
0403e382	313	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
0a82a623 DW	314	tx = async_sum_product(dq, srcs, coef, bytes, submit);
	315
	316	/* Dy = P+Pxy+Dx */
	317	srcs[0] = dp;
	318	srcs[1] = dq;
	319	init_async_submit(submit, flags \| ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
	320	cb_param, scribble);
	321	tx = async_xor(dp, srcs, 0, 2, bytes, submit);
	322
	323	return tx;
	324	}
	325
	326	/**
	327	* async_raid6_2data_recov - asynchronously calculate two missing data blocks
	328	* @disks: number of disks in the RAID-6 array
	329	* @bytes: block size
	330	* @faila: first failed drive index
	331	* @failb: second failed drive index
	332	* @blocks: array of source pointers where the last two entries are p and q
	333	* @submit: submission/completion modifiers
	334	*/
	335	struct dma_async_tx_descriptor *
	336	async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb,
	337	struct page *blocks, struct async_submit_ctl submit)
	338	{
5157b4aa	339	void *scribble = submit->scribble;
da17bf43 DW	340	int non_zero_srcs, i;
da17bf43 DW	341
0a82a623 DW	342	BUG_ON(faila == failb);
	343	if (failb < faila)
	344	swap(faila, failb);
	345
	346	pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
	347
5157b4aa DW	348	/* if a dma resource is not available or a scribble buffer is not
	349	* available punt to the synchronous path. In the 'dma not
	350	* available' case be sure to use the scribble buffer to
	351	* preserve the content of 'blocks' as the caller intended.
0a82a623	352	*/
5157b4aa DW	353	if (!async_dma_find_channel(DMA_PQ) \|\| !scribble) {
5157b4aa DW	354	void ptrs = scribble ? scribble : (void ) blocks;
0a82a623 DW	355
	356	async_tx_quiesce(&submit->depend_tx);
	357	for (i = 0; i < disks; i++)
5dd33c9a	358	if (blocks[i] == NULL)
da17bf43	359	ptrs[i] = (void *) raid6_empty_zero_page;
5dd33c9a N	360	else
5dd33c9a N	361	ptrs[i] = page_address(blocks[i]);
0a82a623 DW	362
	363	raid6_2data_recov(disks, bytes, faila, failb, ptrs);
	364
	365	async_tx_sync_epilog(submit);
	366
	367	return NULL;
	368	}
	369
da17bf43 DW	370	non_zero_srcs = 0;
	371	for (i = 0; i < disks-2 && non_zero_srcs < 4; i++)
	372	if (blocks[i])
	373	non_zero_srcs++;
	374	switch (non_zero_srcs) {
	375	case 0:
	376	case 1:
	377	/* There must be at least 2 sources - the failed devices. */
	378	BUG();
	379
	380	case 2:
0a82a623 DW	381	/* dma devices do not uniformly understand a zero source pq
0a82a623 DW	382	* operation (in contrast to the synchronous case), so
da17bf43 DW	383	* explicitly handle the special case of a 4 disk array with
da17bf43 DW	384	* both data disks missing.
0a82a623	385	*/
da17bf43 DW	386	return __2data_recov_4(disks, bytes, faila, failb, blocks, submit);
da17bf43 DW	387	case 3:
0a82a623 DW	388	/* dma devices do not uniformly understand a single
0a82a623 DW	389	* source pq operation (in contrast to the synchronous
da17bf43 DW	390	* case), so explicitly handle the special case of a 5 disk
da17bf43 DW	391	* array with 2 of 3 data disks missing.
0a82a623	392	*/
da17bf43	393	return __2data_recov_5(disks, bytes, faila, failb, blocks, submit);
0a82a623 DW	394	default:
	395	return __2data_recov_n(disks, bytes, faila, failb, blocks, submit);
	396	}
	397	}
	398	EXPORT_SYMBOL_GPL(async_raid6_2data_recov);
	399
	400	/**
	401	* async_raid6_datap_recov - asynchronously calculate a data and the 'p' block
	402	* @disks: number of disks in the RAID-6 array
	403	* @bytes: block size
	404	* @faila: failed drive index
	405	* @blocks: array of source pointers where the last two entries are p and q
	406	* @submit: submission/completion modifiers
	407	*/
	408	struct dma_async_tx_descriptor *
	409	async_raid6_datap_recov(int disks, size_t bytes, int faila,
	410	struct page *blocks, struct async_submit_ctl submit)
	411	{
	412	struct dma_async_tx_descriptor *tx = NULL;
	413	struct page p, q, *dq;
	414	u8 coef;
	415	enum async_tx_flags flags = submit->flags;
	416	dma_async_tx_callback cb_fn = submit->cb_fn;
	417	void *cb_param = submit->cb_param;
	418	void *scribble = submit->scribble;
da17bf43	419	int good_srcs, good, i;
0a82a623 DW	420	struct page *srcs[2];
	421
	422	pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
	423
5157b4aa DW	424	/* if a dma resource is not available or a scribble buffer is not
	425	* available punt to the synchronous path. In the 'dma not
	426	* available' case be sure to use the scribble buffer to
	427	* preserve the content of 'blocks' as the caller intended.
0a82a623	428	*/
5157b4aa DW	429	if (!async_dma_find_channel(DMA_PQ) \|\| !scribble) {
5157b4aa DW	430	void ptrs = scribble ? scribble : (void ) blocks;
0a82a623 DW	431
	432	async_tx_quiesce(&submit->depend_tx);
	433	for (i = 0; i < disks; i++)
5dd33c9a N	434	if (blocks[i] == NULL)
	435	ptrs[i] = (void*)raid6_empty_zero_page;
	436	else
	437	ptrs[i] = page_address(blocks[i]);
0a82a623 DW	438
	439	raid6_datap_recov(disks, bytes, faila, ptrs);
	440
	441	async_tx_sync_epilog(submit);
	442
	443	return NULL;
	444	}
	445
da17bf43 DW	446	good_srcs = 0;
	447	good = -1;
	448	for (i = 0; i < disks-2; i++) {
	449	if (i == faila)
	450	continue;
	451	if (blocks[i]) {
	452	good = i;
	453	good_srcs++;
	454	if (good_srcs > 1)
	455	break;
	456	}
	457	}
	458	BUG_ON(good_srcs == 0);
	459
0a82a623 DW	460	p = blocks[disks-2];
	461	q = blocks[disks-1];
	462
	463	/* Compute syndrome with zero for the missing data page
	464	* Use the dead data page as temporary storage for delta q
	465	*/
	466	dq = blocks[faila];
5dd33c9a	467	blocks[faila] = NULL;
0a82a623 DW	468	blocks[disks-1] = dq;
0a82a623 DW	469
da17bf43 DW	470	/* in the 4-disk case we only need to perform a single source
da17bf43 DW	471	* multiplication with the one good data block.
0a82a623	472	*/
da17bf43	473	if (good_srcs == 1) {
0a82a623 DW	474	struct page *g = blocks[good];
0a82a623 DW	475
0403e382 DW	476	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
0403e382 DW	477	scribble);
0a82a623 DW	478	tx = async_memcpy(p, g, 0, 0, bytes, submit);
0a82a623 DW	479
0403e382 DW	480	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
0403e382 DW	481	scribble);
0a82a623 DW	482	tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit);
0a82a623 DW	483	} else {
0403e382 DW	484	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
0403e382 DW	485	scribble);
0a82a623 DW	486	tx = async_gen_syndrome(blocks, 0, disks, bytes, submit);
	487	}
	488
	489	/* Restore pointer table */
	490	blocks[faila] = dq;
	491	blocks[disks-1] = q;
	492
	493	/* calculate g^{-faila} */
	494	coef = raid6_gfinv[raid6_gfexp[faila]];
	495
	496	srcs[0] = dq;
	497	srcs[1] = q;
0403e382 DW	498	init_async_submit(submit, ASYNC_TX_FENCE\|ASYNC_TX_XOR_DROP_DST, tx,
0403e382 DW	499	NULL, NULL, scribble);
0a82a623 DW	500	tx = async_xor(dq, srcs, 0, 2, bytes, submit);
0a82a623 DW	501
0403e382	502	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
0a82a623 DW	503	tx = async_mult(dq, dq, coef, bytes, submit);
	504
	505	srcs[0] = p;
	506	srcs[1] = dq;
	507	init_async_submit(submit, flags \| ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
	508	cb_param, scribble);
	509	tx = async_xor(p, srcs, 0, 2, bytes, submit);
	510
	511	return tx;
	512	}
	513	EXPORT_SYMBOL_GPL(async_raid6_datap_recov);
	514
	515	MODULE_AUTHOR("Dan Williams <dan.j.williams@intel.com>");
	516	MODULE_DESCRIPTION("asynchronous RAID-6 recovery api");
	517	MODULE_LICENSE("GPL");