2 * xor offload engine api
4 * Copyright © 2006, Intel Corporation.
6 * Dan Williams <dan.j.williams@intel.com>
8 * with architecture considerations by:
9 * Neil Brown <neilb@suse.de>
10 * Jeff Garzik <jeff@garzik.org>
12 * This program is free software; you can redistribute it and/or modify it
13 * under the terms and conditions of the GNU General Public License,
14 * version 2, as published by the Free Software Foundation.
16 * This program is distributed in the hope it will be useful, but WITHOUT
17 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
21 * You should have received a copy of the GNU General Public License along with
22 * this program; if not, write to the Free Software Foundation, Inc.,
23 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
26 #include <linux/kernel.h>
27 #include <linux/interrupt.h>
29 #include <linux/dma-mapping.h>
30 #include <linux/raid/xor.h>
31 #include <linux/async_tx.h>
33 /* do_async_xor - dma map the pages and perform the xor with an engine.
34 * This routine is marked __always_inline so it can be compiled away
35 * when CONFIG_DMA_ENGINE=n
37 static __always_inline
struct dma_async_tx_descriptor
*
38 do_async_xor(struct dma_device
*device
,
39 struct dma_chan
*chan
, struct page
*dest
, struct page
**src_list
,
40 unsigned int offset
, unsigned int src_cnt
, size_t len
,
41 enum async_tx_flags flags
, struct dma_async_tx_descriptor
*depend_tx
,
42 dma_async_tx_callback cb_fn
, void *cb_param
)
45 dma_addr_t
*dma_src
= (dma_addr_t
*) src_list
;
46 struct dma_async_tx_descriptor
*tx
;
49 pr_debug("%s: len: %zu\n", __FUNCTION__
, len
);
51 dma_dest
= dma_map_page(device
->dev
, dest
, offset
, len
,
54 for (i
= 0; i
< src_cnt
; i
++)
55 dma_src
[i
] = dma_map_page(device
->dev
, src_list
[i
], offset
,
58 /* Since we have clobbered the src_list we are committed
59 * to doing this asynchronously. Drivers force forward progress
60 * in case they can not provide a descriptor
62 tx
= device
->device_prep_dma_xor(chan
, dma_dest
, dma_src
, src_cnt
, len
,
66 dma_wait_for_async_tx(depend_tx
);
69 tx
= device
->device_prep_dma_xor(chan
, dma_dest
,
70 dma_src
, src_cnt
, len
,
74 async_tx_submit(chan
, tx
, flags
, depend_tx
, cb_fn
, cb_param
);
80 do_sync_xor(struct page
*dest
, struct page
**src_list
, unsigned int offset
,
81 unsigned int src_cnt
, size_t len
, enum async_tx_flags flags
,
82 struct dma_async_tx_descriptor
*depend_tx
,
83 dma_async_tx_callback cb_fn
, void *cb_param
)
88 pr_debug("%s: len: %zu\n", __FUNCTION__
, len
);
90 /* reuse the 'src_list' array to convert to buffer pointers */
91 for (i
= 0; i
< src_cnt
; i
++)
92 src_list
[i
] = (struct page
*)
93 (page_address(src_list
[i
]) + offset
);
95 /* set destination address */
96 _dest
= page_address(dest
) + offset
;
98 if (flags
& ASYNC_TX_XOR_ZERO_DST
)
99 memset(_dest
, 0, len
);
101 xor_blocks(src_cnt
, len
, _dest
,
104 async_tx_sync_epilog(flags
, depend_tx
, cb_fn
, cb_param
);
108 * async_xor - attempt to xor a set of blocks with a dma engine.
109 * xor_blocks always uses the dest as a source so the ASYNC_TX_XOR_ZERO_DST
110 * flag must be set to not include dest data in the calculation. The
111 * assumption with dma eninges is that they only use the destination
112 * buffer as a source when it is explicity specified in the source list.
113 * @dest: destination page
114 * @src_list: array of source pages (if the dest is also a source it must be
115 * at index zero). The contents of this array may be overwritten.
116 * @offset: offset in pages to start transaction
117 * @src_cnt: number of source pages
118 * @len: length in bytes
119 * @flags: ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DEST,
120 * ASYNC_TX_ACK, ASYNC_TX_DEP_ACK
121 * @depend_tx: xor depends on the result of this transaction.
122 * @cb_fn: function to call when the xor completes
123 * @cb_param: parameter to pass to the callback routine
125 struct dma_async_tx_descriptor
*
126 async_xor(struct page
*dest
, struct page
**src_list
, unsigned int offset
,
127 int src_cnt
, size_t len
, enum async_tx_flags flags
,
128 struct dma_async_tx_descriptor
*depend_tx
,
129 dma_async_tx_callback cb_fn
, void *cb_param
)
131 struct dma_chan
*chan
= async_tx_find_channel(depend_tx
, DMA_XOR
);
132 struct dma_device
*device
= chan
? chan
->device
: NULL
;
133 struct dma_async_tx_descriptor
*tx
= NULL
;
134 dma_async_tx_callback _cb_fn
;
136 unsigned long local_flags
;
138 int i
= 0, src_off
= 0;
140 BUG_ON(src_cnt
<= 1);
144 if (device
) { /* run the xor asynchronously */
145 xor_src_cnt
= min(src_cnt
, device
->max_xor
);
146 /* if we are submitting additional xors
147 * only set the callback on the last transaction
149 if (src_cnt
> xor_src_cnt
) {
150 local_flags
&= ~ASYNC_TX_ACK
;
155 _cb_param
= cb_param
;
158 tx
= do_async_xor(device
, chan
, dest
,
159 &src_list
[src_off
], offset
,
160 xor_src_cnt
, len
, local_flags
,
161 depend_tx
, _cb_fn
, _cb_param
);
162 } else { /* run the xor synchronously */
163 /* in the sync case the dest is an implied source
164 * (assumes the dest is at the src_off index)
166 if (flags
& ASYNC_TX_XOR_DROP_DST
) {
171 /* process up to 'MAX_XOR_BLOCKS' sources */
172 xor_src_cnt
= min(src_cnt
, MAX_XOR_BLOCKS
);
174 /* if we are submitting additional xors
175 * only set the callback on the last transaction
177 if (src_cnt
> xor_src_cnt
) {
178 local_flags
&= ~ASYNC_TX_ACK
;
183 _cb_param
= cb_param
;
186 /* wait for any prerequisite operations */
188 /* if ack is already set then we cannot be sure
189 * we are referring to the correct operation
191 BUG_ON(depend_tx
->ack
);
192 if (dma_wait_for_async_tx(depend_tx
) ==
194 panic("%s: DMA_ERROR waiting for "
199 do_sync_xor(dest
, &src_list
[src_off
], offset
,
200 xor_src_cnt
, len
, local_flags
, depend_tx
,
204 /* the previous tx is hidden from the client,
208 async_tx_ack(depend_tx
);
212 if (src_cnt
> xor_src_cnt
) {
213 /* drop completed sources */
214 src_cnt
-= xor_src_cnt
;
215 src_off
+= xor_src_cnt
;
217 /* unconditionally preserve the destination */
218 flags
&= ~ASYNC_TX_XOR_ZERO_DST
;
220 /* use the intermediate result a source, but remember
221 * it's dropped, because it's implied, in the sync case
223 src_list
[--src_off
] = dest
;
225 flags
|= ASYNC_TX_XOR_DROP_DST
;
233 EXPORT_SYMBOL_GPL(async_xor
);
235 static int page_is_zero(struct page
*p
, unsigned int offset
, size_t len
)
237 char *a
= page_address(p
) + offset
;
238 return ((*(u32
*) a
) == 0 &&
239 memcmp(a
, a
+ 4, len
- 4) == 0);
243 * async_xor_zero_sum - attempt a xor parity check with a dma engine.
244 * @dest: destination page used if the xor is performed synchronously
245 * @src_list: array of source pages. The dest page must be listed as a source
246 * at index zero. The contents of this array may be overwritten.
247 * @offset: offset in pages to start transaction
248 * @src_cnt: number of source pages
249 * @len: length in bytes
250 * @result: 0 if sum == 0 else non-zero
251 * @flags: ASYNC_TX_ACK, ASYNC_TX_DEP_ACK
252 * @depend_tx: xor depends on the result of this transaction.
253 * @cb_fn: function to call when the xor completes
254 * @cb_param: parameter to pass to the callback routine
256 struct dma_async_tx_descriptor
*
257 async_xor_zero_sum(struct page
*dest
, struct page
**src_list
,
258 unsigned int offset
, int src_cnt
, size_t len
,
259 u32
*result
, enum async_tx_flags flags
,
260 struct dma_async_tx_descriptor
*depend_tx
,
261 dma_async_tx_callback cb_fn
, void *cb_param
)
263 struct dma_chan
*chan
= async_tx_find_channel(depend_tx
, DMA_ZERO_SUM
);
264 struct dma_device
*device
= chan
? chan
->device
: NULL
;
265 struct dma_async_tx_descriptor
*tx
= NULL
;
267 BUG_ON(src_cnt
<= 1);
270 dma_addr_t
*dma_src
= (dma_addr_t
*) src_list
;
273 pr_debug("%s: (async) len: %zu\n", __FUNCTION__
, len
);
275 for (i
= 0; i
< src_cnt
; i
++)
276 dma_src
[i
] = dma_map_page(device
->dev
, src_list
[i
],
277 offset
, len
, DMA_TO_DEVICE
);
279 tx
= device
->device_prep_dma_zero_sum(chan
, dma_src
, src_cnt
,
284 dma_wait_for_async_tx(depend_tx
);
287 tx
= device
->device_prep_dma_zero_sum(chan
,
288 dma_src
, src_cnt
, len
, result
,
292 async_tx_submit(chan
, tx
, flags
, depend_tx
, cb_fn
, cb_param
);
294 unsigned long xor_flags
= flags
;
296 pr_debug("%s: (sync) len: %zu\n", __FUNCTION__
, len
);
298 xor_flags
|= ASYNC_TX_XOR_DROP_DST
;
299 xor_flags
&= ~ASYNC_TX_ACK
;
301 tx
= async_xor(dest
, src_list
, offset
, src_cnt
, len
, xor_flags
,
302 depend_tx
, NULL
, NULL
);
305 if (dma_wait_for_async_tx(tx
) == DMA_ERROR
)
306 panic("%s: DMA_ERROR waiting for tx\n",
311 *result
= page_is_zero(dest
, offset
, len
) ? 0 : 1;
315 async_tx_sync_epilog(flags
, depend_tx
, cb_fn
, cb_param
);
320 EXPORT_SYMBOL_GPL(async_xor_zero_sum
);
322 static int __init
async_xor_init(void)
324 #ifdef CONFIG_DMA_ENGINE
325 /* To conserve stack space the input src_list (array of page pointers)
326 * is reused to hold the array of dma addresses passed to the driver.
327 * This conversion is only possible when dma_addr_t is less than the
328 * the size of a pointer. HIGHMEM64G is known to violate this
331 BUILD_BUG_ON(sizeof(dma_addr_t
) > sizeof(struct page
*));
337 static void __exit
async_xor_exit(void)
342 module_init(async_xor_init
);
343 module_exit(async_xor_exit
);
345 MODULE_AUTHOR("Intel Corporation");
346 MODULE_DESCRIPTION("asynchronous xor/xor-zero-sum api");
347 MODULE_LICENSE("GPL");