1 /* SPDX-License-Identifier: GPL-2.0-only */
3 * Copyright © 2006, Intel Corporation.
7 #include <linux/dmaengine.h>
8 #include <linux/spinlock.h>
9 #include <linux/interrupt.h>
11 /* on architectures without dma-mapping capabilities we need to ensure
12 * that the asynchronous path compiles away
15 #define __async_inline
17 #define __async_inline __always_inline
21 * dma_chan_ref - object used to manage dma channels received from the
23 * @chan - the channel being tracked
24 * @node - node for the channel to be placed on async_tx_master_list
25 * @rcu - for list_del_rcu
26 * @count - number of times this channel is listed in the pool
27 * (for channels with multiple capabiities)
30 struct dma_chan
*chan
;
31 struct list_head node
;
37 * async_tx_flags - modifiers for the async_* calls
38 * @ASYNC_TX_XOR_ZERO_DST: this flag must be used for xor operations where the
39 * the destination address is not a source. The asynchronous case handles this
40 * implicitly, the synchronous case needs to zero the destination block.
41 * @ASYNC_TX_XOR_DROP_DST: this flag must be used if the destination address is
42 * also one of the source addresses. In the synchronous case the destination
43 * address is an implied source, whereas the asynchronous case it must be listed
44 * as a source. The destination address must be the first address in the source
46 * @ASYNC_TX_ACK: immediately ack the descriptor, precludes setting up a
48 * @ASYNC_TX_FENCE: specify that the next operation in the dependency
49 * chain uses this operation's result as an input
50 * @ASYNC_TX_PQ_XOR_DST: do not overwrite the syndrome but XOR it with the
51 * input data. Required for rmw case.
54 ASYNC_TX_XOR_ZERO_DST
= (1 << 0),
55 ASYNC_TX_XOR_DROP_DST
= (1 << 1),
56 ASYNC_TX_ACK
= (1 << 2),
57 ASYNC_TX_FENCE
= (1 << 3),
58 ASYNC_TX_PQ_XOR_DST
= (1 << 4),
62 * struct async_submit_ctl - async_tx submission/completion modifiers
63 * @flags: submission modifiers
64 * @depend_tx: parent dependency of the current operation being submitted
65 * @cb_fn: callback routine to run at operation completion
66 * @cb_param: parameter for the callback routine
67 * @scribble: caller provided space for dma/page address conversions
69 struct async_submit_ctl
{
70 enum async_tx_flags flags
;
71 struct dma_async_tx_descriptor
*depend_tx
;
72 dma_async_tx_callback cb_fn
;
77 #if defined(CONFIG_DMA_ENGINE) && !defined(CONFIG_ASYNC_TX_CHANNEL_SWITCH)
78 #define async_tx_issue_pending_all dma_issue_pending_all
81 * async_tx_issue_pending - send pending descriptor to the hardware channel
82 * @tx: descriptor handle to retrieve hardware context
84 * Note: any dependent operations will have already been issued by
85 * async_tx_channel_switch, or (in the case of no channel switch) will
86 * be already pending on this channel.
88 static inline void async_tx_issue_pending(struct dma_async_tx_descriptor
*tx
)
91 struct dma_chan
*chan
= tx
->chan
;
92 struct dma_device
*dma
= chan
->device
;
94 dma
->device_issue_pending(chan
);
97 #ifdef CONFIG_ARCH_HAS_ASYNC_TX_FIND_CHANNEL
98 #include <asm/async_tx.h>
100 #define async_tx_find_channel(dep, type, dst, dst_count, src, src_count, len) \
101 __async_tx_find_channel(dep, type)
103 __async_tx_find_channel(struct async_submit_ctl
*submit
,
104 enum dma_transaction_type tx_type
);
105 #endif /* CONFIG_ARCH_HAS_ASYNC_TX_FIND_CHANNEL */
107 static inline void async_tx_issue_pending_all(void)
112 static inline void async_tx_issue_pending(struct dma_async_tx_descriptor
*tx
)
117 static inline struct dma_chan
*
118 async_tx_find_channel(struct async_submit_ctl
*submit
,
119 enum dma_transaction_type tx_type
, struct page
**dst
,
120 int dst_count
, struct page
**src
, int src_count
,
128 * async_tx_sync_epilog - actions to take if an operation is run synchronously
129 * @cb_fn: function to call when the transaction completes
130 * @cb_fn_param: parameter to pass to the callback routine
133 async_tx_sync_epilog(struct async_submit_ctl
*submit
)
136 submit
->cb_fn(submit
->cb_param
);
146 init_async_submit(struct async_submit_ctl
*args
, enum async_tx_flags flags
,
147 struct dma_async_tx_descriptor
*tx
,
148 dma_async_tx_callback cb_fn
, void *cb_param
,
149 addr_conv_t
*scribble
)
152 args
->depend_tx
= tx
;
154 args
->cb_param
= cb_param
;
155 args
->scribble
= scribble
;
158 void async_tx_submit(struct dma_chan
*chan
, struct dma_async_tx_descriptor
*tx
,
159 struct async_submit_ctl
*submit
);
161 struct dma_async_tx_descriptor
*
162 async_xor(struct page
*dest
, struct page
**src_list
, unsigned int offset
,
163 int src_cnt
, size_t len
, struct async_submit_ctl
*submit
);
165 struct dma_async_tx_descriptor
*
166 async_xor_val(struct page
*dest
, struct page
**src_list
, unsigned int offset
,
167 int src_cnt
, size_t len
, enum sum_check_flags
*result
,
168 struct async_submit_ctl
*submit
);
170 struct dma_async_tx_descriptor
*
171 async_memcpy(struct page
*dest
, struct page
*src
, unsigned int dest_offset
,
172 unsigned int src_offset
, size_t len
,
173 struct async_submit_ctl
*submit
);
175 struct dma_async_tx_descriptor
*async_trigger_callback(struct async_submit_ctl
*submit
);
177 struct dma_async_tx_descriptor
*
178 async_gen_syndrome(struct page
**blocks
, unsigned int offset
, int src_cnt
,
179 size_t len
, struct async_submit_ctl
*submit
);
181 struct dma_async_tx_descriptor
*
182 async_syndrome_val(struct page
**blocks
, unsigned int offset
, int src_cnt
,
183 size_t len
, enum sum_check_flags
*pqres
, struct page
*spare
,
184 struct async_submit_ctl
*submit
);
186 struct dma_async_tx_descriptor
*
187 async_raid6_2data_recov(int src_num
, size_t bytes
, int faila
, int failb
,
188 struct page
**ptrs
, struct async_submit_ctl
*submit
);
190 struct dma_async_tx_descriptor
*
191 async_raid6_datap_recov(int src_num
, size_t bytes
, int faila
,
192 struct page
**ptrs
, struct async_submit_ctl
*submit
);
194 void async_tx_quiesce(struct dma_async_tx_descriptor
**tx
);
195 #endif /* _ASYNC_TX_H_ */