]>
Commit | Line | Data |
---|---|---|
9f2fd0df RPS |
1 | /* |
2 | * Applied Micro X-Gene SoC DMA engine Driver | |
3 | * | |
4 | * Copyright (c) 2015, Applied Micro Circuits Corporation | |
5 | * Authors: Rameshwar Prasad Sahu <rsahu@apm.com> | |
6 | * Loc Ho <lho@apm.com> | |
7 | * | |
8 | * This program is free software; you can redistribute it and/or modify it | |
9 | * under the terms of the GNU General Public License as published by the | |
10 | * Free Software Foundation; either version 2 of the License, or (at your | |
11 | * option) any later version. | |
12 | * | |
13 | * This program is distributed in the hope that it will be useful, | |
14 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
16 | * GNU General Public License for more details. | |
17 | * | |
18 | * You should have received a copy of the GNU General Public License | |
19 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
20 | * | |
21 | * NOTE: PM support is currently not available. | |
22 | */ | |
23 | ||
89079493 | 24 | #include <linux/acpi.h> |
9f2fd0df RPS |
25 | #include <linux/clk.h> |
26 | #include <linux/delay.h> | |
27 | #include <linux/dma-mapping.h> | |
28 | #include <linux/dmaengine.h> | |
29 | #include <linux/dmapool.h> | |
30 | #include <linux/interrupt.h> | |
31 | #include <linux/io.h> | |
32 | #include <linux/module.h> | |
33 | #include <linux/of_device.h> | |
34 | ||
35 | #include "dmaengine.h" | |
36 | ||
37 | /* X-Gene DMA ring csr registers and bit definations */ | |
38 | #define XGENE_DMA_RING_CONFIG 0x04 | |
39 | #define XGENE_DMA_RING_ENABLE BIT(31) | |
40 | #define XGENE_DMA_RING_ID 0x08 | |
41 | #define XGENE_DMA_RING_ID_SETUP(v) ((v) | BIT(31)) | |
42 | #define XGENE_DMA_RING_ID_BUF 0x0C | |
43 | #define XGENE_DMA_RING_ID_BUF_SETUP(v) (((v) << 9) | BIT(21)) | |
44 | #define XGENE_DMA_RING_THRESLD0_SET1 0x30 | |
45 | #define XGENE_DMA_RING_THRESLD0_SET1_VAL 0X64 | |
46 | #define XGENE_DMA_RING_THRESLD1_SET1 0x34 | |
47 | #define XGENE_DMA_RING_THRESLD1_SET1_VAL 0xC8 | |
48 | #define XGENE_DMA_RING_HYSTERESIS 0x68 | |
49 | #define XGENE_DMA_RING_HYSTERESIS_VAL 0xFFFFFFFF | |
50 | #define XGENE_DMA_RING_STATE 0x6C | |
51 | #define XGENE_DMA_RING_STATE_WR_BASE 0x70 | |
52 | #define XGENE_DMA_RING_NE_INT_MODE 0x017C | |
53 | #define XGENE_DMA_RING_NE_INT_MODE_SET(m, v) \ | |
54 | ((m) = ((m) & ~BIT(31 - (v))) | BIT(31 - (v))) | |
55 | #define XGENE_DMA_RING_NE_INT_MODE_RESET(m, v) \ | |
56 | ((m) &= (~BIT(31 - (v)))) | |
57 | #define XGENE_DMA_RING_CLKEN 0xC208 | |
58 | #define XGENE_DMA_RING_SRST 0xC200 | |
59 | #define XGENE_DMA_RING_MEM_RAM_SHUTDOWN 0xD070 | |
60 | #define XGENE_DMA_RING_BLK_MEM_RDY 0xD074 | |
61 | #define XGENE_DMA_RING_BLK_MEM_RDY_VAL 0xFFFFFFFF | |
9f2fd0df RPS |
62 | #define XGENE_DMA_RING_ID_GET(owner, num) (((owner) << 6) | (num)) |
63 | #define XGENE_DMA_RING_DST_ID(v) ((1 << 10) | (v)) | |
64 | #define XGENE_DMA_RING_CMD_OFFSET 0x2C | |
65 | #define XGENE_DMA_RING_CMD_BASE_OFFSET(v) ((v) << 6) | |
66 | #define XGENE_DMA_RING_COHERENT_SET(m) \ | |
67 | (((u32 *)(m))[2] |= BIT(4)) | |
68 | #define XGENE_DMA_RING_ADDRL_SET(m, v) \ | |
69 | (((u32 *)(m))[2] |= (((v) >> 8) << 5)) | |
70 | #define XGENE_DMA_RING_ADDRH_SET(m, v) \ | |
71 | (((u32 *)(m))[3] |= ((v) >> 35)) | |
72 | #define XGENE_DMA_RING_ACCEPTLERR_SET(m) \ | |
73 | (((u32 *)(m))[3] |= BIT(19)) | |
74 | #define XGENE_DMA_RING_SIZE_SET(m, v) \ | |
75 | (((u32 *)(m))[3] |= ((v) << 23)) | |
76 | #define XGENE_DMA_RING_RECOMBBUF_SET(m) \ | |
77 | (((u32 *)(m))[3] |= BIT(27)) | |
78 | #define XGENE_DMA_RING_RECOMTIMEOUTL_SET(m) \ | |
79 | (((u32 *)(m))[3] |= (0x7 << 28)) | |
80 | #define XGENE_DMA_RING_RECOMTIMEOUTH_SET(m) \ | |
81 | (((u32 *)(m))[4] |= 0x3) | |
82 | #define XGENE_DMA_RING_SELTHRSH_SET(m) \ | |
83 | (((u32 *)(m))[4] |= BIT(3)) | |
84 | #define XGENE_DMA_RING_TYPE_SET(m, v) \ | |
85 | (((u32 *)(m))[4] |= ((v) << 19)) | |
86 | ||
87 | /* X-Gene DMA device csr registers and bit definitions */ | |
88 | #define XGENE_DMA_IPBRR 0x0 | |
89 | #define XGENE_DMA_DEV_ID_RD(v) ((v) & 0x00000FFF) | |
90 | #define XGENE_DMA_BUS_ID_RD(v) (((v) >> 12) & 3) | |
91 | #define XGENE_DMA_REV_NO_RD(v) (((v) >> 14) & 3) | |
92 | #define XGENE_DMA_GCR 0x10 | |
93 | #define XGENE_DMA_CH_SETUP(v) \ | |
94 | ((v) = ((v) & ~0x000FFFFF) | 0x000AAFFF) | |
95 | #define XGENE_DMA_ENABLE(v) ((v) |= BIT(31)) | |
96 | #define XGENE_DMA_DISABLE(v) ((v) &= ~BIT(31)) | |
97 | #define XGENE_DMA_RAID6_CONT 0x14 | |
98 | #define XGENE_DMA_RAID6_MULTI_CTRL(v) ((v) << 24) | |
99 | #define XGENE_DMA_INT 0x70 | |
100 | #define XGENE_DMA_INT_MASK 0x74 | |
101 | #define XGENE_DMA_INT_ALL_MASK 0xFFFFFFFF | |
102 | #define XGENE_DMA_INT_ALL_UNMASK 0x0 | |
103 | #define XGENE_DMA_INT_MASK_SHIFT 0x14 | |
104 | #define XGENE_DMA_RING_INT0_MASK 0x90A0 | |
105 | #define XGENE_DMA_RING_INT1_MASK 0x90A8 | |
106 | #define XGENE_DMA_RING_INT2_MASK 0x90B0 | |
107 | #define XGENE_DMA_RING_INT3_MASK 0x90B8 | |
108 | #define XGENE_DMA_RING_INT4_MASK 0x90C0 | |
109 | #define XGENE_DMA_CFG_RING_WQ_ASSOC 0x90E0 | |
110 | #define XGENE_DMA_ASSOC_RING_MNGR1 0xFFFFFFFF | |
111 | #define XGENE_DMA_MEM_RAM_SHUTDOWN 0xD070 | |
112 | #define XGENE_DMA_BLK_MEM_RDY 0xD074 | |
113 | #define XGENE_DMA_BLK_MEM_RDY_VAL 0xFFFFFFFF | |
cda8e937 | 114 | #define XGENE_DMA_RING_CMD_SM_OFFSET 0x8000 |
9f2fd0df RPS |
115 | |
116 | /* X-Gene SoC EFUSE csr register and bit defination */ | |
117 | #define XGENE_SOC_JTAG1_SHADOW 0x18 | |
118 | #define XGENE_DMA_PQ_DISABLE_MASK BIT(13) | |
119 | ||
120 | /* X-Gene DMA Descriptor format */ | |
121 | #define XGENE_DMA_DESC_NV_BIT BIT_ULL(50) | |
122 | #define XGENE_DMA_DESC_IN_BIT BIT_ULL(55) | |
123 | #define XGENE_DMA_DESC_C_BIT BIT_ULL(63) | |
124 | #define XGENE_DMA_DESC_DR_BIT BIT_ULL(61) | |
125 | #define XGENE_DMA_DESC_ELERR_POS 46 | |
126 | #define XGENE_DMA_DESC_RTYPE_POS 56 | |
127 | #define XGENE_DMA_DESC_LERR_POS 60 | |
9f2fd0df RPS |
128 | #define XGENE_DMA_DESC_BUFLEN_POS 48 |
129 | #define XGENE_DMA_DESC_HOENQ_NUM_POS 48 | |
9f2fd0df RPS |
130 | #define XGENE_DMA_DESC_ELERR_RD(m) \ |
131 | (((m) >> XGENE_DMA_DESC_ELERR_POS) & 0x3) | |
132 | #define XGENE_DMA_DESC_LERR_RD(m) \ | |
133 | (((m) >> XGENE_DMA_DESC_LERR_POS) & 0x7) | |
134 | #define XGENE_DMA_DESC_STATUS(elerr, lerr) \ | |
135 | (((elerr) << 4) | (lerr)) | |
136 | ||
137 | /* X-Gene DMA descriptor empty s/w signature */ | |
9f2fd0df | 138 | #define XGENE_DMA_DESC_EMPTY_SIGNATURE ~0ULL |
9f2fd0df RPS |
139 | |
140 | /* X-Gene DMA configurable parameters defines */ | |
141 | #define XGENE_DMA_RING_NUM 512 | |
142 | #define XGENE_DMA_BUFNUM 0x0 | |
143 | #define XGENE_DMA_CPU_BUFNUM 0x18 | |
144 | #define XGENE_DMA_RING_OWNER_DMA 0x03 | |
145 | #define XGENE_DMA_RING_OWNER_CPU 0x0F | |
146 | #define XGENE_DMA_RING_TYPE_REGULAR 0x01 | |
147 | #define XGENE_DMA_RING_WQ_DESC_SIZE 32 /* 32 Bytes */ | |
148 | #define XGENE_DMA_RING_NUM_CONFIG 5 | |
149 | #define XGENE_DMA_MAX_CHANNEL 4 | |
150 | #define XGENE_DMA_XOR_CHANNEL 0 | |
151 | #define XGENE_DMA_PQ_CHANNEL 1 | |
152 | #define XGENE_DMA_MAX_BYTE_CNT 0x4000 /* 16 KB */ | |
153 | #define XGENE_DMA_MAX_64B_DESC_BYTE_CNT 0x14000 /* 80 KB */ | |
9f2fd0df RPS |
154 | #define XGENE_DMA_MAX_XOR_SRC 5 |
155 | #define XGENE_DMA_16K_BUFFER_LEN_CODE 0x0 | |
6d0767c1 | 156 | #define XGENE_DMA_INVALID_LEN_CODE 0x7800000000000000ULL |
9f2fd0df RPS |
157 | |
158 | /* X-Gene DMA descriptor error codes */ | |
159 | #define ERR_DESC_AXI 0x01 | |
160 | #define ERR_BAD_DESC 0x02 | |
161 | #define ERR_READ_DATA_AXI 0x03 | |
162 | #define ERR_WRITE_DATA_AXI 0x04 | |
163 | #define ERR_FBP_TIMEOUT 0x05 | |
164 | #define ERR_ECC 0x06 | |
165 | #define ERR_DIFF_SIZE 0x08 | |
166 | #define ERR_SCT_GAT_LEN 0x09 | |
167 | #define ERR_CRC_ERR 0x11 | |
168 | #define ERR_CHKSUM 0x12 | |
169 | #define ERR_DIF 0x13 | |
170 | ||
171 | /* X-Gene DMA error interrupt codes */ | |
172 | #define ERR_DIF_SIZE_INT 0x0 | |
173 | #define ERR_GS_ERR_INT 0x1 | |
174 | #define ERR_FPB_TIMEO_INT 0x2 | |
175 | #define ERR_WFIFO_OVF_INT 0x3 | |
176 | #define ERR_RFIFO_OVF_INT 0x4 | |
177 | #define ERR_WR_TIMEO_INT 0x5 | |
178 | #define ERR_RD_TIMEO_INT 0x6 | |
179 | #define ERR_WR_ERR_INT 0x7 | |
180 | #define ERR_RD_ERR_INT 0x8 | |
181 | #define ERR_BAD_DESC_INT 0x9 | |
182 | #define ERR_DESC_DST_INT 0xA | |
183 | #define ERR_DESC_SRC_INT 0xB | |
184 | ||
185 | /* X-Gene DMA flyby operation code */ | |
6d0767c1 RPS |
186 | #define FLYBY_2SRC_XOR 0x80 |
187 | #define FLYBY_3SRC_XOR 0x90 | |
188 | #define FLYBY_4SRC_XOR 0xA0 | |
189 | #define FLYBY_5SRC_XOR 0xB0 | |
9f2fd0df RPS |
190 | |
191 | /* X-Gene DMA SW descriptor flags */ | |
192 | #define XGENE_DMA_FLAG_64B_DESC BIT(0) | |
193 | ||
194 | /* Define to dump X-Gene DMA descriptor */ | |
195 | #define XGENE_DMA_DESC_DUMP(desc, m) \ | |
196 | print_hex_dump(KERN_ERR, (m), \ | |
197 | DUMP_PREFIX_ADDRESS, 16, 8, (desc), 32, 0) | |
198 | ||
199 | #define to_dma_desc_sw(tx) \ | |
200 | container_of(tx, struct xgene_dma_desc_sw, tx) | |
201 | #define to_dma_chan(dchan) \ | |
202 | container_of(dchan, struct xgene_dma_chan, dma_chan) | |
203 | ||
204 | #define chan_dbg(chan, fmt, arg...) \ | |
205 | dev_dbg(chan->dev, "%s: " fmt, chan->name, ##arg) | |
206 | #define chan_err(chan, fmt, arg...) \ | |
207 | dev_err(chan->dev, "%s: " fmt, chan->name, ##arg) | |
208 | ||
209 | struct xgene_dma_desc_hw { | |
6d0767c1 RPS |
210 | __le64 m0; |
211 | __le64 m1; | |
212 | __le64 m2; | |
213 | __le64 m3; | |
9f2fd0df RPS |
214 | }; |
215 | ||
216 | enum xgene_dma_ring_cfgsize { | |
217 | XGENE_DMA_RING_CFG_SIZE_512B, | |
218 | XGENE_DMA_RING_CFG_SIZE_2KB, | |
219 | XGENE_DMA_RING_CFG_SIZE_16KB, | |
220 | XGENE_DMA_RING_CFG_SIZE_64KB, | |
221 | XGENE_DMA_RING_CFG_SIZE_512KB, | |
222 | XGENE_DMA_RING_CFG_SIZE_INVALID | |
223 | }; | |
224 | ||
225 | struct xgene_dma_ring { | |
226 | struct xgene_dma *pdma; | |
227 | u8 buf_num; | |
228 | u16 id; | |
229 | u16 num; | |
230 | u16 head; | |
231 | u16 owner; | |
232 | u16 slots; | |
233 | u16 dst_ring_num; | |
234 | u32 size; | |
235 | void __iomem *cmd; | |
236 | void __iomem *cmd_base; | |
237 | dma_addr_t desc_paddr; | |
238 | u32 state[XGENE_DMA_RING_NUM_CONFIG]; | |
239 | enum xgene_dma_ring_cfgsize cfgsize; | |
240 | union { | |
241 | void *desc_vaddr; | |
242 | struct xgene_dma_desc_hw *desc_hw; | |
243 | }; | |
244 | }; | |
245 | ||
246 | struct xgene_dma_desc_sw { | |
247 | struct xgene_dma_desc_hw desc1; | |
248 | struct xgene_dma_desc_hw desc2; | |
249 | u32 flags; | |
250 | struct list_head node; | |
251 | struct list_head tx_list; | |
252 | struct dma_async_tx_descriptor tx; | |
253 | }; | |
254 | ||
255 | /** | |
256 | * struct xgene_dma_chan - internal representation of an X-Gene DMA channel | |
257 | * @dma_chan: dmaengine channel object member | |
258 | * @pdma: X-Gene DMA device structure reference | |
259 | * @dev: struct device reference for dma mapping api | |
260 | * @id: raw id of this channel | |
261 | * @rx_irq: channel IRQ | |
262 | * @name: name of X-Gene DMA channel | |
263 | * @lock: serializes enqueue/dequeue operations to the descriptor pool | |
264 | * @pending: number of transaction request pushed to DMA controller for | |
265 | * execution, but still waiting for completion, | |
266 | * @max_outstanding: max number of outstanding request we can push to channel | |
267 | * @ld_pending: descriptors which are queued to run, but have not yet been | |
268 | * submitted to the hardware for execution | |
269 | * @ld_running: descriptors which are currently being executing by the hardware | |
270 | * @ld_completed: descriptors which have finished execution by the hardware. | |
271 | * These descriptors have already had their cleanup actions run. They | |
272 | * are waiting for the ACK bit to be set by the async tx API. | |
273 | * @desc_pool: descriptor pool for DMA operations | |
274 | * @tasklet: bottom half where all completed descriptors cleans | |
275 | * @tx_ring: transmit ring descriptor that we use to prepare actual | |
276 | * descriptors for further executions | |
277 | * @rx_ring: receive ring descriptor that we use to get completed DMA | |
278 | * descriptors during cleanup time | |
279 | */ | |
280 | struct xgene_dma_chan { | |
281 | struct dma_chan dma_chan; | |
282 | struct xgene_dma *pdma; | |
283 | struct device *dev; | |
284 | int id; | |
285 | int rx_irq; | |
ed1f0418 | 286 | char name[10]; |
9f2fd0df RPS |
287 | spinlock_t lock; |
288 | int pending; | |
289 | int max_outstanding; | |
290 | struct list_head ld_pending; | |
291 | struct list_head ld_running; | |
292 | struct list_head ld_completed; | |
293 | struct dma_pool *desc_pool; | |
294 | struct tasklet_struct tasklet; | |
295 | struct xgene_dma_ring tx_ring; | |
296 | struct xgene_dma_ring rx_ring; | |
297 | }; | |
298 | ||
299 | /** | |
300 | * struct xgene_dma - internal representation of an X-Gene DMA device | |
301 | * @err_irq: DMA error irq number | |
302 | * @ring_num: start id number for DMA ring | |
303 | * @csr_dma: base for DMA register access | |
304 | * @csr_ring: base for DMA ring register access | |
305 | * @csr_ring_cmd: base for DMA ring command register access | |
306 | * @csr_efuse: base for efuse register access | |
307 | * @dma_dev: embedded struct dma_device | |
308 | * @chan: reference to X-Gene DMA channels | |
309 | */ | |
310 | struct xgene_dma { | |
311 | struct device *dev; | |
312 | struct clk *clk; | |
313 | int err_irq; | |
314 | int ring_num; | |
315 | void __iomem *csr_dma; | |
316 | void __iomem *csr_ring; | |
317 | void __iomem *csr_ring_cmd; | |
318 | void __iomem *csr_efuse; | |
319 | struct dma_device dma_dev[XGENE_DMA_MAX_CHANNEL]; | |
320 | struct xgene_dma_chan chan[XGENE_DMA_MAX_CHANNEL]; | |
321 | }; | |
322 | ||
323 | static const char * const xgene_dma_desc_err[] = { | |
324 | [ERR_DESC_AXI] = "AXI error when reading src/dst link list", | |
325 | [ERR_BAD_DESC] = "ERR or El_ERR fields not set to zero in desc", | |
326 | [ERR_READ_DATA_AXI] = "AXI error when reading data", | |
327 | [ERR_WRITE_DATA_AXI] = "AXI error when writing data", | |
328 | [ERR_FBP_TIMEOUT] = "Timeout on bufpool fetch", | |
329 | [ERR_ECC] = "ECC double bit error", | |
330 | [ERR_DIFF_SIZE] = "Bufpool too small to hold all the DIF result", | |
331 | [ERR_SCT_GAT_LEN] = "Gather and scatter data length not same", | |
332 | [ERR_CRC_ERR] = "CRC error", | |
333 | [ERR_CHKSUM] = "Checksum error", | |
334 | [ERR_DIF] = "DIF error", | |
335 | }; | |
336 | ||
337 | static const char * const xgene_dma_err[] = { | |
338 | [ERR_DIF_SIZE_INT] = "DIF size error", | |
339 | [ERR_GS_ERR_INT] = "Gather scatter not same size error", | |
340 | [ERR_FPB_TIMEO_INT] = "Free pool time out error", | |
341 | [ERR_WFIFO_OVF_INT] = "Write FIFO over flow error", | |
342 | [ERR_RFIFO_OVF_INT] = "Read FIFO over flow error", | |
343 | [ERR_WR_TIMEO_INT] = "Write time out error", | |
344 | [ERR_RD_TIMEO_INT] = "Read time out error", | |
345 | [ERR_WR_ERR_INT] = "HBF bus write error", | |
346 | [ERR_RD_ERR_INT] = "HBF bus read error", | |
347 | [ERR_BAD_DESC_INT] = "Ring descriptor HE0 not set error", | |
348 | [ERR_DESC_DST_INT] = "HFB reading dst link address error", | |
349 | [ERR_DESC_SRC_INT] = "HFB reading src link address error", | |
350 | }; | |
351 | ||
352 | static bool is_pq_enabled(struct xgene_dma *pdma) | |
353 | { | |
354 | u32 val; | |
355 | ||
356 | val = ioread32(pdma->csr_efuse + XGENE_SOC_JTAG1_SHADOW); | |
357 | return !(val & XGENE_DMA_PQ_DISABLE_MASK); | |
358 | } | |
359 | ||
6d0767c1 | 360 | static u64 xgene_dma_encode_len(size_t len) |
9f2fd0df RPS |
361 | { |
362 | return (len < XGENE_DMA_MAX_BYTE_CNT) ? | |
6d0767c1 RPS |
363 | ((u64)len << XGENE_DMA_DESC_BUFLEN_POS) : |
364 | XGENE_DMA_16K_BUFFER_LEN_CODE; | |
9f2fd0df RPS |
365 | } |
366 | ||
367 | static u8 xgene_dma_encode_xor_flyby(u32 src_cnt) | |
368 | { | |
369 | static u8 flyby_type[] = { | |
370 | FLYBY_2SRC_XOR, /* Dummy */ | |
371 | FLYBY_2SRC_XOR, /* Dummy */ | |
372 | FLYBY_2SRC_XOR, | |
373 | FLYBY_3SRC_XOR, | |
374 | FLYBY_4SRC_XOR, | |
375 | FLYBY_5SRC_XOR | |
376 | }; | |
377 | ||
378 | return flyby_type[src_cnt]; | |
379 | } | |
380 | ||
6d0767c1 | 381 | static void xgene_dma_set_src_buffer(__le64 *ext8, size_t *len, |
9f2fd0df RPS |
382 | dma_addr_t *paddr) |
383 | { | |
384 | size_t nbytes = (*len < XGENE_DMA_MAX_BYTE_CNT) ? | |
385 | *len : XGENE_DMA_MAX_BYTE_CNT; | |
386 | ||
6d0767c1 RPS |
387 | *ext8 |= cpu_to_le64(*paddr); |
388 | *ext8 |= cpu_to_le64(xgene_dma_encode_len(nbytes)); | |
9f2fd0df RPS |
389 | *len -= nbytes; |
390 | *paddr += nbytes; | |
391 | } | |
392 | ||
6d0767c1 | 393 | static void xgene_dma_invalidate_buffer(__le64 *ext8) |
9f2fd0df | 394 | { |
6d0767c1 | 395 | *ext8 |= cpu_to_le64(XGENE_DMA_INVALID_LEN_CODE); |
9f2fd0df RPS |
396 | } |
397 | ||
6d0767c1 | 398 | static __le64 *xgene_dma_lookup_ext8(struct xgene_dma_desc_hw *desc, int idx) |
9f2fd0df | 399 | { |
6d0767c1 RPS |
400 | switch (idx) { |
401 | case 0: | |
402 | return &desc->m1; | |
403 | case 1: | |
404 | return &desc->m0; | |
405 | case 2: | |
406 | return &desc->m3; | |
407 | case 3: | |
408 | return &desc->m2; | |
409 | default: | |
410 | pr_err("Invalid dma descriptor index\n"); | |
411 | } | |
412 | ||
413 | return NULL; | |
9f2fd0df RPS |
414 | } |
415 | ||
6d0767c1 RPS |
416 | static void xgene_dma_init_desc(struct xgene_dma_desc_hw *desc, |
417 | u16 dst_ring_num) | |
9f2fd0df | 418 | { |
6d0767c1 RPS |
419 | desc->m0 |= cpu_to_le64(XGENE_DMA_DESC_IN_BIT); |
420 | desc->m0 |= cpu_to_le64((u64)XGENE_DMA_RING_OWNER_DMA << | |
421 | XGENE_DMA_DESC_RTYPE_POS); | |
422 | desc->m1 |= cpu_to_le64(XGENE_DMA_DESC_C_BIT); | |
423 | desc->m3 |= cpu_to_le64((u64)dst_ring_num << | |
424 | XGENE_DMA_DESC_HOENQ_NUM_POS); | |
9f2fd0df RPS |
425 | } |
426 | ||
427 | static void xgene_dma_prep_cpy_desc(struct xgene_dma_chan *chan, | |
428 | struct xgene_dma_desc_sw *desc_sw, | |
429 | dma_addr_t dst, dma_addr_t src, | |
430 | size_t len) | |
431 | { | |
6d0767c1 | 432 | struct xgene_dma_desc_hw *desc1, *desc2; |
9f2fd0df RPS |
433 | int i; |
434 | ||
435 | /* Get 1st descriptor */ | |
436 | desc1 = &desc_sw->desc1; | |
437 | xgene_dma_init_desc(desc1, chan->tx_ring.dst_ring_num); | |
438 | ||
439 | /* Set destination address */ | |
6d0767c1 RPS |
440 | desc1->m2 |= cpu_to_le64(XGENE_DMA_DESC_DR_BIT); |
441 | desc1->m3 |= cpu_to_le64(dst); | |
9f2fd0df RPS |
442 | |
443 | /* Set 1st source address */ | |
6d0767c1 | 444 | xgene_dma_set_src_buffer(&desc1->m1, &len, &src); |
9f2fd0df | 445 | |
6d0767c1 RPS |
446 | if (!len) |
447 | return; | |
9f2fd0df RPS |
448 | |
449 | /* | |
450 | * We need to split this source buffer, | |
451 | * and need to use 2nd descriptor | |
452 | */ | |
453 | desc2 = &desc_sw->desc2; | |
6d0767c1 | 454 | desc1->m0 |= cpu_to_le64(XGENE_DMA_DESC_NV_BIT); |
9f2fd0df RPS |
455 | |
456 | /* Set 2nd to 5th source address */ | |
457 | for (i = 0; i < 4 && len; i++) | |
458 | xgene_dma_set_src_buffer(xgene_dma_lookup_ext8(desc2, i), | |
459 | &len, &src); | |
460 | ||
461 | /* Invalidate unused source address field */ | |
462 | for (; i < 4; i++) | |
463 | xgene_dma_invalidate_buffer(xgene_dma_lookup_ext8(desc2, i)); | |
464 | ||
465 | /* Updated flag that we have prepared 64B descriptor */ | |
466 | desc_sw->flags |= XGENE_DMA_FLAG_64B_DESC; | |
9f2fd0df RPS |
467 | } |
468 | ||
469 | static void xgene_dma_prep_xor_desc(struct xgene_dma_chan *chan, | |
470 | struct xgene_dma_desc_sw *desc_sw, | |
471 | dma_addr_t *dst, dma_addr_t *src, | |
472 | u32 src_cnt, size_t *nbytes, | |
473 | const u8 *scf) | |
474 | { | |
6d0767c1 | 475 | struct xgene_dma_desc_hw *desc1, *desc2; |
9f2fd0df RPS |
476 | size_t len = *nbytes; |
477 | int i; | |
478 | ||
479 | desc1 = &desc_sw->desc1; | |
480 | desc2 = &desc_sw->desc2; | |
481 | ||
482 | /* Initialize DMA descriptor */ | |
483 | xgene_dma_init_desc(desc1, chan->tx_ring.dst_ring_num); | |
484 | ||
485 | /* Set destination address */ | |
6d0767c1 RPS |
486 | desc1->m2 |= cpu_to_le64(XGENE_DMA_DESC_DR_BIT); |
487 | desc1->m3 |= cpu_to_le64(*dst); | |
9f2fd0df RPS |
488 | |
489 | /* We have multiple source addresses, so need to set NV bit*/ | |
6d0767c1 | 490 | desc1->m0 |= cpu_to_le64(XGENE_DMA_DESC_NV_BIT); |
9f2fd0df RPS |
491 | |
492 | /* Set flyby opcode */ | |
6d0767c1 | 493 | desc1->m2 |= cpu_to_le64(xgene_dma_encode_xor_flyby(src_cnt)); |
9f2fd0df RPS |
494 | |
495 | /* Set 1st to 5th source addresses */ | |
496 | for (i = 0; i < src_cnt; i++) { | |
497 | len = *nbytes; | |
6d0767c1 | 498 | xgene_dma_set_src_buffer((i == 0) ? &desc1->m1 : |
9f2fd0df RPS |
499 | xgene_dma_lookup_ext8(desc2, i - 1), |
500 | &len, &src[i]); | |
6d0767c1 | 501 | desc1->m2 |= cpu_to_le64((scf[i] << ((i + 1) * 8))); |
9f2fd0df RPS |
502 | } |
503 | ||
9f2fd0df RPS |
504 | /* Update meta data */ |
505 | *nbytes = len; | |
506 | *dst += XGENE_DMA_MAX_BYTE_CNT; | |
507 | ||
508 | /* We need always 64B descriptor to perform xor or pq operations */ | |
509 | desc_sw->flags |= XGENE_DMA_FLAG_64B_DESC; | |
510 | } | |
511 | ||
512 | static dma_cookie_t xgene_dma_tx_submit(struct dma_async_tx_descriptor *tx) | |
513 | { | |
514 | struct xgene_dma_desc_sw *desc; | |
515 | struct xgene_dma_chan *chan; | |
516 | dma_cookie_t cookie; | |
517 | ||
518 | if (unlikely(!tx)) | |
519 | return -EINVAL; | |
520 | ||
521 | chan = to_dma_chan(tx->chan); | |
522 | desc = to_dma_desc_sw(tx); | |
523 | ||
524 | spin_lock_bh(&chan->lock); | |
525 | ||
526 | cookie = dma_cookie_assign(tx); | |
527 | ||
528 | /* Add this transaction list onto the tail of the pending queue */ | |
529 | list_splice_tail_init(&desc->tx_list, &chan->ld_pending); | |
530 | ||
531 | spin_unlock_bh(&chan->lock); | |
532 | ||
533 | return cookie; | |
534 | } | |
535 | ||
536 | static void xgene_dma_clean_descriptor(struct xgene_dma_chan *chan, | |
537 | struct xgene_dma_desc_sw *desc) | |
538 | { | |
539 | list_del(&desc->node); | |
540 | chan_dbg(chan, "LD %p free\n", desc); | |
541 | dma_pool_free(chan->desc_pool, desc, desc->tx.phys); | |
542 | } | |
543 | ||
544 | static struct xgene_dma_desc_sw *xgene_dma_alloc_descriptor( | |
545 | struct xgene_dma_chan *chan) | |
546 | { | |
547 | struct xgene_dma_desc_sw *desc; | |
548 | dma_addr_t phys; | |
549 | ||
550 | desc = dma_pool_alloc(chan->desc_pool, GFP_NOWAIT, &phys); | |
551 | if (!desc) { | |
552 | chan_err(chan, "Failed to allocate LDs\n"); | |
553 | return NULL; | |
554 | } | |
555 | ||
556 | memset(desc, 0, sizeof(*desc)); | |
557 | ||
558 | INIT_LIST_HEAD(&desc->tx_list); | |
559 | desc->tx.phys = phys; | |
560 | desc->tx.tx_submit = xgene_dma_tx_submit; | |
561 | dma_async_tx_descriptor_init(&desc->tx, &chan->dma_chan); | |
562 | ||
563 | chan_dbg(chan, "LD %p allocated\n", desc); | |
564 | ||
565 | return desc; | |
566 | } | |
567 | ||
568 | /** | |
569 | * xgene_dma_clean_completed_descriptor - free all descriptors which | |
570 | * has been completed and acked | |
571 | * @chan: X-Gene DMA channel | |
572 | * | |
573 | * This function is used on all completed and acked descriptors. | |
574 | */ | |
575 | static void xgene_dma_clean_completed_descriptor(struct xgene_dma_chan *chan) | |
576 | { | |
577 | struct xgene_dma_desc_sw *desc, *_desc; | |
578 | ||
579 | /* Run the callback for each descriptor, in order */ | |
580 | list_for_each_entry_safe(desc, _desc, &chan->ld_completed, node) { | |
581 | if (async_tx_test_ack(&desc->tx)) | |
582 | xgene_dma_clean_descriptor(chan, desc); | |
583 | } | |
584 | } | |
585 | ||
586 | /** | |
587 | * xgene_dma_run_tx_complete_actions - cleanup a single link descriptor | |
588 | * @chan: X-Gene DMA channel | |
589 | * @desc: descriptor to cleanup and free | |
590 | * | |
591 | * This function is used on a descriptor which has been executed by the DMA | |
592 | * controller. It will run any callbacks, submit any dependencies. | |
593 | */ | |
594 | static void xgene_dma_run_tx_complete_actions(struct xgene_dma_chan *chan, | |
595 | struct xgene_dma_desc_sw *desc) | |
596 | { | |
597 | struct dma_async_tx_descriptor *tx = &desc->tx; | |
598 | ||
599 | /* | |
600 | * If this is not the last transaction in the group, | |
601 | * then no need to complete cookie and run any callback as | |
602 | * this is not the tx_descriptor which had been sent to caller | |
603 | * of this DMA request | |
604 | */ | |
605 | ||
606 | if (tx->cookie == 0) | |
607 | return; | |
608 | ||
609 | dma_cookie_complete(tx); | |
610 | ||
611 | /* Run the link descriptor callback function */ | |
612 | if (tx->callback) | |
613 | tx->callback(tx->callback_param); | |
614 | ||
615 | dma_descriptor_unmap(tx); | |
616 | ||
617 | /* Run any dependencies */ | |
618 | dma_run_dependencies(tx); | |
619 | } | |
620 | ||
621 | /** | |
622 | * xgene_dma_clean_running_descriptor - move the completed descriptor from | |
623 | * ld_running to ld_completed | |
624 | * @chan: X-Gene DMA channel | |
625 | * @desc: the descriptor which is completed | |
626 | * | |
627 | * Free the descriptor directly if acked by async_tx api, | |
628 | * else move it to queue ld_completed. | |
629 | */ | |
630 | static void xgene_dma_clean_running_descriptor(struct xgene_dma_chan *chan, | |
631 | struct xgene_dma_desc_sw *desc) | |
632 | { | |
633 | /* Remove from the list of running transactions */ | |
634 | list_del(&desc->node); | |
635 | ||
636 | /* | |
637 | * the client is allowed to attach dependent operations | |
638 | * until 'ack' is set | |
639 | */ | |
640 | if (!async_tx_test_ack(&desc->tx)) { | |
641 | /* | |
642 | * Move this descriptor to the list of descriptors which is | |
643 | * completed, but still awaiting the 'ack' bit to be set. | |
644 | */ | |
645 | list_add_tail(&desc->node, &chan->ld_completed); | |
646 | return; | |
647 | } | |
648 | ||
649 | chan_dbg(chan, "LD %p free\n", desc); | |
650 | dma_pool_free(chan->desc_pool, desc, desc->tx.phys); | |
651 | } | |
652 | ||
ee08b59d RPS |
653 | static void xgene_chan_xfer_request(struct xgene_dma_chan *chan, |
654 | struct xgene_dma_desc_sw *desc_sw) | |
9f2fd0df | 655 | { |
ee08b59d | 656 | struct xgene_dma_ring *ring = &chan->tx_ring; |
9f2fd0df RPS |
657 | struct xgene_dma_desc_hw *desc_hw; |
658 | ||
9f2fd0df RPS |
659 | /* Get hw descriptor from DMA tx ring */ |
660 | desc_hw = &ring->desc_hw[ring->head]; | |
661 | ||
662 | /* | |
663 | * Increment the head count to point next | |
664 | * descriptor for next time | |
665 | */ | |
666 | if (++ring->head == ring->slots) | |
667 | ring->head = 0; | |
668 | ||
669 | /* Copy prepared sw descriptor data to hw descriptor */ | |
670 | memcpy(desc_hw, &desc_sw->desc1, sizeof(*desc_hw)); | |
671 | ||
672 | /* | |
673 | * Check if we have prepared 64B descriptor, | |
674 | * in this case we need one more hw descriptor | |
675 | */ | |
676 | if (desc_sw->flags & XGENE_DMA_FLAG_64B_DESC) { | |
677 | desc_hw = &ring->desc_hw[ring->head]; | |
678 | ||
679 | if (++ring->head == ring->slots) | |
680 | ring->head = 0; | |
681 | ||
682 | memcpy(desc_hw, &desc_sw->desc2, sizeof(*desc_hw)); | |
683 | } | |
684 | ||
ee08b59d RPS |
685 | /* Increment the pending transaction count */ |
686 | chan->pending += ((desc_sw->flags & | |
687 | XGENE_DMA_FLAG_64B_DESC) ? 2 : 1); | |
688 | ||
9f2fd0df RPS |
689 | /* Notify the hw that we have descriptor ready for execution */ |
690 | iowrite32((desc_sw->flags & XGENE_DMA_FLAG_64B_DESC) ? | |
691 | 2 : 1, ring->cmd); | |
9f2fd0df RPS |
692 | } |
693 | ||
694 | /** | |
695 | * xgene_chan_xfer_ld_pending - push any pending transactions to hw | |
696 | * @chan : X-Gene DMA channel | |
697 | * | |
6d0767c1 | 698 | * LOCKING: must hold chan->lock |
9f2fd0df RPS |
699 | */ |
700 | static void xgene_chan_xfer_ld_pending(struct xgene_dma_chan *chan) | |
701 | { | |
702 | struct xgene_dma_desc_sw *desc_sw, *_desc_sw; | |
9f2fd0df RPS |
703 | |
704 | /* | |
705 | * If the list of pending descriptors is empty, then we | |
706 | * don't need to do any work at all | |
707 | */ | |
708 | if (list_empty(&chan->ld_pending)) { | |
709 | chan_dbg(chan, "No pending LDs\n"); | |
710 | return; | |
711 | } | |
712 | ||
713 | /* | |
714 | * Move elements from the queue of pending transactions onto the list | |
715 | * of running transactions and push it to hw for further executions | |
716 | */ | |
717 | list_for_each_entry_safe(desc_sw, _desc_sw, &chan->ld_pending, node) { | |
718 | /* | |
719 | * Check if have pushed max number of transactions to hw | |
720 | * as capable, so let's stop here and will push remaining | |
721 | * elements from pening ld queue after completing some | |
722 | * descriptors that we have already pushed | |
723 | */ | |
724 | if (chan->pending >= chan->max_outstanding) | |
725 | return; | |
726 | ||
ee08b59d | 727 | xgene_chan_xfer_request(chan, desc_sw); |
9f2fd0df RPS |
728 | |
729 | /* | |
730 | * Delete this element from ld pending queue and append it to | |
731 | * ld running queue | |
732 | */ | |
733 | list_move_tail(&desc_sw->node, &chan->ld_running); | |
9f2fd0df RPS |
734 | } |
735 | } | |
736 | ||
737 | /** | |
738 | * xgene_dma_cleanup_descriptors - cleanup link descriptors which are completed | |
739 | * and move them to ld_completed to free until flag 'ack' is set | |
740 | * @chan: X-Gene DMA channel | |
741 | * | |
742 | * This function is used on descriptors which have been executed by the DMA | |
743 | * controller. It will run any callbacks, submit any dependencies, then | |
744 | * free these descriptors if flag 'ack' is set. | |
745 | */ | |
746 | static void xgene_dma_cleanup_descriptors(struct xgene_dma_chan *chan) | |
747 | { | |
748 | struct xgene_dma_ring *ring = &chan->rx_ring; | |
749 | struct xgene_dma_desc_sw *desc_sw, *_desc_sw; | |
750 | struct xgene_dma_desc_hw *desc_hw; | |
005ce70b | 751 | struct list_head ld_completed; |
9f2fd0df RPS |
752 | u8 status; |
753 | ||
005ce70b RPS |
754 | INIT_LIST_HEAD(&ld_completed); |
755 | ||
756 | spin_lock_bh(&chan->lock); | |
757 | ||
9f2fd0df RPS |
758 | /* Clean already completed and acked descriptors */ |
759 | xgene_dma_clean_completed_descriptor(chan); | |
760 | ||
005ce70b | 761 | /* Move all completed descriptors to ld completed queue, in order */ |
9f2fd0df RPS |
762 | list_for_each_entry_safe(desc_sw, _desc_sw, &chan->ld_running, node) { |
763 | /* Get subsequent hw descriptor from DMA rx ring */ | |
764 | desc_hw = &ring->desc_hw[ring->head]; | |
765 | ||
766 | /* Check if this descriptor has been completed */ | |
6d0767c1 RPS |
767 | if (unlikely(le64_to_cpu(desc_hw->m0) == |
768 | XGENE_DMA_DESC_EMPTY_SIGNATURE)) | |
9f2fd0df RPS |
769 | break; |
770 | ||
771 | if (++ring->head == ring->slots) | |
772 | ring->head = 0; | |
773 | ||
774 | /* Check if we have any error with DMA transactions */ | |
775 | status = XGENE_DMA_DESC_STATUS( | |
776 | XGENE_DMA_DESC_ELERR_RD(le64_to_cpu( | |
777 | desc_hw->m0)), | |
778 | XGENE_DMA_DESC_LERR_RD(le64_to_cpu( | |
779 | desc_hw->m0))); | |
780 | if (status) { | |
781 | /* Print the DMA error type */ | |
782 | chan_err(chan, "%s\n", xgene_dma_desc_err[status]); | |
783 | ||
784 | /* | |
785 | * We have DMA transactions error here. Dump DMA Tx | |
786 | * and Rx descriptors for this request */ | |
787 | XGENE_DMA_DESC_DUMP(&desc_sw->desc1, | |
788 | "X-Gene DMA TX DESC1: "); | |
789 | ||
790 | if (desc_sw->flags & XGENE_DMA_FLAG_64B_DESC) | |
791 | XGENE_DMA_DESC_DUMP(&desc_sw->desc2, | |
792 | "X-Gene DMA TX DESC2: "); | |
793 | ||
794 | XGENE_DMA_DESC_DUMP(desc_hw, | |
795 | "X-Gene DMA RX ERR DESC: "); | |
796 | } | |
797 | ||
798 | /* Notify the hw about this completed descriptor */ | |
799 | iowrite32(-1, ring->cmd); | |
800 | ||
801 | /* Mark this hw descriptor as processed */ | |
6d0767c1 | 802 | desc_hw->m0 = cpu_to_le64(XGENE_DMA_DESC_EMPTY_SIGNATURE); |
9f2fd0df | 803 | |
9f2fd0df RPS |
804 | /* |
805 | * Decrement the pending transaction count | |
806 | * as we have processed one | |
807 | */ | |
ee08b59d RPS |
808 | chan->pending -= ((desc_sw->flags & |
809 | XGENE_DMA_FLAG_64B_DESC) ? 2 : 1); | |
005ce70b RPS |
810 | |
811 | /* | |
812 | * Delete this node from ld running queue and append it to | |
813 | * ld completed queue for further processing | |
814 | */ | |
815 | list_move_tail(&desc_sw->node, &ld_completed); | |
9f2fd0df RPS |
816 | } |
817 | ||
818 | /* | |
819 | * Start any pending transactions automatically | |
820 | * In the ideal case, we keep the DMA controller busy while we go | |
821 | * ahead and free the descriptors below. | |
822 | */ | |
823 | xgene_chan_xfer_ld_pending(chan); | |
005ce70b RPS |
824 | |
825 | spin_unlock_bh(&chan->lock); | |
826 | ||
827 | /* Run the callback for each descriptor, in order */ | |
828 | list_for_each_entry_safe(desc_sw, _desc_sw, &ld_completed, node) { | |
829 | xgene_dma_run_tx_complete_actions(chan, desc_sw); | |
830 | xgene_dma_clean_running_descriptor(chan, desc_sw); | |
831 | } | |
9f2fd0df RPS |
832 | } |
833 | ||
834 | static int xgene_dma_alloc_chan_resources(struct dma_chan *dchan) | |
835 | { | |
836 | struct xgene_dma_chan *chan = to_dma_chan(dchan); | |
837 | ||
838 | /* Has this channel already been allocated? */ | |
839 | if (chan->desc_pool) | |
840 | return 1; | |
841 | ||
842 | chan->desc_pool = dma_pool_create(chan->name, chan->dev, | |
843 | sizeof(struct xgene_dma_desc_sw), | |
844 | 0, 0); | |
845 | if (!chan->desc_pool) { | |
846 | chan_err(chan, "Failed to allocate descriptor pool\n"); | |
847 | return -ENOMEM; | |
848 | } | |
849 | ||
850 | chan_dbg(chan, "Allocate descripto pool\n"); | |
851 | ||
852 | return 1; | |
853 | } | |
854 | ||
855 | /** | |
856 | * xgene_dma_free_desc_list - Free all descriptors in a queue | |
857 | * @chan: X-Gene DMA channel | |
858 | * @list: the list to free | |
859 | * | |
6d0767c1 | 860 | * LOCKING: must hold chan->lock |
9f2fd0df RPS |
861 | */ |
862 | static void xgene_dma_free_desc_list(struct xgene_dma_chan *chan, | |
863 | struct list_head *list) | |
864 | { | |
865 | struct xgene_dma_desc_sw *desc, *_desc; | |
866 | ||
867 | list_for_each_entry_safe(desc, _desc, list, node) | |
868 | xgene_dma_clean_descriptor(chan, desc); | |
869 | } | |
870 | ||
9f2fd0df RPS |
871 | static void xgene_dma_free_chan_resources(struct dma_chan *dchan) |
872 | { | |
873 | struct xgene_dma_chan *chan = to_dma_chan(dchan); | |
874 | ||
875 | chan_dbg(chan, "Free all resources\n"); | |
876 | ||
877 | if (!chan->desc_pool) | |
878 | return; | |
879 | ||
9f2fd0df RPS |
880 | /* Process all running descriptor */ |
881 | xgene_dma_cleanup_descriptors(chan); | |
882 | ||
005ce70b RPS |
883 | spin_lock_bh(&chan->lock); |
884 | ||
9f2fd0df RPS |
885 | /* Clean all link descriptor queues */ |
886 | xgene_dma_free_desc_list(chan, &chan->ld_pending); | |
887 | xgene_dma_free_desc_list(chan, &chan->ld_running); | |
888 | xgene_dma_free_desc_list(chan, &chan->ld_completed); | |
889 | ||
890 | spin_unlock_bh(&chan->lock); | |
891 | ||
892 | /* Delete this channel DMA pool */ | |
893 | dma_pool_destroy(chan->desc_pool); | |
894 | chan->desc_pool = NULL; | |
895 | } | |
896 | ||
897 | static struct dma_async_tx_descriptor *xgene_dma_prep_memcpy( | |
898 | struct dma_chan *dchan, dma_addr_t dst, dma_addr_t src, | |
899 | size_t len, unsigned long flags) | |
900 | { | |
901 | struct xgene_dma_desc_sw *first = NULL, *new; | |
902 | struct xgene_dma_chan *chan; | |
903 | size_t copy; | |
904 | ||
905 | if (unlikely(!dchan || !len)) | |
906 | return NULL; | |
907 | ||
908 | chan = to_dma_chan(dchan); | |
909 | ||
910 | do { | |
911 | /* Allocate the link descriptor from DMA pool */ | |
912 | new = xgene_dma_alloc_descriptor(chan); | |
913 | if (!new) | |
914 | goto fail; | |
915 | ||
916 | /* Create the largest transaction possible */ | |
917 | copy = min_t(size_t, len, XGENE_DMA_MAX_64B_DESC_BYTE_CNT); | |
918 | ||
919 | /* Prepare DMA descriptor */ | |
920 | xgene_dma_prep_cpy_desc(chan, new, dst, src, copy); | |
921 | ||
922 | if (!first) | |
923 | first = new; | |
924 | ||
925 | new->tx.cookie = 0; | |
926 | async_tx_ack(&new->tx); | |
927 | ||
928 | /* Update metadata */ | |
929 | len -= copy; | |
930 | dst += copy; | |
931 | src += copy; | |
932 | ||
933 | /* Insert the link descriptor to the LD ring */ | |
934 | list_add_tail(&new->node, &first->tx_list); | |
935 | } while (len); | |
936 | ||
937 | new->tx.flags = flags; /* client is in control of this ack */ | |
938 | new->tx.cookie = -EBUSY; | |
939 | list_splice(&first->tx_list, &new->tx_list); | |
940 | ||
941 | return &new->tx; | |
942 | ||
943 | fail: | |
944 | if (!first) | |
945 | return NULL; | |
946 | ||
6d0767c1 | 947 | xgene_dma_free_desc_list(chan, &first->tx_list); |
9f2fd0df RPS |
948 | return NULL; |
949 | } | |
950 | ||
951 | static struct dma_async_tx_descriptor *xgene_dma_prep_sg( | |
952 | struct dma_chan *dchan, struct scatterlist *dst_sg, | |
953 | u32 dst_nents, struct scatterlist *src_sg, | |
954 | u32 src_nents, unsigned long flags) | |
955 | { | |
956 | struct xgene_dma_desc_sw *first = NULL, *new = NULL; | |
957 | struct xgene_dma_chan *chan; | |
958 | size_t dst_avail, src_avail; | |
959 | dma_addr_t dst, src; | |
960 | size_t len; | |
961 | ||
962 | if (unlikely(!dchan)) | |
963 | return NULL; | |
964 | ||
965 | if (unlikely(!dst_nents || !src_nents)) | |
966 | return NULL; | |
967 | ||
968 | if (unlikely(!dst_sg || !src_sg)) | |
969 | return NULL; | |
970 | ||
971 | chan = to_dma_chan(dchan); | |
972 | ||
973 | /* Get prepared for the loop */ | |
974 | dst_avail = sg_dma_len(dst_sg); | |
975 | src_avail = sg_dma_len(src_sg); | |
976 | dst_nents--; | |
977 | src_nents--; | |
978 | ||
979 | /* Run until we are out of scatterlist entries */ | |
980 | while (true) { | |
981 | /* Create the largest transaction possible */ | |
982 | len = min_t(size_t, src_avail, dst_avail); | |
983 | len = min_t(size_t, len, XGENE_DMA_MAX_64B_DESC_BYTE_CNT); | |
984 | if (len == 0) | |
985 | goto fetch; | |
986 | ||
987 | dst = sg_dma_address(dst_sg) + sg_dma_len(dst_sg) - dst_avail; | |
988 | src = sg_dma_address(src_sg) + sg_dma_len(src_sg) - src_avail; | |
989 | ||
990 | /* Allocate the link descriptor from DMA pool */ | |
991 | new = xgene_dma_alloc_descriptor(chan); | |
992 | if (!new) | |
993 | goto fail; | |
994 | ||
995 | /* Prepare DMA descriptor */ | |
996 | xgene_dma_prep_cpy_desc(chan, new, dst, src, len); | |
997 | ||
998 | if (!first) | |
999 | first = new; | |
1000 | ||
1001 | new->tx.cookie = 0; | |
1002 | async_tx_ack(&new->tx); | |
1003 | ||
1004 | /* update metadata */ | |
1005 | dst_avail -= len; | |
1006 | src_avail -= len; | |
1007 | ||
1008 | /* Insert the link descriptor to the LD ring */ | |
1009 | list_add_tail(&new->node, &first->tx_list); | |
1010 | ||
1011 | fetch: | |
1012 | /* fetch the next dst scatterlist entry */ | |
1013 | if (dst_avail == 0) { | |
1014 | /* no more entries: we're done */ | |
1015 | if (dst_nents == 0) | |
1016 | break; | |
1017 | ||
1018 | /* fetch the next entry: if there are no more: done */ | |
1019 | dst_sg = sg_next(dst_sg); | |
1020 | if (!dst_sg) | |
1021 | break; | |
1022 | ||
1023 | dst_nents--; | |
1024 | dst_avail = sg_dma_len(dst_sg); | |
1025 | } | |
1026 | ||
1027 | /* fetch the next src scatterlist entry */ | |
1028 | if (src_avail == 0) { | |
1029 | /* no more entries: we're done */ | |
1030 | if (src_nents == 0) | |
1031 | break; | |
1032 | ||
1033 | /* fetch the next entry: if there are no more: done */ | |
1034 | src_sg = sg_next(src_sg); | |
1035 | if (!src_sg) | |
1036 | break; | |
1037 | ||
1038 | src_nents--; | |
1039 | src_avail = sg_dma_len(src_sg); | |
1040 | } | |
1041 | } | |
1042 | ||
1043 | if (!new) | |
1044 | return NULL; | |
1045 | ||
1046 | new->tx.flags = flags; /* client is in control of this ack */ | |
1047 | new->tx.cookie = -EBUSY; | |
1048 | list_splice(&first->tx_list, &new->tx_list); | |
1049 | ||
1050 | return &new->tx; | |
1051 | fail: | |
1052 | if (!first) | |
1053 | return NULL; | |
1054 | ||
6d0767c1 | 1055 | xgene_dma_free_desc_list(chan, &first->tx_list); |
9f2fd0df RPS |
1056 | return NULL; |
1057 | } | |
1058 | ||
1059 | static struct dma_async_tx_descriptor *xgene_dma_prep_xor( | |
1060 | struct dma_chan *dchan, dma_addr_t dst, dma_addr_t *src, | |
1061 | u32 src_cnt, size_t len, unsigned long flags) | |
1062 | { | |
1063 | struct xgene_dma_desc_sw *first = NULL, *new; | |
1064 | struct xgene_dma_chan *chan; | |
1065 | static u8 multi[XGENE_DMA_MAX_XOR_SRC] = { | |
1066 | 0x01, 0x01, 0x01, 0x01, 0x01}; | |
1067 | ||
1068 | if (unlikely(!dchan || !len)) | |
1069 | return NULL; | |
1070 | ||
1071 | chan = to_dma_chan(dchan); | |
1072 | ||
1073 | do { | |
1074 | /* Allocate the link descriptor from DMA pool */ | |
1075 | new = xgene_dma_alloc_descriptor(chan); | |
1076 | if (!new) | |
1077 | goto fail; | |
1078 | ||
1079 | /* Prepare xor DMA descriptor */ | |
1080 | xgene_dma_prep_xor_desc(chan, new, &dst, src, | |
1081 | src_cnt, &len, multi); | |
1082 | ||
1083 | if (!first) | |
1084 | first = new; | |
1085 | ||
1086 | new->tx.cookie = 0; | |
1087 | async_tx_ack(&new->tx); | |
1088 | ||
1089 | /* Insert the link descriptor to the LD ring */ | |
1090 | list_add_tail(&new->node, &first->tx_list); | |
1091 | } while (len); | |
1092 | ||
1093 | new->tx.flags = flags; /* client is in control of this ack */ | |
1094 | new->tx.cookie = -EBUSY; | |
1095 | list_splice(&first->tx_list, &new->tx_list); | |
1096 | ||
1097 | return &new->tx; | |
1098 | ||
1099 | fail: | |
1100 | if (!first) | |
1101 | return NULL; | |
1102 | ||
6d0767c1 | 1103 | xgene_dma_free_desc_list(chan, &first->tx_list); |
9f2fd0df RPS |
1104 | return NULL; |
1105 | } | |
1106 | ||
1107 | static struct dma_async_tx_descriptor *xgene_dma_prep_pq( | |
1108 | struct dma_chan *dchan, dma_addr_t *dst, dma_addr_t *src, | |
1109 | u32 src_cnt, const u8 *scf, size_t len, unsigned long flags) | |
1110 | { | |
1111 | struct xgene_dma_desc_sw *first = NULL, *new; | |
1112 | struct xgene_dma_chan *chan; | |
1113 | size_t _len = len; | |
1114 | dma_addr_t _src[XGENE_DMA_MAX_XOR_SRC]; | |
1115 | static u8 multi[XGENE_DMA_MAX_XOR_SRC] = {0x01, 0x01, 0x01, 0x01, 0x01}; | |
1116 | ||
1117 | if (unlikely(!dchan || !len)) | |
1118 | return NULL; | |
1119 | ||
1120 | chan = to_dma_chan(dchan); | |
1121 | ||
1122 | /* | |
1123 | * Save source addresses on local variable, may be we have to | |
1124 | * prepare two descriptor to generate P and Q if both enabled | |
1125 | * in the flags by client | |
1126 | */ | |
1127 | memcpy(_src, src, sizeof(*src) * src_cnt); | |
1128 | ||
1129 | if (flags & DMA_PREP_PQ_DISABLE_P) | |
1130 | len = 0; | |
1131 | ||
1132 | if (flags & DMA_PREP_PQ_DISABLE_Q) | |
1133 | _len = 0; | |
1134 | ||
1135 | do { | |
1136 | /* Allocate the link descriptor from DMA pool */ | |
1137 | new = xgene_dma_alloc_descriptor(chan); | |
1138 | if (!new) | |
1139 | goto fail; | |
1140 | ||
1141 | if (!first) | |
1142 | first = new; | |
1143 | ||
1144 | new->tx.cookie = 0; | |
1145 | async_tx_ack(&new->tx); | |
1146 | ||
1147 | /* Insert the link descriptor to the LD ring */ | |
1148 | list_add_tail(&new->node, &first->tx_list); | |
1149 | ||
1150 | /* | |
1151 | * Prepare DMA descriptor to generate P, | |
1152 | * if DMA_PREP_PQ_DISABLE_P flag is not set | |
1153 | */ | |
1154 | if (len) { | |
1155 | xgene_dma_prep_xor_desc(chan, new, &dst[0], src, | |
1156 | src_cnt, &len, multi); | |
1157 | continue; | |
1158 | } | |
1159 | ||
1160 | /* | |
1161 | * Prepare DMA descriptor to generate Q, | |
1162 | * if DMA_PREP_PQ_DISABLE_Q flag is not set | |
1163 | */ | |
1164 | if (_len) { | |
1165 | xgene_dma_prep_xor_desc(chan, new, &dst[1], _src, | |
1166 | src_cnt, &_len, scf); | |
1167 | } | |
1168 | } while (len || _len); | |
1169 | ||
1170 | new->tx.flags = flags; /* client is in control of this ack */ | |
1171 | new->tx.cookie = -EBUSY; | |
1172 | list_splice(&first->tx_list, &new->tx_list); | |
1173 | ||
1174 | return &new->tx; | |
1175 | ||
1176 | fail: | |
1177 | if (!first) | |
1178 | return NULL; | |
1179 | ||
6d0767c1 | 1180 | xgene_dma_free_desc_list(chan, &first->tx_list); |
9f2fd0df RPS |
1181 | return NULL; |
1182 | } | |
1183 | ||
1184 | static void xgene_dma_issue_pending(struct dma_chan *dchan) | |
1185 | { | |
1186 | struct xgene_dma_chan *chan = to_dma_chan(dchan); | |
1187 | ||
1188 | spin_lock_bh(&chan->lock); | |
1189 | xgene_chan_xfer_ld_pending(chan); | |
1190 | spin_unlock_bh(&chan->lock); | |
1191 | } | |
1192 | ||
1193 | static enum dma_status xgene_dma_tx_status(struct dma_chan *dchan, | |
1194 | dma_cookie_t cookie, | |
1195 | struct dma_tx_state *txstate) | |
1196 | { | |
1197 | return dma_cookie_status(dchan, cookie, txstate); | |
1198 | } | |
1199 | ||
1200 | static void xgene_dma_tasklet_cb(unsigned long data) | |
1201 | { | |
1202 | struct xgene_dma_chan *chan = (struct xgene_dma_chan *)data; | |
1203 | ||
9f2fd0df RPS |
1204 | /* Run all cleanup for descriptors which have been completed */ |
1205 | xgene_dma_cleanup_descriptors(chan); | |
1206 | ||
1207 | /* Re-enable DMA channel IRQ */ | |
1208 | enable_irq(chan->rx_irq); | |
9f2fd0df RPS |
1209 | } |
1210 | ||
1211 | static irqreturn_t xgene_dma_chan_ring_isr(int irq, void *id) | |
1212 | { | |
1213 | struct xgene_dma_chan *chan = (struct xgene_dma_chan *)id; | |
1214 | ||
1215 | BUG_ON(!chan); | |
1216 | ||
1217 | /* | |
1218 | * Disable DMA channel IRQ until we process completed | |
1219 | * descriptors | |
1220 | */ | |
1221 | disable_irq_nosync(chan->rx_irq); | |
1222 | ||
1223 | /* | |
1224 | * Schedule the tasklet to handle all cleanup of the current | |
1225 | * transaction. It will start a new transaction if there is | |
1226 | * one pending. | |
1227 | */ | |
1228 | tasklet_schedule(&chan->tasklet); | |
1229 | ||
1230 | return IRQ_HANDLED; | |
1231 | } | |
1232 | ||
1233 | static irqreturn_t xgene_dma_err_isr(int irq, void *id) | |
1234 | { | |
1235 | struct xgene_dma *pdma = (struct xgene_dma *)id; | |
1236 | unsigned long int_mask; | |
1237 | u32 val, i; | |
1238 | ||
1239 | val = ioread32(pdma->csr_dma + XGENE_DMA_INT); | |
1240 | ||
1241 | /* Clear DMA interrupts */ | |
1242 | iowrite32(val, pdma->csr_dma + XGENE_DMA_INT); | |
1243 | ||
1244 | /* Print DMA error info */ | |
1245 | int_mask = val >> XGENE_DMA_INT_MASK_SHIFT; | |
1246 | for_each_set_bit(i, &int_mask, ARRAY_SIZE(xgene_dma_err)) | |
1247 | dev_err(pdma->dev, | |
1248 | "Interrupt status 0x%08X %s\n", val, xgene_dma_err[i]); | |
1249 | ||
1250 | return IRQ_HANDLED; | |
1251 | } | |
1252 | ||
1253 | static void xgene_dma_wr_ring_state(struct xgene_dma_ring *ring) | |
1254 | { | |
1255 | int i; | |
1256 | ||
1257 | iowrite32(ring->num, ring->pdma->csr_ring + XGENE_DMA_RING_STATE); | |
1258 | ||
1259 | for (i = 0; i < XGENE_DMA_RING_NUM_CONFIG; i++) | |
1260 | iowrite32(ring->state[i], ring->pdma->csr_ring + | |
1261 | XGENE_DMA_RING_STATE_WR_BASE + (i * 4)); | |
1262 | } | |
1263 | ||
1264 | static void xgene_dma_clr_ring_state(struct xgene_dma_ring *ring) | |
1265 | { | |
1266 | memset(ring->state, 0, sizeof(u32) * XGENE_DMA_RING_NUM_CONFIG); | |
1267 | xgene_dma_wr_ring_state(ring); | |
1268 | } | |
1269 | ||
1270 | static void xgene_dma_setup_ring(struct xgene_dma_ring *ring) | |
1271 | { | |
1272 | void *ring_cfg = ring->state; | |
1273 | u64 addr = ring->desc_paddr; | |
9f2fd0df RPS |
1274 | u32 i, val; |
1275 | ||
1276 | ring->slots = ring->size / XGENE_DMA_RING_WQ_DESC_SIZE; | |
1277 | ||
1278 | /* Clear DMA ring state */ | |
1279 | xgene_dma_clr_ring_state(ring); | |
1280 | ||
1281 | /* Set DMA ring type */ | |
1282 | XGENE_DMA_RING_TYPE_SET(ring_cfg, XGENE_DMA_RING_TYPE_REGULAR); | |
1283 | ||
1284 | if (ring->owner == XGENE_DMA_RING_OWNER_DMA) { | |
1285 | /* Set recombination buffer and timeout */ | |
1286 | XGENE_DMA_RING_RECOMBBUF_SET(ring_cfg); | |
1287 | XGENE_DMA_RING_RECOMTIMEOUTL_SET(ring_cfg); | |
1288 | XGENE_DMA_RING_RECOMTIMEOUTH_SET(ring_cfg); | |
1289 | } | |
1290 | ||
1291 | /* Initialize DMA ring state */ | |
1292 | XGENE_DMA_RING_SELTHRSH_SET(ring_cfg); | |
1293 | XGENE_DMA_RING_ACCEPTLERR_SET(ring_cfg); | |
1294 | XGENE_DMA_RING_COHERENT_SET(ring_cfg); | |
1295 | XGENE_DMA_RING_ADDRL_SET(ring_cfg, addr); | |
1296 | XGENE_DMA_RING_ADDRH_SET(ring_cfg, addr); | |
1297 | XGENE_DMA_RING_SIZE_SET(ring_cfg, ring->cfgsize); | |
1298 | ||
1299 | /* Write DMA ring configurations */ | |
1300 | xgene_dma_wr_ring_state(ring); | |
1301 | ||
1302 | /* Set DMA ring id */ | |
1303 | iowrite32(XGENE_DMA_RING_ID_SETUP(ring->id), | |
1304 | ring->pdma->csr_ring + XGENE_DMA_RING_ID); | |
1305 | ||
1306 | /* Set DMA ring buffer */ | |
1307 | iowrite32(XGENE_DMA_RING_ID_BUF_SETUP(ring->num), | |
1308 | ring->pdma->csr_ring + XGENE_DMA_RING_ID_BUF); | |
1309 | ||
1310 | if (ring->owner != XGENE_DMA_RING_OWNER_CPU) | |
1311 | return; | |
1312 | ||
1313 | /* Set empty signature to DMA Rx ring descriptors */ | |
1314 | for (i = 0; i < ring->slots; i++) { | |
6d0767c1 RPS |
1315 | struct xgene_dma_desc_hw *desc; |
1316 | ||
9f2fd0df | 1317 | desc = &ring->desc_hw[i]; |
6d0767c1 | 1318 | desc->m0 = cpu_to_le64(XGENE_DMA_DESC_EMPTY_SIGNATURE); |
9f2fd0df RPS |
1319 | } |
1320 | ||
1321 | /* Enable DMA Rx ring interrupt */ | |
1322 | val = ioread32(ring->pdma->csr_ring + XGENE_DMA_RING_NE_INT_MODE); | |
1323 | XGENE_DMA_RING_NE_INT_MODE_SET(val, ring->buf_num); | |
1324 | iowrite32(val, ring->pdma->csr_ring + XGENE_DMA_RING_NE_INT_MODE); | |
1325 | } | |
1326 | ||
1327 | static void xgene_dma_clear_ring(struct xgene_dma_ring *ring) | |
1328 | { | |
1329 | u32 ring_id, val; | |
1330 | ||
1331 | if (ring->owner == XGENE_DMA_RING_OWNER_CPU) { | |
1332 | /* Disable DMA Rx ring interrupt */ | |
1333 | val = ioread32(ring->pdma->csr_ring + | |
1334 | XGENE_DMA_RING_NE_INT_MODE); | |
1335 | XGENE_DMA_RING_NE_INT_MODE_RESET(val, ring->buf_num); | |
1336 | iowrite32(val, ring->pdma->csr_ring + | |
1337 | XGENE_DMA_RING_NE_INT_MODE); | |
1338 | } | |
1339 | ||
1340 | /* Clear DMA ring state */ | |
1341 | ring_id = XGENE_DMA_RING_ID_SETUP(ring->id); | |
1342 | iowrite32(ring_id, ring->pdma->csr_ring + XGENE_DMA_RING_ID); | |
1343 | ||
1344 | iowrite32(0, ring->pdma->csr_ring + XGENE_DMA_RING_ID_BUF); | |
1345 | xgene_dma_clr_ring_state(ring); | |
1346 | } | |
1347 | ||
1348 | static void xgene_dma_set_ring_cmd(struct xgene_dma_ring *ring) | |
1349 | { | |
1350 | ring->cmd_base = ring->pdma->csr_ring_cmd + | |
1351 | XGENE_DMA_RING_CMD_BASE_OFFSET((ring->num - | |
1352 | XGENE_DMA_RING_NUM)); | |
1353 | ||
1354 | ring->cmd = ring->cmd_base + XGENE_DMA_RING_CMD_OFFSET; | |
1355 | } | |
1356 | ||
1357 | static int xgene_dma_get_ring_size(struct xgene_dma_chan *chan, | |
1358 | enum xgene_dma_ring_cfgsize cfgsize) | |
1359 | { | |
1360 | int size; | |
1361 | ||
1362 | switch (cfgsize) { | |
1363 | case XGENE_DMA_RING_CFG_SIZE_512B: | |
1364 | size = 0x200; | |
1365 | break; | |
1366 | case XGENE_DMA_RING_CFG_SIZE_2KB: | |
1367 | size = 0x800; | |
1368 | break; | |
1369 | case XGENE_DMA_RING_CFG_SIZE_16KB: | |
1370 | size = 0x4000; | |
1371 | break; | |
1372 | case XGENE_DMA_RING_CFG_SIZE_64KB: | |
1373 | size = 0x10000; | |
1374 | break; | |
1375 | case XGENE_DMA_RING_CFG_SIZE_512KB: | |
1376 | size = 0x80000; | |
1377 | break; | |
1378 | default: | |
1379 | chan_err(chan, "Unsupported cfg ring size %d\n", cfgsize); | |
1380 | return -EINVAL; | |
1381 | } | |
1382 | ||
1383 | return size; | |
1384 | } | |
1385 | ||
1386 | static void xgene_dma_delete_ring_one(struct xgene_dma_ring *ring) | |
1387 | { | |
1388 | /* Clear DMA ring configurations */ | |
1389 | xgene_dma_clear_ring(ring); | |
1390 | ||
1391 | /* De-allocate DMA ring descriptor */ | |
1392 | if (ring->desc_vaddr) { | |
1393 | dma_free_coherent(ring->pdma->dev, ring->size, | |
1394 | ring->desc_vaddr, ring->desc_paddr); | |
1395 | ring->desc_vaddr = NULL; | |
1396 | } | |
1397 | } | |
1398 | ||
1399 | static void xgene_dma_delete_chan_rings(struct xgene_dma_chan *chan) | |
1400 | { | |
1401 | xgene_dma_delete_ring_one(&chan->rx_ring); | |
1402 | xgene_dma_delete_ring_one(&chan->tx_ring); | |
1403 | } | |
1404 | ||
1405 | static int xgene_dma_create_ring_one(struct xgene_dma_chan *chan, | |
1406 | struct xgene_dma_ring *ring, | |
1407 | enum xgene_dma_ring_cfgsize cfgsize) | |
1408 | { | |
c1492b4c AH |
1409 | int ret; |
1410 | ||
9f2fd0df RPS |
1411 | /* Setup DMA ring descriptor variables */ |
1412 | ring->pdma = chan->pdma; | |
1413 | ring->cfgsize = cfgsize; | |
1414 | ring->num = chan->pdma->ring_num++; | |
1415 | ring->id = XGENE_DMA_RING_ID_GET(ring->owner, ring->buf_num); | |
1416 | ||
c1492b4c AH |
1417 | ret = xgene_dma_get_ring_size(chan, cfgsize); |
1418 | if (ret <= 0) | |
1419 | return ret; | |
1420 | ring->size = ret; | |
9f2fd0df RPS |
1421 | |
1422 | /* Allocate memory for DMA ring descriptor */ | |
1423 | ring->desc_vaddr = dma_zalloc_coherent(chan->dev, ring->size, | |
1424 | &ring->desc_paddr, GFP_KERNEL); | |
1425 | if (!ring->desc_vaddr) { | |
1426 | chan_err(chan, "Failed to allocate ring desc\n"); | |
1427 | return -ENOMEM; | |
1428 | } | |
1429 | ||
1430 | /* Configure and enable DMA ring */ | |
1431 | xgene_dma_set_ring_cmd(ring); | |
1432 | xgene_dma_setup_ring(ring); | |
1433 | ||
1434 | return 0; | |
1435 | } | |
1436 | ||
1437 | static int xgene_dma_create_chan_rings(struct xgene_dma_chan *chan) | |
1438 | { | |
1439 | struct xgene_dma_ring *rx_ring = &chan->rx_ring; | |
1440 | struct xgene_dma_ring *tx_ring = &chan->tx_ring; | |
1441 | int ret; | |
1442 | ||
1443 | /* Create DMA Rx ring descriptor */ | |
1444 | rx_ring->owner = XGENE_DMA_RING_OWNER_CPU; | |
1445 | rx_ring->buf_num = XGENE_DMA_CPU_BUFNUM + chan->id; | |
1446 | ||
1447 | ret = xgene_dma_create_ring_one(chan, rx_ring, | |
1448 | XGENE_DMA_RING_CFG_SIZE_64KB); | |
1449 | if (ret) | |
1450 | return ret; | |
1451 | ||
1452 | chan_dbg(chan, "Rx ring id 0x%X num %d desc 0x%p\n", | |
1453 | rx_ring->id, rx_ring->num, rx_ring->desc_vaddr); | |
1454 | ||
1455 | /* Create DMA Tx ring descriptor */ | |
1456 | tx_ring->owner = XGENE_DMA_RING_OWNER_DMA; | |
1457 | tx_ring->buf_num = XGENE_DMA_BUFNUM + chan->id; | |
1458 | ||
1459 | ret = xgene_dma_create_ring_one(chan, tx_ring, | |
1460 | XGENE_DMA_RING_CFG_SIZE_64KB); | |
1461 | if (ret) { | |
1462 | xgene_dma_delete_ring_one(rx_ring); | |
1463 | return ret; | |
1464 | } | |
1465 | ||
1466 | tx_ring->dst_ring_num = XGENE_DMA_RING_DST_ID(rx_ring->num); | |
1467 | ||
1468 | chan_dbg(chan, | |
1469 | "Tx ring id 0x%X num %d desc 0x%p\n", | |
1470 | tx_ring->id, tx_ring->num, tx_ring->desc_vaddr); | |
1471 | ||
1472 | /* Set the max outstanding request possible to this channel */ | |
ee08b59d | 1473 | chan->max_outstanding = tx_ring->slots; |
9f2fd0df RPS |
1474 | |
1475 | return ret; | |
1476 | } | |
1477 | ||
1478 | static int xgene_dma_init_rings(struct xgene_dma *pdma) | |
1479 | { | |
1480 | int ret, i, j; | |
1481 | ||
1482 | for (i = 0; i < XGENE_DMA_MAX_CHANNEL; i++) { | |
1483 | ret = xgene_dma_create_chan_rings(&pdma->chan[i]); | |
1484 | if (ret) { | |
1485 | for (j = 0; j < i; j++) | |
1486 | xgene_dma_delete_chan_rings(&pdma->chan[j]); | |
1487 | return ret; | |
1488 | } | |
1489 | } | |
1490 | ||
1491 | return ret; | |
1492 | } | |
1493 | ||
1494 | static void xgene_dma_enable(struct xgene_dma *pdma) | |
1495 | { | |
1496 | u32 val; | |
1497 | ||
1498 | /* Configure and enable DMA engine */ | |
1499 | val = ioread32(pdma->csr_dma + XGENE_DMA_GCR); | |
1500 | XGENE_DMA_CH_SETUP(val); | |
1501 | XGENE_DMA_ENABLE(val); | |
1502 | iowrite32(val, pdma->csr_dma + XGENE_DMA_GCR); | |
1503 | } | |
1504 | ||
1505 | static void xgene_dma_disable(struct xgene_dma *pdma) | |
1506 | { | |
1507 | u32 val; | |
1508 | ||
1509 | val = ioread32(pdma->csr_dma + XGENE_DMA_GCR); | |
1510 | XGENE_DMA_DISABLE(val); | |
1511 | iowrite32(val, pdma->csr_dma + XGENE_DMA_GCR); | |
1512 | } | |
1513 | ||
1514 | static void xgene_dma_mask_interrupts(struct xgene_dma *pdma) | |
1515 | { | |
1516 | /* | |
1517 | * Mask DMA ring overflow, underflow and | |
1518 | * AXI write/read error interrupts | |
1519 | */ | |
1520 | iowrite32(XGENE_DMA_INT_ALL_MASK, | |
1521 | pdma->csr_dma + XGENE_DMA_RING_INT0_MASK); | |
1522 | iowrite32(XGENE_DMA_INT_ALL_MASK, | |
1523 | pdma->csr_dma + XGENE_DMA_RING_INT1_MASK); | |
1524 | iowrite32(XGENE_DMA_INT_ALL_MASK, | |
1525 | pdma->csr_dma + XGENE_DMA_RING_INT2_MASK); | |
1526 | iowrite32(XGENE_DMA_INT_ALL_MASK, | |
1527 | pdma->csr_dma + XGENE_DMA_RING_INT3_MASK); | |
1528 | iowrite32(XGENE_DMA_INT_ALL_MASK, | |
1529 | pdma->csr_dma + XGENE_DMA_RING_INT4_MASK); | |
1530 | ||
1531 | /* Mask DMA error interrupts */ | |
1532 | iowrite32(XGENE_DMA_INT_ALL_MASK, pdma->csr_dma + XGENE_DMA_INT_MASK); | |
1533 | } | |
1534 | ||
1535 | static void xgene_dma_unmask_interrupts(struct xgene_dma *pdma) | |
1536 | { | |
1537 | /* | |
1538 | * Unmask DMA ring overflow, underflow and | |
1539 | * AXI write/read error interrupts | |
1540 | */ | |
1541 | iowrite32(XGENE_DMA_INT_ALL_UNMASK, | |
1542 | pdma->csr_dma + XGENE_DMA_RING_INT0_MASK); | |
1543 | iowrite32(XGENE_DMA_INT_ALL_UNMASK, | |
1544 | pdma->csr_dma + XGENE_DMA_RING_INT1_MASK); | |
1545 | iowrite32(XGENE_DMA_INT_ALL_UNMASK, | |
1546 | pdma->csr_dma + XGENE_DMA_RING_INT2_MASK); | |
1547 | iowrite32(XGENE_DMA_INT_ALL_UNMASK, | |
1548 | pdma->csr_dma + XGENE_DMA_RING_INT3_MASK); | |
1549 | iowrite32(XGENE_DMA_INT_ALL_UNMASK, | |
1550 | pdma->csr_dma + XGENE_DMA_RING_INT4_MASK); | |
1551 | ||
1552 | /* Unmask DMA error interrupts */ | |
1553 | iowrite32(XGENE_DMA_INT_ALL_UNMASK, | |
1554 | pdma->csr_dma + XGENE_DMA_INT_MASK); | |
1555 | } | |
1556 | ||
1557 | static void xgene_dma_init_hw(struct xgene_dma *pdma) | |
1558 | { | |
1559 | u32 val; | |
1560 | ||
1561 | /* Associate DMA ring to corresponding ring HW */ | |
1562 | iowrite32(XGENE_DMA_ASSOC_RING_MNGR1, | |
1563 | pdma->csr_dma + XGENE_DMA_CFG_RING_WQ_ASSOC); | |
1564 | ||
1565 | /* Configure RAID6 polynomial control setting */ | |
1566 | if (is_pq_enabled(pdma)) | |
1567 | iowrite32(XGENE_DMA_RAID6_MULTI_CTRL(0x1D), | |
1568 | pdma->csr_dma + XGENE_DMA_RAID6_CONT); | |
1569 | else | |
1570 | dev_info(pdma->dev, "PQ is disabled in HW\n"); | |
1571 | ||
1572 | xgene_dma_enable(pdma); | |
1573 | xgene_dma_unmask_interrupts(pdma); | |
1574 | ||
1575 | /* Get DMA id and version info */ | |
1576 | val = ioread32(pdma->csr_dma + XGENE_DMA_IPBRR); | |
1577 | ||
1578 | /* DMA device info */ | |
1579 | dev_info(pdma->dev, | |
1580 | "X-Gene DMA v%d.%02d.%02d driver registered %d channels", | |
1581 | XGENE_DMA_REV_NO_RD(val), XGENE_DMA_BUS_ID_RD(val), | |
1582 | XGENE_DMA_DEV_ID_RD(val), XGENE_DMA_MAX_CHANNEL); | |
1583 | } | |
1584 | ||
a3f92e8e | 1585 | static int xgene_dma_init_ring_mngr(struct xgene_dma *pdma) |
9f2fd0df RPS |
1586 | { |
1587 | if (ioread32(pdma->csr_ring + XGENE_DMA_RING_CLKEN) && | |
1588 | (!ioread32(pdma->csr_ring + XGENE_DMA_RING_SRST))) | |
1589 | return 0; | |
1590 | ||
1591 | iowrite32(0x3, pdma->csr_ring + XGENE_DMA_RING_CLKEN); | |
1592 | iowrite32(0x0, pdma->csr_ring + XGENE_DMA_RING_SRST); | |
1593 | ||
1594 | /* Bring up memory */ | |
1595 | iowrite32(0x0, pdma->csr_ring + XGENE_DMA_RING_MEM_RAM_SHUTDOWN); | |
1596 | ||
1597 | /* Force a barrier */ | |
1598 | ioread32(pdma->csr_ring + XGENE_DMA_RING_MEM_RAM_SHUTDOWN); | |
1599 | ||
1600 | /* reset may take up to 1ms */ | |
1601 | usleep_range(1000, 1100); | |
1602 | ||
1603 | if (ioread32(pdma->csr_ring + XGENE_DMA_RING_BLK_MEM_RDY) | |
1604 | != XGENE_DMA_RING_BLK_MEM_RDY_VAL) { | |
1605 | dev_err(pdma->dev, | |
1606 | "Failed to release ring mngr memory from shutdown\n"); | |
1607 | return -ENODEV; | |
1608 | } | |
1609 | ||
1610 | /* program threshold set 1 and all hysteresis */ | |
1611 | iowrite32(XGENE_DMA_RING_THRESLD0_SET1_VAL, | |
1612 | pdma->csr_ring + XGENE_DMA_RING_THRESLD0_SET1); | |
1613 | iowrite32(XGENE_DMA_RING_THRESLD1_SET1_VAL, | |
1614 | pdma->csr_ring + XGENE_DMA_RING_THRESLD1_SET1); | |
1615 | iowrite32(XGENE_DMA_RING_HYSTERESIS_VAL, | |
1616 | pdma->csr_ring + XGENE_DMA_RING_HYSTERESIS); | |
1617 | ||
1618 | /* Enable QPcore and assign error queue */ | |
1619 | iowrite32(XGENE_DMA_RING_ENABLE, | |
1620 | pdma->csr_ring + XGENE_DMA_RING_CONFIG); | |
1621 | ||
1622 | return 0; | |
1623 | } | |
1624 | ||
1625 | static int xgene_dma_init_mem(struct xgene_dma *pdma) | |
1626 | { | |
1627 | int ret; | |
1628 | ||
1629 | ret = xgene_dma_init_ring_mngr(pdma); | |
1630 | if (ret) | |
1631 | return ret; | |
1632 | ||
1633 | /* Bring up memory */ | |
1634 | iowrite32(0x0, pdma->csr_dma + XGENE_DMA_MEM_RAM_SHUTDOWN); | |
1635 | ||
1636 | /* Force a barrier */ | |
1637 | ioread32(pdma->csr_dma + XGENE_DMA_MEM_RAM_SHUTDOWN); | |
1638 | ||
1639 | /* reset may take up to 1ms */ | |
1640 | usleep_range(1000, 1100); | |
1641 | ||
1642 | if (ioread32(pdma->csr_dma + XGENE_DMA_BLK_MEM_RDY) | |
1643 | != XGENE_DMA_BLK_MEM_RDY_VAL) { | |
1644 | dev_err(pdma->dev, | |
1645 | "Failed to release DMA memory from shutdown\n"); | |
1646 | return -ENODEV; | |
1647 | } | |
1648 | ||
1649 | return 0; | |
1650 | } | |
1651 | ||
1652 | static int xgene_dma_request_irqs(struct xgene_dma *pdma) | |
1653 | { | |
1654 | struct xgene_dma_chan *chan; | |
1655 | int ret, i, j; | |
1656 | ||
1657 | /* Register DMA error irq */ | |
1658 | ret = devm_request_irq(pdma->dev, pdma->err_irq, xgene_dma_err_isr, | |
1659 | 0, "dma_error", pdma); | |
1660 | if (ret) { | |
1661 | dev_err(pdma->dev, | |
1662 | "Failed to register error IRQ %d\n", pdma->err_irq); | |
1663 | return ret; | |
1664 | } | |
1665 | ||
1666 | /* Register DMA channel rx irq */ | |
1667 | for (i = 0; i < XGENE_DMA_MAX_CHANNEL; i++) { | |
1668 | chan = &pdma->chan[i]; | |
1669 | ret = devm_request_irq(chan->dev, chan->rx_irq, | |
1670 | xgene_dma_chan_ring_isr, | |
1671 | 0, chan->name, chan); | |
1672 | if (ret) { | |
1673 | chan_err(chan, "Failed to register Rx IRQ %d\n", | |
1674 | chan->rx_irq); | |
1675 | devm_free_irq(pdma->dev, pdma->err_irq, pdma); | |
1676 | ||
1677 | for (j = 0; j < i; j++) { | |
1678 | chan = &pdma->chan[i]; | |
1679 | devm_free_irq(chan->dev, chan->rx_irq, chan); | |
1680 | } | |
1681 | ||
1682 | return ret; | |
1683 | } | |
1684 | } | |
1685 | ||
1686 | return 0; | |
1687 | } | |
1688 | ||
1689 | static void xgene_dma_free_irqs(struct xgene_dma *pdma) | |
1690 | { | |
1691 | struct xgene_dma_chan *chan; | |
1692 | int i; | |
1693 | ||
1694 | /* Free DMA device error irq */ | |
1695 | devm_free_irq(pdma->dev, pdma->err_irq, pdma); | |
1696 | ||
1697 | for (i = 0; i < XGENE_DMA_MAX_CHANNEL; i++) { | |
1698 | chan = &pdma->chan[i]; | |
1699 | devm_free_irq(chan->dev, chan->rx_irq, chan); | |
1700 | } | |
1701 | } | |
1702 | ||
1703 | static void xgene_dma_set_caps(struct xgene_dma_chan *chan, | |
1704 | struct dma_device *dma_dev) | |
1705 | { | |
1706 | /* Initialize DMA device capability mask */ | |
1707 | dma_cap_zero(dma_dev->cap_mask); | |
1708 | ||
1709 | /* Set DMA device capability */ | |
1710 | dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); | |
1711 | dma_cap_set(DMA_SG, dma_dev->cap_mask); | |
1712 | ||
1713 | /* Basically here, the X-Gene SoC DMA engine channel 0 supports XOR | |
1714 | * and channel 1 supports XOR, PQ both. First thing here is we have | |
1715 | * mechanism in hw to enable/disable PQ/XOR supports on channel 1, | |
1716 | * we can make sure this by reading SoC Efuse register. | |
1717 | * Second thing, we have hw errata that if we run channel 0 and | |
1718 | * channel 1 simultaneously with executing XOR and PQ request, | |
1719 | * suddenly DMA engine hangs, So here we enable XOR on channel 0 only | |
1720 | * if XOR and PQ supports on channel 1 is disabled. | |
1721 | */ | |
1722 | if ((chan->id == XGENE_DMA_PQ_CHANNEL) && | |
1723 | is_pq_enabled(chan->pdma)) { | |
1724 | dma_cap_set(DMA_PQ, dma_dev->cap_mask); | |
1725 | dma_cap_set(DMA_XOR, dma_dev->cap_mask); | |
1726 | } else if ((chan->id == XGENE_DMA_XOR_CHANNEL) && | |
1727 | !is_pq_enabled(chan->pdma)) { | |
1728 | dma_cap_set(DMA_XOR, dma_dev->cap_mask); | |
1729 | } | |
1730 | ||
1731 | /* Set base and prep routines */ | |
1732 | dma_dev->dev = chan->dev; | |
1733 | dma_dev->device_alloc_chan_resources = xgene_dma_alloc_chan_resources; | |
1734 | dma_dev->device_free_chan_resources = xgene_dma_free_chan_resources; | |
1735 | dma_dev->device_issue_pending = xgene_dma_issue_pending; | |
1736 | dma_dev->device_tx_status = xgene_dma_tx_status; | |
1737 | dma_dev->device_prep_dma_memcpy = xgene_dma_prep_memcpy; | |
1738 | dma_dev->device_prep_dma_sg = xgene_dma_prep_sg; | |
1739 | ||
1740 | if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { | |
1741 | dma_dev->device_prep_dma_xor = xgene_dma_prep_xor; | |
1742 | dma_dev->max_xor = XGENE_DMA_MAX_XOR_SRC; | |
77a68e56 | 1743 | dma_dev->xor_align = DMAENGINE_ALIGN_64_BYTES; |
9f2fd0df RPS |
1744 | } |
1745 | ||
1746 | if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { | |
1747 | dma_dev->device_prep_dma_pq = xgene_dma_prep_pq; | |
1748 | dma_dev->max_pq = XGENE_DMA_MAX_XOR_SRC; | |
77a68e56 | 1749 | dma_dev->pq_align = DMAENGINE_ALIGN_64_BYTES; |
9f2fd0df RPS |
1750 | } |
1751 | } | |
1752 | ||
1753 | static int xgene_dma_async_register(struct xgene_dma *pdma, int id) | |
1754 | { | |
1755 | struct xgene_dma_chan *chan = &pdma->chan[id]; | |
1756 | struct dma_device *dma_dev = &pdma->dma_dev[id]; | |
1757 | int ret; | |
1758 | ||
1759 | chan->dma_chan.device = dma_dev; | |
1760 | ||
1761 | spin_lock_init(&chan->lock); | |
1762 | INIT_LIST_HEAD(&chan->ld_pending); | |
1763 | INIT_LIST_HEAD(&chan->ld_running); | |
1764 | INIT_LIST_HEAD(&chan->ld_completed); | |
1765 | tasklet_init(&chan->tasklet, xgene_dma_tasklet_cb, | |
1766 | (unsigned long)chan); | |
1767 | ||
1768 | chan->pending = 0; | |
1769 | chan->desc_pool = NULL; | |
1770 | dma_cookie_init(&chan->dma_chan); | |
1771 | ||
1772 | /* Setup dma device capabilities and prep routines */ | |
1773 | xgene_dma_set_caps(chan, dma_dev); | |
1774 | ||
1775 | /* Initialize DMA device list head */ | |
1776 | INIT_LIST_HEAD(&dma_dev->channels); | |
1777 | list_add_tail(&chan->dma_chan.device_node, &dma_dev->channels); | |
1778 | ||
1779 | /* Register with Linux async DMA framework*/ | |
1780 | ret = dma_async_device_register(dma_dev); | |
1781 | if (ret) { | |
1782 | chan_err(chan, "Failed to register async device %d", ret); | |
1783 | tasklet_kill(&chan->tasklet); | |
1784 | ||
1785 | return ret; | |
1786 | } | |
1787 | ||
1788 | /* DMA capability info */ | |
1789 | dev_info(pdma->dev, | |
1790 | "%s: CAPABILITY ( %s%s%s%s)\n", dma_chan_name(&chan->dma_chan), | |
1791 | dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "MEMCPY " : "", | |
1792 | dma_has_cap(DMA_SG, dma_dev->cap_mask) ? "SGCPY " : "", | |
1793 | dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "XOR " : "", | |
1794 | dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "PQ " : ""); | |
1795 | ||
1796 | return 0; | |
1797 | } | |
1798 | ||
1799 | static int xgene_dma_init_async(struct xgene_dma *pdma) | |
1800 | { | |
1801 | int ret, i, j; | |
1802 | ||
1803 | for (i = 0; i < XGENE_DMA_MAX_CHANNEL ; i++) { | |
1804 | ret = xgene_dma_async_register(pdma, i); | |
1805 | if (ret) { | |
1806 | for (j = 0; j < i; j++) { | |
1807 | dma_async_device_unregister(&pdma->dma_dev[j]); | |
1808 | tasklet_kill(&pdma->chan[j].tasklet); | |
1809 | } | |
1810 | ||
1811 | return ret; | |
1812 | } | |
1813 | } | |
1814 | ||
1815 | return ret; | |
1816 | } | |
1817 | ||
1818 | static void xgene_dma_async_unregister(struct xgene_dma *pdma) | |
1819 | { | |
1820 | int i; | |
1821 | ||
1822 | for (i = 0; i < XGENE_DMA_MAX_CHANNEL; i++) | |
1823 | dma_async_device_unregister(&pdma->dma_dev[i]); | |
1824 | } | |
1825 | ||
1826 | static void xgene_dma_init_channels(struct xgene_dma *pdma) | |
1827 | { | |
1828 | struct xgene_dma_chan *chan; | |
1829 | int i; | |
1830 | ||
1831 | pdma->ring_num = XGENE_DMA_RING_NUM; | |
1832 | ||
1833 | for (i = 0; i < XGENE_DMA_MAX_CHANNEL; i++) { | |
1834 | chan = &pdma->chan[i]; | |
1835 | chan->dev = pdma->dev; | |
1836 | chan->pdma = pdma; | |
1837 | chan->id = i; | |
ed1f0418 | 1838 | snprintf(chan->name, sizeof(chan->name), "dmachan%d", chan->id); |
9f2fd0df RPS |
1839 | } |
1840 | } | |
1841 | ||
1842 | static int xgene_dma_get_resources(struct platform_device *pdev, | |
1843 | struct xgene_dma *pdma) | |
1844 | { | |
1845 | struct resource *res; | |
1846 | int irq, i; | |
1847 | ||
1848 | /* Get DMA csr region */ | |
1849 | res = platform_get_resource(pdev, IORESOURCE_MEM, 0); | |
1850 | if (!res) { | |
1851 | dev_err(&pdev->dev, "Failed to get csr region\n"); | |
1852 | return -ENXIO; | |
1853 | } | |
1854 | ||
1855 | pdma->csr_dma = devm_ioremap(&pdev->dev, res->start, | |
1856 | resource_size(res)); | |
9c361b1a | 1857 | if (!pdma->csr_dma) { |
9f2fd0df | 1858 | dev_err(&pdev->dev, "Failed to ioremap csr region"); |
9c361b1a | 1859 | return -ENOMEM; |
9f2fd0df RPS |
1860 | } |
1861 | ||
1862 | /* Get DMA ring csr region */ | |
1863 | res = platform_get_resource(pdev, IORESOURCE_MEM, 1); | |
1864 | if (!res) { | |
1865 | dev_err(&pdev->dev, "Failed to get ring csr region\n"); | |
1866 | return -ENXIO; | |
1867 | } | |
1868 | ||
1869 | pdma->csr_ring = devm_ioremap(&pdev->dev, res->start, | |
1870 | resource_size(res)); | |
9c361b1a | 1871 | if (!pdma->csr_ring) { |
9f2fd0df | 1872 | dev_err(&pdev->dev, "Failed to ioremap ring csr region"); |
9c361b1a | 1873 | return -ENOMEM; |
9f2fd0df RPS |
1874 | } |
1875 | ||
1876 | /* Get DMA ring cmd csr region */ | |
1877 | res = platform_get_resource(pdev, IORESOURCE_MEM, 2); | |
1878 | if (!res) { | |
1879 | dev_err(&pdev->dev, "Failed to get ring cmd csr region\n"); | |
1880 | return -ENXIO; | |
1881 | } | |
1882 | ||
1883 | pdma->csr_ring_cmd = devm_ioremap(&pdev->dev, res->start, | |
1884 | resource_size(res)); | |
9c361b1a | 1885 | if (!pdma->csr_ring_cmd) { |
9f2fd0df | 1886 | dev_err(&pdev->dev, "Failed to ioremap ring cmd csr region"); |
9c361b1a | 1887 | return -ENOMEM; |
9f2fd0df RPS |
1888 | } |
1889 | ||
cda8e937 RPS |
1890 | pdma->csr_ring_cmd += XGENE_DMA_RING_CMD_SM_OFFSET; |
1891 | ||
9f2fd0df RPS |
1892 | /* Get efuse csr region */ |
1893 | res = platform_get_resource(pdev, IORESOURCE_MEM, 3); | |
1894 | if (!res) { | |
1895 | dev_err(&pdev->dev, "Failed to get efuse csr region\n"); | |
1896 | return -ENXIO; | |
1897 | } | |
1898 | ||
1899 | pdma->csr_efuse = devm_ioremap(&pdev->dev, res->start, | |
1900 | resource_size(res)); | |
9c361b1a | 1901 | if (!pdma->csr_efuse) { |
9f2fd0df | 1902 | dev_err(&pdev->dev, "Failed to ioremap efuse csr region"); |
9c361b1a | 1903 | return -ENOMEM; |
9f2fd0df RPS |
1904 | } |
1905 | ||
1906 | /* Get DMA error interrupt */ | |
1907 | irq = platform_get_irq(pdev, 0); | |
1908 | if (irq <= 0) { | |
1909 | dev_err(&pdev->dev, "Failed to get Error IRQ\n"); | |
1910 | return -ENXIO; | |
1911 | } | |
1912 | ||
1913 | pdma->err_irq = irq; | |
1914 | ||
1915 | /* Get DMA Rx ring descriptor interrupts for all DMA channels */ | |
1916 | for (i = 1; i <= XGENE_DMA_MAX_CHANNEL; i++) { | |
1917 | irq = platform_get_irq(pdev, i); | |
1918 | if (irq <= 0) { | |
1919 | dev_err(&pdev->dev, "Failed to get Rx IRQ\n"); | |
1920 | return -ENXIO; | |
1921 | } | |
1922 | ||
1923 | pdma->chan[i - 1].rx_irq = irq; | |
1924 | } | |
1925 | ||
1926 | return 0; | |
1927 | } | |
1928 | ||
1929 | static int xgene_dma_probe(struct platform_device *pdev) | |
1930 | { | |
1931 | struct xgene_dma *pdma; | |
1932 | int ret, i; | |
1933 | ||
1934 | pdma = devm_kzalloc(&pdev->dev, sizeof(*pdma), GFP_KERNEL); | |
1935 | if (!pdma) | |
1936 | return -ENOMEM; | |
1937 | ||
1938 | pdma->dev = &pdev->dev; | |
1939 | platform_set_drvdata(pdev, pdma); | |
1940 | ||
1941 | ret = xgene_dma_get_resources(pdev, pdma); | |
1942 | if (ret) | |
1943 | return ret; | |
1944 | ||
1945 | pdma->clk = devm_clk_get(&pdev->dev, NULL); | |
89079493 | 1946 | if (IS_ERR(pdma->clk) && !ACPI_COMPANION(&pdev->dev)) { |
9f2fd0df RPS |
1947 | dev_err(&pdev->dev, "Failed to get clk\n"); |
1948 | return PTR_ERR(pdma->clk); | |
1949 | } | |
1950 | ||
1951 | /* Enable clk before accessing registers */ | |
89079493 RPS |
1952 | if (!IS_ERR(pdma->clk)) { |
1953 | ret = clk_prepare_enable(pdma->clk); | |
1954 | if (ret) { | |
1955 | dev_err(&pdev->dev, "Failed to enable clk %d\n", ret); | |
1956 | return ret; | |
1957 | } | |
9f2fd0df RPS |
1958 | } |
1959 | ||
1960 | /* Remove DMA RAM out of shutdown */ | |
1961 | ret = xgene_dma_init_mem(pdma); | |
1962 | if (ret) | |
1963 | goto err_clk_enable; | |
1964 | ||
1965 | ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(42)); | |
1966 | if (ret) { | |
1967 | dev_err(&pdev->dev, "No usable DMA configuration\n"); | |
1968 | goto err_dma_mask; | |
1969 | } | |
1970 | ||
1971 | /* Initialize DMA channels software state */ | |
1972 | xgene_dma_init_channels(pdma); | |
1973 | ||
1974 | /* Configue DMA rings */ | |
1975 | ret = xgene_dma_init_rings(pdma); | |
1976 | if (ret) | |
1977 | goto err_clk_enable; | |
1978 | ||
1979 | ret = xgene_dma_request_irqs(pdma); | |
1980 | if (ret) | |
1981 | goto err_request_irq; | |
1982 | ||
1983 | /* Configure and enable DMA engine */ | |
1984 | xgene_dma_init_hw(pdma); | |
1985 | ||
1986 | /* Register DMA device with linux async framework */ | |
1987 | ret = xgene_dma_init_async(pdma); | |
1988 | if (ret) | |
1989 | goto err_async_init; | |
1990 | ||
1991 | return 0; | |
1992 | ||
1993 | err_async_init: | |
1994 | xgene_dma_free_irqs(pdma); | |
1995 | ||
1996 | err_request_irq: | |
1997 | for (i = 0; i < XGENE_DMA_MAX_CHANNEL; i++) | |
1998 | xgene_dma_delete_chan_rings(&pdma->chan[i]); | |
1999 | ||
2000 | err_dma_mask: | |
2001 | err_clk_enable: | |
89079493 RPS |
2002 | if (!IS_ERR(pdma->clk)) |
2003 | clk_disable_unprepare(pdma->clk); | |
9f2fd0df RPS |
2004 | |
2005 | return ret; | |
2006 | } | |
2007 | ||
2008 | static int xgene_dma_remove(struct platform_device *pdev) | |
2009 | { | |
2010 | struct xgene_dma *pdma = platform_get_drvdata(pdev); | |
2011 | struct xgene_dma_chan *chan; | |
2012 | int i; | |
2013 | ||
2014 | xgene_dma_async_unregister(pdma); | |
2015 | ||
2016 | /* Mask interrupts and disable DMA engine */ | |
2017 | xgene_dma_mask_interrupts(pdma); | |
2018 | xgene_dma_disable(pdma); | |
2019 | xgene_dma_free_irqs(pdma); | |
2020 | ||
2021 | for (i = 0; i < XGENE_DMA_MAX_CHANNEL; i++) { | |
2022 | chan = &pdma->chan[i]; | |
2023 | tasklet_kill(&chan->tasklet); | |
2024 | xgene_dma_delete_chan_rings(chan); | |
2025 | } | |
2026 | ||
89079493 RPS |
2027 | if (!IS_ERR(pdma->clk)) |
2028 | clk_disable_unprepare(pdma->clk); | |
9f2fd0df RPS |
2029 | |
2030 | return 0; | |
2031 | } | |
2032 | ||
89079493 RPS |
2033 | #ifdef CONFIG_ACPI |
2034 | static const struct acpi_device_id xgene_dma_acpi_match_ptr[] = { | |
2035 | {"APMC0D43", 0}, | |
2036 | {}, | |
2037 | }; | |
2038 | MODULE_DEVICE_TABLE(acpi, xgene_dma_acpi_match_ptr); | |
2039 | #endif | |
2040 | ||
9f2fd0df RPS |
2041 | static const struct of_device_id xgene_dma_of_match_ptr[] = { |
2042 | {.compatible = "apm,xgene-storm-dma",}, | |
2043 | {}, | |
2044 | }; | |
2045 | MODULE_DEVICE_TABLE(of, xgene_dma_of_match_ptr); | |
2046 | ||
2047 | static struct platform_driver xgene_dma_driver = { | |
2048 | .probe = xgene_dma_probe, | |
2049 | .remove = xgene_dma_remove, | |
2050 | .driver = { | |
2051 | .name = "X-Gene-DMA", | |
9f2fd0df | 2052 | .of_match_table = xgene_dma_of_match_ptr, |
89079493 | 2053 | .acpi_match_table = ACPI_PTR(xgene_dma_acpi_match_ptr), |
9f2fd0df RPS |
2054 | }, |
2055 | }; | |
2056 | ||
2057 | module_platform_driver(xgene_dma_driver); | |
2058 | ||
2059 | MODULE_DESCRIPTION("APM X-Gene SoC DMA driver"); | |
2060 | MODULE_AUTHOR("Rameshwar Prasad Sahu <rsahu@apm.com>"); | |
2061 | MODULE_AUTHOR("Loc Ho <lho@apm.com>"); | |
2062 | MODULE_LICENSE("GPL"); | |
2063 | MODULE_VERSION("1.0"); |