]>
Commit | Line | Data |
---|---|---|
3277e8aa GS |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * TI K3 NAVSS Ring Accelerator subsystem driver | |
4 | * | |
5 | * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com | |
6 | */ | |
7 | ||
8 | #include <linux/dma-mapping.h> | |
9 | #include <linux/io.h> | |
10 | #include <linux/init.h> | |
11 | #include <linux/of.h> | |
a8fc8e5b | 12 | #include <linux/of_device.h> |
3277e8aa | 13 | #include <linux/platform_device.h> |
95e7be06 | 14 | #include <linux/sys_soc.h> |
d782298c | 15 | #include <linux/dma/ti-cppi5.h> |
3277e8aa GS |
16 | #include <linux/soc/ti/k3-ringacc.h> |
17 | #include <linux/soc/ti/ti_sci_protocol.h> | |
18 | #include <linux/soc/ti/ti_sci_inta_msi.h> | |
19 | #include <linux/of_irq.h> | |
20 | #include <linux/irqdomain.h> | |
21 | ||
22 | static LIST_HEAD(k3_ringacc_list); | |
23 | static DEFINE_MUTEX(k3_ringacc_list_lock); | |
24 | ||
25 | #define K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK GENMASK(19, 0) | |
d782298c | 26 | #define K3_DMARING_CFG_RING_SIZE_ELCNT_MASK GENMASK(15, 0) |
3277e8aa GS |
27 | |
28 | /** | |
29 | * struct k3_ring_rt_regs - The RA realtime Control/Status Registers region | |
30 | * | |
31 | * @resv_16: Reserved | |
32 | * @db: Ring Doorbell Register | |
33 | * @resv_4: Reserved | |
34 | * @occ: Ring Occupancy Register | |
35 | * @indx: Ring Current Index Register | |
36 | * @hwocc: Ring Hardware Occupancy Register | |
37 | * @hwindx: Ring Hardware Current Index Register | |
38 | */ | |
39 | struct k3_ring_rt_regs { | |
40 | u32 resv_16[4]; | |
41 | u32 db; | |
42 | u32 resv_4[1]; | |
43 | u32 occ; | |
44 | u32 indx; | |
45 | u32 hwocc; | |
46 | u32 hwindx; | |
47 | }; | |
48 | ||
d782298c GS |
49 | #define K3_RINGACC_RT_REGS_STEP 0x1000 |
50 | #define K3_DMARING_RT_REGS_STEP 0x2000 | |
51 | #define K3_DMARING_RT_REGS_REVERSE_OFS 0x1000 | |
52 | #define K3_RINGACC_RT_OCC_MASK GENMASK(20, 0) | |
53 | #define K3_DMARING_RT_OCC_TDOWN_COMPLETE BIT(31) | |
54 | #define K3_DMARING_RT_DB_ENTRY_MASK GENMASK(7, 0) | |
55 | #define K3_DMARING_RT_DB_TDOWN_ACK BIT(31) | |
3277e8aa GS |
56 | |
57 | /** | |
58 | * struct k3_ring_fifo_regs - The Ring Accelerator Queues Registers region | |
59 | * | |
60 | * @head_data: Ring Head Entry Data Registers | |
61 | * @tail_data: Ring Tail Entry Data Registers | |
62 | * @peek_head_data: Ring Peek Head Entry Data Regs | |
63 | * @peek_tail_data: Ring Peek Tail Entry Data Regs | |
64 | */ | |
65 | struct k3_ring_fifo_regs { | |
66 | u32 head_data[128]; | |
67 | u32 tail_data[128]; | |
68 | u32 peek_head_data[128]; | |
69 | u32 peek_tail_data[128]; | |
70 | }; | |
71 | ||
72 | /** | |
73 | * struct k3_ringacc_proxy_gcfg_regs - RA Proxy Global Config MMIO Region | |
74 | * | |
75 | * @revision: Revision Register | |
76 | * @config: Config Register | |
77 | */ | |
78 | struct k3_ringacc_proxy_gcfg_regs { | |
79 | u32 revision; | |
80 | u32 config; | |
81 | }; | |
82 | ||
83 | #define K3_RINGACC_PROXY_CFG_THREADS_MASK GENMASK(15, 0) | |
84 | ||
85 | /** | |
86 | * struct k3_ringacc_proxy_target_regs - Proxy Datapath MMIO Region | |
87 | * | |
88 | * @control: Proxy Control Register | |
89 | * @status: Proxy Status Register | |
90 | * @resv_512: Reserved | |
91 | * @data: Proxy Data Register | |
92 | */ | |
93 | struct k3_ringacc_proxy_target_regs { | |
94 | u32 control; | |
95 | u32 status; | |
96 | u8 resv_512[504]; | |
97 | u32 data[128]; | |
98 | }; | |
99 | ||
100 | #define K3_RINGACC_PROXY_TARGET_STEP 0x1000 | |
101 | #define K3_RINGACC_PROXY_NOT_USED (-1) | |
102 | ||
103 | enum k3_ringacc_proxy_access_mode { | |
104 | PROXY_ACCESS_MODE_HEAD = 0, | |
105 | PROXY_ACCESS_MODE_TAIL = 1, | |
106 | PROXY_ACCESS_MODE_PEEK_HEAD = 2, | |
107 | PROXY_ACCESS_MODE_PEEK_TAIL = 3, | |
108 | }; | |
109 | ||
110 | #define K3_RINGACC_FIFO_WINDOW_SIZE_BYTES (512U) | |
111 | #define K3_RINGACC_FIFO_REGS_STEP 0x1000 | |
112 | #define K3_RINGACC_MAX_DB_RING_CNT (127U) | |
113 | ||
114 | struct k3_ring_ops { | |
115 | int (*push_tail)(struct k3_ring *ring, void *elm); | |
116 | int (*push_head)(struct k3_ring *ring, void *elm); | |
117 | int (*pop_tail)(struct k3_ring *ring, void *elm); | |
118 | int (*pop_head)(struct k3_ring *ring, void *elm); | |
119 | }; | |
120 | ||
6b3da0b4 PU |
121 | /** |
122 | * struct k3_ring_state - Internal state tracking structure | |
123 | * | |
124 | * @free: Number of free entries | |
125 | * @occ: Occupancy | |
126 | * @windex: Write index | |
127 | * @rindex: Read index | |
128 | */ | |
129 | struct k3_ring_state { | |
130 | u32 free; | |
131 | u32 occ; | |
132 | u32 windex; | |
133 | u32 rindex; | |
d782298c | 134 | u32 tdown_complete:1; |
6b3da0b4 PU |
135 | }; |
136 | ||
3277e8aa GS |
137 | /** |
138 | * struct k3_ring - RA Ring descriptor | |
139 | * | |
140 | * @rt: Ring control/status registers | |
141 | * @fifos: Ring queues registers | |
142 | * @proxy: Ring Proxy Datapath registers | |
143 | * @ring_mem_dma: Ring buffer dma address | |
144 | * @ring_mem_virt: Ring buffer virt address | |
145 | * @ops: Ring operations | |
146 | * @size: Ring size in elements | |
147 | * @elm_size: Size of the ring element | |
148 | * @mode: Ring mode | |
149 | * @flags: flags | |
50883aff | 150 | * @state: Ring state |
3277e8aa GS |
151 | * @ring_id: Ring Id |
152 | * @parent: Pointer on struct @k3_ringacc | |
153 | * @use_count: Use count for shared rings | |
154 | * @proxy_id: RA Ring Proxy Id (only if @K3_RINGACC_RING_USE_PROXY) | |
8c42379e | 155 | * @dma_dev: device to be used for DMA API (allocation, mapping) |
d782298c | 156 | * @asel: Address Space Select value for physical addresses |
3277e8aa GS |
157 | */ |
158 | struct k3_ring { | |
159 | struct k3_ring_rt_regs __iomem *rt; | |
160 | struct k3_ring_fifo_regs __iomem *fifos; | |
161 | struct k3_ringacc_proxy_target_regs __iomem *proxy; | |
162 | dma_addr_t ring_mem_dma; | |
163 | void *ring_mem_virt; | |
164 | struct k3_ring_ops *ops; | |
165 | u32 size; | |
166 | enum k3_ring_size elm_size; | |
167 | enum k3_ring_mode mode; | |
168 | u32 flags; | |
169 | #define K3_RING_FLAG_BUSY BIT(1) | |
170 | #define K3_RING_FLAG_SHARED BIT(2) | |
d782298c | 171 | #define K3_RING_FLAG_REVERSE BIT(3) |
6b3da0b4 | 172 | struct k3_ring_state state; |
3277e8aa GS |
173 | u32 ring_id; |
174 | struct k3_ringacc *parent; | |
175 | u32 use_count; | |
176 | int proxy_id; | |
8c42379e | 177 | struct device *dma_dev; |
d782298c GS |
178 | u32 asel; |
179 | #define K3_ADDRESS_ASEL_SHIFT 48 | |
3277e8aa GS |
180 | }; |
181 | ||
40a2a7c3 GS |
182 | struct k3_ringacc_ops { |
183 | int (*init)(struct platform_device *pdev, struct k3_ringacc *ringacc); | |
184 | }; | |
185 | ||
3277e8aa GS |
186 | /** |
187 | * struct k3_ringacc - Rings accelerator descriptor | |
188 | * | |
189 | * @dev: pointer on RA device | |
190 | * @proxy_gcfg: RA proxy global config registers | |
191 | * @proxy_target_base: RA proxy datapath region | |
192 | * @num_rings: number of ring in RA | |
193 | * @rings_inuse: bitfield for ring usage tracking | |
194 | * @rm_gp_range: general purpose rings range from tisci | |
195 | * @dma_ring_reset_quirk: DMA reset w/a enable | |
196 | * @num_proxies: number of RA proxies | |
197 | * @proxy_inuse: bitfield for proxy usage tracking | |
198 | * @rings: array of rings descriptors (struct @k3_ring) | |
199 | * @list: list of RAs in the system | |
200 | * @req_lock: protect rings allocation | |
201 | * @tisci: pointer ti-sci handle | |
202 | * @tisci_ring_ops: ti-sci rings ops | |
203 | * @tisci_dev_id: ti-sci device id | |
40a2a7c3 | 204 | * @ops: SoC specific ringacc operation |
d782298c | 205 | * @dma_rings: indicate DMA ring (dual ring within BCDMA/PKTDMA) |
3277e8aa GS |
206 | */ |
207 | struct k3_ringacc { | |
208 | struct device *dev; | |
209 | struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg; | |
210 | void __iomem *proxy_target_base; | |
211 | u32 num_rings; /* number of rings in Ringacc module */ | |
212 | unsigned long *rings_inuse; | |
213 | struct ti_sci_resource *rm_gp_range; | |
214 | ||
215 | bool dma_ring_reset_quirk; | |
216 | u32 num_proxies; | |
217 | unsigned long *proxy_inuse; | |
218 | ||
219 | struct k3_ring *rings; | |
220 | struct list_head list; | |
221 | struct mutex req_lock; /* protect rings allocation */ | |
222 | ||
223 | const struct ti_sci_handle *tisci; | |
224 | const struct ti_sci_rm_ringacc_ops *tisci_ring_ops; | |
225 | u32 tisci_dev_id; | |
40a2a7c3 GS |
226 | |
227 | const struct k3_ringacc_ops *ops; | |
d782298c | 228 | bool dma_rings; |
3277e8aa GS |
229 | }; |
230 | ||
95e7be06 GS |
231 | /** |
232 | * struct k3_ringacc - Rings accelerator SoC data | |
233 | * | |
234 | * @dma_ring_reset_quirk: DMA reset w/a enable | |
235 | */ | |
236 | struct k3_ringacc_soc_data { | |
237 | unsigned dma_ring_reset_quirk:1; | |
238 | }; | |
239 | ||
d782298c GS |
240 | static int k3_ringacc_ring_read_occ(struct k3_ring *ring) |
241 | { | |
242 | return readl(&ring->rt->occ) & K3_RINGACC_RT_OCC_MASK; | |
243 | } | |
244 | ||
245 | static void k3_ringacc_ring_update_occ(struct k3_ring *ring) | |
246 | { | |
247 | u32 val; | |
248 | ||
249 | val = readl(&ring->rt->occ); | |
250 | ||
251 | ring->state.occ = val & K3_RINGACC_RT_OCC_MASK; | |
252 | ring->state.tdown_complete = !!(val & K3_DMARING_RT_OCC_TDOWN_COMPLETE); | |
253 | } | |
254 | ||
3277e8aa GS |
255 | static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring) |
256 | { | |
257 | return K3_RINGACC_FIFO_WINDOW_SIZE_BYTES - | |
258 | (4 << ring->elm_size); | |
259 | } | |
260 | ||
261 | static void *k3_ringacc_get_elm_addr(struct k3_ring *ring, u32 idx) | |
262 | { | |
263 | return (ring->ring_mem_virt + idx * (4 << ring->elm_size)); | |
264 | } | |
265 | ||
266 | static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem); | |
267 | static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem); | |
d782298c GS |
268 | static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem); |
269 | static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem); | |
3277e8aa GS |
270 | |
271 | static struct k3_ring_ops k3_ring_mode_ring_ops = { | |
272 | .push_tail = k3_ringacc_ring_push_mem, | |
273 | .pop_head = k3_ringacc_ring_pop_mem, | |
274 | }; | |
275 | ||
d782298c GS |
276 | static struct k3_ring_ops k3_dmaring_fwd_ops = { |
277 | .push_tail = k3_ringacc_ring_push_mem, | |
278 | .pop_head = k3_dmaring_fwd_pop, | |
279 | }; | |
280 | ||
281 | static struct k3_ring_ops k3_dmaring_reverse_ops = { | |
282 | /* Reverse side of the DMA ring can only be popped by SW */ | |
283 | .pop_head = k3_dmaring_reverse_pop, | |
284 | }; | |
285 | ||
3277e8aa GS |
286 | static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem); |
287 | static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem); | |
288 | static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem); | |
289 | static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem); | |
290 | ||
291 | static struct k3_ring_ops k3_ring_mode_msg_ops = { | |
292 | .push_tail = k3_ringacc_ring_push_io, | |
293 | .push_head = k3_ringacc_ring_push_head_io, | |
294 | .pop_tail = k3_ringacc_ring_pop_tail_io, | |
295 | .pop_head = k3_ringacc_ring_pop_io, | |
296 | }; | |
297 | ||
298 | static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem); | |
299 | static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem); | |
300 | static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem); | |
301 | static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem); | |
302 | ||
303 | static struct k3_ring_ops k3_ring_mode_proxy_ops = { | |
304 | .push_tail = k3_ringacc_ring_push_tail_proxy, | |
305 | .push_head = k3_ringacc_ring_push_head_proxy, | |
306 | .pop_tail = k3_ringacc_ring_pop_tail_proxy, | |
307 | .pop_head = k3_ringacc_ring_pop_head_proxy, | |
308 | }; | |
309 | ||
310 | static void k3_ringacc_ring_dump(struct k3_ring *ring) | |
311 | { | |
312 | struct device *dev = ring->parent->dev; | |
313 | ||
314 | dev_dbg(dev, "dump ring: %d\n", ring->ring_id); | |
315 | dev_dbg(dev, "dump mem virt %p, dma %pad\n", ring->ring_mem_virt, | |
316 | &ring->ring_mem_dma); | |
317 | dev_dbg(dev, "dump elmsize %d, size %d, mode %d, proxy_id %d\n", | |
318 | ring->elm_size, ring->size, ring->mode, ring->proxy_id); | |
175e663f | 319 | dev_dbg(dev, "dump flags %08X\n", ring->flags); |
3277e8aa GS |
320 | |
321 | dev_dbg(dev, "dump ring_rt_regs: db%08x\n", readl(&ring->rt->db)); | |
322 | dev_dbg(dev, "dump occ%08x\n", readl(&ring->rt->occ)); | |
323 | dev_dbg(dev, "dump indx%08x\n", readl(&ring->rt->indx)); | |
324 | dev_dbg(dev, "dump hwocc%08x\n", readl(&ring->rt->hwocc)); | |
325 | dev_dbg(dev, "dump hwindx%08x\n", readl(&ring->rt->hwindx)); | |
326 | ||
327 | if (ring->ring_mem_virt) | |
328 | print_hex_dump_debug("dump ring_mem_virt ", DUMP_PREFIX_NONE, | |
329 | 16, 1, ring->ring_mem_virt, 16 * 8, false); | |
330 | } | |
331 | ||
332 | struct k3_ring *k3_ringacc_request_ring(struct k3_ringacc *ringacc, | |
333 | int id, u32 flags) | |
334 | { | |
335 | int proxy_id = K3_RINGACC_PROXY_NOT_USED; | |
336 | ||
337 | mutex_lock(&ringacc->req_lock); | |
338 | ||
339 | if (id == K3_RINGACC_RING_ID_ANY) { | |
340 | /* Request for any general purpose ring */ | |
341 | struct ti_sci_resource_desc *gp_rings = | |
342 | &ringacc->rm_gp_range->desc[0]; | |
343 | unsigned long size; | |
344 | ||
345 | size = gp_rings->start + gp_rings->num; | |
346 | id = find_next_zero_bit(ringacc->rings_inuse, size, | |
347 | gp_rings->start); | |
348 | if (id == size) | |
349 | goto error; | |
350 | } else if (id < 0) { | |
351 | goto error; | |
352 | } | |
353 | ||
354 | if (test_bit(id, ringacc->rings_inuse) && | |
355 | !(ringacc->rings[id].flags & K3_RING_FLAG_SHARED)) | |
356 | goto error; | |
357 | else if (ringacc->rings[id].flags & K3_RING_FLAG_SHARED) | |
358 | goto out; | |
359 | ||
360 | if (flags & K3_RINGACC_RING_USE_PROXY) { | |
361 | proxy_id = find_next_zero_bit(ringacc->proxy_inuse, | |
362 | ringacc->num_proxies, 0); | |
363 | if (proxy_id == ringacc->num_proxies) | |
364 | goto error; | |
365 | } | |
366 | ||
367 | if (proxy_id != K3_RINGACC_PROXY_NOT_USED) { | |
368 | set_bit(proxy_id, ringacc->proxy_inuse); | |
369 | ringacc->rings[id].proxy_id = proxy_id; | |
370 | dev_dbg(ringacc->dev, "Giving ring#%d proxy#%d\n", id, | |
371 | proxy_id); | |
372 | } else { | |
373 | dev_dbg(ringacc->dev, "Giving ring#%d\n", id); | |
374 | } | |
375 | ||
376 | set_bit(id, ringacc->rings_inuse); | |
377 | out: | |
378 | ringacc->rings[id].use_count++; | |
379 | mutex_unlock(&ringacc->req_lock); | |
380 | return &ringacc->rings[id]; | |
381 | ||
382 | error: | |
383 | mutex_unlock(&ringacc->req_lock); | |
384 | return NULL; | |
385 | } | |
386 | EXPORT_SYMBOL_GPL(k3_ringacc_request_ring); | |
387 | ||
d782298c GS |
388 | static int k3_dmaring_request_dual_ring(struct k3_ringacc *ringacc, int fwd_id, |
389 | struct k3_ring **fwd_ring, | |
390 | struct k3_ring **compl_ring) | |
391 | { | |
392 | int ret = 0; | |
393 | ||
394 | /* | |
395 | * DMA rings must be requested by ID, completion ring is the reverse | |
396 | * side of the forward ring | |
397 | */ | |
398 | if (fwd_id < 0) | |
399 | return -EINVAL; | |
400 | ||
401 | mutex_lock(&ringacc->req_lock); | |
402 | ||
403 | if (test_bit(fwd_id, ringacc->rings_inuse)) { | |
404 | ret = -EBUSY; | |
405 | goto error; | |
406 | } | |
407 | ||
408 | *fwd_ring = &ringacc->rings[fwd_id]; | |
409 | *compl_ring = &ringacc->rings[fwd_id + ringacc->num_rings]; | |
410 | set_bit(fwd_id, ringacc->rings_inuse); | |
411 | ringacc->rings[fwd_id].use_count++; | |
412 | dev_dbg(ringacc->dev, "Giving ring#%d\n", fwd_id); | |
413 | ||
414 | mutex_unlock(&ringacc->req_lock); | |
415 | return 0; | |
416 | ||
417 | error: | |
418 | mutex_unlock(&ringacc->req_lock); | |
419 | return ret; | |
420 | } | |
421 | ||
43148b1c GS |
422 | int k3_ringacc_request_rings_pair(struct k3_ringacc *ringacc, |
423 | int fwd_id, int compl_id, | |
424 | struct k3_ring **fwd_ring, | |
425 | struct k3_ring **compl_ring) | |
426 | { | |
427 | int ret = 0; | |
428 | ||
429 | if (!fwd_ring || !compl_ring) | |
430 | return -EINVAL; | |
431 | ||
d782298c GS |
432 | if (ringacc->dma_rings) |
433 | return k3_dmaring_request_dual_ring(ringacc, fwd_id, | |
434 | fwd_ring, compl_ring); | |
435 | ||
43148b1c GS |
436 | *fwd_ring = k3_ringacc_request_ring(ringacc, fwd_id, 0); |
437 | if (!(*fwd_ring)) | |
438 | return -ENODEV; | |
439 | ||
440 | *compl_ring = k3_ringacc_request_ring(ringacc, compl_id, 0); | |
441 | if (!(*compl_ring)) { | |
442 | k3_ringacc_ring_free(*fwd_ring); | |
443 | ret = -ENODEV; | |
444 | } | |
445 | ||
446 | return ret; | |
447 | } | |
448 | EXPORT_SYMBOL_GPL(k3_ringacc_request_rings_pair); | |
449 | ||
3277e8aa GS |
450 | static void k3_ringacc_ring_reset_sci(struct k3_ring *ring) |
451 | { | |
bb49ca00 | 452 | struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; |
3277e8aa GS |
453 | struct k3_ringacc *ringacc = ring->parent; |
454 | int ret; | |
455 | ||
bb49ca00 PU |
456 | ring_cfg.nav_id = ringacc->tisci_dev_id; |
457 | ring_cfg.index = ring->ring_id; | |
458 | ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID; | |
459 | ring_cfg.count = ring->size; | |
460 | ||
461 | ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); | |
3277e8aa GS |
462 | if (ret) |
463 | dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n", | |
464 | ret, ring->ring_id); | |
465 | } | |
466 | ||
467 | void k3_ringacc_ring_reset(struct k3_ring *ring) | |
468 | { | |
469 | if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) | |
470 | return; | |
471 | ||
6b3da0b4 | 472 | memset(&ring->state, 0, sizeof(ring->state)); |
3277e8aa GS |
473 | |
474 | k3_ringacc_ring_reset_sci(ring); | |
475 | } | |
476 | EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset); | |
477 | ||
478 | static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_ring *ring, | |
479 | enum k3_ring_mode mode) | |
480 | { | |
bb49ca00 | 481 | struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; |
3277e8aa GS |
482 | struct k3_ringacc *ringacc = ring->parent; |
483 | int ret; | |
484 | ||
bb49ca00 PU |
485 | ring_cfg.nav_id = ringacc->tisci_dev_id; |
486 | ring_cfg.index = ring->ring_id; | |
487 | ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_RING_MODE_VALID; | |
488 | ring_cfg.mode = mode; | |
489 | ||
490 | ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); | |
3277e8aa GS |
491 | if (ret) |
492 | dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n", | |
493 | ret, ring->ring_id); | |
494 | } | |
495 | ||
496 | void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ) | |
497 | { | |
498 | if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) | |
499 | return; | |
500 | ||
501 | if (!ring->parent->dma_ring_reset_quirk) | |
502 | goto reset; | |
503 | ||
504 | if (!occ) | |
d782298c | 505 | occ = k3_ringacc_ring_read_occ(ring); |
3277e8aa GS |
506 | |
507 | if (occ) { | |
508 | u32 db_ring_cnt, db_ring_cnt_cur; | |
509 | ||
510 | dev_dbg(ring->parent->dev, "%s %u occ: %u\n", __func__, | |
511 | ring->ring_id, occ); | |
512 | /* TI-SCI ring reset */ | |
513 | k3_ringacc_ring_reset_sci(ring); | |
514 | ||
515 | /* | |
516 | * Setup the ring in ring/doorbell mode (if not already in this | |
517 | * mode) | |
518 | */ | |
519 | if (ring->mode != K3_RINGACC_RING_MODE_RING) | |
520 | k3_ringacc_ring_reconfig_qmode_sci( | |
521 | ring, K3_RINGACC_RING_MODE_RING); | |
522 | /* | |
523 | * Ring the doorbell 2**22 – ringOcc times. | |
524 | * This will wrap the internal UDMAP ring state occupancy | |
525 | * counter (which is 21-bits wide) to 0. | |
526 | */ | |
527 | db_ring_cnt = (1U << 22) - occ; | |
528 | ||
529 | while (db_ring_cnt != 0) { | |
530 | /* | |
531 | * Ring the doorbell with the maximum count each | |
532 | * iteration if possible to minimize the total | |
533 | * of writes | |
534 | */ | |
535 | if (db_ring_cnt > K3_RINGACC_MAX_DB_RING_CNT) | |
536 | db_ring_cnt_cur = K3_RINGACC_MAX_DB_RING_CNT; | |
537 | else | |
538 | db_ring_cnt_cur = db_ring_cnt; | |
539 | ||
540 | writel(db_ring_cnt_cur, &ring->rt->db); | |
541 | db_ring_cnt -= db_ring_cnt_cur; | |
542 | } | |
543 | ||
544 | /* Restore the original ring mode (if not ring mode) */ | |
545 | if (ring->mode != K3_RINGACC_RING_MODE_RING) | |
546 | k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode); | |
547 | } | |
548 | ||
549 | reset: | |
550 | /* Reset the ring */ | |
551 | k3_ringacc_ring_reset(ring); | |
552 | } | |
553 | EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset_dma); | |
554 | ||
555 | static void k3_ringacc_ring_free_sci(struct k3_ring *ring) | |
556 | { | |
bb49ca00 | 557 | struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; |
3277e8aa GS |
558 | struct k3_ringacc *ringacc = ring->parent; |
559 | int ret; | |
560 | ||
bb49ca00 PU |
561 | ring_cfg.nav_id = ringacc->tisci_dev_id; |
562 | ring_cfg.index = ring->ring_id; | |
563 | ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER; | |
564 | ||
565 | ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); | |
3277e8aa GS |
566 | if (ret) |
567 | dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n", | |
568 | ret, ring->ring_id); | |
569 | } | |
570 | ||
571 | int k3_ringacc_ring_free(struct k3_ring *ring) | |
572 | { | |
573 | struct k3_ringacc *ringacc; | |
574 | ||
575 | if (!ring) | |
576 | return -EINVAL; | |
577 | ||
578 | ringacc = ring->parent; | |
579 | ||
d782298c GS |
580 | /* |
581 | * DMA rings: rings shared memory and configuration, only forward ring | |
582 | * is configured and reverse ring considered as slave. | |
583 | */ | |
584 | if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE)) | |
585 | return 0; | |
586 | ||
3277e8aa GS |
587 | dev_dbg(ring->parent->dev, "flags: 0x%08x\n", ring->flags); |
588 | ||
589 | if (!test_bit(ring->ring_id, ringacc->rings_inuse)) | |
590 | return -EINVAL; | |
591 | ||
592 | mutex_lock(&ringacc->req_lock); | |
593 | ||
594 | if (--ring->use_count) | |
595 | goto out; | |
596 | ||
597 | if (!(ring->flags & K3_RING_FLAG_BUSY)) | |
598 | goto no_init; | |
599 | ||
600 | k3_ringacc_ring_free_sci(ring); | |
601 | ||
8c42379e | 602 | dma_free_coherent(ring->dma_dev, |
3277e8aa GS |
603 | ring->size * (4 << ring->elm_size), |
604 | ring->ring_mem_virt, ring->ring_mem_dma); | |
605 | ring->flags = 0; | |
606 | ring->ops = NULL; | |
8c42379e | 607 | ring->dma_dev = NULL; |
d782298c GS |
608 | ring->asel = 0; |
609 | ||
3277e8aa GS |
610 | if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) { |
611 | clear_bit(ring->proxy_id, ringacc->proxy_inuse); | |
612 | ring->proxy = NULL; | |
613 | ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; | |
614 | } | |
615 | ||
616 | no_init: | |
617 | clear_bit(ring->ring_id, ringacc->rings_inuse); | |
618 | ||
619 | out: | |
620 | mutex_unlock(&ringacc->req_lock); | |
621 | return 0; | |
622 | } | |
623 | EXPORT_SYMBOL_GPL(k3_ringacc_ring_free); | |
624 | ||
625 | u32 k3_ringacc_get_ring_id(struct k3_ring *ring) | |
626 | { | |
627 | if (!ring) | |
628 | return -EINVAL; | |
629 | ||
630 | return ring->ring_id; | |
631 | } | |
632 | EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_id); | |
633 | ||
634 | u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring) | |
635 | { | |
636 | if (!ring) | |
637 | return -EINVAL; | |
638 | ||
639 | return ring->parent->tisci_dev_id; | |
640 | } | |
641 | EXPORT_SYMBOL_GPL(k3_ringacc_get_tisci_dev_id); | |
642 | ||
643 | int k3_ringacc_get_ring_irq_num(struct k3_ring *ring) | |
644 | { | |
645 | int irq_num; | |
646 | ||
647 | if (!ring) | |
648 | return -EINVAL; | |
649 | ||
650 | irq_num = ti_sci_inta_msi_get_virq(ring->parent->dev, ring->ring_id); | |
651 | if (irq_num <= 0) | |
652 | irq_num = -EINVAL; | |
653 | return irq_num; | |
654 | } | |
655 | EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_irq_num); | |
656 | ||
657 | static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring) | |
658 | { | |
bb49ca00 | 659 | struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; |
3277e8aa | 660 | struct k3_ringacc *ringacc = ring->parent; |
3277e8aa GS |
661 | int ret; |
662 | ||
663 | if (!ringacc->tisci) | |
664 | return -EINVAL; | |
665 | ||
bb49ca00 PU |
666 | ring_cfg.nav_id = ringacc->tisci_dev_id; |
667 | ring_cfg.index = ring->ring_id; | |
668 | ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER; | |
669 | ring_cfg.addr_lo = lower_32_bits(ring->ring_mem_dma); | |
670 | ring_cfg.addr_hi = upper_32_bits(ring->ring_mem_dma); | |
671 | ring_cfg.count = ring->size; | |
672 | ring_cfg.mode = ring->mode; | |
673 | ring_cfg.size = ring->elm_size; | |
d782298c | 674 | ring_cfg.asel = ring->asel; |
bb49ca00 PU |
675 | |
676 | ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); | |
3277e8aa GS |
677 | if (ret) |
678 | dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n", | |
bb49ca00 | 679 | ret, ring->ring_id); |
3277e8aa GS |
680 | |
681 | return ret; | |
682 | } | |
683 | ||
d782298c GS |
684 | static int k3_dmaring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg) |
685 | { | |
686 | struct k3_ringacc *ringacc; | |
687 | struct k3_ring *reverse_ring; | |
688 | int ret = 0; | |
689 | ||
690 | if (cfg->elm_size != K3_RINGACC_RING_ELSIZE_8 || | |
691 | cfg->mode != K3_RINGACC_RING_MODE_RING || | |
692 | cfg->size & ~K3_DMARING_CFG_RING_SIZE_ELCNT_MASK) | |
693 | return -EINVAL; | |
694 | ||
695 | ringacc = ring->parent; | |
696 | ||
697 | /* | |
698 | * DMA rings: rings shared memory and configuration, only forward ring | |
699 | * is configured and reverse ring considered as slave. | |
700 | */ | |
701 | if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE)) | |
702 | return 0; | |
703 | ||
704 | if (!test_bit(ring->ring_id, ringacc->rings_inuse)) | |
705 | return -EINVAL; | |
706 | ||
707 | ring->size = cfg->size; | |
708 | ring->elm_size = cfg->elm_size; | |
709 | ring->mode = cfg->mode; | |
710 | ring->asel = cfg->asel; | |
711 | ring->dma_dev = cfg->dma_dev; | |
712 | if (!ring->dma_dev) { | |
713 | dev_warn(ringacc->dev, "dma_dev is not provided for ring%d\n", | |
714 | ring->ring_id); | |
715 | ring->dma_dev = ringacc->dev; | |
716 | } | |
717 | ||
718 | memset(&ring->state, 0, sizeof(ring->state)); | |
719 | ||
720 | ring->ops = &k3_dmaring_fwd_ops; | |
721 | ||
722 | ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev, | |
723 | ring->size * (4 << ring->elm_size), | |
724 | &ring->ring_mem_dma, GFP_KERNEL); | |
725 | if (!ring->ring_mem_virt) { | |
726 | dev_err(ringacc->dev, "Failed to alloc ring mem\n"); | |
727 | ret = -ENOMEM; | |
728 | goto err_free_ops; | |
729 | } | |
730 | ||
731 | ret = k3_ringacc_ring_cfg_sci(ring); | |
732 | if (ret) | |
733 | goto err_free_mem; | |
734 | ||
735 | ring->flags |= K3_RING_FLAG_BUSY; | |
736 | ||
737 | k3_ringacc_ring_dump(ring); | |
738 | ||
739 | /* DMA rings: configure reverse ring */ | |
740 | reverse_ring = &ringacc->rings[ring->ring_id + ringacc->num_rings]; | |
741 | reverse_ring->size = cfg->size; | |
742 | reverse_ring->elm_size = cfg->elm_size; | |
743 | reverse_ring->mode = cfg->mode; | |
744 | reverse_ring->asel = cfg->asel; | |
745 | memset(&reverse_ring->state, 0, sizeof(reverse_ring->state)); | |
746 | reverse_ring->ops = &k3_dmaring_reverse_ops; | |
747 | ||
748 | reverse_ring->ring_mem_virt = ring->ring_mem_virt; | |
749 | reverse_ring->ring_mem_dma = ring->ring_mem_dma; | |
750 | reverse_ring->flags |= K3_RING_FLAG_BUSY; | |
751 | k3_ringacc_ring_dump(reverse_ring); | |
752 | ||
753 | return 0; | |
754 | ||
755 | err_free_mem: | |
756 | dma_free_coherent(ring->dma_dev, | |
757 | ring->size * (4 << ring->elm_size), | |
758 | ring->ring_mem_virt, | |
759 | ring->ring_mem_dma); | |
760 | err_free_ops: | |
761 | ring->ops = NULL; | |
762 | ring->proxy = NULL; | |
763 | ring->dma_dev = NULL; | |
764 | ring->asel = 0; | |
765 | return ret; | |
766 | } | |
767 | ||
3277e8aa GS |
768 | int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg) |
769 | { | |
80ff73f7 | 770 | struct k3_ringacc *ringacc; |
3277e8aa GS |
771 | int ret = 0; |
772 | ||
773 | if (!ring || !cfg) | |
774 | return -EINVAL; | |
d782298c | 775 | |
80ff73f7 GS |
776 | ringacc = ring->parent; |
777 | ||
d782298c GS |
778 | if (ringacc->dma_rings) |
779 | return k3_dmaring_cfg(ring, cfg); | |
780 | ||
3277e8aa GS |
781 | if (cfg->elm_size > K3_RINGACC_RING_ELSIZE_256 || |
782 | cfg->mode >= K3_RINGACC_RING_MODE_INVALID || | |
783 | cfg->size & ~K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK || | |
784 | !test_bit(ring->ring_id, ringacc->rings_inuse)) | |
785 | return -EINVAL; | |
786 | ||
787 | if (cfg->mode == K3_RINGACC_RING_MODE_MESSAGE && | |
788 | ring->proxy_id == K3_RINGACC_PROXY_NOT_USED && | |
789 | cfg->elm_size > K3_RINGACC_RING_ELSIZE_8) { | |
790 | dev_err(ringacc->dev, | |
791 | "Message mode must use proxy for %u element size\n", | |
792 | 4 << ring->elm_size); | |
793 | return -EINVAL; | |
794 | } | |
795 | ||
796 | /* | |
797 | * In case of shared ring only the first user (master user) can | |
798 | * configure the ring. The sequence should be by the client: | |
799 | * ring = k3_ringacc_request_ring(ringacc, ring_id, 0); # master user | |
800 | * k3_ringacc_ring_cfg(ring, cfg); # master configuration | |
801 | * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED); | |
802 | * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED); | |
803 | */ | |
804 | if (ring->use_count != 1) | |
805 | return 0; | |
806 | ||
807 | ring->size = cfg->size; | |
808 | ring->elm_size = cfg->elm_size; | |
809 | ring->mode = cfg->mode; | |
6b3da0b4 | 810 | memset(&ring->state, 0, sizeof(ring->state)); |
3277e8aa GS |
811 | |
812 | if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) | |
813 | ring->proxy = ringacc->proxy_target_base + | |
814 | ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP; | |
815 | ||
816 | switch (ring->mode) { | |
817 | case K3_RINGACC_RING_MODE_RING: | |
818 | ring->ops = &k3_ring_mode_ring_ops; | |
8c42379e PU |
819 | ring->dma_dev = cfg->dma_dev; |
820 | if (!ring->dma_dev) | |
821 | ring->dma_dev = ringacc->dev; | |
3277e8aa GS |
822 | break; |
823 | case K3_RINGACC_RING_MODE_MESSAGE: | |
8c42379e | 824 | ring->dma_dev = ringacc->dev; |
3277e8aa GS |
825 | if (ring->proxy) |
826 | ring->ops = &k3_ring_mode_proxy_ops; | |
827 | else | |
828 | ring->ops = &k3_ring_mode_msg_ops; | |
829 | break; | |
830 | default: | |
831 | ring->ops = NULL; | |
832 | ret = -EINVAL; | |
833 | goto err_free_proxy; | |
1d036016 | 834 | } |
3277e8aa | 835 | |
8c42379e PU |
836 | ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev, |
837 | ring->size * (4 << ring->elm_size), | |
838 | &ring->ring_mem_dma, GFP_KERNEL); | |
3277e8aa GS |
839 | if (!ring->ring_mem_virt) { |
840 | dev_err(ringacc->dev, "Failed to alloc ring mem\n"); | |
841 | ret = -ENOMEM; | |
842 | goto err_free_ops; | |
843 | } | |
844 | ||
845 | ret = k3_ringacc_ring_cfg_sci(ring); | |
846 | ||
847 | if (ret) | |
848 | goto err_free_mem; | |
849 | ||
850 | ring->flags |= K3_RING_FLAG_BUSY; | |
851 | ring->flags |= (cfg->flags & K3_RINGACC_RING_SHARED) ? | |
852 | K3_RING_FLAG_SHARED : 0; | |
853 | ||
854 | k3_ringacc_ring_dump(ring); | |
855 | ||
856 | return 0; | |
857 | ||
858 | err_free_mem: | |
8c42379e | 859 | dma_free_coherent(ring->dma_dev, |
3277e8aa GS |
860 | ring->size * (4 << ring->elm_size), |
861 | ring->ring_mem_virt, | |
862 | ring->ring_mem_dma); | |
863 | err_free_ops: | |
864 | ring->ops = NULL; | |
8c42379e | 865 | ring->dma_dev = NULL; |
3277e8aa GS |
866 | err_free_proxy: |
867 | ring->proxy = NULL; | |
868 | return ret; | |
869 | } | |
870 | EXPORT_SYMBOL_GPL(k3_ringacc_ring_cfg); | |
871 | ||
872 | u32 k3_ringacc_ring_get_size(struct k3_ring *ring) | |
873 | { | |
874 | if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) | |
875 | return -EINVAL; | |
876 | ||
877 | return ring->size; | |
878 | } | |
879 | EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_size); | |
880 | ||
881 | u32 k3_ringacc_ring_get_free(struct k3_ring *ring) | |
882 | { | |
883 | if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) | |
884 | return -EINVAL; | |
885 | ||
6b3da0b4 | 886 | if (!ring->state.free) |
d782298c | 887 | ring->state.free = ring->size - k3_ringacc_ring_read_occ(ring); |
3277e8aa | 888 | |
6b3da0b4 | 889 | return ring->state.free; |
3277e8aa GS |
890 | } |
891 | EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_free); | |
892 | ||
893 | u32 k3_ringacc_ring_get_occ(struct k3_ring *ring) | |
894 | { | |
895 | if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) | |
896 | return -EINVAL; | |
897 | ||
d782298c | 898 | return k3_ringacc_ring_read_occ(ring); |
3277e8aa GS |
899 | } |
900 | EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_occ); | |
901 | ||
902 | u32 k3_ringacc_ring_is_full(struct k3_ring *ring) | |
903 | { | |
904 | return !k3_ringacc_ring_get_free(ring); | |
905 | } | |
906 | EXPORT_SYMBOL_GPL(k3_ringacc_ring_is_full); | |
907 | ||
908 | enum k3_ringacc_access_mode { | |
909 | K3_RINGACC_ACCESS_MODE_PUSH_HEAD, | |
910 | K3_RINGACC_ACCESS_MODE_POP_HEAD, | |
911 | K3_RINGACC_ACCESS_MODE_PUSH_TAIL, | |
912 | K3_RINGACC_ACCESS_MODE_POP_TAIL, | |
913 | K3_RINGACC_ACCESS_MODE_PEEK_HEAD, | |
914 | K3_RINGACC_ACCESS_MODE_PEEK_TAIL, | |
915 | }; | |
916 | ||
917 | #define K3_RINGACC_PROXY_MODE(x) (((x) & 0x3) << 16) | |
918 | #define K3_RINGACC_PROXY_ELSIZE(x) (((x) & 0x7) << 24) | |
919 | static int k3_ringacc_ring_cfg_proxy(struct k3_ring *ring, | |
920 | enum k3_ringacc_proxy_access_mode mode) | |
921 | { | |
922 | u32 val; | |
923 | ||
924 | val = ring->ring_id; | |
925 | val |= K3_RINGACC_PROXY_MODE(mode); | |
926 | val |= K3_RINGACC_PROXY_ELSIZE(ring->elm_size); | |
927 | writel(val, &ring->proxy->control); | |
928 | return 0; | |
929 | } | |
930 | ||
931 | static int k3_ringacc_ring_access_proxy(struct k3_ring *ring, void *elem, | |
932 | enum k3_ringacc_access_mode access_mode) | |
933 | { | |
934 | void __iomem *ptr; | |
935 | ||
936 | ptr = (void __iomem *)&ring->proxy->data; | |
937 | ||
938 | switch (access_mode) { | |
939 | case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: | |
940 | case K3_RINGACC_ACCESS_MODE_POP_HEAD: | |
941 | k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD); | |
942 | break; | |
943 | case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: | |
944 | case K3_RINGACC_ACCESS_MODE_POP_TAIL: | |
945 | k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL); | |
946 | break; | |
947 | default: | |
948 | return -EINVAL; | |
949 | } | |
950 | ||
951 | ptr += k3_ringacc_ring_get_fifo_pos(ring); | |
952 | ||
953 | switch (access_mode) { | |
954 | case K3_RINGACC_ACCESS_MODE_POP_HEAD: | |
955 | case K3_RINGACC_ACCESS_MODE_POP_TAIL: | |
956 | dev_dbg(ring->parent->dev, | |
957 | "proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr, | |
958 | access_mode); | |
959 | memcpy_fromio(elem, ptr, (4 << ring->elm_size)); | |
6b3da0b4 | 960 | ring->state.occ--; |
3277e8aa GS |
961 | break; |
962 | case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: | |
963 | case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: | |
964 | dev_dbg(ring->parent->dev, | |
965 | "proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr, | |
966 | access_mode); | |
967 | memcpy_toio(ptr, elem, (4 << ring->elm_size)); | |
6b3da0b4 | 968 | ring->state.free--; |
3277e8aa GS |
969 | break; |
970 | default: | |
971 | return -EINVAL; | |
972 | } | |
973 | ||
6b3da0b4 PU |
974 | dev_dbg(ring->parent->dev, "proxy: free%d occ%d\n", ring->state.free, |
975 | ring->state.occ); | |
3277e8aa GS |
976 | return 0; |
977 | } | |
978 | ||
979 | static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem) | |
980 | { | |
981 | return k3_ringacc_ring_access_proxy(ring, elem, | |
982 | K3_RINGACC_ACCESS_MODE_PUSH_HEAD); | |
983 | } | |
984 | ||
985 | static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem) | |
986 | { | |
987 | return k3_ringacc_ring_access_proxy(ring, elem, | |
988 | K3_RINGACC_ACCESS_MODE_PUSH_TAIL); | |
989 | } | |
990 | ||
991 | static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem) | |
992 | { | |
993 | return k3_ringacc_ring_access_proxy(ring, elem, | |
994 | K3_RINGACC_ACCESS_MODE_POP_HEAD); | |
995 | } | |
996 | ||
997 | static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem) | |
998 | { | |
999 | return k3_ringacc_ring_access_proxy(ring, elem, | |
1000 | K3_RINGACC_ACCESS_MODE_POP_HEAD); | |
1001 | } | |
1002 | ||
1003 | static int k3_ringacc_ring_access_io(struct k3_ring *ring, void *elem, | |
1004 | enum k3_ringacc_access_mode access_mode) | |
1005 | { | |
1006 | void __iomem *ptr; | |
1007 | ||
1008 | switch (access_mode) { | |
1009 | case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: | |
1010 | case K3_RINGACC_ACCESS_MODE_POP_HEAD: | |
1011 | ptr = (void __iomem *)&ring->fifos->head_data; | |
1012 | break; | |
1013 | case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: | |
1014 | case K3_RINGACC_ACCESS_MODE_POP_TAIL: | |
1015 | ptr = (void __iomem *)&ring->fifos->tail_data; | |
1016 | break; | |
1017 | default: | |
1018 | return -EINVAL; | |
1019 | } | |
1020 | ||
1021 | ptr += k3_ringacc_ring_get_fifo_pos(ring); | |
1022 | ||
1023 | switch (access_mode) { | |
1024 | case K3_RINGACC_ACCESS_MODE_POP_HEAD: | |
1025 | case K3_RINGACC_ACCESS_MODE_POP_TAIL: | |
1026 | dev_dbg(ring->parent->dev, | |
1027 | "memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr, | |
1028 | access_mode); | |
1029 | memcpy_fromio(elem, ptr, (4 << ring->elm_size)); | |
6b3da0b4 | 1030 | ring->state.occ--; |
3277e8aa GS |
1031 | break; |
1032 | case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: | |
1033 | case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: | |
1034 | dev_dbg(ring->parent->dev, | |
1035 | "memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr, | |
1036 | access_mode); | |
1037 | memcpy_toio(ptr, elem, (4 << ring->elm_size)); | |
6b3da0b4 | 1038 | ring->state.free--; |
3277e8aa GS |
1039 | break; |
1040 | default: | |
1041 | return -EINVAL; | |
1042 | } | |
1043 | ||
6b3da0b4 PU |
1044 | dev_dbg(ring->parent->dev, "free%d index%d occ%d index%d\n", |
1045 | ring->state.free, ring->state.windex, ring->state.occ, | |
1046 | ring->state.rindex); | |
3277e8aa GS |
1047 | return 0; |
1048 | } | |
1049 | ||
1050 | static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem) | |
1051 | { | |
1052 | return k3_ringacc_ring_access_io(ring, elem, | |
1053 | K3_RINGACC_ACCESS_MODE_PUSH_HEAD); | |
1054 | } | |
1055 | ||
1056 | static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem) | |
1057 | { | |
1058 | return k3_ringacc_ring_access_io(ring, elem, | |
1059 | K3_RINGACC_ACCESS_MODE_PUSH_TAIL); | |
1060 | } | |
1061 | ||
1062 | static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem) | |
1063 | { | |
1064 | return k3_ringacc_ring_access_io(ring, elem, | |
1065 | K3_RINGACC_ACCESS_MODE_POP_HEAD); | |
1066 | } | |
1067 | ||
1068 | static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem) | |
1069 | { | |
1070 | return k3_ringacc_ring_access_io(ring, elem, | |
1071 | K3_RINGACC_ACCESS_MODE_POP_HEAD); | |
1072 | } | |
1073 | ||
d782298c GS |
1074 | /* |
1075 | * The element is 48 bits of address + ASEL bits in the ring. | |
1076 | * ASEL is used by the DMAs and should be removed for the kernel as it is not | |
1077 | * part of the physical memory address. | |
1078 | */ | |
1079 | static void k3_dmaring_remove_asel_from_elem(u64 *elem) | |
1080 | { | |
1081 | *elem &= GENMASK_ULL(K3_ADDRESS_ASEL_SHIFT - 1, 0); | |
1082 | } | |
1083 | ||
1084 | static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem) | |
1085 | { | |
1086 | void *elem_ptr; | |
1087 | u32 elem_idx; | |
1088 | ||
1089 | /* | |
1090 | * DMA rings: forward ring is always tied DMA channel and HW does not | |
1091 | * maintain any state data required for POP operation and its unknown | |
1092 | * how much elements were consumed by HW. So, to actually | |
1093 | * do POP, the read pointer has to be recalculated every time. | |
1094 | */ | |
1095 | ring->state.occ = k3_ringacc_ring_read_occ(ring); | |
1096 | if (ring->state.windex >= ring->state.occ) | |
1097 | elem_idx = ring->state.windex - ring->state.occ; | |
1098 | else | |
1099 | elem_idx = ring->size - (ring->state.occ - ring->state.windex); | |
1100 | ||
1101 | elem_ptr = k3_ringacc_get_elm_addr(ring, elem_idx); | |
1102 | memcpy(elem, elem_ptr, (4 << ring->elm_size)); | |
1103 | k3_dmaring_remove_asel_from_elem(elem); | |
1104 | ||
1105 | ring->state.occ--; | |
1106 | writel(-1, &ring->rt->db); | |
1107 | ||
1108 | dev_dbg(ring->parent->dev, "%s: occ%d Windex%d Rindex%d pos_ptr%px\n", | |
1109 | __func__, ring->state.occ, ring->state.windex, elem_idx, | |
1110 | elem_ptr); | |
1111 | return 0; | |
1112 | } | |
1113 | ||
1114 | static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem) | |
1115 | { | |
1116 | void *elem_ptr; | |
1117 | ||
1118 | elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex); | |
1119 | ||
1120 | if (ring->state.occ) { | |
1121 | memcpy(elem, elem_ptr, (4 << ring->elm_size)); | |
1122 | k3_dmaring_remove_asel_from_elem(elem); | |
1123 | ||
1124 | ring->state.rindex = (ring->state.rindex + 1) % ring->size; | |
1125 | ring->state.occ--; | |
1126 | writel(-1 & K3_DMARING_RT_DB_ENTRY_MASK, &ring->rt->db); | |
1127 | } else if (ring->state.tdown_complete) { | |
1128 | dma_addr_t *value = elem; | |
1129 | ||
1130 | *value = CPPI5_TDCM_MARKER; | |
1131 | writel(K3_DMARING_RT_DB_TDOWN_ACK, &ring->rt->db); | |
1132 | ring->state.tdown_complete = false; | |
1133 | } | |
1134 | ||
1135 | dev_dbg(ring->parent->dev, "%s: occ%d index%d pos_ptr%px\n", | |
1136 | __func__, ring->state.occ, ring->state.rindex, elem_ptr); | |
1137 | return 0; | |
1138 | } | |
1139 | ||
3277e8aa GS |
1140 | static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem) |
1141 | { | |
1142 | void *elem_ptr; | |
1143 | ||
6b3da0b4 | 1144 | elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.windex); |
3277e8aa GS |
1145 | |
1146 | memcpy(elem_ptr, elem, (4 << ring->elm_size)); | |
d782298c GS |
1147 | if (ring->parent->dma_rings) { |
1148 | u64 *addr = elem_ptr; | |
1149 | ||
1150 | *addr |= ((u64)ring->asel << K3_ADDRESS_ASEL_SHIFT); | |
1151 | } | |
3277e8aa | 1152 | |
6b3da0b4 PU |
1153 | ring->state.windex = (ring->state.windex + 1) % ring->size; |
1154 | ring->state.free--; | |
3277e8aa GS |
1155 | writel(1, &ring->rt->db); |
1156 | ||
1157 | dev_dbg(ring->parent->dev, "ring_push_mem: free%d index%d\n", | |
6b3da0b4 | 1158 | ring->state.free, ring->state.windex); |
3277e8aa GS |
1159 | |
1160 | return 0; | |
1161 | } | |
1162 | ||
1163 | static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem) | |
1164 | { | |
1165 | void *elem_ptr; | |
1166 | ||
6b3da0b4 | 1167 | elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex); |
3277e8aa GS |
1168 | |
1169 | memcpy(elem, elem_ptr, (4 << ring->elm_size)); | |
1170 | ||
6b3da0b4 PU |
1171 | ring->state.rindex = (ring->state.rindex + 1) % ring->size; |
1172 | ring->state.occ--; | |
3277e8aa GS |
1173 | writel(-1, &ring->rt->db); |
1174 | ||
1175 | dev_dbg(ring->parent->dev, "ring_pop_mem: occ%d index%d pos_ptr%p\n", | |
6b3da0b4 | 1176 | ring->state.occ, ring->state.rindex, elem_ptr); |
3277e8aa GS |
1177 | return 0; |
1178 | } | |
1179 | ||
1180 | int k3_ringacc_ring_push(struct k3_ring *ring, void *elem) | |
1181 | { | |
1182 | int ret = -EOPNOTSUPP; | |
1183 | ||
1184 | if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) | |
1185 | return -EINVAL; | |
1186 | ||
6b3da0b4 PU |
1187 | dev_dbg(ring->parent->dev, "ring_push: free%d index%d\n", |
1188 | ring->state.free, ring->state.windex); | |
3277e8aa GS |
1189 | |
1190 | if (k3_ringacc_ring_is_full(ring)) | |
1191 | return -ENOMEM; | |
1192 | ||
1193 | if (ring->ops && ring->ops->push_tail) | |
1194 | ret = ring->ops->push_tail(ring, elem); | |
1195 | ||
1196 | return ret; | |
1197 | } | |
1198 | EXPORT_SYMBOL_GPL(k3_ringacc_ring_push); | |
1199 | ||
1200 | int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem) | |
1201 | { | |
1202 | int ret = -EOPNOTSUPP; | |
1203 | ||
1204 | if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) | |
1205 | return -EINVAL; | |
1206 | ||
1207 | dev_dbg(ring->parent->dev, "ring_push_head: free%d index%d\n", | |
6b3da0b4 | 1208 | ring->state.free, ring->state.windex); |
3277e8aa GS |
1209 | |
1210 | if (k3_ringacc_ring_is_full(ring)) | |
1211 | return -ENOMEM; | |
1212 | ||
1213 | if (ring->ops && ring->ops->push_head) | |
1214 | ret = ring->ops->push_head(ring, elem); | |
1215 | ||
1216 | return ret; | |
1217 | } | |
1218 | EXPORT_SYMBOL_GPL(k3_ringacc_ring_push_head); | |
1219 | ||
1220 | int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem) | |
1221 | { | |
1222 | int ret = -EOPNOTSUPP; | |
1223 | ||
1224 | if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) | |
1225 | return -EINVAL; | |
1226 | ||
6b3da0b4 | 1227 | if (!ring->state.occ) |
d782298c | 1228 | k3_ringacc_ring_update_occ(ring); |
3277e8aa | 1229 | |
6b3da0b4 PU |
1230 | dev_dbg(ring->parent->dev, "ring_pop: occ%d index%d\n", ring->state.occ, |
1231 | ring->state.rindex); | |
3277e8aa | 1232 | |
d782298c | 1233 | if (!ring->state.occ && !ring->state.tdown_complete) |
3277e8aa GS |
1234 | return -ENODATA; |
1235 | ||
1236 | if (ring->ops && ring->ops->pop_head) | |
1237 | ret = ring->ops->pop_head(ring, elem); | |
1238 | ||
1239 | return ret; | |
1240 | } | |
1241 | EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop); | |
1242 | ||
1243 | int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem) | |
1244 | { | |
1245 | int ret = -EOPNOTSUPP; | |
1246 | ||
1247 | if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) | |
1248 | return -EINVAL; | |
1249 | ||
6b3da0b4 | 1250 | if (!ring->state.occ) |
d782298c | 1251 | k3_ringacc_ring_update_occ(ring); |
3277e8aa | 1252 | |
6b3da0b4 PU |
1253 | dev_dbg(ring->parent->dev, "ring_pop_tail: occ%d index%d\n", |
1254 | ring->state.occ, ring->state.rindex); | |
3277e8aa | 1255 | |
6b3da0b4 | 1256 | if (!ring->state.occ) |
3277e8aa GS |
1257 | return -ENODATA; |
1258 | ||
1259 | if (ring->ops && ring->ops->pop_tail) | |
1260 | ret = ring->ops->pop_tail(ring, elem); | |
1261 | ||
1262 | return ret; | |
1263 | } | |
1264 | EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop_tail); | |
1265 | ||
1266 | struct k3_ringacc *of_k3_ringacc_get_by_phandle(struct device_node *np, | |
1267 | const char *property) | |
1268 | { | |
1269 | struct device_node *ringacc_np; | |
1270 | struct k3_ringacc *ringacc = ERR_PTR(-EPROBE_DEFER); | |
1271 | struct k3_ringacc *entry; | |
1272 | ||
1273 | ringacc_np = of_parse_phandle(np, property, 0); | |
1274 | if (!ringacc_np) | |
1275 | return ERR_PTR(-ENODEV); | |
1276 | ||
1277 | mutex_lock(&k3_ringacc_list_lock); | |
1278 | list_for_each_entry(entry, &k3_ringacc_list, list) | |
1279 | if (entry->dev->of_node == ringacc_np) { | |
1280 | ringacc = entry; | |
1281 | break; | |
1282 | } | |
1283 | mutex_unlock(&k3_ringacc_list_lock); | |
1284 | of_node_put(ringacc_np); | |
1285 | ||
1286 | return ringacc; | |
1287 | } | |
1288 | EXPORT_SYMBOL_GPL(of_k3_ringacc_get_by_phandle); | |
1289 | ||
1290 | static int k3_ringacc_probe_dt(struct k3_ringacc *ringacc) | |
1291 | { | |
1292 | struct device_node *node = ringacc->dev->of_node; | |
1293 | struct device *dev = ringacc->dev; | |
1294 | struct platform_device *pdev = to_platform_device(dev); | |
1295 | int ret; | |
1296 | ||
1297 | if (!node) { | |
1298 | dev_err(dev, "device tree info unavailable\n"); | |
1299 | return -ENODEV; | |
1300 | } | |
1301 | ||
1302 | ret = of_property_read_u32(node, "ti,num-rings", &ringacc->num_rings); | |
1303 | if (ret) { | |
1304 | dev_err(dev, "ti,num-rings read failure %d\n", ret); | |
1305 | return ret; | |
1306 | } | |
1307 | ||
3277e8aa GS |
1308 | ringacc->tisci = ti_sci_get_by_phandle(node, "ti,sci"); |
1309 | if (IS_ERR(ringacc->tisci)) { | |
1310 | ret = PTR_ERR(ringacc->tisci); | |
1311 | if (ret != -EPROBE_DEFER) | |
1312 | dev_err(dev, "ti,sci read fail %d\n", ret); | |
1313 | ringacc->tisci = NULL; | |
1314 | return ret; | |
1315 | } | |
1316 | ||
1317 | ret = of_property_read_u32(node, "ti,sci-dev-id", | |
1318 | &ringacc->tisci_dev_id); | |
1319 | if (ret) { | |
1320 | dev_err(dev, "ti,sci-dev-id read fail %d\n", ret); | |
1321 | return ret; | |
1322 | } | |
1323 | ||
1324 | pdev->id = ringacc->tisci_dev_id; | |
1325 | ||
1326 | ringacc->rm_gp_range = devm_ti_sci_get_of_resource(ringacc->tisci, dev, | |
1327 | ringacc->tisci_dev_id, | |
1328 | "ti,sci-rm-range-gp-rings"); | |
1329 | if (IS_ERR(ringacc->rm_gp_range)) { | |
1330 | dev_err(dev, "Failed to allocate MSI interrupts\n"); | |
1331 | return PTR_ERR(ringacc->rm_gp_range); | |
1332 | } | |
1333 | ||
1334 | return ti_sci_inta_msi_domain_alloc_irqs(ringacc->dev, | |
1335 | ringacc->rm_gp_range); | |
1336 | } | |
1337 | ||
95e7be06 GS |
1338 | static const struct k3_ringacc_soc_data k3_ringacc_soc_data_sr1 = { |
1339 | .dma_ring_reset_quirk = 1, | |
1340 | }; | |
1341 | ||
1342 | static const struct soc_device_attribute k3_ringacc_socinfo[] = { | |
1343 | { .family = "AM65X", | |
1344 | .revision = "SR1.0", | |
1345 | .data = &k3_ringacc_soc_data_sr1 | |
1346 | }, | |
1347 | {/* sentinel */} | |
1348 | }; | |
1349 | ||
40a2a7c3 GS |
1350 | static int k3_ringacc_init(struct platform_device *pdev, |
1351 | struct k3_ringacc *ringacc) | |
3277e8aa | 1352 | { |
95e7be06 | 1353 | const struct soc_device_attribute *soc; |
3277e8aa GS |
1354 | void __iomem *base_fifo, *base_rt; |
1355 | struct device *dev = &pdev->dev; | |
1356 | struct resource *res; | |
1357 | int ret, i; | |
1358 | ||
3277e8aa GS |
1359 | dev->msi_domain = of_msi_get_domain(dev, dev->of_node, |
1360 | DOMAIN_BUS_TI_SCI_INTA_MSI); | |
1361 | if (!dev->msi_domain) { | |
1362 | dev_err(dev, "Failed to get MSI domain\n"); | |
1363 | return -EPROBE_DEFER; | |
1364 | } | |
1365 | ||
1366 | ret = k3_ringacc_probe_dt(ringacc); | |
1367 | if (ret) | |
1368 | return ret; | |
1369 | ||
95e7be06 GS |
1370 | soc = soc_device_match(k3_ringacc_socinfo); |
1371 | if (soc && soc->data) { | |
1372 | const struct k3_ringacc_soc_data *soc_data = soc->data; | |
1373 | ||
1374 | ringacc->dma_ring_reset_quirk = soc_data->dma_ring_reset_quirk; | |
1375 | } | |
1376 | ||
3277e8aa GS |
1377 | res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "rt"); |
1378 | base_rt = devm_ioremap_resource(dev, res); | |
1379 | if (IS_ERR(base_rt)) | |
1380 | return PTR_ERR(base_rt); | |
1381 | ||
1382 | res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "fifos"); | |
1383 | base_fifo = devm_ioremap_resource(dev, res); | |
1384 | if (IS_ERR(base_fifo)) | |
1385 | return PTR_ERR(base_fifo); | |
1386 | ||
1387 | res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "proxy_gcfg"); | |
1388 | ringacc->proxy_gcfg = devm_ioremap_resource(dev, res); | |
1389 | if (IS_ERR(ringacc->proxy_gcfg)) | |
1390 | return PTR_ERR(ringacc->proxy_gcfg); | |
1391 | ||
1392 | res = platform_get_resource_byname(pdev, IORESOURCE_MEM, | |
1393 | "proxy_target"); | |
1394 | ringacc->proxy_target_base = devm_ioremap_resource(dev, res); | |
1395 | if (IS_ERR(ringacc->proxy_target_base)) | |
1396 | return PTR_ERR(ringacc->proxy_target_base); | |
1397 | ||
1398 | ringacc->num_proxies = readl(&ringacc->proxy_gcfg->config) & | |
1399 | K3_RINGACC_PROXY_CFG_THREADS_MASK; | |
1400 | ||
1401 | ringacc->rings = devm_kzalloc(dev, | |
1402 | sizeof(*ringacc->rings) * | |
1403 | ringacc->num_rings, | |
1404 | GFP_KERNEL); | |
1405 | ringacc->rings_inuse = devm_kcalloc(dev, | |
1406 | BITS_TO_LONGS(ringacc->num_rings), | |
1407 | sizeof(unsigned long), GFP_KERNEL); | |
1408 | ringacc->proxy_inuse = devm_kcalloc(dev, | |
1409 | BITS_TO_LONGS(ringacc->num_proxies), | |
1410 | sizeof(unsigned long), GFP_KERNEL); | |
1411 | ||
1412 | if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse) | |
1413 | return -ENOMEM; | |
1414 | ||
1415 | for (i = 0; i < ringacc->num_rings; i++) { | |
1416 | ringacc->rings[i].rt = base_rt + | |
1417 | K3_RINGACC_RT_REGS_STEP * i; | |
1418 | ringacc->rings[i].fifos = base_fifo + | |
1419 | K3_RINGACC_FIFO_REGS_STEP * i; | |
1420 | ringacc->rings[i].parent = ringacc; | |
1421 | ringacc->rings[i].ring_id = i; | |
1422 | ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED; | |
1423 | } | |
3277e8aa GS |
1424 | |
1425 | ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops; | |
1426 | ||
3277e8aa GS |
1427 | dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n", |
1428 | ringacc->num_rings, | |
1429 | ringacc->rm_gp_range->desc[0].start, | |
1430 | ringacc->rm_gp_range->desc[0].num, | |
1431 | ringacc->tisci_dev_id); | |
1432 | dev_info(dev, "dma-ring-reset-quirk: %s\n", | |
1433 | ringacc->dma_ring_reset_quirk ? "enabled" : "disabled"); | |
1434 | dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n", | |
1435 | readl(&ringacc->proxy_gcfg->revision), ringacc->num_proxies); | |
40a2a7c3 | 1436 | |
3277e8aa GS |
1437 | return 0; |
1438 | } | |
1439 | ||
40a2a7c3 GS |
1440 | struct ringacc_match_data { |
1441 | struct k3_ringacc_ops ops; | |
1442 | }; | |
1443 | ||
1444 | static struct ringacc_match_data k3_ringacc_data = { | |
1445 | .ops = { | |
1446 | .init = k3_ringacc_init, | |
1447 | }, | |
1448 | }; | |
1449 | ||
3277e8aa GS |
1450 | /* Match table for of_platform binding */ |
1451 | static const struct of_device_id k3_ringacc_of_match[] = { | |
40a2a7c3 | 1452 | { .compatible = "ti,am654-navss-ringacc", .data = &k3_ringacc_data, }, |
3277e8aa GS |
1453 | {}, |
1454 | }; | |
1455 | ||
d782298c GS |
1456 | struct k3_ringacc *k3_ringacc_dmarings_init(struct platform_device *pdev, |
1457 | struct k3_ringacc_init_data *data) | |
1458 | { | |
1459 | struct device *dev = &pdev->dev; | |
1460 | struct k3_ringacc *ringacc; | |
1461 | void __iomem *base_rt; | |
1462 | struct resource *res; | |
1463 | int i; | |
1464 | ||
1465 | ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL); | |
1466 | if (!ringacc) | |
1467 | return ERR_PTR(-ENOMEM); | |
1468 | ||
1469 | ringacc->dev = dev; | |
1470 | ringacc->dma_rings = true; | |
1471 | ringacc->num_rings = data->num_rings; | |
1472 | ringacc->tisci = data->tisci; | |
1473 | ringacc->tisci_dev_id = data->tisci_dev_id; | |
1474 | ||
1475 | mutex_init(&ringacc->req_lock); | |
1476 | ||
1477 | res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ringrt"); | |
1478 | base_rt = devm_ioremap_resource(dev, res); | |
1479 | if (IS_ERR(base_rt)) | |
115ff12a | 1480 | return ERR_CAST(base_rt); |
d782298c GS |
1481 | |
1482 | ringacc->rings = devm_kzalloc(dev, | |
1483 | sizeof(*ringacc->rings) * | |
1484 | ringacc->num_rings * 2, | |
1485 | GFP_KERNEL); | |
1486 | ringacc->rings_inuse = devm_kcalloc(dev, | |
1487 | BITS_TO_LONGS(ringacc->num_rings), | |
1488 | sizeof(unsigned long), GFP_KERNEL); | |
1489 | ||
1490 | if (!ringacc->rings || !ringacc->rings_inuse) | |
1491 | return ERR_PTR(-ENOMEM); | |
1492 | ||
1493 | for (i = 0; i < ringacc->num_rings; i++) { | |
1494 | struct k3_ring *ring = &ringacc->rings[i]; | |
1495 | ||
1496 | ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i; | |
1497 | ring->parent = ringacc; | |
1498 | ring->ring_id = i; | |
1499 | ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; | |
1500 | ||
1501 | ring = &ringacc->rings[ringacc->num_rings + i]; | |
1502 | ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i + | |
1503 | K3_DMARING_RT_REGS_REVERSE_OFS; | |
1504 | ring->parent = ringacc; | |
1505 | ring->ring_id = i; | |
1506 | ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; | |
1507 | ring->flags = K3_RING_FLAG_REVERSE; | |
1508 | } | |
1509 | ||
1510 | ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops; | |
1511 | ||
1512 | dev_info(dev, "Number of rings: %u\n", ringacc->num_rings); | |
1513 | ||
1514 | return ringacc; | |
1515 | } | |
1516 | EXPORT_SYMBOL_GPL(k3_ringacc_dmarings_init); | |
1517 | ||
40a2a7c3 GS |
1518 | static int k3_ringacc_probe(struct platform_device *pdev) |
1519 | { | |
1520 | const struct ringacc_match_data *match_data; | |
40a2a7c3 GS |
1521 | struct device *dev = &pdev->dev; |
1522 | struct k3_ringacc *ringacc; | |
1523 | int ret; | |
1524 | ||
a8fc8e5b SA |
1525 | match_data = of_device_get_match_data(&pdev->dev); |
1526 | if (!match_data) | |
40a2a7c3 | 1527 | return -ENODEV; |
40a2a7c3 GS |
1528 | |
1529 | ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL); | |
1530 | if (!ringacc) | |
1531 | return -ENOMEM; | |
1532 | ||
1533 | ringacc->dev = dev; | |
1534 | mutex_init(&ringacc->req_lock); | |
1535 | ringacc->ops = &match_data->ops; | |
1536 | ||
1537 | ret = ringacc->ops->init(pdev, ringacc); | |
1538 | if (ret) | |
1539 | return ret; | |
1540 | ||
1541 | dev_set_drvdata(dev, ringacc); | |
1542 | ||
1543 | mutex_lock(&k3_ringacc_list_lock); | |
1544 | list_add_tail(&ringacc->list, &k3_ringacc_list); | |
1545 | mutex_unlock(&k3_ringacc_list_lock); | |
1546 | ||
1547 | return 0; | |
1548 | } | |
1549 | ||
3277e8aa GS |
1550 | static struct platform_driver k3_ringacc_driver = { |
1551 | .probe = k3_ringacc_probe, | |
1552 | .driver = { | |
1553 | .name = "k3-ringacc", | |
1554 | .of_match_table = k3_ringacc_of_match, | |
1555 | .suppress_bind_attrs = true, | |
1556 | }, | |
1557 | }; | |
1558 | builtin_platform_driver(k3_ringacc_driver); |