]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - drivers/soc/ti/k3-ringacc.c
projects / mirror_ubuntu-jammy-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'for-5.12-rc6-tag' of git://git.kernel.org/pub/scm/linux/kernel/git/kdave...
[mirror_ubuntu-jammy-kernel.git] / drivers / soc / ti / k3-ringacc.c
CommitLineData
3277e8aa
GS		 1// SPDX-License-Identifier: GPL-2.0
2/*
3 * TI K3 NAVSS Ring Accelerator subsystem driver
4 *
5 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
6 */
7
8#include <linux/dma-mapping.h>
9#include <linux/io.h>
10#include <linux/init.h>
11#include <linux/of.h>
a8fc8e5b 12#include <linux/of_device.h>
3277e8aa 13#include <linux/platform_device.h>
95e7be06 14#include <linux/sys_soc.h>
d782298c 15#include <linux/dma/ti-cppi5.h>
3277e8aa
GS		 16#include <linux/soc/ti/k3-ringacc.h>
17#include <linux/soc/ti/ti_sci_protocol.h>
18#include <linux/soc/ti/ti_sci_inta_msi.h>
19#include <linux/of_irq.h>
20#include <linux/irqdomain.h>
21
22static LIST_HEAD(k3_ringacc_list);
23static DEFINE_MUTEX(k3_ringacc_list_lock);
24
25#define K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK GENMASK(19, 0)
d782298c 26#define K3_DMARING_CFG_RING_SIZE_ELCNT_MASK GENMASK(15, 0)
3277e8aa
GS		 27
28/**
29 * struct k3_ring_rt_regs - The RA realtime Control/Status Registers region
30 *
31 * @resv_16: Reserved
32 * @db: Ring Doorbell Register
33 * @resv_4: Reserved
34 * @occ: Ring Occupancy Register
35 * @indx: Ring Current Index Register
36 * @hwocc: Ring Hardware Occupancy Register
37 * @hwindx: Ring Hardware Current Index Register
38 */
39struct k3_ring_rt_regs {
40 u32 resv_16[4];
41 u32 db;
42 u32 resv_4[1];
43 u32 occ;
44 u32 indx;
45 u32 hwocc;
46 u32 hwindx;
47};
48
d782298c
GS		 49#define K3_RINGACC_RT_REGS_STEP 0x1000
50#define K3_DMARING_RT_REGS_STEP 0x2000
51#define K3_DMARING_RT_REGS_REVERSE_OFS 0x1000
52#define K3_RINGACC_RT_OCC_MASK GENMASK(20, 0)
53#define K3_DMARING_RT_OCC_TDOWN_COMPLETE BIT(31)
54#define K3_DMARING_RT_DB_ENTRY_MASK GENMASK(7, 0)
55#define K3_DMARING_RT_DB_TDOWN_ACK BIT(31)
3277e8aa
GS		 56
57/**
58 * struct k3_ring_fifo_regs - The Ring Accelerator Queues Registers region
59 *
60 * @head_data: Ring Head Entry Data Registers
61 * @tail_data: Ring Tail Entry Data Registers
62 * @peek_head_data: Ring Peek Head Entry Data Regs
63 * @peek_tail_data: Ring Peek Tail Entry Data Regs
64 */
65struct k3_ring_fifo_regs {
66 u32 head_data[128];
67 u32 tail_data[128];
68 u32 peek_head_data[128];
69 u32 peek_tail_data[128];
70};
71
72/**
73 * struct k3_ringacc_proxy_gcfg_regs - RA Proxy Global Config MMIO Region
74 *
75 * @revision: Revision Register
76 * @config: Config Register
77 */
78struct k3_ringacc_proxy_gcfg_regs {
79 u32 revision;
80 u32 config;
81};
82
83#define K3_RINGACC_PROXY_CFG_THREADS_MASK GENMASK(15, 0)
84
85/**
86 * struct k3_ringacc_proxy_target_regs - Proxy Datapath MMIO Region
87 *
88 * @control: Proxy Control Register
89 * @status: Proxy Status Register
90 * @resv_512: Reserved
91 * @data: Proxy Data Register
92 */
93struct k3_ringacc_proxy_target_regs {
94 u32 control;
95 u32 status;
96 u8 resv_512[504];
97 u32 data[128];
98};
99
100#define K3_RINGACC_PROXY_TARGET_STEP 0x1000
101#define K3_RINGACC_PROXY_NOT_USED (-1)
102
103enum k3_ringacc_proxy_access_mode {
104 PROXY_ACCESS_MODE_HEAD = 0,
105 PROXY_ACCESS_MODE_TAIL = 1,
106 PROXY_ACCESS_MODE_PEEK_HEAD = 2,
107 PROXY_ACCESS_MODE_PEEK_TAIL = 3,
108};
109
110#define K3_RINGACC_FIFO_WINDOW_SIZE_BYTES (512U)
111#define K3_RINGACC_FIFO_REGS_STEP 0x1000
112#define K3_RINGACC_MAX_DB_RING_CNT (127U)
113
114struct k3_ring_ops {
115 int (*push_tail)(struct k3_ring *ring, void *elm);
116 int (*push_head)(struct k3_ring *ring, void *elm);
117 int (*pop_tail)(struct k3_ring *ring, void *elm);
118 int (*pop_head)(struct k3_ring *ring, void *elm);
119};
120
6b3da0b4
PU		 121/**
122 * struct k3_ring_state - Internal state tracking structure
123 *
124 * @free: Number of free entries
125 * @occ: Occupancy
126 * @windex: Write index
127 * @rindex: Read index
128 */
129struct k3_ring_state {
130 u32 free;
131 u32 occ;
132 u32 windex;
133 u32 rindex;
d782298c 134 u32 tdown_complete:1;
6b3da0b4
PU		 135};
136
3277e8aa
GS		 137/**
138 * struct k3_ring - RA Ring descriptor
139 *
140 * @rt: Ring control/status registers
141 * @fifos: Ring queues registers
142 * @proxy: Ring Proxy Datapath registers
143 * @ring_mem_dma: Ring buffer dma address
144 * @ring_mem_virt: Ring buffer virt address
145 * @ops: Ring operations
146 * @size: Ring size in elements
147 * @elm_size: Size of the ring element
148 * @mode: Ring mode
149 * @flags: flags
50883aff 150 * @state: Ring state
3277e8aa
GS		 151 * @ring_id: Ring Id
152 * @parent: Pointer on struct @k3_ringacc
153 * @use_count: Use count for shared rings
154 * @proxy_id: RA Ring Proxy Id (only if @K3_RINGACC_RING_USE_PROXY)
8c42379e 155 * @dma_dev: device to be used for DMA API (allocation, mapping)
d782298c 156 * @asel: Address Space Select value for physical addresses
3277e8aa
GS		 157 */
158struct k3_ring {
159 struct k3_ring_rt_regs __iomem *rt;
160 struct k3_ring_fifo_regs __iomem *fifos;
161 struct k3_ringacc_proxy_target_regs __iomem *proxy;
162 dma_addr_t ring_mem_dma;
163 void *ring_mem_virt;
164 struct k3_ring_ops *ops;
165 u32 size;
166 enum k3_ring_size elm_size;
167 enum k3_ring_mode mode;
168 u32 flags;
169#define K3_RING_FLAG_BUSY BIT(1)
170#define K3_RING_FLAG_SHARED BIT(2)
d782298c 171#define K3_RING_FLAG_REVERSE BIT(3)
6b3da0b4 172 struct k3_ring_state state;
3277e8aa
GS		 173 u32 ring_id;
174 struct k3_ringacc *parent;
175 u32 use_count;
176 int proxy_id;
8c42379e 177 struct device *dma_dev;
d782298c
GS		 178 u32 asel;
179#define K3_ADDRESS_ASEL_SHIFT 48
3277e8aa
GS		 180};
181
40a2a7c3
GS		 182struct k3_ringacc_ops {
183 int (*init)(struct platform_device *pdev, struct k3_ringacc *ringacc);
184};
185
3277e8aa
GS		 186/**
187 * struct k3_ringacc - Rings accelerator descriptor
188 *
189 * @dev: pointer on RA device
190 * @proxy_gcfg: RA proxy global config registers
191 * @proxy_target_base: RA proxy datapath region
192 * @num_rings: number of ring in RA
193 * @rings_inuse: bitfield for ring usage tracking
194 * @rm_gp_range: general purpose rings range from tisci
195 * @dma_ring_reset_quirk: DMA reset w/a enable
196 * @num_proxies: number of RA proxies
197 * @proxy_inuse: bitfield for proxy usage tracking
198 * @rings: array of rings descriptors (struct @k3_ring)
199 * @list: list of RAs in the system
200 * @req_lock: protect rings allocation
201 * @tisci: pointer ti-sci handle
202 * @tisci_ring_ops: ti-sci rings ops
203 * @tisci_dev_id: ti-sci device id
40a2a7c3 204 * @ops: SoC specific ringacc operation
d782298c 205 * @dma_rings: indicate DMA ring (dual ring within BCDMA/PKTDMA)
3277e8aa
GS		 206 */
207struct k3_ringacc {
208 struct device *dev;
209 struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg;
210 void __iomem *proxy_target_base;
211 u32 num_rings; /* number of rings in Ringacc module */
212 unsigned long *rings_inuse;
213 struct ti_sci_resource *rm_gp_range;
214
215 bool dma_ring_reset_quirk;
216 u32 num_proxies;
217 unsigned long *proxy_inuse;
218
219 struct k3_ring *rings;
220 struct list_head list;
221 struct mutex req_lock; /* protect rings allocation */
222
223 const struct ti_sci_handle *tisci;
224 const struct ti_sci_rm_ringacc_ops *tisci_ring_ops;
225 u32 tisci_dev_id;
40a2a7c3
GS		 226
227 const struct k3_ringacc_ops *ops;
d782298c 228 bool dma_rings;
3277e8aa
GS		 229};
230
95e7be06
GS		 231/**
232 * struct k3_ringacc - Rings accelerator SoC data
233 *
234 * @dma_ring_reset_quirk: DMA reset w/a enable
235 */
236struct k3_ringacc_soc_data {
237 unsigned dma_ring_reset_quirk:1;
238};
239
d782298c
GS		 240static int k3_ringacc_ring_read_occ(struct k3_ring *ring)
241{
242 return readl(&ring->rt->occ) & K3_RINGACC_RT_OCC_MASK;
243}
244
245static void k3_ringacc_ring_update_occ(struct k3_ring *ring)
246{
247 u32 val;
248
249 val = readl(&ring->rt->occ);
250
251 ring->state.occ = val & K3_RINGACC_RT_OCC_MASK;
252 ring->state.tdown_complete = !!(val & K3_DMARING_RT_OCC_TDOWN_COMPLETE);
253}
254
3277e8aa
GS		 255static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring)
256{
257 return K3_RINGACC_FIFO_WINDOW_SIZE_BYTES -
258 (4 << ring->elm_size);
259}
260
261static void *k3_ringacc_get_elm_addr(struct k3_ring *ring, u32 idx)
262{
263 return (ring->ring_mem_virt + idx * (4 << ring->elm_size));
264}
265
266static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem);
267static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem);
d782298c
GS		 268static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem);
269static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem);
3277e8aa
GS		 270
271static struct k3_ring_ops k3_ring_mode_ring_ops = {
272 .push_tail = k3_ringacc_ring_push_mem,
273 .pop_head = k3_ringacc_ring_pop_mem,
274};
275
d782298c
GS		 276static struct k3_ring_ops k3_dmaring_fwd_ops = {
277 .push_tail = k3_ringacc_ring_push_mem,
278 .pop_head = k3_dmaring_fwd_pop,
279};
280
281static struct k3_ring_ops k3_dmaring_reverse_ops = {
282 /* Reverse side of the DMA ring can only be popped by SW */
283 .pop_head = k3_dmaring_reverse_pop,
284};
285
3277e8aa
GS		 286static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem);
287static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem);
288static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem);
289static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem);
290
291static struct k3_ring_ops k3_ring_mode_msg_ops = {
292 .push_tail = k3_ringacc_ring_push_io,
293 .push_head = k3_ringacc_ring_push_head_io,
294 .pop_tail = k3_ringacc_ring_pop_tail_io,
295 .pop_head = k3_ringacc_ring_pop_io,
296};
297
298static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem);
299static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem);
300static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem);
301static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem);
302
303static struct k3_ring_ops k3_ring_mode_proxy_ops = {
304 .push_tail = k3_ringacc_ring_push_tail_proxy,
305 .push_head = k3_ringacc_ring_push_head_proxy,
306 .pop_tail = k3_ringacc_ring_pop_tail_proxy,
307 .pop_head = k3_ringacc_ring_pop_head_proxy,
308};
309
310static void k3_ringacc_ring_dump(struct k3_ring *ring)
311{
312 struct device *dev = ring->parent->dev;
313
314 dev_dbg(dev, "dump ring: %d\n", ring->ring_id);
315 dev_dbg(dev, "dump mem virt %p, dma %pad\n", ring->ring_mem_virt,
316 &ring->ring_mem_dma);
317 dev_dbg(dev, "dump elmsize %d, size %d, mode %d, proxy_id %d\n",
318 ring->elm_size, ring->size, ring->mode, ring->proxy_id);
175e663f 319 dev_dbg(dev, "dump flags %08X\n", ring->flags);
3277e8aa
GS		 320
321 dev_dbg(dev, "dump ring_rt_regs: db%08x\n", readl(&ring->rt->db));
322 dev_dbg(dev, "dump occ%08x\n", readl(&ring->rt->occ));
323 dev_dbg(dev, "dump indx%08x\n", readl(&ring->rt->indx));
324 dev_dbg(dev, "dump hwocc%08x\n", readl(&ring->rt->hwocc));
325 dev_dbg(dev, "dump hwindx%08x\n", readl(&ring->rt->hwindx));
326
327 if (ring->ring_mem_virt)
328 print_hex_dump_debug("dump ring_mem_virt ", DUMP_PREFIX_NONE,
329 16, 1, ring->ring_mem_virt, 16 * 8, false);
330}
331
332struct k3_ring *k3_ringacc_request_ring(struct k3_ringacc *ringacc,
333 int id, u32 flags)
334{
335 int proxy_id = K3_RINGACC_PROXY_NOT_USED;
336
337 mutex_lock(&ringacc->req_lock);
338
339 if (id == K3_RINGACC_RING_ID_ANY) {
340 /* Request for any general purpose ring */
341 struct ti_sci_resource_desc *gp_rings =
342 &ringacc->rm_gp_range->desc[0];
343 unsigned long size;
344
345 size = gp_rings->start + gp_rings->num;
346 id = find_next_zero_bit(ringacc->rings_inuse, size,
347 gp_rings->start);
348 if (id == size)
349 goto error;
350 } else if (id < 0) {
351 goto error;
352 }
353
354 if (test_bit(id, ringacc->rings_inuse) &&
355 !(ringacc->rings[id].flags & K3_RING_FLAG_SHARED))
356 goto error;
357 else if (ringacc->rings[id].flags & K3_RING_FLAG_SHARED)
358 goto out;
359
360 if (flags & K3_RINGACC_RING_USE_PROXY) {
361 proxy_id = find_next_zero_bit(ringacc->proxy_inuse,
362 ringacc->num_proxies, 0);
363 if (proxy_id == ringacc->num_proxies)
364 goto error;
365 }
366
367 if (proxy_id != K3_RINGACC_PROXY_NOT_USED) {
368 set_bit(proxy_id, ringacc->proxy_inuse);
369 ringacc->rings[id].proxy_id = proxy_id;
370 dev_dbg(ringacc->dev, "Giving ring#%d proxy#%d\n", id,
371 proxy_id);
372 } else {
373 dev_dbg(ringacc->dev, "Giving ring#%d\n", id);
374 }
375
376 set_bit(id, ringacc->rings_inuse);
377out:
378 ringacc->rings[id].use_count++;
379 mutex_unlock(&ringacc->req_lock);
380 return &ringacc->rings[id];
381
382error:
383 mutex_unlock(&ringacc->req_lock);
384 return NULL;
385}
386EXPORT_SYMBOL_GPL(k3_ringacc_request_ring);
387
d782298c
GS		 388static int k3_dmaring_request_dual_ring(struct k3_ringacc *ringacc, int fwd_id,
389 struct k3_ring **fwd_ring,
390 struct k3_ring **compl_ring)
391{
392 int ret = 0;
393
394 /*
395 * DMA rings must be requested by ID, completion ring is the reverse
396 * side of the forward ring
397 */
398 if (fwd_id < 0)
399 return -EINVAL;
400
401 mutex_lock(&ringacc->req_lock);
402
403 if (test_bit(fwd_id, ringacc->rings_inuse)) {
404 ret = -EBUSY;
405 goto error;
406 }
407
408 *fwd_ring = &ringacc->rings[fwd_id];
409 *compl_ring = &ringacc->rings[fwd_id + ringacc->num_rings];
410 set_bit(fwd_id, ringacc->rings_inuse);
411 ringacc->rings[fwd_id].use_count++;
412 dev_dbg(ringacc->dev, "Giving ring#%d\n", fwd_id);
413
414 mutex_unlock(&ringacc->req_lock);
415 return 0;
416
417error:
418 mutex_unlock(&ringacc->req_lock);
419 return ret;
420}
421
43148b1c
GS		 422int k3_ringacc_request_rings_pair(struct k3_ringacc *ringacc,
423 int fwd_id, int compl_id,
424 struct k3_ring **fwd_ring,
425 struct k3_ring **compl_ring)
426{
427 int ret = 0;
428
429 if (!fwd_ring || !compl_ring)
430 return -EINVAL;
431
d782298c
GS		 432 if (ringacc->dma_rings)
433 return k3_dmaring_request_dual_ring(ringacc, fwd_id,
434 fwd_ring, compl_ring);
435
43148b1c
GS		 436 *fwd_ring = k3_ringacc_request_ring(ringacc, fwd_id, 0);
437 if (!(*fwd_ring))
438 return -ENODEV;
439
440 *compl_ring = k3_ringacc_request_ring(ringacc, compl_id, 0);
441 if (!(*compl_ring)) {
442 k3_ringacc_ring_free(*fwd_ring);
443 ret = -ENODEV;
444 }
445
446 return ret;
447}
448EXPORT_SYMBOL_GPL(k3_ringacc_request_rings_pair);
449
3277e8aa
GS		 450static void k3_ringacc_ring_reset_sci(struct k3_ring *ring)
451{
bb49ca00 452 struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 };
3277e8aa
GS		 453 struct k3_ringacc *ringacc = ring->parent;
454 int ret;
455
bb49ca00
PU		 456 ring_cfg.nav_id = ringacc->tisci_dev_id;
457 ring_cfg.index = ring->ring_id;
458 ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID;
459 ring_cfg.count = ring->size;
460
461 ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg);
3277e8aa
GS		 462 if (ret)
463 dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n",
464 ret, ring->ring_id);
465}
466
467void k3_ringacc_ring_reset(struct k3_ring *ring)
468{
469 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
470 return;
471
6b3da0b4 472 memset(&ring->state, 0, sizeof(ring->state));
3277e8aa
GS		 473
474 k3_ringacc_ring_reset_sci(ring);
475}
476EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset);
477
478static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_ring *ring,
479 enum k3_ring_mode mode)
480{
bb49ca00 481 struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 };
3277e8aa
GS		 482 struct k3_ringacc *ringacc = ring->parent;
483 int ret;
484
bb49ca00
PU		 485 ring_cfg.nav_id = ringacc->tisci_dev_id;
486 ring_cfg.index = ring->ring_id;
487 ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_RING_MODE_VALID;
488 ring_cfg.mode = mode;
489
490 ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg);
3277e8aa
GS		 491 if (ret)
492 dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n",
493 ret, ring->ring_id);
494}
495
496void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ)
497{
498 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
499 return;
500
501 if (!ring->parent->dma_ring_reset_quirk)
502 goto reset;
503
504 if (!occ)
d782298c 505 occ = k3_ringacc_ring_read_occ(ring);
3277e8aa
GS		 506
507 if (occ) {
508 u32 db_ring_cnt, db_ring_cnt_cur;
509
510 dev_dbg(ring->parent->dev, "%s %u occ: %u\n", __func__,
511 ring->ring_id, occ);
512 /* TI-SCI ring reset */
513 k3_ringacc_ring_reset_sci(ring);
514
515 /*
516 * Setup the ring in ring/doorbell mode (if not already in this
517 * mode)
518 */
519 if (ring->mode != K3_RINGACC_RING_MODE_RING)
520 k3_ringacc_ring_reconfig_qmode_sci(
521 ring, K3_RINGACC_RING_MODE_RING);
522 /*
523 * Ring the doorbell 2**22 – ringOcc times.
524 * This will wrap the internal UDMAP ring state occupancy
525 * counter (which is 21-bits wide) to 0.
526 */
527 db_ring_cnt = (1U << 22) - occ;
528
529 while (db_ring_cnt != 0) {
530 /*
531 * Ring the doorbell with the maximum count each
532 * iteration if possible to minimize the total
533 * of writes
534 */
535 if (db_ring_cnt > K3_RINGACC_MAX_DB_RING_CNT)
536 db_ring_cnt_cur = K3_RINGACC_MAX_DB_RING_CNT;
537 else
538 db_ring_cnt_cur = db_ring_cnt;
539
540 writel(db_ring_cnt_cur, &ring->rt->db);
541 db_ring_cnt -= db_ring_cnt_cur;
542 }
543
544 /* Restore the original ring mode (if not ring mode) */
545 if (ring->mode != K3_RINGACC_RING_MODE_RING)
546 k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode);
547 }
548
549reset:
550 /* Reset the ring */
551 k3_ringacc_ring_reset(ring);
552}
553EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset_dma);
554
555static void k3_ringacc_ring_free_sci(struct k3_ring *ring)
556{
bb49ca00 557 struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 };
3277e8aa
GS		 558 struct k3_ringacc *ringacc = ring->parent;
559 int ret;
560
bb49ca00
PU		 561 ring_cfg.nav_id = ringacc->tisci_dev_id;
562 ring_cfg.index = ring->ring_id;
563 ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER;
564
565 ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg);
3277e8aa
GS		 566 if (ret)
567 dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n",
568 ret, ring->ring_id);
569}
570
571int k3_ringacc_ring_free(struct k3_ring *ring)
572{
573 struct k3_ringacc *ringacc;
574
575 if (!ring)
576 return -EINVAL;
577
578 ringacc = ring->parent;
579
d782298c
GS		 580 /*
581 * DMA rings: rings shared memory and configuration, only forward ring
582 * is configured and reverse ring considered as slave.
583 */
584 if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE))
585 return 0;
586
3277e8aa
GS		 587 dev_dbg(ring->parent->dev, "flags: 0x%08x\n", ring->flags);
588
589 if (!test_bit(ring->ring_id, ringacc->rings_inuse))
590 return -EINVAL;
591
592 mutex_lock(&ringacc->req_lock);
593
594 if (--ring->use_count)
595 goto out;
596
597 if (!(ring->flags & K3_RING_FLAG_BUSY))
598 goto no_init;
599
600 k3_ringacc_ring_free_sci(ring);
601
8c42379e 602 dma_free_coherent(ring->dma_dev,
3277e8aa
GS		 603 ring->size * (4 << ring->elm_size),
604 ring->ring_mem_virt, ring->ring_mem_dma);
605 ring->flags = 0;
606 ring->ops = NULL;
8c42379e 607 ring->dma_dev = NULL;
d782298c
GS		 608 ring->asel = 0;
609
3277e8aa
GS		 610 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) {
611 clear_bit(ring->proxy_id, ringacc->proxy_inuse);
612 ring->proxy = NULL;
613 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
614 }
615
616no_init:
617 clear_bit(ring->ring_id, ringacc->rings_inuse);
618
619out:
620 mutex_unlock(&ringacc->req_lock);
621 return 0;
622}
623EXPORT_SYMBOL_GPL(k3_ringacc_ring_free);
624
625u32 k3_ringacc_get_ring_id(struct k3_ring *ring)
626{
627 if (!ring)
628 return -EINVAL;
629
630 return ring->ring_id;
631}
632EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_id);
633
634u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring)
635{
636 if (!ring)
637 return -EINVAL;
638
639 return ring->parent->tisci_dev_id;
640}
641EXPORT_SYMBOL_GPL(k3_ringacc_get_tisci_dev_id);
642
643int k3_ringacc_get_ring_irq_num(struct k3_ring *ring)
644{
645 int irq_num;
646
647 if (!ring)
648 return -EINVAL;
649
650 irq_num = ti_sci_inta_msi_get_virq(ring->parent->dev, ring->ring_id);
651 if (irq_num <= 0)
652 irq_num = -EINVAL;
653 return irq_num;
654}
655EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_irq_num);
656
657static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring)
658{
bb49ca00 659 struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 };
3277e8aa 660 struct k3_ringacc *ringacc = ring->parent;
3277e8aa
GS		 661 int ret;
662
663 if (!ringacc->tisci)
664 return -EINVAL;
665
bb49ca00
PU		 666 ring_cfg.nav_id = ringacc->tisci_dev_id;
667 ring_cfg.index = ring->ring_id;
668 ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER;
669 ring_cfg.addr_lo = lower_32_bits(ring->ring_mem_dma);
670 ring_cfg.addr_hi = upper_32_bits(ring->ring_mem_dma);
671 ring_cfg.count = ring->size;
672 ring_cfg.mode = ring->mode;
673 ring_cfg.size = ring->elm_size;
d782298c 674 ring_cfg.asel = ring->asel;
bb49ca00
PU		 675
676 ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg);
3277e8aa
GS		 677 if (ret)
678 dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n",
bb49ca00 679 ret, ring->ring_id);
3277e8aa
GS		 680
681 return ret;
682}
683
d782298c
GS		 684static int k3_dmaring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg)
685{
686 struct k3_ringacc *ringacc;
687 struct k3_ring *reverse_ring;
688 int ret = 0;
689
690 if (cfg->elm_size != K3_RINGACC_RING_ELSIZE_8 ||
691 cfg->mode != K3_RINGACC_RING_MODE_RING ||
692 cfg->size & ~K3_DMARING_CFG_RING_SIZE_ELCNT_MASK)
693 return -EINVAL;
694
695 ringacc = ring->parent;
696
697 /*
698 * DMA rings: rings shared memory and configuration, only forward ring
699 * is configured and reverse ring considered as slave.
700 */
701 if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE))
702 return 0;
703
704 if (!test_bit(ring->ring_id, ringacc->rings_inuse))
705 return -EINVAL;
706
707 ring->size = cfg->size;
708 ring->elm_size = cfg->elm_size;
709 ring->mode = cfg->mode;
710 ring->asel = cfg->asel;
711 ring->dma_dev = cfg->dma_dev;
712 if (!ring->dma_dev) {
713 dev_warn(ringacc->dev, "dma_dev is not provided for ring%d\n",
714 ring->ring_id);
715 ring->dma_dev = ringacc->dev;
716 }
717
718 memset(&ring->state, 0, sizeof(ring->state));
719
720 ring->ops = &k3_dmaring_fwd_ops;
721
722 ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev,
723 ring->size * (4 << ring->elm_size),
724 &ring->ring_mem_dma, GFP_KERNEL);
725 if (!ring->ring_mem_virt) {
726 dev_err(ringacc->dev, "Failed to alloc ring mem\n");
727 ret = -ENOMEM;
728 goto err_free_ops;
729 }
730
731 ret = k3_ringacc_ring_cfg_sci(ring);
732 if (ret)
733 goto err_free_mem;
734
735 ring->flags |= K3_RING_FLAG_BUSY;
736
737 k3_ringacc_ring_dump(ring);
738
739 /* DMA rings: configure reverse ring */
740 reverse_ring = &ringacc->rings[ring->ring_id + ringacc->num_rings];
741 reverse_ring->size = cfg->size;
742 reverse_ring->elm_size = cfg->elm_size;
743 reverse_ring->mode = cfg->mode;
744 reverse_ring->asel = cfg->asel;
745 memset(&reverse_ring->state, 0, sizeof(reverse_ring->state));
746 reverse_ring->ops = &k3_dmaring_reverse_ops;
747
748 reverse_ring->ring_mem_virt = ring->ring_mem_virt;
749 reverse_ring->ring_mem_dma = ring->ring_mem_dma;
750 reverse_ring->flags |= K3_RING_FLAG_BUSY;
751 k3_ringacc_ring_dump(reverse_ring);
752
753 return 0;
754
755err_free_mem:
756 dma_free_coherent(ring->dma_dev,
757 ring->size * (4 << ring->elm_size),
758 ring->ring_mem_virt,
759 ring->ring_mem_dma);
760err_free_ops:
761 ring->ops = NULL;
762 ring->proxy = NULL;
763 ring->dma_dev = NULL;
764 ring->asel = 0;
765 return ret;
766}
767
3277e8aa
GS		 768int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg)
769{
80ff73f7 770 struct k3_ringacc *ringacc;
3277e8aa
GS		 771 int ret = 0;
772
773 if (!ring || !cfg)
774 return -EINVAL;
d782298c 775
80ff73f7
GS		 776 ringacc = ring->parent;
777
d782298c
GS		 778 if (ringacc->dma_rings)
779 return k3_dmaring_cfg(ring, cfg);
780
3277e8aa
GS		 781 if (cfg->elm_size > K3_RINGACC_RING_ELSIZE_256 ||
782 cfg->mode >= K3_RINGACC_RING_MODE_INVALID ||
783 cfg->size & ~K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK ||
784 !test_bit(ring->ring_id, ringacc->rings_inuse))
785 return -EINVAL;
786
787 if (cfg->mode == K3_RINGACC_RING_MODE_MESSAGE &&
788 ring->proxy_id == K3_RINGACC_PROXY_NOT_USED &&
789 cfg->elm_size > K3_RINGACC_RING_ELSIZE_8) {
790 dev_err(ringacc->dev,
791 "Message mode must use proxy for %u element size\n",
792 4 << ring->elm_size);
793 return -EINVAL;
794 }
795
796 /*
797 * In case of shared ring only the first user (master user) can
798 * configure the ring. The sequence should be by the client:
799 * ring = k3_ringacc_request_ring(ringacc, ring_id, 0); # master user
800 * k3_ringacc_ring_cfg(ring, cfg); # master configuration
801 * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED);
802 * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED);
803 */
804 if (ring->use_count != 1)
805 return 0;
806
807 ring->size = cfg->size;
808 ring->elm_size = cfg->elm_size;
809 ring->mode = cfg->mode;
6b3da0b4 810 memset(&ring->state, 0, sizeof(ring->state));
3277e8aa
GS		 811
812 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED)
813 ring->proxy = ringacc->proxy_target_base +
814 ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP;
815
816 switch (ring->mode) {
817 case K3_RINGACC_RING_MODE_RING:
818 ring->ops = &k3_ring_mode_ring_ops;
8c42379e
PU		 819 ring->dma_dev = cfg->dma_dev;
820 if (!ring->dma_dev)
821 ring->dma_dev = ringacc->dev;
3277e8aa
GS		 822 break;
823 case K3_RINGACC_RING_MODE_MESSAGE:
8c42379e 824 ring->dma_dev = ringacc->dev;
3277e8aa
GS		 825 if (ring->proxy)
826 ring->ops = &k3_ring_mode_proxy_ops;
827 else
828 ring->ops = &k3_ring_mode_msg_ops;
829 break;
830 default:
831 ring->ops = NULL;
832 ret = -EINVAL;
833 goto err_free_proxy;
1d036016 834 }
3277e8aa 835
8c42379e
PU		 836 ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev,
837 ring->size * (4 << ring->elm_size),
838 &ring->ring_mem_dma, GFP_KERNEL);
3277e8aa
GS		 839 if (!ring->ring_mem_virt) {
840 dev_err(ringacc->dev, "Failed to alloc ring mem\n");
841 ret = -ENOMEM;
842 goto err_free_ops;
843 }
844
845 ret = k3_ringacc_ring_cfg_sci(ring);
846
847 if (ret)
848 goto err_free_mem;
849
850 ring->flags |= K3_RING_FLAG_BUSY;
851 ring->flags |= (cfg->flags & K3_RINGACC_RING_SHARED) ?
852 K3_RING_FLAG_SHARED : 0;
853
854 k3_ringacc_ring_dump(ring);
855
856 return 0;
857
858err_free_mem:
8c42379e 859 dma_free_coherent(ring->dma_dev,
3277e8aa
GS		 860 ring->size * (4 << ring->elm_size),
861 ring->ring_mem_virt,
862 ring->ring_mem_dma);
863err_free_ops:
864 ring->ops = NULL;
8c42379e 865 ring->dma_dev = NULL;
3277e8aa
GS		 866err_free_proxy:
867 ring->proxy = NULL;
868 return ret;
869}
870EXPORT_SYMBOL_GPL(k3_ringacc_ring_cfg);
871
872u32 k3_ringacc_ring_get_size(struct k3_ring *ring)
873{
874 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
875 return -EINVAL;
876
877 return ring->size;
878}
879EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_size);
880
881u32 k3_ringacc_ring_get_free(struct k3_ring *ring)
882{
883 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
884 return -EINVAL;
885
6b3da0b4 886 if (!ring->state.free)
d782298c 887 ring->state.free = ring->size - k3_ringacc_ring_read_occ(ring);
3277e8aa 888
6b3da0b4 889 return ring->state.free;
3277e8aa
GS		 890}
891EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_free);
892
893u32 k3_ringacc_ring_get_occ(struct k3_ring *ring)
894{
895 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
896 return -EINVAL;
897
d782298c 898 return k3_ringacc_ring_read_occ(ring);
3277e8aa
GS		 899}
900EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_occ);
901
902u32 k3_ringacc_ring_is_full(struct k3_ring *ring)
903{
904 return !k3_ringacc_ring_get_free(ring);
905}
906EXPORT_SYMBOL_GPL(k3_ringacc_ring_is_full);
907
908enum k3_ringacc_access_mode {
909 K3_RINGACC_ACCESS_MODE_PUSH_HEAD,
910 K3_RINGACC_ACCESS_MODE_POP_HEAD,
911 K3_RINGACC_ACCESS_MODE_PUSH_TAIL,
912 K3_RINGACC_ACCESS_MODE_POP_TAIL,
913 K3_RINGACC_ACCESS_MODE_PEEK_HEAD,
914 K3_RINGACC_ACCESS_MODE_PEEK_TAIL,
915};
916
917#define K3_RINGACC_PROXY_MODE(x) (((x) & 0x3) << 16)
918#define K3_RINGACC_PROXY_ELSIZE(x) (((x) & 0x7) << 24)
919static int k3_ringacc_ring_cfg_proxy(struct k3_ring *ring,
920 enum k3_ringacc_proxy_access_mode mode)
921{
922 u32 val;
923
924 val = ring->ring_id;
925 val |= K3_RINGACC_PROXY_MODE(mode);
926 val |= K3_RINGACC_PROXY_ELSIZE(ring->elm_size);
927 writel(val, &ring->proxy->control);
928 return 0;
929}
930
931static int k3_ringacc_ring_access_proxy(struct k3_ring *ring, void *elem,
932 enum k3_ringacc_access_mode access_mode)
933{
934 void __iomem *ptr;
935
936 ptr = (void __iomem *)&ring->proxy->data;
937
938 switch (access_mode) {
939 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
940 case K3_RINGACC_ACCESS_MODE_POP_HEAD:
941 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD);
942 break;
943 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
944 case K3_RINGACC_ACCESS_MODE_POP_TAIL:
945 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL);
946 break;
947 default:
948 return -EINVAL;
949 }
950
951 ptr += k3_ringacc_ring_get_fifo_pos(ring);
952
953 switch (access_mode) {
954 case K3_RINGACC_ACCESS_MODE_POP_HEAD:
955 case K3_RINGACC_ACCESS_MODE_POP_TAIL:
956 dev_dbg(ring->parent->dev,
957 "proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr,
958 access_mode);
959 memcpy_fromio(elem, ptr, (4 << ring->elm_size));
6b3da0b4 960 ring->state.occ--;
3277e8aa
GS		 961 break;
962 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
963 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
964 dev_dbg(ring->parent->dev,
965 "proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr,
966 access_mode);
967 memcpy_toio(ptr, elem, (4 << ring->elm_size));
6b3da0b4 968 ring->state.free--;
3277e8aa
GS		 969 break;
970 default:
971 return -EINVAL;
972 }
973
6b3da0b4
PU		 974 dev_dbg(ring->parent->dev, "proxy: free%d occ%d\n", ring->state.free,
975 ring->state.occ);
3277e8aa
GS		 976 return 0;
977}
978
979static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem)
980{
981 return k3_ringacc_ring_access_proxy(ring, elem,
982 K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
983}
984
985static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem)
986{
987 return k3_ringacc_ring_access_proxy(ring, elem,
988 K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
989}
990
991static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem)
992{
993 return k3_ringacc_ring_access_proxy(ring, elem,
994 K3_RINGACC_ACCESS_MODE_POP_HEAD);
995}
996
997static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem)
998{
999 return k3_ringacc_ring_access_proxy(ring, elem,
1000 K3_RINGACC_ACCESS_MODE_POP_HEAD);
1001}
1002
1003static int k3_ringacc_ring_access_io(struct k3_ring *ring, void *elem,
1004 enum k3_ringacc_access_mode access_mode)
1005{
1006 void __iomem *ptr;
1007
1008 switch (access_mode) {
1009 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
1010 case K3_RINGACC_ACCESS_MODE_POP_HEAD:
1011 ptr = (void __iomem *)&ring->fifos->head_data;
1012 break;
1013 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
1014 case K3_RINGACC_ACCESS_MODE_POP_TAIL:
1015 ptr = (void __iomem *)&ring->fifos->tail_data;
1016 break;
1017 default:
1018 return -EINVAL;
1019 }
1020
1021 ptr += k3_ringacc_ring_get_fifo_pos(ring);
1022
1023 switch (access_mode) {
1024 case K3_RINGACC_ACCESS_MODE_POP_HEAD:
1025 case K3_RINGACC_ACCESS_MODE_POP_TAIL:
1026 dev_dbg(ring->parent->dev,
1027 "memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr,
1028 access_mode);
1029 memcpy_fromio(elem, ptr, (4 << ring->elm_size));
6b3da0b4 1030 ring->state.occ--;
3277e8aa
GS		 1031 break;
1032 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
1033 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
1034 dev_dbg(ring->parent->dev,
1035 "memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr,
1036 access_mode);
1037 memcpy_toio(ptr, elem, (4 << ring->elm_size));
6b3da0b4 1038 ring->state.free--;
3277e8aa
GS		 1039 break;
1040 default:
1041 return -EINVAL;
1042 }
1043
6b3da0b4
PU		 1044 dev_dbg(ring->parent->dev, "free%d index%d occ%d index%d\n",
1045 ring->state.free, ring->state.windex, ring->state.occ,
1046 ring->state.rindex);
3277e8aa
GS		 1047 return 0;
1048}
1049
1050static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem)
1051{
1052 return k3_ringacc_ring_access_io(ring, elem,
1053 K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
1054}
1055
1056static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem)
1057{
1058 return k3_ringacc_ring_access_io(ring, elem,
1059 K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
1060}
1061
1062static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem)
1063{
1064 return k3_ringacc_ring_access_io(ring, elem,
1065 K3_RINGACC_ACCESS_MODE_POP_HEAD);
1066}
1067
1068static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem)
1069{
1070 return k3_ringacc_ring_access_io(ring, elem,
1071 K3_RINGACC_ACCESS_MODE_POP_HEAD);
1072}
1073
d782298c
GS		 1074/*
1075 * The element is 48 bits of address + ASEL bits in the ring.
1076 * ASEL is used by the DMAs and should be removed for the kernel as it is not
1077 * part of the physical memory address.
1078 */
1079static void k3_dmaring_remove_asel_from_elem(u64 *elem)
1080{
1081 *elem &= GENMASK_ULL(K3_ADDRESS_ASEL_SHIFT - 1, 0);
1082}
1083
1084static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem)
1085{
1086 void *elem_ptr;
1087 u32 elem_idx;
1088
1089 /*
1090 * DMA rings: forward ring is always tied DMA channel and HW does not
1091 * maintain any state data required for POP operation and its unknown
1092 * how much elements were consumed by HW. So, to actually
1093 * do POP, the read pointer has to be recalculated every time.
1094 */
1095 ring->state.occ = k3_ringacc_ring_read_occ(ring);
1096 if (ring->state.windex >= ring->state.occ)
1097 elem_idx = ring->state.windex - ring->state.occ;
1098 else
1099 elem_idx = ring->size - (ring->state.occ - ring->state.windex);
1100
1101 elem_ptr = k3_ringacc_get_elm_addr(ring, elem_idx);
1102 memcpy(elem, elem_ptr, (4 << ring->elm_size));
1103 k3_dmaring_remove_asel_from_elem(elem);
1104
1105 ring->state.occ--;
1106 writel(-1, &ring->rt->db);
1107
1108 dev_dbg(ring->parent->dev, "%s: occ%d Windex%d Rindex%d pos_ptr%px\n",
1109 __func__, ring->state.occ, ring->state.windex, elem_idx,
1110 elem_ptr);
1111 return 0;
1112}
1113
1114static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem)
1115{
1116 void *elem_ptr;
1117
1118 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex);
1119
1120 if (ring->state.occ) {
1121 memcpy(elem, elem_ptr, (4 << ring->elm_size));
1122 k3_dmaring_remove_asel_from_elem(elem);
1123
1124 ring->state.rindex = (ring->state.rindex + 1) % ring->size;
1125 ring->state.occ--;
1126 writel(-1 & K3_DMARING_RT_DB_ENTRY_MASK, &ring->rt->db);
1127 } else if (ring->state.tdown_complete) {
1128 dma_addr_t *value = elem;
1129
1130 *value = CPPI5_TDCM_MARKER;
1131 writel(K3_DMARING_RT_DB_TDOWN_ACK, &ring->rt->db);
1132 ring->state.tdown_complete = false;
1133 }
1134
1135 dev_dbg(ring->parent->dev, "%s: occ%d index%d pos_ptr%px\n",
1136 __func__, ring->state.occ, ring->state.rindex, elem_ptr);
1137 return 0;
1138}
1139
3277e8aa
GS		 1140static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem)
1141{
1142 void *elem_ptr;
1143
6b3da0b4 1144 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.windex);
3277e8aa
GS		 1145
1146 memcpy(elem_ptr, elem, (4 << ring->elm_size));
d782298c
GS		 1147 if (ring->parent->dma_rings) {
1148 u64 *addr = elem_ptr;
1149
1150 *addr |= ((u64)ring->asel << K3_ADDRESS_ASEL_SHIFT);
1151 }
3277e8aa 1152
6b3da0b4
PU		 1153 ring->state.windex = (ring->state.windex + 1) % ring->size;
1154 ring->state.free--;
3277e8aa
GS		 1155 writel(1, &ring->rt->db);
1156
1157 dev_dbg(ring->parent->dev, "ring_push_mem: free%d index%d\n",
6b3da0b4 1158 ring->state.free, ring->state.windex);
3277e8aa
GS		 1159
1160 return 0;
1161}
1162
1163static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem)
1164{
1165 void *elem_ptr;
1166
6b3da0b4 1167 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex);
3277e8aa
GS		 1168
1169 memcpy(elem, elem_ptr, (4 << ring->elm_size));
1170
6b3da0b4
PU		 1171 ring->state.rindex = (ring->state.rindex + 1) % ring->size;
1172 ring->state.occ--;
3277e8aa
GS		 1173 writel(-1, &ring->rt->db);
1174
1175 dev_dbg(ring->parent->dev, "ring_pop_mem: occ%d index%d pos_ptr%p\n",
6b3da0b4 1176 ring->state.occ, ring->state.rindex, elem_ptr);
3277e8aa
GS		 1177 return 0;
1178}
1179
1180int k3_ringacc_ring_push(struct k3_ring *ring, void *elem)
1181{
1182 int ret = -EOPNOTSUPP;
1183
1184 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
1185 return -EINVAL;
1186
6b3da0b4
PU		 1187 dev_dbg(ring->parent->dev, "ring_push: free%d index%d\n",
1188 ring->state.free, ring->state.windex);
3277e8aa
GS		 1189
1190 if (k3_ringacc_ring_is_full(ring))
1191 return -ENOMEM;
1192
1193 if (ring->ops && ring->ops->push_tail)
1194 ret = ring->ops->push_tail(ring, elem);
1195
1196 return ret;
1197}
1198EXPORT_SYMBOL_GPL(k3_ringacc_ring_push);
1199
1200int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem)
1201{
1202 int ret = -EOPNOTSUPP;
1203
1204 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
1205 return -EINVAL;
1206
1207 dev_dbg(ring->parent->dev, "ring_push_head: free%d index%d\n",
6b3da0b4 1208 ring->state.free, ring->state.windex);
3277e8aa
GS		 1209
1210 if (k3_ringacc_ring_is_full(ring))
1211 return -ENOMEM;
1212
1213 if (ring->ops && ring->ops->push_head)
1214 ret = ring->ops->push_head(ring, elem);
1215
1216 return ret;
1217}
1218EXPORT_SYMBOL_GPL(k3_ringacc_ring_push_head);
1219
1220int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem)
1221{
1222 int ret = -EOPNOTSUPP;
1223
1224 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
1225 return -EINVAL;
1226
6b3da0b4 1227 if (!ring->state.occ)
d782298c 1228 k3_ringacc_ring_update_occ(ring);
3277e8aa 1229
6b3da0b4
PU		 1230 dev_dbg(ring->parent->dev, "ring_pop: occ%d index%d\n", ring->state.occ,
1231 ring->state.rindex);
3277e8aa 1232
d782298c 1233 if (!ring->state.occ && !ring->state.tdown_complete)
3277e8aa
GS		 1234 return -ENODATA;
1235
1236 if (ring->ops && ring->ops->pop_head)
1237 ret = ring->ops->pop_head(ring, elem);
1238
1239 return ret;
1240}
1241EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop);
1242
1243int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem)
1244{
1245 int ret = -EOPNOTSUPP;
1246
1247 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
1248 return -EINVAL;
1249
6b3da0b4 1250 if (!ring->state.occ)
d782298c 1251 k3_ringacc_ring_update_occ(ring);
3277e8aa 1252
6b3da0b4
PU		 1253 dev_dbg(ring->parent->dev, "ring_pop_tail: occ%d index%d\n",
1254 ring->state.occ, ring->state.rindex);
3277e8aa 1255
6b3da0b4 1256 if (!ring->state.occ)
3277e8aa
GS		 1257 return -ENODATA;
1258
1259 if (ring->ops && ring->ops->pop_tail)
1260 ret = ring->ops->pop_tail(ring, elem);
1261
1262 return ret;
1263}
1264EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop_tail);
1265
1266struct k3_ringacc *of_k3_ringacc_get_by_phandle(struct device_node *np,
1267 const char *property)
1268{
1269 struct device_node *ringacc_np;
1270 struct k3_ringacc *ringacc = ERR_PTR(-EPROBE_DEFER);
1271 struct k3_ringacc *entry;
1272
1273 ringacc_np = of_parse_phandle(np, property, 0);
1274 if (!ringacc_np)
1275 return ERR_PTR(-ENODEV);
1276
1277 mutex_lock(&k3_ringacc_list_lock);
1278 list_for_each_entry(entry, &k3_ringacc_list, list)
1279 if (entry->dev->of_node == ringacc_np) {
1280 ringacc = entry;
1281 break;
1282 }
1283 mutex_unlock(&k3_ringacc_list_lock);
1284 of_node_put(ringacc_np);
1285
1286 return ringacc;
1287}
1288EXPORT_SYMBOL_GPL(of_k3_ringacc_get_by_phandle);
1289
1290static int k3_ringacc_probe_dt(struct k3_ringacc *ringacc)
1291{
1292 struct device_node *node = ringacc->dev->of_node;
1293 struct device *dev = ringacc->dev;
1294 struct platform_device *pdev = to_platform_device(dev);
1295 int ret;
1296
1297 if (!node) {
1298 dev_err(dev, "device tree info unavailable\n");
1299 return -ENODEV;
1300 }
1301
1302 ret = of_property_read_u32(node, "ti,num-rings", &ringacc->num_rings);
1303 if (ret) {
1304 dev_err(dev, "ti,num-rings read failure %d\n", ret);
1305 return ret;
1306 }
1307
3277e8aa
GS		 1308 ringacc->tisci = ti_sci_get_by_phandle(node, "ti,sci");
1309 if (IS_ERR(ringacc->tisci)) {
1310 ret = PTR_ERR(ringacc->tisci);
1311 if (ret != -EPROBE_DEFER)
1312 dev_err(dev, "ti,sci read fail %d\n", ret);
1313 ringacc->tisci = NULL;
1314 return ret;
1315 }
1316
1317 ret = of_property_read_u32(node, "ti,sci-dev-id",
1318 &ringacc->tisci_dev_id);
1319 if (ret) {
1320 dev_err(dev, "ti,sci-dev-id read fail %d\n", ret);
1321 return ret;
1322 }
1323
1324 pdev->id = ringacc->tisci_dev_id;
1325
1326 ringacc->rm_gp_range = devm_ti_sci_get_of_resource(ringacc->tisci, dev,
1327 ringacc->tisci_dev_id,
1328 "ti,sci-rm-range-gp-rings");
1329 if (IS_ERR(ringacc->rm_gp_range)) {
1330 dev_err(dev, "Failed to allocate MSI interrupts\n");
1331 return PTR_ERR(ringacc->rm_gp_range);
1332 }
1333
1334 return ti_sci_inta_msi_domain_alloc_irqs(ringacc->dev,
1335 ringacc->rm_gp_range);
1336}
1337
95e7be06
GS		 1338static const struct k3_ringacc_soc_data k3_ringacc_soc_data_sr1 = {
1339 .dma_ring_reset_quirk = 1,
1340};
1341
1342static const struct soc_device_attribute k3_ringacc_socinfo[] = {
1343 { .family = "AM65X",
1344 .revision = "SR1.0",
1345 .data = &k3_ringacc_soc_data_sr1
1346 },
1347 {/* sentinel */}
1348};
1349
40a2a7c3
GS		 1350static int k3_ringacc_init(struct platform_device *pdev,
1351 struct k3_ringacc *ringacc)
3277e8aa 1352{
95e7be06 1353 const struct soc_device_attribute *soc;
3277e8aa
GS		 1354 void __iomem *base_fifo, *base_rt;
1355 struct device *dev = &pdev->dev;
1356 struct resource *res;
1357 int ret, i;
1358
3277e8aa
GS		 1359 dev->msi_domain = of_msi_get_domain(dev, dev->of_node,
1360 DOMAIN_BUS_TI_SCI_INTA_MSI);
1361 if (!dev->msi_domain) {
1362 dev_err(dev, "Failed to get MSI domain\n");
1363 return -EPROBE_DEFER;
1364 }
1365
1366 ret = k3_ringacc_probe_dt(ringacc);
1367 if (ret)
1368 return ret;
1369
95e7be06
GS		 1370 soc = soc_device_match(k3_ringacc_socinfo);
1371 if (soc && soc->data) {
1372 const struct k3_ringacc_soc_data *soc_data = soc->data;
1373
1374 ringacc->dma_ring_reset_quirk = soc_data->dma_ring_reset_quirk;
1375 }
1376
3277e8aa
GS		 1377 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "rt");
1378 base_rt = devm_ioremap_resource(dev, res);
1379 if (IS_ERR(base_rt))
1380 return PTR_ERR(base_rt);
1381
1382 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "fifos");
1383 base_fifo = devm_ioremap_resource(dev, res);
1384 if (IS_ERR(base_fifo))
1385 return PTR_ERR(base_fifo);
1386
1387 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "proxy_gcfg");
1388 ringacc->proxy_gcfg = devm_ioremap_resource(dev, res);
1389 if (IS_ERR(ringacc->proxy_gcfg))
1390 return PTR_ERR(ringacc->proxy_gcfg);
1391
1392 res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
1393 "proxy_target");
1394 ringacc->proxy_target_base = devm_ioremap_resource(dev, res);
1395 if (IS_ERR(ringacc->proxy_target_base))
1396 return PTR_ERR(ringacc->proxy_target_base);
1397
1398 ringacc->num_proxies = readl(&ringacc->proxy_gcfg->config) &
1399 K3_RINGACC_PROXY_CFG_THREADS_MASK;
1400
1401 ringacc->rings = devm_kzalloc(dev,
1402 sizeof(*ringacc->rings) *
1403 ringacc->num_rings,
1404 GFP_KERNEL);
1405 ringacc->rings_inuse = devm_kcalloc(dev,
1406 BITS_TO_LONGS(ringacc->num_rings),
1407 sizeof(unsigned long), GFP_KERNEL);
1408 ringacc->proxy_inuse = devm_kcalloc(dev,
1409 BITS_TO_LONGS(ringacc->num_proxies),
1410 sizeof(unsigned long), GFP_KERNEL);
1411
1412 if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse)
1413 return -ENOMEM;
1414
1415 for (i = 0; i < ringacc->num_rings; i++) {
1416 ringacc->rings[i].rt = base_rt +
1417 K3_RINGACC_RT_REGS_STEP * i;
1418 ringacc->rings[i].fifos = base_fifo +
1419 K3_RINGACC_FIFO_REGS_STEP * i;
1420 ringacc->rings[i].parent = ringacc;
1421 ringacc->rings[i].ring_id = i;
1422 ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED;
1423 }
3277e8aa
GS		 1424
1425 ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
1426
3277e8aa
GS		 1427 dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n",
1428 ringacc->num_rings,
1429 ringacc->rm_gp_range->desc[0].start,
1430 ringacc->rm_gp_range->desc[0].num,
1431 ringacc->tisci_dev_id);
1432 dev_info(dev, "dma-ring-reset-quirk: %s\n",
1433 ringacc->dma_ring_reset_quirk ? "enabled" : "disabled");
1434 dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n",
1435 readl(&ringacc->proxy_gcfg->revision), ringacc->num_proxies);
40a2a7c3 1436
3277e8aa
GS		 1437 return 0;
1438}
1439
40a2a7c3
GS		 1440struct ringacc_match_data {
1441 struct k3_ringacc_ops ops;
1442};
1443
1444static struct ringacc_match_data k3_ringacc_data = {
1445 .ops = {
1446 .init = k3_ringacc_init,
1447 },
1448};
1449
3277e8aa
GS		 1450/* Match table for of_platform binding */
1451static const struct of_device_id k3_ringacc_of_match[] = {
40a2a7c3 1452 { .compatible = "ti,am654-navss-ringacc", .data = &k3_ringacc_data, },
3277e8aa
GS		 1453 {},
1454};
1455
d782298c
GS		 1456struct k3_ringacc *k3_ringacc_dmarings_init(struct platform_device *pdev,
1457 struct k3_ringacc_init_data *data)
1458{
1459 struct device *dev = &pdev->dev;
1460 struct k3_ringacc *ringacc;
1461 void __iomem *base_rt;
1462 struct resource *res;
1463 int i;
1464
1465 ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL);
1466 if (!ringacc)
1467 return ERR_PTR(-ENOMEM);
1468
1469 ringacc->dev = dev;
1470 ringacc->dma_rings = true;
1471 ringacc->num_rings = data->num_rings;
1472 ringacc->tisci = data->tisci;
1473 ringacc->tisci_dev_id = data->tisci_dev_id;
1474
1475 mutex_init(&ringacc->req_lock);
1476
1477 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ringrt");
1478 base_rt = devm_ioremap_resource(dev, res);
1479 if (IS_ERR(base_rt))
115ff12a 1480 return ERR_CAST(base_rt);
d782298c
GS		 1481
1482 ringacc->rings = devm_kzalloc(dev,
1483 sizeof(*ringacc->rings) *
1484 ringacc->num_rings * 2,
1485 GFP_KERNEL);
1486 ringacc->rings_inuse = devm_kcalloc(dev,
1487 BITS_TO_LONGS(ringacc->num_rings),
1488 sizeof(unsigned long), GFP_KERNEL);
1489
1490 if (!ringacc->rings || !ringacc->rings_inuse)
1491 return ERR_PTR(-ENOMEM);
1492
1493 for (i = 0; i < ringacc->num_rings; i++) {
1494 struct k3_ring *ring = &ringacc->rings[i];
1495
1496 ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i;
1497 ring->parent = ringacc;
1498 ring->ring_id = i;
1499 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
1500
1501 ring = &ringacc->rings[ringacc->num_rings + i];
1502 ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i +
1503 K3_DMARING_RT_REGS_REVERSE_OFS;
1504 ring->parent = ringacc;
1505 ring->ring_id = i;
1506 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
1507 ring->flags = K3_RING_FLAG_REVERSE;
1508 }
1509
1510 ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
1511
1512 dev_info(dev, "Number of rings: %u\n", ringacc->num_rings);
1513
1514 return ringacc;
1515}
1516EXPORT_SYMBOL_GPL(k3_ringacc_dmarings_init);
1517
40a2a7c3
GS		 1518static int k3_ringacc_probe(struct platform_device *pdev)
1519{
1520 const struct ringacc_match_data *match_data;
40a2a7c3
GS		 1521 struct device *dev = &pdev->dev;
1522 struct k3_ringacc *ringacc;
1523 int ret;
1524
a8fc8e5b
SA		 1525 match_data = of_device_get_match_data(&pdev->dev);
1526 if (!match_data)
40a2a7c3 1527 return -ENODEV;
40a2a7c3
GS		 1528
1529 ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL);
1530 if (!ringacc)
1531 return -ENOMEM;
1532
1533 ringacc->dev = dev;
1534 mutex_init(&ringacc->req_lock);
1535 ringacc->ops = &match_data->ops;
1536
1537 ret = ringacc->ops->init(pdev, ringacc);
1538 if (ret)
1539 return ret;
1540
1541 dev_set_drvdata(dev, ringacc);
1542
1543 mutex_lock(&k3_ringacc_list_lock);
1544 list_add_tail(&ringacc->list, &k3_ringacc_list);
1545 mutex_unlock(&k3_ringacc_list_lock);
1546
1547 return 0;
1548}
1549
3277e8aa
GS		 1550static struct platform_driver k3_ringacc_driver = {
1551 .probe = k3_ringacc_probe,
1552 .driver = {
1553 .name = "k3-ringacc",
1554 .of_match_table = k3_ringacc_of_match,
1555 .suppress_bind_attrs = true,
1556 },
1557};
1558builtin_platform_driver(k3_ringacc_driver);
Ubuntu Jammy Linux kernel mirror