2 * Texas Instruments CPDMA Driver
4 * Copyright (C) 2010 Texas Instruments
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License as
8 * published by the Free Software Foundation version 2.
10 * This program is distributed "as is" WITHOUT ANY WARRANTY of any
11 * kind, whether express or implied; without even the implied warranty
12 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 #include <linux/kernel.h>
16 #include <linux/spinlock.h>
17 #include <linux/device.h>
18 #include <linux/module.h>
19 #include <linux/slab.h>
20 #include <linux/err.h>
21 #include <linux/dma-mapping.h>
23 #include <linux/delay.h>
24 #include <linux/genalloc.h>
25 #include "davinci_cpdma.h"
28 #define CPDMA_TXIDVER 0x00
29 #define CPDMA_TXCONTROL 0x04
30 #define CPDMA_TXTEARDOWN 0x08
31 #define CPDMA_RXIDVER 0x10
32 #define CPDMA_RXCONTROL 0x14
33 #define CPDMA_SOFTRESET 0x1c
34 #define CPDMA_RXTEARDOWN 0x18
35 #define CPDMA_TX_PRI0_RATE 0x30
36 #define CPDMA_TXINTSTATRAW 0x80
37 #define CPDMA_TXINTSTATMASKED 0x84
38 #define CPDMA_TXINTMASKSET 0x88
39 #define CPDMA_TXINTMASKCLEAR 0x8c
40 #define CPDMA_MACINVECTOR 0x90
41 #define CPDMA_MACEOIVECTOR 0x94
42 #define CPDMA_RXINTSTATRAW 0xa0
43 #define CPDMA_RXINTSTATMASKED 0xa4
44 #define CPDMA_RXINTMASKSET 0xa8
45 #define CPDMA_RXINTMASKCLEAR 0xac
46 #define CPDMA_DMAINTSTATRAW 0xb0
47 #define CPDMA_DMAINTSTATMASKED 0xb4
48 #define CPDMA_DMAINTMASKSET 0xb8
49 #define CPDMA_DMAINTMASKCLEAR 0xbc
50 #define CPDMA_DMAINT_HOSTERR BIT(1)
52 /* the following exist only if has_ext_regs is set */
53 #define CPDMA_DMACONTROL 0x20
54 #define CPDMA_DMASTATUS 0x24
55 #define CPDMA_RXBUFFOFS 0x28
56 #define CPDMA_EM_CONTROL 0x2c
58 /* Descriptor mode bits */
59 #define CPDMA_DESC_SOP BIT(31)
60 #define CPDMA_DESC_EOP BIT(30)
61 #define CPDMA_DESC_OWNER BIT(29)
62 #define CPDMA_DESC_EOQ BIT(28)
63 #define CPDMA_DESC_TD_COMPLETE BIT(27)
64 #define CPDMA_DESC_PASS_CRC BIT(26)
65 #define CPDMA_DESC_TO_PORT_EN BIT(20)
66 #define CPDMA_TO_PORT_SHIFT 16
67 #define CPDMA_DESC_PORT_MASK (BIT(18) | BIT(17) | BIT(16))
68 #define CPDMA_DESC_CRC_LEN 4
70 #define CPDMA_TEARDOWN_VALUE 0xfffffffc
72 #define CPDMA_MAX_RLIM_CNT 16384
86 struct cpdma_desc_pool
{
89 void __iomem
*iomap
; /* ioremap map */
90 void *cpumap
; /* dma_alloc map */
91 int desc_size
, mem_size
;
94 struct gen_pool
*gen_pool
;
100 CPDMA_STATE_TEARDOWN
,
104 enum cpdma_state state
;
105 struct cpdma_params params
;
107 struct cpdma_desc_pool
*pool
;
109 struct cpdma_chan
*channels
[2 * CPDMA_MAX_CHANNELS
];
111 int num_rx_desc
; /* RX descriptors number */
112 int num_tx_desc
; /* TX descriptors number */
116 struct cpdma_desc __iomem
*head
, *tail
;
117 void __iomem
*hdp
, *cp
, *rxfree
;
118 enum cpdma_state state
;
119 struct cpdma_ctlr
*ctlr
;
125 cpdma_handler_fn handler
;
126 enum dma_data_direction dir
;
127 struct cpdma_chan_stats stats
;
128 /* offsets into dmaregs */
129 int int_set
, int_clear
, td
;
135 struct cpdma_control_info
{
139 #define ACCESS_RO BIT(0)
140 #define ACCESS_WO BIT(1)
141 #define ACCESS_RW (ACCESS_RO | ACCESS_WO)
144 static struct cpdma_control_info controls
[] = {
145 [CPDMA_TX_RLIM
] = {CPDMA_DMACONTROL
, 8, 0xffff, ACCESS_RW
},
146 [CPDMA_CMD_IDLE
] = {CPDMA_DMACONTROL
, 3, 1, ACCESS_WO
},
147 [CPDMA_COPY_ERROR_FRAMES
] = {CPDMA_DMACONTROL
, 4, 1, ACCESS_RW
},
148 [CPDMA_RX_OFF_LEN_UPDATE
] = {CPDMA_DMACONTROL
, 2, 1, ACCESS_RW
},
149 [CPDMA_RX_OWNERSHIP_FLIP
] = {CPDMA_DMACONTROL
, 1, 1, ACCESS_RW
},
150 [CPDMA_TX_PRIO_FIXED
] = {CPDMA_DMACONTROL
, 0, 1, ACCESS_RW
},
151 [CPDMA_STAT_IDLE
] = {CPDMA_DMASTATUS
, 31, 1, ACCESS_RO
},
152 [CPDMA_STAT_TX_ERR_CODE
] = {CPDMA_DMASTATUS
, 20, 0xf, ACCESS_RW
},
153 [CPDMA_STAT_TX_ERR_CHAN
] = {CPDMA_DMASTATUS
, 16, 0x7, ACCESS_RW
},
154 [CPDMA_STAT_RX_ERR_CODE
] = {CPDMA_DMASTATUS
, 12, 0xf, ACCESS_RW
},
155 [CPDMA_STAT_RX_ERR_CHAN
] = {CPDMA_DMASTATUS
, 8, 0x7, ACCESS_RW
},
156 [CPDMA_RX_BUFFER_OFFSET
] = {CPDMA_RXBUFFOFS
, 0, 0xffff, ACCESS_RW
},
159 #define tx_chan_num(chan) (chan)
160 #define rx_chan_num(chan) ((chan) + CPDMA_MAX_CHANNELS)
161 #define is_rx_chan(chan) ((chan)->chan_num >= CPDMA_MAX_CHANNELS)
162 #define is_tx_chan(chan) (!is_rx_chan(chan))
163 #define __chan_linear(chan_num) ((chan_num) & (CPDMA_MAX_CHANNELS - 1))
164 #define chan_linear(chan) __chan_linear((chan)->chan_num)
166 /* The following make access to common cpdma_ctlr params more readable */
167 #define dmaregs params.dmaregs
168 #define num_chan params.num_chan
170 /* various accessors */
171 #define dma_reg_read(ctlr, ofs) readl((ctlr)->dmaregs + (ofs))
172 #define chan_read(chan, fld) readl((chan)->fld)
173 #define desc_read(desc, fld) readl(&(desc)->fld)
174 #define dma_reg_write(ctlr, ofs, v) writel(v, (ctlr)->dmaregs + (ofs))
175 #define chan_write(chan, fld, v) writel(v, (chan)->fld)
176 #define desc_write(desc, fld, v) writel((u32)(v), &(desc)->fld)
178 #define cpdma_desc_to_port(chan, mode, directed) \
180 if (!is_rx_chan(chan) && ((directed == 1) || \
182 mode |= (CPDMA_DESC_TO_PORT_EN | \
183 (directed << CPDMA_TO_PORT_SHIFT)); \
186 static void cpdma_desc_pool_destroy(struct cpdma_ctlr
*ctlr
)
188 struct cpdma_desc_pool
*pool
= ctlr
->pool
;
193 WARN(gen_pool_size(pool
->gen_pool
) != gen_pool_avail(pool
->gen_pool
),
194 "cpdma_desc_pool size %d != avail %d",
195 gen_pool_size(pool
->gen_pool
),
196 gen_pool_avail(pool
->gen_pool
));
198 dma_free_coherent(ctlr
->dev
, pool
->mem_size
, pool
->cpumap
,
203 * Utility constructs for a cpdma descriptor pool. Some devices (e.g. davinci
204 * emac) have dedicated on-chip memory for these descriptors. Some other
205 * devices (e.g. cpsw switches) use plain old memory. Descriptor pools
206 * abstract out these details
208 int cpdma_desc_pool_create(struct cpdma_ctlr
*ctlr
)
210 struct cpdma_params
*cpdma_params
= &ctlr
->params
;
211 struct cpdma_desc_pool
*pool
;
214 pool
= devm_kzalloc(ctlr
->dev
, sizeof(*pool
), GFP_KERNEL
);
216 goto gen_pool_create_fail
;
219 pool
->mem_size
= cpdma_params
->desc_mem_size
;
220 pool
->desc_size
= ALIGN(sizeof(struct cpdma_desc
),
221 cpdma_params
->desc_align
);
222 pool
->num_desc
= pool
->mem_size
/ pool
->desc_size
;
224 if (cpdma_params
->descs_pool_size
) {
225 /* recalculate memory size required cpdma descriptor pool
226 * basing on number of descriptors specified by user and
227 * if memory size > CPPI internal RAM size (desc_mem_size)
228 * then switch to use DDR
230 pool
->num_desc
= cpdma_params
->descs_pool_size
;
231 pool
->mem_size
= pool
->desc_size
* pool
->num_desc
;
232 if (pool
->mem_size
> cpdma_params
->desc_mem_size
)
233 cpdma_params
->desc_mem_phys
= 0;
236 pool
->gen_pool
= devm_gen_pool_create(ctlr
->dev
, ilog2(pool
->desc_size
),
238 if (IS_ERR(pool
->gen_pool
)) {
239 ret
= PTR_ERR(pool
->gen_pool
);
240 dev_err(ctlr
->dev
, "pool create failed %d\n", ret
);
241 goto gen_pool_create_fail
;
244 if (cpdma_params
->desc_mem_phys
) {
245 pool
->phys
= cpdma_params
->desc_mem_phys
;
246 pool
->iomap
= devm_ioremap(ctlr
->dev
, pool
->phys
,
248 pool
->hw_addr
= cpdma_params
->desc_hw_addr
;
250 pool
->cpumap
= dma_alloc_coherent(ctlr
->dev
, pool
->mem_size
,
251 &pool
->hw_addr
, GFP_KERNEL
);
252 pool
->iomap
= (void __iomem __force
*)pool
->cpumap
;
253 pool
->phys
= pool
->hw_addr
; /* assumes no IOMMU, don't use this value */
257 goto gen_pool_create_fail
;
259 ret
= gen_pool_add_virt(pool
->gen_pool
, (unsigned long)pool
->iomap
,
260 pool
->phys
, pool
->mem_size
, -1);
262 dev_err(ctlr
->dev
, "pool add failed %d\n", ret
);
263 goto gen_pool_add_virt_fail
;
268 gen_pool_add_virt_fail
:
269 cpdma_desc_pool_destroy(ctlr
);
270 gen_pool_create_fail
:
275 static inline dma_addr_t
desc_phys(struct cpdma_desc_pool
*pool
,
276 struct cpdma_desc __iomem
*desc
)
280 return pool
->hw_addr
+ (__force
long)desc
- (__force
long)pool
->iomap
;
283 static inline struct cpdma_desc __iomem
*
284 desc_from_phys(struct cpdma_desc_pool
*pool
, dma_addr_t dma
)
286 return dma
? pool
->iomap
+ dma
- pool
->hw_addr
: NULL
;
289 static struct cpdma_desc __iomem
*
290 cpdma_desc_alloc(struct cpdma_desc_pool
*pool
)
292 return (struct cpdma_desc __iomem
*)
293 gen_pool_alloc(pool
->gen_pool
, pool
->desc_size
);
296 static void cpdma_desc_free(struct cpdma_desc_pool
*pool
,
297 struct cpdma_desc __iomem
*desc
, int num_desc
)
299 gen_pool_free(pool
->gen_pool
, (unsigned long)desc
, pool
->desc_size
);
302 static int _cpdma_control_set(struct cpdma_ctlr
*ctlr
, int control
, int value
)
304 struct cpdma_control_info
*info
= &controls
[control
];
307 if (!ctlr
->params
.has_ext_regs
)
310 if (ctlr
->state
!= CPDMA_STATE_ACTIVE
)
313 if (control
< 0 || control
>= ARRAY_SIZE(controls
))
316 if ((info
->access
& ACCESS_WO
) != ACCESS_WO
)
319 val
= dma_reg_read(ctlr
, info
->reg
);
320 val
&= ~(info
->mask
<< info
->shift
);
321 val
|= (value
& info
->mask
) << info
->shift
;
322 dma_reg_write(ctlr
, info
->reg
, val
);
327 static int _cpdma_control_get(struct cpdma_ctlr
*ctlr
, int control
)
329 struct cpdma_control_info
*info
= &controls
[control
];
332 if (!ctlr
->params
.has_ext_regs
)
335 if (ctlr
->state
!= CPDMA_STATE_ACTIVE
)
338 if (control
< 0 || control
>= ARRAY_SIZE(controls
))
341 if ((info
->access
& ACCESS_RO
) != ACCESS_RO
)
344 ret
= (dma_reg_read(ctlr
, info
->reg
) >> info
->shift
) & info
->mask
;
348 /* cpdma_chan_set_chan_shaper - set shaper for a channel
349 * Has to be called under ctlr lock
351 static int cpdma_chan_set_chan_shaper(struct cpdma_chan
*chan
)
353 struct cpdma_ctlr
*ctlr
= chan
->ctlr
;
361 rate_reg
= CPDMA_TX_PRI0_RATE
+ 4 * chan
->chan_num
;
362 dma_reg_write(ctlr
, rate_reg
, chan
->rate_factor
);
364 rmask
= _cpdma_control_get(ctlr
, CPDMA_TX_RLIM
);
367 ret
= _cpdma_control_set(ctlr
, CPDMA_TX_RLIM
, rmask
);
371 static int cpdma_chan_on(struct cpdma_chan
*chan
)
373 struct cpdma_ctlr
*ctlr
= chan
->ctlr
;
374 struct cpdma_desc_pool
*pool
= ctlr
->pool
;
377 spin_lock_irqsave(&chan
->lock
, flags
);
378 if (chan
->state
!= CPDMA_STATE_IDLE
) {
379 spin_unlock_irqrestore(&chan
->lock
, flags
);
382 if (ctlr
->state
!= CPDMA_STATE_ACTIVE
) {
383 spin_unlock_irqrestore(&chan
->lock
, flags
);
386 dma_reg_write(ctlr
, chan
->int_set
, chan
->mask
);
387 chan
->state
= CPDMA_STATE_ACTIVE
;
389 chan_write(chan
, hdp
, desc_phys(pool
, chan
->head
));
391 chan_write(chan
, rxfree
, chan
->count
);
394 spin_unlock_irqrestore(&chan
->lock
, flags
);
398 /* cpdma_chan_fit_rate - set rate for a channel and check if it's possible.
399 * rmask - mask of rate limited channels
400 * Returns min rate in Kb/s
402 static int cpdma_chan_fit_rate(struct cpdma_chan
*ch
, u32 rate
,
403 u32
*rmask
, int *prio_mode
)
405 struct cpdma_ctlr
*ctlr
= ch
->ctlr
;
406 struct cpdma_chan
*chan
;
407 u32 old_rate
= ch
->rate
;
413 for (i
= tx_chan_num(0); i
< tx_chan_num(CPDMA_MAX_CHANNELS
); i
++) {
414 chan
= ctlr
->channels
[i
];
425 new_rmask
|= chan
->mask
;
428 dev_err(ctlr
->dev
, "Prev channel of %dch is not rate limited\n",
442 static u32
cpdma_chan_set_factors(struct cpdma_ctlr
*ctlr
,
443 struct cpdma_chan
*ch
)
445 u32 delta
= UINT_MAX
, prev_delta
= UINT_MAX
, best_delta
= UINT_MAX
;
446 u32 best_send_cnt
= 0, best_idle_cnt
= 0;
447 u32 new_rate
, best_rate
= 0, rate_reg
;
448 u64 send_cnt
, idle_cnt
;
449 u32 min_send_cnt
, freq
;
450 u64 divident
, divisor
;
457 freq
= ctlr
->params
.bus_freq_mhz
* 1000 * 32;
459 dev_err(ctlr
->dev
, "The bus frequency is not set\n");
463 min_send_cnt
= freq
- ch
->rate
;
464 send_cnt
= DIV_ROUND_UP(min_send_cnt
, ch
->rate
);
465 while (send_cnt
<= CPDMA_MAX_RLIM_CNT
) {
466 divident
= ch
->rate
* send_cnt
;
467 divisor
= min_send_cnt
;
468 idle_cnt
= DIV_ROUND_CLOSEST_ULL(divident
, divisor
);
470 divident
= freq
* idle_cnt
;
471 divisor
= idle_cnt
+ send_cnt
;
472 new_rate
= DIV_ROUND_CLOSEST_ULL(divident
, divisor
);
474 delta
= new_rate
>= ch
->rate
? new_rate
- ch
->rate
: delta
;
475 if (delta
< best_delta
) {
477 best_send_cnt
= send_cnt
;
478 best_idle_cnt
= idle_cnt
;
479 best_rate
= new_rate
;
485 if (prev_delta
>= delta
) {
492 divident
= freq
* idle_cnt
;
493 send_cnt
= DIV_ROUND_CLOSEST_ULL(divident
, ch
->rate
);
494 send_cnt
-= idle_cnt
;
495 prev_delta
= UINT_MAX
;
498 ch
->rate
= best_rate
;
499 ch
->rate_factor
= best_send_cnt
| (best_idle_cnt
<< 16);
502 rate_reg
= CPDMA_TX_PRI0_RATE
+ 4 * ch
->chan_num
;
503 dma_reg_write(ctlr
, rate_reg
, ch
->rate_factor
);
507 struct cpdma_ctlr
*cpdma_ctlr_create(struct cpdma_params
*params
)
509 struct cpdma_ctlr
*ctlr
;
511 ctlr
= devm_kzalloc(params
->dev
, sizeof(*ctlr
), GFP_KERNEL
);
515 ctlr
->state
= CPDMA_STATE_IDLE
;
516 ctlr
->params
= *params
;
517 ctlr
->dev
= params
->dev
;
519 spin_lock_init(&ctlr
->lock
);
521 if (cpdma_desc_pool_create(ctlr
))
523 /* split pool equally between RX/TX by default */
524 ctlr
->num_tx_desc
= ctlr
->pool
->num_desc
/ 2;
525 ctlr
->num_rx_desc
= ctlr
->pool
->num_desc
- ctlr
->num_tx_desc
;
527 if (WARN_ON(ctlr
->num_chan
> CPDMA_MAX_CHANNELS
))
528 ctlr
->num_chan
= CPDMA_MAX_CHANNELS
;
531 EXPORT_SYMBOL_GPL(cpdma_ctlr_create
);
533 int cpdma_ctlr_start(struct cpdma_ctlr
*ctlr
)
535 struct cpdma_chan
*chan
;
539 spin_lock_irqsave(&ctlr
->lock
, flags
);
540 if (ctlr
->state
!= CPDMA_STATE_IDLE
) {
541 spin_unlock_irqrestore(&ctlr
->lock
, flags
);
545 if (ctlr
->params
.has_soft_reset
) {
546 unsigned timeout
= 10 * 100;
548 dma_reg_write(ctlr
, CPDMA_SOFTRESET
, 1);
550 if (dma_reg_read(ctlr
, CPDMA_SOFTRESET
) == 0)
558 for (i
= 0; i
< ctlr
->num_chan
; i
++) {
559 writel(0, ctlr
->params
.txhdp
+ 4 * i
);
560 writel(0, ctlr
->params
.rxhdp
+ 4 * i
);
561 writel(0, ctlr
->params
.txcp
+ 4 * i
);
562 writel(0, ctlr
->params
.rxcp
+ 4 * i
);
565 dma_reg_write(ctlr
, CPDMA_RXINTMASKCLEAR
, 0xffffffff);
566 dma_reg_write(ctlr
, CPDMA_TXINTMASKCLEAR
, 0xffffffff);
568 dma_reg_write(ctlr
, CPDMA_TXCONTROL
, 1);
569 dma_reg_write(ctlr
, CPDMA_RXCONTROL
, 1);
571 ctlr
->state
= CPDMA_STATE_ACTIVE
;
574 for (i
= 0; i
< ARRAY_SIZE(ctlr
->channels
); i
++) {
575 chan
= ctlr
->channels
[i
];
577 cpdma_chan_set_chan_shaper(chan
);
580 /* off prio mode if all tx channels are rate limited */
581 if (is_tx_chan(chan
) && !chan
->rate
)
586 _cpdma_control_set(ctlr
, CPDMA_TX_PRIO_FIXED
, prio_mode
);
587 _cpdma_control_set(ctlr
, CPDMA_RX_BUFFER_OFFSET
, 0);
589 spin_unlock_irqrestore(&ctlr
->lock
, flags
);
592 EXPORT_SYMBOL_GPL(cpdma_ctlr_start
);
594 int cpdma_ctlr_stop(struct cpdma_ctlr
*ctlr
)
599 spin_lock_irqsave(&ctlr
->lock
, flags
);
600 if (ctlr
->state
!= CPDMA_STATE_ACTIVE
) {
601 spin_unlock_irqrestore(&ctlr
->lock
, flags
);
605 ctlr
->state
= CPDMA_STATE_TEARDOWN
;
606 spin_unlock_irqrestore(&ctlr
->lock
, flags
);
608 for (i
= 0; i
< ARRAY_SIZE(ctlr
->channels
); i
++) {
609 if (ctlr
->channels
[i
])
610 cpdma_chan_stop(ctlr
->channels
[i
]);
613 spin_lock_irqsave(&ctlr
->lock
, flags
);
614 dma_reg_write(ctlr
, CPDMA_RXINTMASKCLEAR
, 0xffffffff);
615 dma_reg_write(ctlr
, CPDMA_TXINTMASKCLEAR
, 0xffffffff);
617 dma_reg_write(ctlr
, CPDMA_TXCONTROL
, 0);
618 dma_reg_write(ctlr
, CPDMA_RXCONTROL
, 0);
620 ctlr
->state
= CPDMA_STATE_IDLE
;
622 spin_unlock_irqrestore(&ctlr
->lock
, flags
);
625 EXPORT_SYMBOL_GPL(cpdma_ctlr_stop
);
627 int cpdma_ctlr_destroy(struct cpdma_ctlr
*ctlr
)
634 if (ctlr
->state
!= CPDMA_STATE_IDLE
)
635 cpdma_ctlr_stop(ctlr
);
637 for (i
= 0; i
< ARRAY_SIZE(ctlr
->channels
); i
++)
638 cpdma_chan_destroy(ctlr
->channels
[i
]);
640 cpdma_desc_pool_destroy(ctlr
);
643 EXPORT_SYMBOL_GPL(cpdma_ctlr_destroy
);
645 int cpdma_ctlr_int_ctrl(struct cpdma_ctlr
*ctlr
, bool enable
)
650 spin_lock_irqsave(&ctlr
->lock
, flags
);
651 if (ctlr
->state
!= CPDMA_STATE_ACTIVE
) {
652 spin_unlock_irqrestore(&ctlr
->lock
, flags
);
656 for (i
= 0; i
< ARRAY_SIZE(ctlr
->channels
); i
++) {
657 if (ctlr
->channels
[i
])
658 cpdma_chan_int_ctrl(ctlr
->channels
[i
], enable
);
661 spin_unlock_irqrestore(&ctlr
->lock
, flags
);
664 EXPORT_SYMBOL_GPL(cpdma_ctlr_int_ctrl
);
666 void cpdma_ctlr_eoi(struct cpdma_ctlr
*ctlr
, u32 value
)
668 dma_reg_write(ctlr
, CPDMA_MACEOIVECTOR
, value
);
670 EXPORT_SYMBOL_GPL(cpdma_ctlr_eoi
);
672 u32
cpdma_ctrl_rxchs_state(struct cpdma_ctlr
*ctlr
)
674 return dma_reg_read(ctlr
, CPDMA_RXINTSTATMASKED
);
676 EXPORT_SYMBOL_GPL(cpdma_ctrl_rxchs_state
);
678 u32
cpdma_ctrl_txchs_state(struct cpdma_ctlr
*ctlr
)
680 return dma_reg_read(ctlr
, CPDMA_TXINTSTATMASKED
);
682 EXPORT_SYMBOL_GPL(cpdma_ctrl_txchs_state
);
684 static void cpdma_chan_set_descs(struct cpdma_ctlr
*ctlr
,
685 int rx
, int desc_num
,
688 struct cpdma_chan
*chan
, *most_chan
= NULL
;
689 int desc_cnt
= desc_num
;
697 min
= rx_chan_num(0);
698 max
= rx_chan_num(CPDMA_MAX_CHANNELS
);
700 min
= tx_chan_num(0);
701 max
= tx_chan_num(CPDMA_MAX_CHANNELS
);
704 for (i
= min
; i
< max
; i
++) {
705 chan
= ctlr
->channels
[i
];
710 chan
->desc_num
= (chan
->weight
* desc_num
) / 100;
712 chan
->desc_num
= per_ch_desc
;
714 desc_cnt
-= chan
->desc_num
;
716 if (most_dnum
< chan
->desc_num
) {
717 most_dnum
= chan
->desc_num
;
723 most_chan
->desc_num
+= desc_cnt
;
727 * cpdma_chan_split_pool - Splits ctrl pool between all channels.
728 * Has to be called under ctlr lock
730 int cpdma_chan_split_pool(struct cpdma_ctlr
*ctlr
)
732 int tx_per_ch_desc
= 0, rx_per_ch_desc
= 0;
733 int free_rx_num
= 0, free_tx_num
= 0;
734 int rx_weight
= 0, tx_weight
= 0;
735 int tx_desc_num
, rx_desc_num
;
736 struct cpdma_chan
*chan
;
742 for (i
= 0; i
< ARRAY_SIZE(ctlr
->channels
); i
++) {
743 chan
= ctlr
->channels
[i
];
747 if (is_rx_chan(chan
)) {
750 rx_weight
+= chan
->weight
;
754 tx_weight
+= chan
->weight
;
758 if (rx_weight
> 100 || tx_weight
> 100)
761 tx_desc_num
= ctlr
->num_tx_desc
;
762 rx_desc_num
= ctlr
->num_rx_desc
;
765 tx_per_ch_desc
= tx_desc_num
- (tx_weight
* tx_desc_num
) / 100;
766 tx_per_ch_desc
/= free_tx_num
;
769 rx_per_ch_desc
= rx_desc_num
- (rx_weight
* rx_desc_num
) / 100;
770 rx_per_ch_desc
/= free_rx_num
;
773 cpdma_chan_set_descs(ctlr
, 0, tx_desc_num
, tx_per_ch_desc
);
774 cpdma_chan_set_descs(ctlr
, 1, rx_desc_num
, rx_per_ch_desc
);
778 EXPORT_SYMBOL_GPL(cpdma_chan_split_pool
);
781 /* cpdma_chan_set_weight - set weight of a channel in percentage.
782 * Tx and Rx channels have separate weights. That is 100% for RX
783 * and 100% for Tx. The weight is used to split cpdma resources
784 * in correct proportion required by the channels, including number
785 * of descriptors. The channel rate is not enough to know the
786 * weight of a channel as the maximum rate of an interface is needed.
787 * If weight = 0, then channel uses rest of descriptors leaved by
790 int cpdma_chan_set_weight(struct cpdma_chan
*ch
, int weight
)
792 struct cpdma_ctlr
*ctlr
= ch
->ctlr
;
793 unsigned long flags
, ch_flags
;
796 spin_lock_irqsave(&ctlr
->lock
, flags
);
797 spin_lock_irqsave(&ch
->lock
, ch_flags
);
798 if (ch
->weight
== weight
) {
799 spin_unlock_irqrestore(&ch
->lock
, ch_flags
);
800 spin_unlock_irqrestore(&ctlr
->lock
, flags
);
804 spin_unlock_irqrestore(&ch
->lock
, ch_flags
);
806 /* re-split pool using new channel weight */
807 ret
= cpdma_chan_split_pool(ctlr
);
808 spin_unlock_irqrestore(&ctlr
->lock
, flags
);
811 EXPORT_SYMBOL_GPL(cpdma_chan_set_weight
);
813 /* cpdma_chan_get_min_rate - get minimum allowed rate for channel
814 * Should be called before cpdma_chan_set_rate.
815 * Returns min rate in Kb/s
817 u32
cpdma_chan_get_min_rate(struct cpdma_ctlr
*ctlr
)
819 unsigned int divident
, divisor
;
821 divident
= ctlr
->params
.bus_freq_mhz
* 32 * 1000;
822 divisor
= 1 + CPDMA_MAX_RLIM_CNT
;
824 return DIV_ROUND_UP(divident
, divisor
);
826 EXPORT_SYMBOL_GPL(cpdma_chan_get_min_rate
);
828 /* cpdma_chan_set_rate - limits bandwidth for transmit channel.
829 * The bandwidth * limited channels have to be in order beginning from lowest.
830 * ch - transmit channel the bandwidth is configured for
831 * rate - bandwidth in Kb/s, if 0 - then off shaper
833 int cpdma_chan_set_rate(struct cpdma_chan
*ch
, u32 rate
)
835 unsigned long flags
, ch_flags
;
836 struct cpdma_ctlr
*ctlr
;
840 if (!ch
|| !is_tx_chan(ch
))
843 if (ch
->rate
== rate
)
847 spin_lock_irqsave(&ctlr
->lock
, flags
);
848 spin_lock_irqsave(&ch
->lock
, ch_flags
);
850 ret
= cpdma_chan_fit_rate(ch
, rate
, &rmask
, &prio_mode
);
854 ret
= cpdma_chan_set_factors(ctlr
, ch
);
858 spin_unlock_irqrestore(&ch
->lock
, ch_flags
);
861 _cpdma_control_set(ctlr
, CPDMA_TX_RLIM
, rmask
);
862 _cpdma_control_set(ctlr
, CPDMA_TX_PRIO_FIXED
, prio_mode
);
863 spin_unlock_irqrestore(&ctlr
->lock
, flags
);
867 spin_unlock_irqrestore(&ch
->lock
, ch_flags
);
868 spin_unlock_irqrestore(&ctlr
->lock
, flags
);
871 EXPORT_SYMBOL_GPL(cpdma_chan_set_rate
);
873 u32
cpdma_chan_get_rate(struct cpdma_chan
*ch
)
878 spin_lock_irqsave(&ch
->lock
, flags
);
880 spin_unlock_irqrestore(&ch
->lock
, flags
);
884 EXPORT_SYMBOL_GPL(cpdma_chan_get_rate
);
886 struct cpdma_chan
*cpdma_chan_create(struct cpdma_ctlr
*ctlr
, int chan_num
,
887 cpdma_handler_fn handler
, int rx_type
)
889 int offset
= chan_num
* 4;
890 struct cpdma_chan
*chan
;
893 chan_num
= rx_type
? rx_chan_num(chan_num
) : tx_chan_num(chan_num
);
895 if (__chan_linear(chan_num
) >= ctlr
->num_chan
)
898 chan
= devm_kzalloc(ctlr
->dev
, sizeof(*chan
), GFP_KERNEL
);
900 return ERR_PTR(-ENOMEM
);
902 spin_lock_irqsave(&ctlr
->lock
, flags
);
903 if (ctlr
->channels
[chan_num
]) {
904 spin_unlock_irqrestore(&ctlr
->lock
, flags
);
905 devm_kfree(ctlr
->dev
, chan
);
906 return ERR_PTR(-EBUSY
);
910 chan
->state
= CPDMA_STATE_IDLE
;
911 chan
->chan_num
= chan_num
;
912 chan
->handler
= handler
;
916 if (is_rx_chan(chan
)) {
917 chan
->hdp
= ctlr
->params
.rxhdp
+ offset
;
918 chan
->cp
= ctlr
->params
.rxcp
+ offset
;
919 chan
->rxfree
= ctlr
->params
.rxfree
+ offset
;
920 chan
->int_set
= CPDMA_RXINTMASKSET
;
921 chan
->int_clear
= CPDMA_RXINTMASKCLEAR
;
922 chan
->td
= CPDMA_RXTEARDOWN
;
923 chan
->dir
= DMA_FROM_DEVICE
;
925 chan
->hdp
= ctlr
->params
.txhdp
+ offset
;
926 chan
->cp
= ctlr
->params
.txcp
+ offset
;
927 chan
->int_set
= CPDMA_TXINTMASKSET
;
928 chan
->int_clear
= CPDMA_TXINTMASKCLEAR
;
929 chan
->td
= CPDMA_TXTEARDOWN
;
930 chan
->dir
= DMA_TO_DEVICE
;
932 chan
->mask
= BIT(chan_linear(chan
));
934 spin_lock_init(&chan
->lock
);
936 ctlr
->channels
[chan_num
] = chan
;
939 cpdma_chan_split_pool(ctlr
);
941 spin_unlock_irqrestore(&ctlr
->lock
, flags
);
944 EXPORT_SYMBOL_GPL(cpdma_chan_create
);
946 int cpdma_chan_get_rx_buf_num(struct cpdma_chan
*chan
)
951 spin_lock_irqsave(&chan
->lock
, flags
);
952 desc_num
= chan
->desc_num
;
953 spin_unlock_irqrestore(&chan
->lock
, flags
);
957 EXPORT_SYMBOL_GPL(cpdma_chan_get_rx_buf_num
);
959 int cpdma_chan_destroy(struct cpdma_chan
*chan
)
961 struct cpdma_ctlr
*ctlr
;
968 spin_lock_irqsave(&ctlr
->lock
, flags
);
969 if (chan
->state
!= CPDMA_STATE_IDLE
)
970 cpdma_chan_stop(chan
);
971 ctlr
->channels
[chan
->chan_num
] = NULL
;
973 devm_kfree(ctlr
->dev
, chan
);
974 cpdma_chan_split_pool(ctlr
);
976 spin_unlock_irqrestore(&ctlr
->lock
, flags
);
979 EXPORT_SYMBOL_GPL(cpdma_chan_destroy
);
981 int cpdma_chan_get_stats(struct cpdma_chan
*chan
,
982 struct cpdma_chan_stats
*stats
)
987 spin_lock_irqsave(&chan
->lock
, flags
);
988 memcpy(stats
, &chan
->stats
, sizeof(*stats
));
989 spin_unlock_irqrestore(&chan
->lock
, flags
);
992 EXPORT_SYMBOL_GPL(cpdma_chan_get_stats
);
994 static void __cpdma_chan_submit(struct cpdma_chan
*chan
,
995 struct cpdma_desc __iomem
*desc
)
997 struct cpdma_ctlr
*ctlr
= chan
->ctlr
;
998 struct cpdma_desc __iomem
*prev
= chan
->tail
;
999 struct cpdma_desc_pool
*pool
= ctlr
->pool
;
1000 dma_addr_t desc_dma
;
1003 desc_dma
= desc_phys(pool
, desc
);
1005 /* simple case - idle channel */
1007 chan
->stats
.head_enqueue
++;
1010 if (chan
->state
== CPDMA_STATE_ACTIVE
)
1011 chan_write(chan
, hdp
, desc_dma
);
1015 /* first chain the descriptor at the tail of the list */
1016 desc_write(prev
, hw_next
, desc_dma
);
1018 chan
->stats
.tail_enqueue
++;
1020 /* next check if EOQ has been triggered already */
1021 mode
= desc_read(prev
, hw_mode
);
1022 if (((mode
& (CPDMA_DESC_EOQ
| CPDMA_DESC_OWNER
)) == CPDMA_DESC_EOQ
) &&
1023 (chan
->state
== CPDMA_STATE_ACTIVE
)) {
1024 desc_write(prev
, hw_mode
, mode
& ~CPDMA_DESC_EOQ
);
1025 chan_write(chan
, hdp
, desc_dma
);
1026 chan
->stats
.misqueued
++;
1030 int cpdma_chan_submit(struct cpdma_chan
*chan
, void *token
, void *data
,
1031 int len
, int directed
)
1033 struct cpdma_ctlr
*ctlr
= chan
->ctlr
;
1034 struct cpdma_desc __iomem
*desc
;
1036 unsigned long flags
;
1040 spin_lock_irqsave(&chan
->lock
, flags
);
1042 if (chan
->state
== CPDMA_STATE_TEARDOWN
) {
1047 if (chan
->count
>= chan
->desc_num
) {
1048 chan
->stats
.desc_alloc_fail
++;
1053 desc
= cpdma_desc_alloc(ctlr
->pool
);
1055 chan
->stats
.desc_alloc_fail
++;
1060 if (len
< ctlr
->params
.min_packet_size
) {
1061 len
= ctlr
->params
.min_packet_size
;
1062 chan
->stats
.runt_transmit_buff
++;
1065 buffer
= dma_map_single(ctlr
->dev
, data
, len
, chan
->dir
);
1066 ret
= dma_mapping_error(ctlr
->dev
, buffer
);
1068 cpdma_desc_free(ctlr
->pool
, desc
, 1);
1073 mode
= CPDMA_DESC_OWNER
| CPDMA_DESC_SOP
| CPDMA_DESC_EOP
;
1074 cpdma_desc_to_port(chan
, mode
, directed
);
1076 /* Relaxed IO accessors can be used here as there is read barrier
1077 * at the end of write sequence.
1079 writel_relaxed(0, &desc
->hw_next
);
1080 writel_relaxed(buffer
, &desc
->hw_buffer
);
1081 writel_relaxed(len
, &desc
->hw_len
);
1082 writel_relaxed(mode
| len
, &desc
->hw_mode
);
1083 writel_relaxed(token
, &desc
->sw_token
);
1084 writel_relaxed(buffer
, &desc
->sw_buffer
);
1085 writel_relaxed(len
, &desc
->sw_len
);
1086 desc_read(desc
, sw_len
);
1088 __cpdma_chan_submit(chan
, desc
);
1090 if (chan
->state
== CPDMA_STATE_ACTIVE
&& chan
->rxfree
)
1091 chan_write(chan
, rxfree
, 1);
1096 spin_unlock_irqrestore(&chan
->lock
, flags
);
1099 EXPORT_SYMBOL_GPL(cpdma_chan_submit
);
1101 bool cpdma_check_free_tx_desc(struct cpdma_chan
*chan
)
1103 struct cpdma_ctlr
*ctlr
= chan
->ctlr
;
1104 struct cpdma_desc_pool
*pool
= ctlr
->pool
;
1106 unsigned long flags
;
1108 spin_lock_irqsave(&chan
->lock
, flags
);
1109 free_tx_desc
= (chan
->count
< chan
->desc_num
) &&
1110 gen_pool_avail(pool
->gen_pool
);
1111 spin_unlock_irqrestore(&chan
->lock
, flags
);
1112 return free_tx_desc
;
1114 EXPORT_SYMBOL_GPL(cpdma_check_free_tx_desc
);
1116 static void __cpdma_chan_free(struct cpdma_chan
*chan
,
1117 struct cpdma_desc __iomem
*desc
,
1118 int outlen
, int status
)
1120 struct cpdma_ctlr
*ctlr
= chan
->ctlr
;
1121 struct cpdma_desc_pool
*pool
= ctlr
->pool
;
1122 dma_addr_t buff_dma
;
1126 token
= (void *)desc_read(desc
, sw_token
);
1127 buff_dma
= desc_read(desc
, sw_buffer
);
1128 origlen
= desc_read(desc
, sw_len
);
1130 dma_unmap_single(ctlr
->dev
, buff_dma
, origlen
, chan
->dir
);
1131 cpdma_desc_free(pool
, desc
, 1);
1132 (*chan
->handler
)(token
, outlen
, status
);
1135 static int __cpdma_chan_process(struct cpdma_chan
*chan
)
1137 struct cpdma_ctlr
*ctlr
= chan
->ctlr
;
1138 struct cpdma_desc __iomem
*desc
;
1141 struct cpdma_desc_pool
*pool
= ctlr
->pool
;
1142 dma_addr_t desc_dma
;
1143 unsigned long flags
;
1145 spin_lock_irqsave(&chan
->lock
, flags
);
1149 chan
->stats
.empty_dequeue
++;
1153 desc_dma
= desc_phys(pool
, desc
);
1155 status
= desc_read(desc
, hw_mode
);
1156 outlen
= status
& 0x7ff;
1157 if (status
& CPDMA_DESC_OWNER
) {
1158 chan
->stats
.busy_dequeue
++;
1163 if (status
& CPDMA_DESC_PASS_CRC
)
1164 outlen
-= CPDMA_DESC_CRC_LEN
;
1166 status
= status
& (CPDMA_DESC_EOQ
| CPDMA_DESC_TD_COMPLETE
|
1167 CPDMA_DESC_PORT_MASK
);
1169 chan
->head
= desc_from_phys(pool
, desc_read(desc
, hw_next
));
1170 chan_write(chan
, cp
, desc_dma
);
1172 chan
->stats
.good_dequeue
++;
1174 if ((status
& CPDMA_DESC_EOQ
) && chan
->head
) {
1175 chan
->stats
.requeue
++;
1176 chan_write(chan
, hdp
, desc_phys(pool
, chan
->head
));
1179 spin_unlock_irqrestore(&chan
->lock
, flags
);
1180 if (unlikely(status
& CPDMA_DESC_TD_COMPLETE
))
1181 cb_status
= -ENOSYS
;
1185 __cpdma_chan_free(chan
, desc
, outlen
, cb_status
);
1189 spin_unlock_irqrestore(&chan
->lock
, flags
);
1193 int cpdma_chan_process(struct cpdma_chan
*chan
, int quota
)
1195 int used
= 0, ret
= 0;
1197 if (chan
->state
!= CPDMA_STATE_ACTIVE
)
1200 while (used
< quota
) {
1201 ret
= __cpdma_chan_process(chan
);
1208 EXPORT_SYMBOL_GPL(cpdma_chan_process
);
1210 int cpdma_chan_start(struct cpdma_chan
*chan
)
1212 struct cpdma_ctlr
*ctlr
= chan
->ctlr
;
1213 unsigned long flags
;
1216 spin_lock_irqsave(&ctlr
->lock
, flags
);
1217 ret
= cpdma_chan_set_chan_shaper(chan
);
1218 spin_unlock_irqrestore(&ctlr
->lock
, flags
);
1222 ret
= cpdma_chan_on(chan
);
1228 EXPORT_SYMBOL_GPL(cpdma_chan_start
);
1230 int cpdma_chan_stop(struct cpdma_chan
*chan
)
1232 struct cpdma_ctlr
*ctlr
= chan
->ctlr
;
1233 struct cpdma_desc_pool
*pool
= ctlr
->pool
;
1234 unsigned long flags
;
1238 spin_lock_irqsave(&chan
->lock
, flags
);
1239 if (chan
->state
== CPDMA_STATE_TEARDOWN
) {
1240 spin_unlock_irqrestore(&chan
->lock
, flags
);
1244 chan
->state
= CPDMA_STATE_TEARDOWN
;
1245 dma_reg_write(ctlr
, chan
->int_clear
, chan
->mask
);
1247 /* trigger teardown */
1248 dma_reg_write(ctlr
, chan
->td
, chan_linear(chan
));
1250 /* wait for teardown complete */
1251 timeout
= 100 * 100; /* 100 ms */
1253 u32 cp
= chan_read(chan
, cp
);
1254 if ((cp
& CPDMA_TEARDOWN_VALUE
) == CPDMA_TEARDOWN_VALUE
)
1260 chan_write(chan
, cp
, CPDMA_TEARDOWN_VALUE
);
1262 /* handle completed packets */
1263 spin_unlock_irqrestore(&chan
->lock
, flags
);
1265 ret
= __cpdma_chan_process(chan
);
1268 } while ((ret
& CPDMA_DESC_TD_COMPLETE
) == 0);
1269 spin_lock_irqsave(&chan
->lock
, flags
);
1271 /* remaining packets haven't been tx/rx'ed, clean them up */
1272 while (chan
->head
) {
1273 struct cpdma_desc __iomem
*desc
= chan
->head
;
1274 dma_addr_t next_dma
;
1276 next_dma
= desc_read(desc
, hw_next
);
1277 chan
->head
= desc_from_phys(pool
, next_dma
);
1279 chan
->stats
.teardown_dequeue
++;
1281 /* issue callback without locks held */
1282 spin_unlock_irqrestore(&chan
->lock
, flags
);
1283 __cpdma_chan_free(chan
, desc
, 0, -ENOSYS
);
1284 spin_lock_irqsave(&chan
->lock
, flags
);
1287 chan
->state
= CPDMA_STATE_IDLE
;
1288 spin_unlock_irqrestore(&chan
->lock
, flags
);
1291 EXPORT_SYMBOL_GPL(cpdma_chan_stop
);
1293 int cpdma_chan_int_ctrl(struct cpdma_chan
*chan
, bool enable
)
1295 unsigned long flags
;
1297 spin_lock_irqsave(&chan
->lock
, flags
);
1298 if (chan
->state
!= CPDMA_STATE_ACTIVE
) {
1299 spin_unlock_irqrestore(&chan
->lock
, flags
);
1303 dma_reg_write(chan
->ctlr
, enable
? chan
->int_set
: chan
->int_clear
,
1305 spin_unlock_irqrestore(&chan
->lock
, flags
);
1310 int cpdma_control_get(struct cpdma_ctlr
*ctlr
, int control
)
1312 unsigned long flags
;
1315 spin_lock_irqsave(&ctlr
->lock
, flags
);
1316 ret
= _cpdma_control_get(ctlr
, control
);
1317 spin_unlock_irqrestore(&ctlr
->lock
, flags
);
1322 int cpdma_control_set(struct cpdma_ctlr
*ctlr
, int control
, int value
)
1324 unsigned long flags
;
1327 spin_lock_irqsave(&ctlr
->lock
, flags
);
1328 ret
= _cpdma_control_set(ctlr
, control
, value
);
1329 spin_unlock_irqrestore(&ctlr
->lock
, flags
);
1333 EXPORT_SYMBOL_GPL(cpdma_control_set
);
1335 int cpdma_get_num_rx_descs(struct cpdma_ctlr
*ctlr
)
1337 return ctlr
->num_rx_desc
;
1339 EXPORT_SYMBOL_GPL(cpdma_get_num_rx_descs
);
1341 int cpdma_get_num_tx_descs(struct cpdma_ctlr
*ctlr
)
1343 return ctlr
->num_tx_desc
;
1345 EXPORT_SYMBOL_GPL(cpdma_get_num_tx_descs
);
1347 void cpdma_set_num_rx_descs(struct cpdma_ctlr
*ctlr
, int num_rx_desc
)
1349 ctlr
->num_rx_desc
= num_rx_desc
;
1350 ctlr
->num_tx_desc
= ctlr
->pool
->num_desc
- ctlr
->num_rx_desc
;
1352 EXPORT_SYMBOL_GPL(cpdma_set_num_rx_descs
);
1354 MODULE_LICENSE("GPL");