2 * CAAM/SEC 4.x transport/backend driver
3 * JobR backend functionality
5 * Copyright 2008-2012 Freescale Semiconductor, Inc.
14 /* Main per-ring interrupt handler */
15 static irqreturn_t
caam_jr_interrupt(int irq
, void *st_dev
)
17 struct device
*dev
= st_dev
;
18 struct caam_drv_private_jr
*jrp
= dev_get_drvdata(dev
);
22 * Check the output ring for ready responses, kick
23 * tasklet if jobs done.
25 irqstate
= rd_reg32(&jrp
->rregs
->jrintstatus
);
30 * If JobR error, we got more development work to do
31 * Flag a bug now, but we really need to shut down and
32 * restart the queue (and fix code).
34 if (irqstate
& JRINT_JR_ERROR
) {
35 dev_err(dev
, "job ring error: irqstate: %08x\n", irqstate
);
39 /* mask valid interrupts */
40 setbits32(&jrp
->rregs
->rconfig_lo
, JRCFG_IMSK
);
42 /* Have valid interrupt at this point, just ACK and trigger */
43 wr_reg32(&jrp
->rregs
->jrintstatus
, irqstate
);
46 tasklet_schedule(&jrp
->irqtask
);
52 /* Deferred service handler, run as interrupt-fired tasklet */
53 static void caam_jr_dequeue(unsigned long devarg
)
55 int hw_idx
, sw_idx
, i
, head
, tail
;
56 struct device
*dev
= (struct device
*)devarg
;
57 struct caam_drv_private_jr
*jrp
= dev_get_drvdata(dev
);
58 void (*usercall
)(struct device
*dev
, u32
*desc
, u32 status
, void *arg
);
59 u32
*userdesc
, userstatus
;
62 while (rd_reg32(&jrp
->rregs
->outring_used
)) {
64 head
= ACCESS_ONCE(jrp
->head
);
66 spin_lock(&jrp
->outlock
);
68 sw_idx
= tail
= jrp
->tail
;
69 hw_idx
= jrp
->out_ring_read_index
;
71 for (i
= 0; CIRC_CNT(head
, tail
+ i
, JOBR_DEPTH
) >= 1; i
++) {
72 sw_idx
= (tail
+ i
) & (JOBR_DEPTH
- 1);
74 smp_read_barrier_depends();
76 if (jrp
->outring
[hw_idx
].desc
==
77 jrp
->entinfo
[sw_idx
].desc_addr_dma
)
80 /* we should never fail to find a matching descriptor */
81 BUG_ON(CIRC_CNT(head
, tail
+ i
, JOBR_DEPTH
) <= 0);
83 /* Unmap just-run descriptor so we can post-process */
84 dma_unmap_single(dev
, jrp
->outring
[hw_idx
].desc
,
85 jrp
->entinfo
[sw_idx
].desc_size
,
88 /* mark completed, avoid matching on a recycled desc addr */
89 jrp
->entinfo
[sw_idx
].desc_addr_dma
= 0;
91 /* Stash callback params for use outside of lock */
92 usercall
= jrp
->entinfo
[sw_idx
].callbk
;
93 userarg
= jrp
->entinfo
[sw_idx
].cbkarg
;
94 userdesc
= jrp
->entinfo
[sw_idx
].desc_addr_virt
;
95 userstatus
= jrp
->outring
[hw_idx
].jrstatus
;
98 wr_reg32(&jrp
->rregs
->outring_rmvd
, 1);
100 jrp
->out_ring_read_index
= (jrp
->out_ring_read_index
+ 1) &
104 * if this job completed out-of-order, do not increment
105 * the tail. Otherwise, increment tail by 1 plus the
106 * number of subsequent jobs already completed out-of-order
108 if (sw_idx
== tail
) {
110 tail
= (tail
+ 1) & (JOBR_DEPTH
- 1);
111 smp_read_barrier_depends();
112 } while (CIRC_CNT(head
, tail
, JOBR_DEPTH
) >= 1 &&
113 jrp
->entinfo
[tail
].desc_addr_dma
== 0);
118 spin_unlock(&jrp
->outlock
);
120 /* Finally, execute user's callback */
121 usercall(dev
, userdesc
, userstatus
, userarg
);
124 /* reenable / unmask IRQs */
125 clrbits32(&jrp
->rregs
->rconfig_lo
, JRCFG_IMSK
);
129 * caam_jr_register() - Alloc a ring for someone to use as needed. Returns
130 * an ordinal of the rings allocated, else returns -ENODEV if no rings
132 * @ctrldev: points to the controller level dev (parent) that
133 * owns rings available for use.
134 * @dev: points to where a pointer to the newly allocated queue's
135 * dev can be written to if successful.
137 int caam_jr_register(struct device
*ctrldev
, struct device
**rdev
)
139 struct caam_drv_private
*ctrlpriv
= dev_get_drvdata(ctrldev
);
140 struct caam_drv_private_jr
*jrpriv
= NULL
;
143 /* Lock, if free ring - assign, unlock */
144 spin_lock(&ctrlpriv
->jr_alloc_lock
);
145 for (ring
= 0; ring
< ctrlpriv
->total_jobrs
; ring
++) {
146 jrpriv
= dev_get_drvdata(ctrlpriv
->jrdev
[ring
]);
147 if (jrpriv
->assign
== JOBR_UNASSIGNED
) {
148 jrpriv
->assign
= JOBR_ASSIGNED
;
149 *rdev
= ctrlpriv
->jrdev
[ring
];
150 spin_unlock(&ctrlpriv
->jr_alloc_lock
);
155 /* If assigned, write dev where caller needs it */
156 spin_unlock(&ctrlpriv
->jr_alloc_lock
);
161 EXPORT_SYMBOL(caam_jr_register
);
164 * caam_jr_deregister() - Deregister an API and release the queue.
165 * Returns 0 if OK, -EBUSY if queue still contains pending entries
166 * or unprocessed results at the time of the call
167 * @dev - points to the dev that identifies the queue to
170 int caam_jr_deregister(struct device
*rdev
)
172 struct caam_drv_private_jr
*jrpriv
= dev_get_drvdata(rdev
);
173 struct caam_drv_private
*ctrlpriv
;
175 /* Get the owning controller's private space */
176 ctrlpriv
= dev_get_drvdata(jrpriv
->parentdev
);
179 * Make sure ring empty before release
181 if (rd_reg32(&jrpriv
->rregs
->outring_used
) ||
182 (rd_reg32(&jrpriv
->rregs
->inpring_avail
) != JOBR_DEPTH
))
186 spin_lock(&ctrlpriv
->jr_alloc_lock
);
187 jrpriv
->assign
= JOBR_UNASSIGNED
;
188 spin_unlock(&ctrlpriv
->jr_alloc_lock
);
192 EXPORT_SYMBOL(caam_jr_deregister
);
195 * caam_jr_enqueue() - Enqueue a job descriptor head. Returns 0 if OK,
196 * -EBUSY if the queue is full, -EIO if it cannot map the caller's
198 * @dev: device of the job ring to be used. This device should have
199 * been assigned prior by caam_jr_register().
200 * @desc: points to a job descriptor that execute our request. All
201 * descriptors (and all referenced data) must be in a DMAable
202 * region, and all data references must be physical addresses
203 * accessible to CAAM (i.e. within a PAMU window granted
205 * @cbk: pointer to a callback function to be invoked upon completion
206 * of this request. This has the form:
207 * callback(struct device *dev, u32 *desc, u32 stat, void *arg)
209 * @dev: contains the job ring device that processed this
211 * @desc: descriptor that initiated the request, same as
212 * "desc" being argued to caam_jr_enqueue().
213 * @status: untranslated status received from CAAM. See the
214 * reference manual for a detailed description of
215 * error meaning, or see the JRSTA definitions in the
216 * register header file
217 * @areq: optional pointer to an argument passed with the
219 * @areq: optional pointer to a user argument for use at callback
222 int caam_jr_enqueue(struct device
*dev
, u32
*desc
,
223 void (*cbk
)(struct device
*dev
, u32
*desc
,
224 u32 status
, void *areq
),
227 struct caam_drv_private_jr
*jrp
= dev_get_drvdata(dev
);
228 struct caam_jrentry_info
*head_entry
;
229 int head
, tail
, desc_size
;
232 desc_size
= (*desc
& HDR_JD_LENGTH_MASK
) * sizeof(u32
);
233 desc_dma
= dma_map_single(dev
, desc
, desc_size
, DMA_TO_DEVICE
);
234 if (dma_mapping_error(dev
, desc_dma
)) {
235 dev_err(dev
, "caam_jr_enqueue(): can't map jobdesc\n");
239 spin_lock_bh(&jrp
->inplock
);
242 tail
= ACCESS_ONCE(jrp
->tail
);
244 if (!rd_reg32(&jrp
->rregs
->inpring_avail
) ||
245 CIRC_SPACE(head
, tail
, JOBR_DEPTH
) <= 0) {
246 spin_unlock_bh(&jrp
->inplock
);
247 dma_unmap_single(dev
, desc_dma
, desc_size
, DMA_TO_DEVICE
);
251 head_entry
= &jrp
->entinfo
[head
];
252 head_entry
->desc_addr_virt
= desc
;
253 head_entry
->desc_size
= desc_size
;
254 head_entry
->callbk
= (void *)cbk
;
255 head_entry
->cbkarg
= areq
;
256 head_entry
->desc_addr_dma
= desc_dma
;
258 jrp
->inpring
[jrp
->inp_ring_write_index
] = desc_dma
;
262 jrp
->inp_ring_write_index
= (jrp
->inp_ring_write_index
+ 1) &
264 jrp
->head
= (head
+ 1) & (JOBR_DEPTH
- 1);
266 wr_reg32(&jrp
->rregs
->inpring_jobadd
, 1);
268 spin_unlock_bh(&jrp
->inplock
);
272 EXPORT_SYMBOL(caam_jr_enqueue
);
274 static int caam_reset_hw_jr(struct device
*dev
)
276 struct caam_drv_private_jr
*jrp
= dev_get_drvdata(dev
);
277 unsigned int timeout
= 100000;
280 * mask interrupts since we are going to poll
281 * for reset completion status
283 setbits32(&jrp
->rregs
->rconfig_lo
, JRCFG_IMSK
);
285 /* initiate flush (required prior to reset) */
286 wr_reg32(&jrp
->rregs
->jrcommand
, JRCR_RESET
);
287 while (((rd_reg32(&jrp
->rregs
->jrintstatus
) & JRINT_ERR_HALT_MASK
) ==
288 JRINT_ERR_HALT_INPROGRESS
) && --timeout
)
291 if ((rd_reg32(&jrp
->rregs
->jrintstatus
) & JRINT_ERR_HALT_MASK
) !=
292 JRINT_ERR_HALT_COMPLETE
|| timeout
== 0) {
293 dev_err(dev
, "failed to flush job ring %d\n", jrp
->ridx
);
299 wr_reg32(&jrp
->rregs
->jrcommand
, JRCR_RESET
);
300 while ((rd_reg32(&jrp
->rregs
->jrcommand
) & JRCR_RESET
) && --timeout
)
304 dev_err(dev
, "failed to reset job ring %d\n", jrp
->ridx
);
308 /* unmask interrupts */
309 clrbits32(&jrp
->rregs
->rconfig_lo
, JRCFG_IMSK
);
315 * Init JobR independent of platform property detection
317 static int caam_jr_init(struct device
*dev
)
319 struct caam_drv_private_jr
*jrp
;
320 dma_addr_t inpbusaddr
, outbusaddr
;
323 jrp
= dev_get_drvdata(dev
);
325 tasklet_init(&jrp
->irqtask
, caam_jr_dequeue
, (unsigned long)dev
);
327 /* Connect job ring interrupt handler. */
328 error
= request_irq(jrp
->irq
, caam_jr_interrupt
, IRQF_SHARED
,
331 dev_err(dev
, "can't connect JobR %d interrupt (%d)\n",
332 jrp
->ridx
, jrp
->irq
);
333 irq_dispose_mapping(jrp
->irq
);
338 error
= caam_reset_hw_jr(dev
);
342 jrp
->inpring
= dma_alloc_coherent(dev
, sizeof(dma_addr_t
) * JOBR_DEPTH
,
343 &inpbusaddr
, GFP_KERNEL
);
345 jrp
->outring
= dma_alloc_coherent(dev
, sizeof(struct jr_outentry
) *
346 JOBR_DEPTH
, &outbusaddr
, GFP_KERNEL
);
348 jrp
->entinfo
= kzalloc(sizeof(struct caam_jrentry_info
) * JOBR_DEPTH
,
351 if ((jrp
->inpring
== NULL
) || (jrp
->outring
== NULL
) ||
352 (jrp
->entinfo
== NULL
)) {
353 dev_err(dev
, "can't allocate job rings for %d\n",
358 for (i
= 0; i
< JOBR_DEPTH
; i
++)
359 jrp
->entinfo
[i
].desc_addr_dma
= !0;
362 jrp
->inp_ring_write_index
= 0;
363 jrp
->out_ring_read_index
= 0;
367 wr_reg64(&jrp
->rregs
->inpring_base
, inpbusaddr
);
368 wr_reg64(&jrp
->rregs
->outring_base
, outbusaddr
);
369 wr_reg32(&jrp
->rregs
->inpring_size
, JOBR_DEPTH
);
370 wr_reg32(&jrp
->rregs
->outring_size
, JOBR_DEPTH
);
372 jrp
->ringsize
= JOBR_DEPTH
;
374 spin_lock_init(&jrp
->inplock
);
375 spin_lock_init(&jrp
->outlock
);
377 /* Select interrupt coalescing parameters */
378 setbits32(&jrp
->rregs
->rconfig_lo
, JOBR_INTC
|
379 (JOBR_INTC_COUNT_THLD
<< JRCFG_ICDCT_SHIFT
) |
380 (JOBR_INTC_TIME_THLD
<< JRCFG_ICTT_SHIFT
));
382 jrp
->assign
= JOBR_UNASSIGNED
;
387 * Shutdown JobR independent of platform property code
389 int caam_jr_shutdown(struct device
*dev
)
391 struct caam_drv_private_jr
*jrp
= dev_get_drvdata(dev
);
392 dma_addr_t inpbusaddr
, outbusaddr
;
395 ret
= caam_reset_hw_jr(dev
);
397 tasklet_kill(&jrp
->irqtask
);
399 /* Release interrupt */
400 free_irq(jrp
->irq
, dev
);
403 inpbusaddr
= rd_reg64(&jrp
->rregs
->inpring_base
);
404 outbusaddr
= rd_reg64(&jrp
->rregs
->outring_base
);
405 dma_free_coherent(dev
, sizeof(dma_addr_t
) * JOBR_DEPTH
,
406 jrp
->inpring
, inpbusaddr
);
407 dma_free_coherent(dev
, sizeof(struct jr_outentry
) * JOBR_DEPTH
,
408 jrp
->outring
, outbusaddr
);
410 of_device_unregister(jrp
->jr_pdev
);
416 * Probe routine for each detected JobR subsystem. It assumes that
417 * property detection was picked up externally.
419 int caam_jr_probe(struct platform_device
*pdev
, struct device_node
*np
,
422 struct device
*ctrldev
, *jrdev
;
423 struct platform_device
*jr_pdev
;
424 struct caam_drv_private
*ctrlpriv
;
425 struct caam_drv_private_jr
*jrpriv
;
429 ctrldev
= &pdev
->dev
;
430 ctrlpriv
= dev_get_drvdata(ctrldev
);
432 jrpriv
= kmalloc(sizeof(struct caam_drv_private_jr
),
434 if (jrpriv
== NULL
) {
435 dev_err(ctrldev
, "can't alloc private mem for job ring %d\n",
439 jrpriv
->parentdev
= ctrldev
; /* point back to parent */
440 jrpriv
->ridx
= ring
; /* save ring identity relative to detection */
443 * Derive a pointer to the detected JobRs regs
444 * Driver has already iomapped the entire space, we just
445 * need to add in the offset to this JobR. Don't know if I
446 * like this long-term, but it'll run
448 jroffset
= (u32
*)of_get_property(np
, "reg", NULL
);
449 jrpriv
->rregs
= (struct caam_job_ring __iomem
*)((void *)ctrlpriv
->ctrl
452 /* Build a local dev for each detected queue */
453 jr_pdev
= of_platform_device_create(np
, NULL
, ctrldev
);
454 if (jr_pdev
== NULL
) {
459 jrpriv
->jr_pdev
= jr_pdev
;
460 jrdev
= &jr_pdev
->dev
;
461 dev_set_drvdata(jrdev
, jrpriv
);
462 ctrlpriv
->jrdev
[ring
] = jrdev
;
464 if (sizeof(dma_addr_t
) == sizeof(u64
))
465 if (of_device_is_compatible(np
, "fsl,sec-v5.0-job-ring"))
466 dma_set_mask(jrdev
, DMA_BIT_MASK(40));
468 dma_set_mask(jrdev
, DMA_BIT_MASK(36));
470 dma_set_mask(jrdev
, DMA_BIT_MASK(32));
472 /* Identify the interrupt */
473 jrpriv
->irq
= of_irq_to_resource(np
, 0, NULL
);
475 /* Now do the platform independent part */
476 error
= caam_jr_init(jrdev
); /* now turn on hardware */
478 of_device_unregister(jr_pdev
);