2 * CAAM/SEC 4.x transport/backend driver
3 * JobR backend functionality
5 * Copyright 2008-2012 Freescale Semiconductor, Inc.
8 #include <linux/of_irq.h>
16 struct jr_driver_data
{
17 /* List of Physical JobR's with the Driver */
18 struct list_head jr_list
;
19 spinlock_t jr_alloc_lock
; /* jr_list lock */
20 } ____cacheline_aligned
;
22 static struct jr_driver_data driver_data
;
24 static int caam_reset_hw_jr(struct device
*dev
)
26 struct caam_drv_private_jr
*jrp
= dev_get_drvdata(dev
);
27 unsigned int timeout
= 100000;
30 * mask interrupts since we are going to poll
31 * for reset completion status
33 setbits32(&jrp
->rregs
->rconfig_lo
, JRCFG_IMSK
);
35 /* initiate flush (required prior to reset) */
36 wr_reg32(&jrp
->rregs
->jrcommand
, JRCR_RESET
);
37 while (((rd_reg32(&jrp
->rregs
->jrintstatus
) & JRINT_ERR_HALT_MASK
) ==
38 JRINT_ERR_HALT_INPROGRESS
) && --timeout
)
41 if ((rd_reg32(&jrp
->rregs
->jrintstatus
) & JRINT_ERR_HALT_MASK
) !=
42 JRINT_ERR_HALT_COMPLETE
|| timeout
== 0) {
43 dev_err(dev
, "failed to flush job ring %d\n", jrp
->ridx
);
49 wr_reg32(&jrp
->rregs
->jrcommand
, JRCR_RESET
);
50 while ((rd_reg32(&jrp
->rregs
->jrcommand
) & JRCR_RESET
) && --timeout
)
54 dev_err(dev
, "failed to reset job ring %d\n", jrp
->ridx
);
58 /* unmask interrupts */
59 clrbits32(&jrp
->rregs
->rconfig_lo
, JRCFG_IMSK
);
65 * Shutdown JobR independent of platform property code
67 int caam_jr_shutdown(struct device
*dev
)
69 struct caam_drv_private_jr
*jrp
= dev_get_drvdata(dev
);
70 dma_addr_t inpbusaddr
, outbusaddr
;
73 ret
= caam_reset_hw_jr(dev
);
75 tasklet_kill(&jrp
->irqtask
);
77 /* Release interrupt */
78 free_irq(jrp
->irq
, dev
);
81 inpbusaddr
= rd_reg64(&jrp
->rregs
->inpring_base
);
82 outbusaddr
= rd_reg64(&jrp
->rregs
->outring_base
);
83 dma_free_coherent(dev
, sizeof(dma_addr_t
) * JOBR_DEPTH
,
84 jrp
->inpring
, inpbusaddr
);
85 dma_free_coherent(dev
, sizeof(struct jr_outentry
) * JOBR_DEPTH
,
86 jrp
->outring
, outbusaddr
);
92 static int caam_jr_remove(struct platform_device
*pdev
)
96 struct caam_drv_private_jr
*jrpriv
;
99 jrpriv
= dev_get_drvdata(jrdev
);
102 * Return EBUSY if job ring already allocated.
104 if (atomic_read(&jrpriv
->tfm_count
)) {
105 dev_err(jrdev
, "Device is busy\n");
109 /* Remove the node from Physical JobR list maintained by driver */
110 spin_lock(&driver_data
.jr_alloc_lock
);
111 list_del(&jrpriv
->list_node
);
112 spin_unlock(&driver_data
.jr_alloc_lock
);
115 ret
= caam_jr_shutdown(jrdev
);
117 dev_err(jrdev
, "Failed to shut down job ring\n");
118 irq_dispose_mapping(jrpriv
->irq
);
123 /* Main per-ring interrupt handler */
124 static irqreturn_t
caam_jr_interrupt(int irq
, void *st_dev
)
126 struct device
*dev
= st_dev
;
127 struct caam_drv_private_jr
*jrp
= dev_get_drvdata(dev
);
131 * Check the output ring for ready responses, kick
132 * tasklet if jobs done.
134 irqstate
= rd_reg32(&jrp
->rregs
->jrintstatus
);
139 * If JobR error, we got more development work to do
140 * Flag a bug now, but we really need to shut down and
141 * restart the queue (and fix code).
143 if (irqstate
& JRINT_JR_ERROR
) {
144 dev_err(dev
, "job ring error: irqstate: %08x\n", irqstate
);
148 /* mask valid interrupts */
149 setbits32(&jrp
->rregs
->rconfig_lo
, JRCFG_IMSK
);
151 /* Have valid interrupt at this point, just ACK and trigger */
152 wr_reg32(&jrp
->rregs
->jrintstatus
, irqstate
);
155 tasklet_schedule(&jrp
->irqtask
);
161 /* Deferred service handler, run as interrupt-fired tasklet */
162 static void caam_jr_dequeue(unsigned long devarg
)
164 int hw_idx
, sw_idx
, i
, head
, tail
;
165 struct device
*dev
= (struct device
*)devarg
;
166 struct caam_drv_private_jr
*jrp
= dev_get_drvdata(dev
);
167 void (*usercall
)(struct device
*dev
, u32
*desc
, u32 status
, void *arg
);
168 u32
*userdesc
, userstatus
;
171 while (rd_reg32(&jrp
->rregs
->outring_used
)) {
173 head
= ACCESS_ONCE(jrp
->head
);
175 spin_lock(&jrp
->outlock
);
177 sw_idx
= tail
= jrp
->tail
;
178 hw_idx
= jrp
->out_ring_read_index
;
180 for (i
= 0; CIRC_CNT(head
, tail
+ i
, JOBR_DEPTH
) >= 1; i
++) {
181 sw_idx
= (tail
+ i
) & (JOBR_DEPTH
- 1);
183 smp_read_barrier_depends();
185 if (jrp
->outring
[hw_idx
].desc
==
186 jrp
->entinfo
[sw_idx
].desc_addr_dma
)
189 /* we should never fail to find a matching descriptor */
190 BUG_ON(CIRC_CNT(head
, tail
+ i
, JOBR_DEPTH
) <= 0);
192 /* Unmap just-run descriptor so we can post-process */
193 dma_unmap_single(dev
, jrp
->outring
[hw_idx
].desc
,
194 jrp
->entinfo
[sw_idx
].desc_size
,
197 /* mark completed, avoid matching on a recycled desc addr */
198 jrp
->entinfo
[sw_idx
].desc_addr_dma
= 0;
200 /* Stash callback params for use outside of lock */
201 usercall
= jrp
->entinfo
[sw_idx
].callbk
;
202 userarg
= jrp
->entinfo
[sw_idx
].cbkarg
;
203 userdesc
= jrp
->entinfo
[sw_idx
].desc_addr_virt
;
204 userstatus
= jrp
->outring
[hw_idx
].jrstatus
;
207 wr_reg32(&jrp
->rregs
->outring_rmvd
, 1);
209 jrp
->out_ring_read_index
= (jrp
->out_ring_read_index
+ 1) &
213 * if this job completed out-of-order, do not increment
214 * the tail. Otherwise, increment tail by 1 plus the
215 * number of subsequent jobs already completed out-of-order
217 if (sw_idx
== tail
) {
219 tail
= (tail
+ 1) & (JOBR_DEPTH
- 1);
220 smp_read_barrier_depends();
221 } while (CIRC_CNT(head
, tail
, JOBR_DEPTH
) >= 1 &&
222 jrp
->entinfo
[tail
].desc_addr_dma
== 0);
227 spin_unlock(&jrp
->outlock
);
229 /* Finally, execute user's callback */
230 usercall(dev
, userdesc
, userstatus
, userarg
);
233 /* reenable / unmask IRQs */
234 clrbits32(&jrp
->rregs
->rconfig_lo
, JRCFG_IMSK
);
238 * caam_jr_alloc() - Alloc a job ring for someone to use as needed.
240 * returns : pointer to the newly allocated physical
241 * JobR dev can be written to if successful.
243 struct device
*caam_jr_alloc(void)
245 struct caam_drv_private_jr
*jrpriv
, *min_jrpriv
= NULL
;
246 struct device
*dev
= NULL
;
247 int min_tfm_cnt
= INT_MAX
;
250 spin_lock(&driver_data
.jr_alloc_lock
);
252 if (list_empty(&driver_data
.jr_list
)) {
253 spin_unlock(&driver_data
.jr_alloc_lock
);
254 return ERR_PTR(-ENODEV
);
257 list_for_each_entry(jrpriv
, &driver_data
.jr_list
, list_node
) {
258 tfm_cnt
= atomic_read(&jrpriv
->tfm_count
);
259 if (tfm_cnt
< min_tfm_cnt
) {
260 min_tfm_cnt
= tfm_cnt
;
268 atomic_inc(&min_jrpriv
->tfm_count
);
269 dev
= min_jrpriv
->dev
;
271 spin_unlock(&driver_data
.jr_alloc_lock
);
275 EXPORT_SYMBOL(caam_jr_alloc
);
278 * caam_jr_free() - Free the Job Ring
279 * @rdev - points to the dev that identifies the Job ring to
282 void caam_jr_free(struct device
*rdev
)
284 struct caam_drv_private_jr
*jrpriv
= dev_get_drvdata(rdev
);
286 atomic_dec(&jrpriv
->tfm_count
);
288 EXPORT_SYMBOL(caam_jr_free
);
291 * caam_jr_enqueue() - Enqueue a job descriptor head. Returns 0 if OK,
292 * -EBUSY if the queue is full, -EIO if it cannot map the caller's
294 * @dev: device of the job ring to be used. This device should have
295 * been assigned prior by caam_jr_register().
296 * @desc: points to a job descriptor that execute our request. All
297 * descriptors (and all referenced data) must be in a DMAable
298 * region, and all data references must be physical addresses
299 * accessible to CAAM (i.e. within a PAMU window granted
301 * @cbk: pointer to a callback function to be invoked upon completion
302 * of this request. This has the form:
303 * callback(struct device *dev, u32 *desc, u32 stat, void *arg)
305 * @dev: contains the job ring device that processed this
307 * @desc: descriptor that initiated the request, same as
308 * "desc" being argued to caam_jr_enqueue().
309 * @status: untranslated status received from CAAM. See the
310 * reference manual for a detailed description of
311 * error meaning, or see the JRSTA definitions in the
312 * register header file
313 * @areq: optional pointer to an argument passed with the
315 * @areq: optional pointer to a user argument for use at callback
318 int caam_jr_enqueue(struct device
*dev
, u32
*desc
,
319 void (*cbk
)(struct device
*dev
, u32
*desc
,
320 u32 status
, void *areq
),
323 struct caam_drv_private_jr
*jrp
= dev_get_drvdata(dev
);
324 struct caam_jrentry_info
*head_entry
;
325 int head
, tail
, desc_size
;
328 desc_size
= (*desc
& HDR_JD_LENGTH_MASK
) * sizeof(u32
);
329 desc_dma
= dma_map_single(dev
, desc
, desc_size
, DMA_TO_DEVICE
);
330 if (dma_mapping_error(dev
, desc_dma
)) {
331 dev_err(dev
, "caam_jr_enqueue(): can't map jobdesc\n");
335 spin_lock_bh(&jrp
->inplock
);
338 tail
= ACCESS_ONCE(jrp
->tail
);
340 if (!rd_reg32(&jrp
->rregs
->inpring_avail
) ||
341 CIRC_SPACE(head
, tail
, JOBR_DEPTH
) <= 0) {
342 spin_unlock_bh(&jrp
->inplock
);
343 dma_unmap_single(dev
, desc_dma
, desc_size
, DMA_TO_DEVICE
);
347 head_entry
= &jrp
->entinfo
[head
];
348 head_entry
->desc_addr_virt
= desc
;
349 head_entry
->desc_size
= desc_size
;
350 head_entry
->callbk
= (void *)cbk
;
351 head_entry
->cbkarg
= areq
;
352 head_entry
->desc_addr_dma
= desc_dma
;
354 jrp
->inpring
[jrp
->inp_ring_write_index
] = desc_dma
;
358 jrp
->inp_ring_write_index
= (jrp
->inp_ring_write_index
+ 1) &
360 jrp
->head
= (head
+ 1) & (JOBR_DEPTH
- 1);
362 wr_reg32(&jrp
->rregs
->inpring_jobadd
, 1);
364 spin_unlock_bh(&jrp
->inplock
);
368 EXPORT_SYMBOL(caam_jr_enqueue
);
371 * Init JobR independent of platform property detection
373 static int caam_jr_init(struct device
*dev
)
375 struct caam_drv_private_jr
*jrp
;
376 dma_addr_t inpbusaddr
, outbusaddr
;
379 jrp
= dev_get_drvdata(dev
);
381 tasklet_init(&jrp
->irqtask
, caam_jr_dequeue
, (unsigned long)dev
);
383 /* Connect job ring interrupt handler. */
384 error
= request_irq(jrp
->irq
, caam_jr_interrupt
, IRQF_SHARED
,
387 dev_err(dev
, "can't connect JobR %d interrupt (%d)\n",
388 jrp
->ridx
, jrp
->irq
);
389 irq_dispose_mapping(jrp
->irq
);
394 error
= caam_reset_hw_jr(dev
);
398 jrp
->inpring
= dma_alloc_coherent(dev
, sizeof(dma_addr_t
) * JOBR_DEPTH
,
399 &inpbusaddr
, GFP_KERNEL
);
401 jrp
->outring
= dma_alloc_coherent(dev
, sizeof(struct jr_outentry
) *
402 JOBR_DEPTH
, &outbusaddr
, GFP_KERNEL
);
404 jrp
->entinfo
= kzalloc(sizeof(struct caam_jrentry_info
) * JOBR_DEPTH
,
407 if ((jrp
->inpring
== NULL
) || (jrp
->outring
== NULL
) ||
408 (jrp
->entinfo
== NULL
)) {
409 dev_err(dev
, "can't allocate job rings for %d\n",
414 for (i
= 0; i
< JOBR_DEPTH
; i
++)
415 jrp
->entinfo
[i
].desc_addr_dma
= !0;
418 jrp
->inp_ring_write_index
= 0;
419 jrp
->out_ring_read_index
= 0;
423 wr_reg64(&jrp
->rregs
->inpring_base
, inpbusaddr
);
424 wr_reg64(&jrp
->rregs
->outring_base
, outbusaddr
);
425 wr_reg32(&jrp
->rregs
->inpring_size
, JOBR_DEPTH
);
426 wr_reg32(&jrp
->rregs
->outring_size
, JOBR_DEPTH
);
428 jrp
->ringsize
= JOBR_DEPTH
;
430 spin_lock_init(&jrp
->inplock
);
431 spin_lock_init(&jrp
->outlock
);
433 /* Select interrupt coalescing parameters */
434 setbits32(&jrp
->rregs
->rconfig_lo
, JOBR_INTC
|
435 (JOBR_INTC_COUNT_THLD
<< JRCFG_ICDCT_SHIFT
) |
436 (JOBR_INTC_TIME_THLD
<< JRCFG_ICTT_SHIFT
));
443 * Probe routine for each detected JobR subsystem.
445 static int caam_jr_probe(struct platform_device
*pdev
)
447 struct device
*jrdev
;
448 struct device_node
*nprop
;
449 struct caam_job_ring __iomem
*ctrl
;
450 struct caam_drv_private_jr
*jrpriv
;
451 static int total_jobrs
;
455 jrpriv
= kmalloc(sizeof(struct caam_drv_private_jr
),
460 dev_set_drvdata(jrdev
, jrpriv
);
462 /* save ring identity relative to detection */
463 jrpriv
->ridx
= total_jobrs
++;
465 nprop
= pdev
->dev
.of_node
;
466 /* Get configuration properties from device tree */
467 /* First, get register page */
468 ctrl
= of_iomap(nprop
, 0);
470 dev_err(jrdev
, "of_iomap() failed\n");
474 jrpriv
->rregs
= (struct caam_job_ring __force
*)ctrl
;
476 if (sizeof(dma_addr_t
) == sizeof(u64
))
477 if (of_device_is_compatible(nprop
, "fsl,sec-v5.0-job-ring"))
478 dma_set_mask(jrdev
, DMA_BIT_MASK(40));
480 dma_set_mask(jrdev
, DMA_BIT_MASK(36));
482 dma_set_mask(jrdev
, DMA_BIT_MASK(32));
484 /* Identify the interrupt */
485 jrpriv
->irq
= irq_of_parse_and_map(nprop
, 0);
487 /* Now do the platform independent part */
488 error
= caam_jr_init(jrdev
); /* now turn on hardware */
495 spin_lock(&driver_data
.jr_alloc_lock
);
496 list_add_tail(&jrpriv
->list_node
, &driver_data
.jr_list
);
497 spin_unlock(&driver_data
.jr_alloc_lock
);
499 atomic_set(&jrpriv
->tfm_count
, 0);
504 static struct of_device_id caam_jr_match
[] = {
506 .compatible
= "fsl,sec-v4.0-job-ring",
509 .compatible
= "fsl,sec4.0-job-ring",
513 MODULE_DEVICE_TABLE(of
, caam_jr_match
);
515 static struct platform_driver caam_jr_driver
= {
518 .owner
= THIS_MODULE
,
519 .of_match_table
= caam_jr_match
,
521 .probe
= caam_jr_probe
,
522 .remove
= caam_jr_remove
,
525 static int __init
jr_driver_init(void)
527 spin_lock_init(&driver_data
.jr_alloc_lock
);
528 INIT_LIST_HEAD(&driver_data
.jr_list
);
529 return platform_driver_register(&caam_jr_driver
);
532 static void __exit
jr_driver_exit(void)
534 platform_driver_unregister(&caam_jr_driver
);
537 module_init(jr_driver_init
);
538 module_exit(jr_driver_exit
);
540 MODULE_LICENSE("GPL");
541 MODULE_DESCRIPTION("FSL CAAM JR request backend");
542 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");