]>
Commit | Line | Data |
---|---|---|
d2912cb1 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
ea0375af GH |
2 | /* |
3 | * AMD Cryptographic Coprocessor (CCP) driver | |
4 | * | |
68cc652f | 5 | * Copyright (C) 2013,2017 Advanced Micro Devices, Inc. |
ea0375af GH |
6 | * |
7 | * Author: Tom Lendacky <thomas.lendacky@amd.com> | |
fba8855c | 8 | * Author: Gary R Hook <gary.hook@amd.com> |
ea0375af GH |
9 | */ |
10 | ||
11 | #include <linux/module.h> | |
12 | #include <linux/kernel.h> | |
13 | #include <linux/pci.h> | |
14 | #include <linux/kthread.h> | |
15 | #include <linux/interrupt.h> | |
16 | #include <linux/ccp.h> | |
17 | ||
18 | #include "ccp-dev.h" | |
19 | ||
58a690b7 GH |
20 | static u32 ccp_alloc_ksb(struct ccp_cmd_queue *cmd_q, unsigned int count) |
21 | { | |
22 | int start; | |
23 | struct ccp_device *ccp = cmd_q->ccp; | |
24 | ||
25 | for (;;) { | |
26 | mutex_lock(&ccp->sb_mutex); | |
27 | ||
28 | start = (u32)bitmap_find_next_zero_area(ccp->sb, | |
29 | ccp->sb_count, | |
30 | ccp->sb_start, | |
31 | count, 0); | |
32 | if (start <= ccp->sb_count) { | |
33 | bitmap_set(ccp->sb, start, count); | |
34 | ||
35 | mutex_unlock(&ccp->sb_mutex); | |
36 | break; | |
37 | } | |
38 | ||
39 | ccp->sb_avail = 0; | |
40 | ||
41 | mutex_unlock(&ccp->sb_mutex); | |
42 | ||
43 | /* Wait for KSB entries to become available */ | |
44 | if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail)) | |
45 | return 0; | |
46 | } | |
47 | ||
48 | return KSB_START + start; | |
49 | } | |
50 | ||
51 | static void ccp_free_ksb(struct ccp_cmd_queue *cmd_q, unsigned int start, | |
52 | unsigned int count) | |
53 | { | |
54 | struct ccp_device *ccp = cmd_q->ccp; | |
55 | ||
56 | if (!start) | |
57 | return; | |
58 | ||
59 | mutex_lock(&ccp->sb_mutex); | |
60 | ||
61 | bitmap_clear(ccp->sb, start - KSB_START, count); | |
62 | ||
63 | ccp->sb_avail = 1; | |
64 | ||
65 | mutex_unlock(&ccp->sb_mutex); | |
66 | ||
67 | wake_up_interruptible_all(&ccp->sb_queue); | |
68 | } | |
69 | ||
bb4e89b3 GH |
70 | static unsigned int ccp_get_free_slots(struct ccp_cmd_queue *cmd_q) |
71 | { | |
72 | return CMD_Q_DEPTH(ioread32(cmd_q->reg_status)); | |
73 | } | |
74 | ||
ea0375af GH |
75 | static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count) |
76 | { | |
77 | struct ccp_cmd_queue *cmd_q = op->cmd_q; | |
78 | struct ccp_device *ccp = cmd_q->ccp; | |
79 | void __iomem *cr_addr; | |
80 | u32 cr0, cmd; | |
81 | unsigned int i; | |
82 | int ret = 0; | |
83 | ||
84 | /* We could read a status register to see how many free slots | |
85 | * are actually available, but reading that register resets it | |
86 | * and you could lose some error information. | |
87 | */ | |
88 | cmd_q->free_slots--; | |
89 | ||
90 | cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT) | |
91 | | (op->jobid << REQ0_JOBID_SHIFT) | |
92 | | REQ0_WAIT_FOR_WRITE; | |
93 | ||
94 | if (op->soc) | |
95 | cr0 |= REQ0_STOP_ON_COMPLETE | |
96 | | REQ0_INT_ON_COMPLETE; | |
97 | ||
98 | if (op->ioc || !cmd_q->free_slots) | |
99 | cr0 |= REQ0_INT_ON_COMPLETE; | |
100 | ||
101 | /* Start at CMD_REQ1 */ | |
102 | cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR; | |
103 | ||
104 | mutex_lock(&ccp->req_mutex); | |
105 | ||
106 | /* Write CMD_REQ1 through CMD_REQx first */ | |
107 | for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR) | |
108 | iowrite32(*(cr + i), cr_addr); | |
109 | ||
110 | /* Tell the CCP to start */ | |
111 | wmb(); | |
112 | iowrite32(cr0, ccp->io_regs + CMD_REQ0); | |
113 | ||
114 | mutex_unlock(&ccp->req_mutex); | |
115 | ||
116 | if (cr0 & REQ0_INT_ON_COMPLETE) { | |
117 | /* Wait for the job to complete */ | |
118 | ret = wait_event_interruptible(cmd_q->int_queue, | |
119 | cmd_q->int_rcvd); | |
120 | if (ret || cmd_q->cmd_error) { | |
121 | /* On error delete all related jobs from the queue */ | |
122 | cmd = (cmd_q->id << DEL_Q_ID_SHIFT) | |
123 | | op->jobid; | |
81422bad GH |
124 | if (cmd_q->cmd_error) |
125 | ccp_log_error(cmd_q->ccp, | |
126 | cmd_q->cmd_error); | |
ea0375af GH |
127 | |
128 | iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); | |
129 | ||
130 | if (!ret) | |
131 | ret = -EIO; | |
132 | } else if (op->soc) { | |
133 | /* Delete just head job from the queue on SoC */ | |
134 | cmd = DEL_Q_ACTIVE | |
135 | | (cmd_q->id << DEL_Q_ID_SHIFT) | |
136 | | op->jobid; | |
137 | ||
138 | iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); | |
139 | } | |
140 | ||
141 | cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status); | |
142 | ||
143 | cmd_q->int_rcvd = 0; | |
144 | } | |
145 | ||
146 | return ret; | |
147 | } | |
148 | ||
149 | static int ccp_perform_aes(struct ccp_op *op) | |
150 | { | |
151 | u32 cr[6]; | |
152 | ||
153 | /* Fill out the register contents for REQ1 through REQ6 */ | |
154 | cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT) | |
155 | | (op->u.aes.type << REQ1_AES_TYPE_SHIFT) | |
156 | | (op->u.aes.mode << REQ1_AES_MODE_SHIFT) | |
157 | | (op->u.aes.action << REQ1_AES_ACTION_SHIFT) | |
956ee21a | 158 | | (op->sb_key << REQ1_KEY_KSB_SHIFT); |
ea0375af GH |
159 | cr[1] = op->src.u.dma.length - 1; |
160 | cr[2] = ccp_addr_lo(&op->src.u.dma); | |
956ee21a | 161 | cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) |
ea0375af GH |
162 | | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) |
163 | | ccp_addr_hi(&op->src.u.dma); | |
164 | cr[4] = ccp_addr_lo(&op->dst.u.dma); | |
165 | cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) | |
166 | | ccp_addr_hi(&op->dst.u.dma); | |
167 | ||
168 | if (op->u.aes.mode == CCP_AES_MODE_CFB) | |
169 | cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT); | |
170 | ||
171 | if (op->eom) | |
172 | cr[0] |= REQ1_EOM; | |
173 | ||
174 | if (op->init) | |
175 | cr[0] |= REQ1_INIT; | |
176 | ||
177 | return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); | |
178 | } | |
179 | ||
180 | static int ccp_perform_xts_aes(struct ccp_op *op) | |
181 | { | |
182 | u32 cr[6]; | |
183 | ||
184 | /* Fill out the register contents for REQ1 through REQ6 */ | |
185 | cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT) | |
186 | | (op->u.xts.action << REQ1_AES_ACTION_SHIFT) | |
187 | | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT) | |
956ee21a | 188 | | (op->sb_key << REQ1_KEY_KSB_SHIFT); |
ea0375af GH |
189 | cr[1] = op->src.u.dma.length - 1; |
190 | cr[2] = ccp_addr_lo(&op->src.u.dma); | |
956ee21a | 191 | cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) |
ea0375af GH |
192 | | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) |
193 | | ccp_addr_hi(&op->src.u.dma); | |
194 | cr[4] = ccp_addr_lo(&op->dst.u.dma); | |
195 | cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) | |
196 | | ccp_addr_hi(&op->dst.u.dma); | |
197 | ||
198 | if (op->eom) | |
199 | cr[0] |= REQ1_EOM; | |
200 | ||
201 | if (op->init) | |
202 | cr[0] |= REQ1_INIT; | |
203 | ||
204 | return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); | |
205 | } | |
206 | ||
207 | static int ccp_perform_sha(struct ccp_op *op) | |
208 | { | |
209 | u32 cr[6]; | |
210 | ||
211 | /* Fill out the register contents for REQ1 through REQ6 */ | |
212 | cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT) | |
213 | | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT) | |
214 | | REQ1_INIT; | |
215 | cr[1] = op->src.u.dma.length - 1; | |
216 | cr[2] = ccp_addr_lo(&op->src.u.dma); | |
956ee21a | 217 | cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) |
ea0375af GH |
218 | | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) |
219 | | ccp_addr_hi(&op->src.u.dma); | |
220 | ||
221 | if (op->eom) { | |
222 | cr[0] |= REQ1_EOM; | |
223 | cr[4] = lower_32_bits(op->u.sha.msg_bits); | |
224 | cr[5] = upper_32_bits(op->u.sha.msg_bits); | |
225 | } else { | |
226 | cr[4] = 0; | |
227 | cr[5] = 0; | |
228 | } | |
229 | ||
230 | return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); | |
231 | } | |
232 | ||
233 | static int ccp_perform_rsa(struct ccp_op *op) | |
234 | { | |
235 | u32 cr[6]; | |
236 | ||
237 | /* Fill out the register contents for REQ1 through REQ6 */ | |
238 | cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT) | |
239 | | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT) | |
956ee21a | 240 | | (op->sb_key << REQ1_KEY_KSB_SHIFT) |
ea0375af GH |
241 | | REQ1_EOM; |
242 | cr[1] = op->u.rsa.input_len - 1; | |
243 | cr[2] = ccp_addr_lo(&op->src.u.dma); | |
956ee21a | 244 | cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) |
ea0375af GH |
245 | | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) |
246 | | ccp_addr_hi(&op->src.u.dma); | |
247 | cr[4] = ccp_addr_lo(&op->dst.u.dma); | |
248 | cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) | |
249 | | ccp_addr_hi(&op->dst.u.dma); | |
250 | ||
251 | return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); | |
252 | } | |
253 | ||
254 | static int ccp_perform_passthru(struct ccp_op *op) | |
255 | { | |
256 | u32 cr[6]; | |
257 | ||
258 | /* Fill out the register contents for REQ1 through REQ6 */ | |
259 | cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT) | |
260 | | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT) | |
261 | | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT); | |
262 | ||
263 | if (op->src.type == CCP_MEMTYPE_SYSTEM) | |
264 | cr[1] = op->src.u.dma.length - 1; | |
265 | else | |
266 | cr[1] = op->dst.u.dma.length - 1; | |
267 | ||
268 | if (op->src.type == CCP_MEMTYPE_SYSTEM) { | |
269 | cr[2] = ccp_addr_lo(&op->src.u.dma); | |
270 | cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) | |
271 | | ccp_addr_hi(&op->src.u.dma); | |
272 | ||
273 | if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP) | |
956ee21a | 274 | cr[3] |= (op->sb_key << REQ4_KSB_SHIFT); |
ea0375af | 275 | } else { |
956ee21a GH |
276 | cr[2] = op->src.u.sb * CCP_SB_BYTES; |
277 | cr[3] = (CCP_MEMTYPE_SB << REQ4_MEMTYPE_SHIFT); | |
ea0375af GH |
278 | } |
279 | ||
280 | if (op->dst.type == CCP_MEMTYPE_SYSTEM) { | |
281 | cr[4] = ccp_addr_lo(&op->dst.u.dma); | |
282 | cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) | |
283 | | ccp_addr_hi(&op->dst.u.dma); | |
284 | } else { | |
956ee21a GH |
285 | cr[4] = op->dst.u.sb * CCP_SB_BYTES; |
286 | cr[5] = (CCP_MEMTYPE_SB << REQ6_MEMTYPE_SHIFT); | |
ea0375af GH |
287 | } |
288 | ||
289 | if (op->eom) | |
290 | cr[0] |= REQ1_EOM; | |
291 | ||
292 | return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); | |
293 | } | |
294 | ||
295 | static int ccp_perform_ecc(struct ccp_op *op) | |
296 | { | |
297 | u32 cr[6]; | |
298 | ||
299 | /* Fill out the register contents for REQ1 through REQ6 */ | |
300 | cr[0] = REQ1_ECC_AFFINE_CONVERT | |
301 | | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT) | |
302 | | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT) | |
303 | | REQ1_EOM; | |
304 | cr[1] = op->src.u.dma.length - 1; | |
305 | cr[2] = ccp_addr_lo(&op->src.u.dma); | |
306 | cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) | |
307 | | ccp_addr_hi(&op->src.u.dma); | |
308 | cr[4] = ccp_addr_lo(&op->dst.u.dma); | |
309 | cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) | |
310 | | ccp_addr_hi(&op->dst.u.dma); | |
311 | ||
312 | return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); | |
313 | } | |
314 | ||
7b537b24 GH |
315 | static void ccp_disable_queue_interrupts(struct ccp_device *ccp) |
316 | { | |
317 | iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG); | |
318 | } | |
319 | ||
320 | static void ccp_enable_queue_interrupts(struct ccp_device *ccp) | |
321 | { | |
322 | iowrite32(ccp->qim, ccp->io_regs + IRQ_MASK_REG); | |
323 | } | |
324 | ||
325 | static void ccp_irq_bh(unsigned long data) | |
326 | { | |
327 | struct ccp_device *ccp = (struct ccp_device *)data; | |
328 | struct ccp_cmd_queue *cmd_q; | |
329 | u32 q_int, status; | |
330 | unsigned int i; | |
331 | ||
332 | status = ioread32(ccp->io_regs + IRQ_STATUS_REG); | |
333 | ||
334 | for (i = 0; i < ccp->cmd_q_count; i++) { | |
335 | cmd_q = &ccp->cmd_q[i]; | |
336 | ||
337 | q_int = status & (cmd_q->int_ok | cmd_q->int_err); | |
338 | if (q_int) { | |
339 | cmd_q->int_status = status; | |
340 | cmd_q->q_status = ioread32(cmd_q->reg_status); | |
341 | cmd_q->q_int_status = ioread32(cmd_q->reg_int_status); | |
342 | ||
343 | /* On error, only save the first error value */ | |
344 | if ((q_int & cmd_q->int_err) && !cmd_q->cmd_error) | |
345 | cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); | |
346 | ||
347 | cmd_q->int_rcvd = 1; | |
348 | ||
349 | /* Acknowledge the interrupt and wake the kthread */ | |
350 | iowrite32(q_int, ccp->io_regs + IRQ_STATUS_REG); | |
351 | wake_up_interruptible(&cmd_q->int_queue); | |
352 | } | |
353 | } | |
354 | ccp_enable_queue_interrupts(ccp); | |
355 | } | |
356 | ||
357 | static irqreturn_t ccp_irq_handler(int irq, void *data) | |
358 | { | |
720419f0 | 359 | struct ccp_device *ccp = (struct ccp_device *)data; |
7b537b24 GH |
360 | |
361 | ccp_disable_queue_interrupts(ccp); | |
362 | if (ccp->use_tasklet) | |
363 | tasklet_schedule(&ccp->irq_tasklet); | |
364 | else | |
365 | ccp_irq_bh((unsigned long)ccp); | |
366 | ||
367 | return IRQ_HANDLED; | |
368 | } | |
369 | ||
ea0375af GH |
370 | static int ccp_init(struct ccp_device *ccp) |
371 | { | |
372 | struct device *dev = ccp->dev; | |
373 | struct ccp_cmd_queue *cmd_q; | |
374 | struct dma_pool *dma_pool; | |
375 | char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; | |
7b537b24 | 376 | unsigned int qmr, i; |
ea0375af GH |
377 | int ret; |
378 | ||
379 | /* Find available queues */ | |
7b537b24 | 380 | ccp->qim = 0; |
ea0375af GH |
381 | qmr = ioread32(ccp->io_regs + Q_MASK_REG); |
382 | for (i = 0; i < MAX_HW_QUEUES; i++) { | |
383 | if (!(qmr & (1 << i))) | |
384 | continue; | |
385 | ||
386 | /* Allocate a dma pool for this queue */ | |
387 | snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d", | |
388 | ccp->name, i); | |
389 | dma_pool = dma_pool_create(dma_pool_name, dev, | |
390 | CCP_DMAPOOL_MAX_SIZE, | |
391 | CCP_DMAPOOL_ALIGN, 0); | |
392 | if (!dma_pool) { | |
393 | dev_err(dev, "unable to allocate dma pool\n"); | |
394 | ret = -ENOMEM; | |
395 | goto e_pool; | |
396 | } | |
397 | ||
398 | cmd_q = &ccp->cmd_q[ccp->cmd_q_count]; | |
399 | ccp->cmd_q_count++; | |
400 | ||
401 | cmd_q->ccp = ccp; | |
402 | cmd_q->id = i; | |
403 | cmd_q->dma_pool = dma_pool; | |
404 | ||
405 | /* Reserve 2 KSB regions for the queue */ | |
956ee21a GH |
406 | cmd_q->sb_key = KSB_START + ccp->sb_start++; |
407 | cmd_q->sb_ctx = KSB_START + ccp->sb_start++; | |
408 | ccp->sb_count -= 2; | |
ea0375af GH |
409 | |
410 | /* Preset some register values and masks that are queue | |
411 | * number dependent | |
412 | */ | |
413 | cmd_q->reg_status = ccp->io_regs + CMD_Q_STATUS_BASE + | |
414 | (CMD_Q_STATUS_INCR * i); | |
415 | cmd_q->reg_int_status = ccp->io_regs + CMD_Q_INT_STATUS_BASE + | |
416 | (CMD_Q_STATUS_INCR * i); | |
417 | cmd_q->int_ok = 1 << (i * 2); | |
418 | cmd_q->int_err = 1 << ((i * 2) + 1); | |
419 | ||
bb4e89b3 | 420 | cmd_q->free_slots = ccp_get_free_slots(cmd_q); |
ea0375af GH |
421 | |
422 | init_waitqueue_head(&cmd_q->int_queue); | |
423 | ||
424 | /* Build queue interrupt mask (two interrupts per queue) */ | |
7b537b24 | 425 | ccp->qim |= cmd_q->int_ok | cmd_q->int_err; |
ea0375af GH |
426 | |
427 | #ifdef CONFIG_ARM64 | |
428 | /* For arm64 set the recommended queue cache settings */ | |
429 | iowrite32(ccp->axcache, ccp->io_regs + CMD_Q_CACHE_BASE + | |
430 | (CMD_Q_CACHE_INC * i)); | |
431 | #endif | |
432 | ||
433 | dev_dbg(dev, "queue #%u available\n", i); | |
434 | } | |
435 | if (ccp->cmd_q_count == 0) { | |
436 | dev_notice(dev, "no command queues available\n"); | |
437 | ret = -EIO; | |
438 | goto e_pool; | |
439 | } | |
440 | dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count); | |
441 | ||
442 | /* Disable and clear interrupts until ready */ | |
7b537b24 | 443 | ccp_disable_queue_interrupts(ccp); |
ea0375af GH |
444 | for (i = 0; i < ccp->cmd_q_count; i++) { |
445 | cmd_q = &ccp->cmd_q[i]; | |
446 | ||
447 | ioread32(cmd_q->reg_int_status); | |
448 | ioread32(cmd_q->reg_status); | |
449 | } | |
7b537b24 | 450 | iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG); |
ea0375af GH |
451 | |
452 | /* Request an irq */ | |
f4d18d65 | 453 | ret = sp_request_ccp_irq(ccp->sp, ccp_irq_handler, ccp->name, ccp); |
ea0375af GH |
454 | if (ret) { |
455 | dev_err(dev, "unable to allocate an IRQ\n"); | |
456 | goto e_pool; | |
457 | } | |
458 | ||
7b537b24 GH |
459 | /* Initialize the ISR tasklet? */ |
460 | if (ccp->use_tasklet) | |
461 | tasklet_init(&ccp->irq_tasklet, ccp_irq_bh, | |
462 | (unsigned long)ccp); | |
463 | ||
4b394a23 | 464 | dev_dbg(dev, "Starting threads...\n"); |
ea0375af GH |
465 | /* Create a kthread for each queue */ |
466 | for (i = 0; i < ccp->cmd_q_count; i++) { | |
467 | struct task_struct *kthread; | |
468 | ||
469 | cmd_q = &ccp->cmd_q[i]; | |
470 | ||
471 | kthread = kthread_create(ccp_cmd_queue_thread, cmd_q, | |
472 | "%s-q%u", ccp->name, cmd_q->id); | |
473 | if (IS_ERR(kthread)) { | |
474 | dev_err(dev, "error creating queue thread (%ld)\n", | |
475 | PTR_ERR(kthread)); | |
476 | ret = PTR_ERR(kthread); | |
477 | goto e_kthread; | |
478 | } | |
479 | ||
480 | cmd_q->kthread = kthread; | |
481 | wake_up_process(kthread); | |
482 | } | |
483 | ||
4b394a23 GH |
484 | dev_dbg(dev, "Enabling interrupts...\n"); |
485 | /* Enable interrupts */ | |
7b537b24 | 486 | ccp_enable_queue_interrupts(ccp); |
4b394a23 GH |
487 | |
488 | dev_dbg(dev, "Registering device...\n"); | |
489 | ccp_add_device(ccp); | |
490 | ||
084935b2 GH |
491 | ret = ccp_register_rng(ccp); |
492 | if (ret) | |
ea0375af | 493 | goto e_kthread; |
ea0375af | 494 | |
58ea8abf GH |
495 | /* Register the DMA engine support */ |
496 | ret = ccp_dmaengine_register(ccp); | |
497 | if (ret) | |
498 | goto e_hwrng; | |
499 | ||
ea0375af GH |
500 | return 0; |
501 | ||
58ea8abf | 502 | e_hwrng: |
084935b2 | 503 | ccp_unregister_rng(ccp); |
58ea8abf | 504 | |
ea0375af GH |
505 | e_kthread: |
506 | for (i = 0; i < ccp->cmd_q_count; i++) | |
507 | if (ccp->cmd_q[i].kthread) | |
508 | kthread_stop(ccp->cmd_q[i].kthread); | |
509 | ||
f4d18d65 | 510 | sp_free_ccp_irq(ccp->sp, ccp); |
ea0375af GH |
511 | |
512 | e_pool: | |
513 | for (i = 0; i < ccp->cmd_q_count; i++) | |
514 | dma_pool_destroy(ccp->cmd_q[i].dma_pool); | |
515 | ||
516 | return ret; | |
517 | } | |
518 | ||
519 | static void ccp_destroy(struct ccp_device *ccp) | |
520 | { | |
521 | struct ccp_cmd_queue *cmd_q; | |
522 | struct ccp_cmd *cmd; | |
7b537b24 | 523 | unsigned int i; |
ea0375af | 524 | |
4b394a23 GH |
525 | /* Unregister the DMA engine */ |
526 | ccp_dmaengine_unregister(ccp); | |
527 | ||
528 | /* Unregister the RNG */ | |
084935b2 | 529 | ccp_unregister_rng(ccp); |
4b394a23 GH |
530 | |
531 | /* Remove this device from the list of available units */ | |
ea0375af GH |
532 | ccp_del_device(ccp); |
533 | ||
ea0375af | 534 | /* Disable and clear interrupts */ |
7b537b24 | 535 | ccp_disable_queue_interrupts(ccp); |
ea0375af GH |
536 | for (i = 0; i < ccp->cmd_q_count; i++) { |
537 | cmd_q = &ccp->cmd_q[i]; | |
538 | ||
539 | ioread32(cmd_q->reg_int_status); | |
540 | ioread32(cmd_q->reg_status); | |
541 | } | |
7b537b24 | 542 | iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG); |
ea0375af | 543 | |
8256e683 GH |
544 | /* Stop the queue kthreads */ |
545 | for (i = 0; i < ccp->cmd_q_count; i++) | |
546 | if (ccp->cmd_q[i].kthread) | |
547 | kthread_stop(ccp->cmd_q[i].kthread); | |
548 | ||
f4d18d65 | 549 | sp_free_ccp_irq(ccp->sp, ccp); |
ea0375af GH |
550 | |
551 | for (i = 0; i < ccp->cmd_q_count; i++) | |
552 | dma_pool_destroy(ccp->cmd_q[i].dma_pool); | |
553 | ||
554 | /* Flush the cmd and backlog queue */ | |
555 | while (!list_empty(&ccp->cmd)) { | |
556 | /* Invoke the callback directly with an error code */ | |
557 | cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry); | |
558 | list_del(&cmd->entry); | |
559 | cmd->callback(cmd->data, -ENODEV); | |
560 | } | |
561 | while (!list_empty(&ccp->backlog)) { | |
562 | /* Invoke the callback directly with an error code */ | |
563 | cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry); | |
564 | list_del(&cmd->entry); | |
565 | cmd->callback(cmd->data, -ENODEV); | |
566 | } | |
567 | } | |
568 | ||
bc197b2a | 569 | static const struct ccp_actions ccp3_actions = { |
a43eb985 GH |
570 | .aes = ccp_perform_aes, |
571 | .xts_aes = ccp_perform_xts_aes, | |
990672d4 | 572 | .des3 = NULL, |
a43eb985 GH |
573 | .sha = ccp_perform_sha, |
574 | .rsa = ccp_perform_rsa, | |
575 | .passthru = ccp_perform_passthru, | |
576 | .ecc = ccp_perform_ecc, | |
58a690b7 GH |
577 | .sballoc = ccp_alloc_ksb, |
578 | .sbfree = ccp_free_ksb, | |
ea0375af GH |
579 | .init = ccp_init, |
580 | .destroy = ccp_destroy, | |
bb4e89b3 | 581 | .get_free_slots = ccp_get_free_slots, |
ea0375af GH |
582 | .irqhandler = ccp_irq_handler, |
583 | }; | |
584 | ||
970e8303 BS |
585 | const struct ccp_vdata ccpv3_platform = { |
586 | .version = CCP_VERSION(3, 0), | |
587 | .setup = NULL, | |
588 | .perform = &ccp3_actions, | |
589 | .offset = 0, | |
590 | }; | |
591 | ||
9ddb9dc6 | 592 | const struct ccp_vdata ccpv3 = { |
ea0375af | 593 | .version = CCP_VERSION(3, 0), |
4b394a23 | 594 | .setup = NULL, |
ea0375af | 595 | .perform = &ccp3_actions, |
fba8855c | 596 | .offset = 0x20000, |
e28c190d | 597 | .rsamax = CCP_RSA_MAX_WIDTH, |
ea0375af | 598 | }; |