]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - drivers/crypto/ccp/ccp-dev-v5.c
projects / mirror_ubuntu-bionic-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge remote-tracking branches 'asoc/topic/adsp', 'asoc/topic/ak4613', 'asoc/topic...
[mirror_ubuntu-bionic-kernel.git] / drivers / crypto / ccp / ccp-dev-v5.c
1 /*
2 * AMD Cryptographic Coprocessor (CCP) driver
3 *
4 * Copyright (C) 2016 Advanced Micro Devices, Inc.
5 *
6 * Author: Gary R Hook <gary.hook@amd.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12
13 #include <linux/module.h>
14 #include <linux/kernel.h>
15 #include <linux/pci.h>
16 #include <linux/kthread.h>
17 #include <linux/dma-mapping.h>
18 #include <linux/interrupt.h>
19 #include <linux/compiler.h>
20 #include <linux/ccp.h>
21
22 #include "ccp-dev.h"
23
24 /* Allocate the requested number of contiguous LSB slots
25 * from the LSB bitmap. Look in the private range for this
26 * queue first; failing that, check the public area.
27 * If no space is available, wait around.
28 * Return: first slot number
29 */
30 static u32 ccp_lsb_alloc(struct ccp_cmd_queue *cmd_q, unsigned int count)
31 {
32 struct ccp_device *ccp;
33 int start;
34
35 /* First look at the map for the queue */
36 if (cmd_q->lsb >= 0) {
37 start = (u32)bitmap_find_next_zero_area(cmd_q->lsbmap,
38 LSB_SIZE,
39 0, count, 0);
40 if (start < LSB_SIZE) {
41 bitmap_set(cmd_q->lsbmap, start, count);
42 return start + cmd_q->lsb * LSB_SIZE;
43 }
44 }
45
46 /* No joy; try to get an entry from the shared blocks */
47 ccp = cmd_q->ccp;
48 for (;;) {
49 mutex_lock(&ccp->sb_mutex);
50
51 start = (u32)bitmap_find_next_zero_area(ccp->lsbmap,
52 MAX_LSB_CNT * LSB_SIZE,
53 0,
54 count, 0);
55 if (start <= MAX_LSB_CNT * LSB_SIZE) {
56 bitmap_set(ccp->lsbmap, start, count);
57
58 mutex_unlock(&ccp->sb_mutex);
59 return start;
60 }
61
62 ccp->sb_avail = 0;
63
64 mutex_unlock(&ccp->sb_mutex);
65
66 /* Wait for KSB entries to become available */
67 if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail))
68 return 0;
69 }
70 }
71
72 /* Free a number of LSB slots from the bitmap, starting at
73 * the indicated starting slot number.
74 */
75 static void ccp_lsb_free(struct ccp_cmd_queue *cmd_q, unsigned int start,
76 unsigned int count)
77 {
78 if (!start)
79 return;
80
81 if (cmd_q->lsb == start) {
82 /* An entry from the private LSB */
83 bitmap_clear(cmd_q->lsbmap, start, count);
84 } else {
85 /* From the shared LSBs */
86 struct ccp_device *ccp = cmd_q->ccp;
87
88 mutex_lock(&ccp->sb_mutex);
89 bitmap_clear(ccp->lsbmap, start, count);
90 ccp->sb_avail = 1;
91 mutex_unlock(&ccp->sb_mutex);
92 wake_up_interruptible_all(&ccp->sb_queue);
93 }
94 }
95
96 /* CCP version 5: Union to define the function field (cmd_reg1/dword0) */
97 union ccp_function {
98 struct {
99 u16 size:7;
100 u16 encrypt:1;
101 u16 mode:5;
102 u16 type:2;
103 } aes;
104 struct {
105 u16 size:7;
106 u16 encrypt:1;
107 u16 rsvd:5;
108 u16 type:2;
109 } aes_xts;
110 struct {
111 u16 rsvd1:10;
112 u16 type:4;
113 u16 rsvd2:1;
114 } sha;
115 struct {
116 u16 mode:3;
117 u16 size:12;
118 } rsa;
119 struct {
120 u16 byteswap:2;
121 u16 bitwise:3;
122 u16 reflect:2;
123 u16 rsvd:8;
124 } pt;
125 struct {
126 u16 rsvd:13;
127 } zlib;
128 struct {
129 u16 size:10;
130 u16 type:2;
131 u16 mode:3;
132 } ecc;
133 u16 raw;
134 };
135
136 #define CCP_AES_SIZE(p) ((p)->aes.size)
137 #define CCP_AES_ENCRYPT(p) ((p)->aes.encrypt)
138 #define CCP_AES_MODE(p) ((p)->aes.mode)
139 #define CCP_AES_TYPE(p) ((p)->aes.type)
140 #define CCP_XTS_SIZE(p) ((p)->aes_xts.size)
141 #define CCP_XTS_ENCRYPT(p) ((p)->aes_xts.encrypt)
142 #define CCP_SHA_TYPE(p) ((p)->sha.type)
143 #define CCP_RSA_SIZE(p) ((p)->rsa.size)
144 #define CCP_PT_BYTESWAP(p) ((p)->pt.byteswap)
145 #define CCP_PT_BITWISE(p) ((p)->pt.bitwise)
146 #define CCP_ECC_MODE(p) ((p)->ecc.mode)
147 #define CCP_ECC_AFFINE(p) ((p)->ecc.one)
148
149 /* Word 0 */
150 #define CCP5_CMD_DW0(p) ((p)->dw0)
151 #define CCP5_CMD_SOC(p) (CCP5_CMD_DW0(p).soc)
152 #define CCP5_CMD_IOC(p) (CCP5_CMD_DW0(p).ioc)
153 #define CCP5_CMD_INIT(p) (CCP5_CMD_DW0(p).init)
154 #define CCP5_CMD_EOM(p) (CCP5_CMD_DW0(p).eom)
155 #define CCP5_CMD_FUNCTION(p) (CCP5_CMD_DW0(p).function)
156 #define CCP5_CMD_ENGINE(p) (CCP5_CMD_DW0(p).engine)
157 #define CCP5_CMD_PROT(p) (CCP5_CMD_DW0(p).prot)
158
159 /* Word 1 */
160 #define CCP5_CMD_DW1(p) ((p)->length)
161 #define CCP5_CMD_LEN(p) (CCP5_CMD_DW1(p))
162
163 /* Word 2 */
164 #define CCP5_CMD_DW2(p) ((p)->src_lo)
165 #define CCP5_CMD_SRC_LO(p) (CCP5_CMD_DW2(p))
166
167 /* Word 3 */
168 #define CCP5_CMD_DW3(p) ((p)->dw3)
169 #define CCP5_CMD_SRC_MEM(p) ((p)->dw3.src_mem)
170 #define CCP5_CMD_SRC_HI(p) ((p)->dw3.src_hi)
171 #define CCP5_CMD_LSB_ID(p) ((p)->dw3.lsb_cxt_id)
172 #define CCP5_CMD_FIX_SRC(p) ((p)->dw3.fixed)
173
174 /* Words 4/5 */
175 #define CCP5_CMD_DW4(p) ((p)->dw4)
176 #define CCP5_CMD_DST_LO(p) (CCP5_CMD_DW4(p).dst_lo)
177 #define CCP5_CMD_DW5(p) ((p)->dw5.fields.dst_hi)
178 #define CCP5_CMD_DST_HI(p) (CCP5_CMD_DW5(p))
179 #define CCP5_CMD_DST_MEM(p) ((p)->dw5.fields.dst_mem)
180 #define CCP5_CMD_FIX_DST(p) ((p)->dw5.fields.fixed)
181 #define CCP5_CMD_SHA_LO(p) ((p)->dw4.sha_len_lo)
182 #define CCP5_CMD_SHA_HI(p) ((p)->dw5.sha_len_hi)
183
184 /* Word 6/7 */
185 #define CCP5_CMD_DW6(p) ((p)->key_lo)
186 #define CCP5_CMD_KEY_LO(p) (CCP5_CMD_DW6(p))
187 #define CCP5_CMD_DW7(p) ((p)->dw7)
188 #define CCP5_CMD_KEY_HI(p) ((p)->dw7.key_hi)
189 #define CCP5_CMD_KEY_MEM(p) ((p)->dw7.key_mem)
190
191 static inline u32 low_address(unsigned long addr)
192 {
193 return (u64)addr & 0x0ffffffff;
194 }
195
196 static inline u32 high_address(unsigned long addr)
197 {
198 return ((u64)addr >> 32) & 0x00000ffff;
199 }
200
201 static unsigned int ccp5_get_free_slots(struct ccp_cmd_queue *cmd_q)
202 {
203 unsigned int head_idx, n;
204 u32 head_lo, queue_start;
205
206 queue_start = low_address(cmd_q->qdma_tail);
207 head_lo = ioread32(cmd_q->reg_head_lo);
208 head_idx = (head_lo - queue_start) / sizeof(struct ccp5_desc);
209
210 n = head_idx + COMMANDS_PER_QUEUE - cmd_q->qidx - 1;
211
212 return n % COMMANDS_PER_QUEUE; /* Always one unused spot */
213 }
214
215 static int ccp5_do_cmd(struct ccp5_desc *desc,
216 struct ccp_cmd_queue *cmd_q)
217 {
218 u32 *mP;
219 __le32 *dP;
220 u32 tail;
221 int i;
222 int ret = 0;
223
224 if (CCP5_CMD_SOC(desc)) {
225 CCP5_CMD_IOC(desc) = 1;
226 CCP5_CMD_SOC(desc) = 0;
227 }
228 mutex_lock(&cmd_q->q_mutex);
229
230 mP = (u32 *) &cmd_q->qbase[cmd_q->qidx];
231 dP = (__le32 *) desc;
232 for (i = 0; i < 8; i++)
233 mP[i] = cpu_to_le32(dP[i]); /* handle endianness */
234
235 cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
236
237 /* The data used by this command must be flushed to memory */
238 wmb();
239
240 /* Write the new tail address back to the queue register */
241 tail = low_address(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);
242 iowrite32(tail, cmd_q->reg_tail_lo);
243
244 /* Turn the queue back on using our cached control register */
245 iowrite32(cmd_q->qcontrol | CMD5_Q_RUN, cmd_q->reg_control);
246 mutex_unlock(&cmd_q->q_mutex);
247
248 if (CCP5_CMD_IOC(desc)) {
249 /* Wait for the job to complete */
250 ret = wait_event_interruptible(cmd_q->int_queue,
251 cmd_q->int_rcvd);
252 if (ret || cmd_q->cmd_error) {
253 /* Log the error and flush the queue by
254 * moving the head pointer
255 */
256 if (cmd_q->cmd_error)
257 ccp_log_error(cmd_q->ccp,
258 cmd_q->cmd_error);
259 iowrite32(tail, cmd_q->reg_head_lo);
260 if (!ret)
261 ret = -EIO;
262 }
263 cmd_q->int_rcvd = 0;
264 }
265
266 return ret;
267 }
268
269 static int ccp5_perform_aes(struct ccp_op *op)
270 {
271 struct ccp5_desc desc;
272 union ccp_function function;
273 u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
274
275 /* Zero out all the fields of the command desc */
276 memset(&desc, 0, Q_DESC_SIZE);
277
278 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_AES;
279
280 CCP5_CMD_SOC(&desc) = op->soc;
281 CCP5_CMD_IOC(&desc) = 1;
282 CCP5_CMD_INIT(&desc) = op->init;
283 CCP5_CMD_EOM(&desc) = op->eom;
284 CCP5_CMD_PROT(&desc) = 0;
285
286 function.raw = 0;
287 CCP_AES_ENCRYPT(&function) = op->u.aes.action;
288 CCP_AES_MODE(&function) = op->u.aes.mode;
289 CCP_AES_TYPE(&function) = op->u.aes.type;
290 CCP_AES_SIZE(&function) = op->u.aes.size;
291
292 CCP5_CMD_FUNCTION(&desc) = function.raw;
293
294 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
295
296 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
297 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
298 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
299
300 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
301 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
302 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
303
304 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
305 CCP5_CMD_KEY_HI(&desc) = 0;
306 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
307 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
308
309 return ccp5_do_cmd(&desc, op->cmd_q);
310 }
311
312 static int ccp5_perform_xts_aes(struct ccp_op *op)
313 {
314 struct ccp5_desc desc;
315 union ccp_function function;
316 u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
317
318 /* Zero out all the fields of the command desc */
319 memset(&desc, 0, Q_DESC_SIZE);
320
321 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_XTS_AES_128;
322
323 CCP5_CMD_SOC(&desc) = op->soc;
324 CCP5_CMD_IOC(&desc) = 1;
325 CCP5_CMD_INIT(&desc) = op->init;
326 CCP5_CMD_EOM(&desc) = op->eom;
327 CCP5_CMD_PROT(&desc) = 0;
328
329 function.raw = 0;
330 CCP_XTS_ENCRYPT(&function) = op->u.xts.action;
331 CCP_XTS_SIZE(&function) = op->u.xts.unit_size;
332 CCP5_CMD_FUNCTION(&desc) = function.raw;
333
334 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
335
336 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
337 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
338 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
339
340 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
341 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
342 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
343
344 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
345 CCP5_CMD_KEY_HI(&desc) = 0;
346 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
347 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
348
349 return ccp5_do_cmd(&desc, op->cmd_q);
350 }
351
352 static int ccp5_perform_sha(struct ccp_op *op)
353 {
354 struct ccp5_desc desc;
355 union ccp_function function;
356
357 /* Zero out all the fields of the command desc */
358 memset(&desc, 0, Q_DESC_SIZE);
359
360 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_SHA;
361
362 CCP5_CMD_SOC(&desc) = op->soc;
363 CCP5_CMD_IOC(&desc) = 1;
364 CCP5_CMD_INIT(&desc) = 1;
365 CCP5_CMD_EOM(&desc) = op->eom;
366 CCP5_CMD_PROT(&desc) = 0;
367
368 function.raw = 0;
369 CCP_SHA_TYPE(&function) = op->u.sha.type;
370 CCP5_CMD_FUNCTION(&desc) = function.raw;
371
372 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
373
374 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
375 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
376 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
377
378 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
379
380 if (op->eom) {
381 CCP5_CMD_SHA_LO(&desc) = lower_32_bits(op->u.sha.msg_bits);
382 CCP5_CMD_SHA_HI(&desc) = upper_32_bits(op->u.sha.msg_bits);
383 } else {
384 CCP5_CMD_SHA_LO(&desc) = 0;
385 CCP5_CMD_SHA_HI(&desc) = 0;
386 }
387
388 return ccp5_do_cmd(&desc, op->cmd_q);
389 }
390
391 static int ccp5_perform_rsa(struct ccp_op *op)
392 {
393 struct ccp5_desc desc;
394 union ccp_function function;
395
396 /* Zero out all the fields of the command desc */
397 memset(&desc, 0, Q_DESC_SIZE);
398
399 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_RSA;
400
401 CCP5_CMD_SOC(&desc) = op->soc;
402 CCP5_CMD_IOC(&desc) = 1;
403 CCP5_CMD_INIT(&desc) = 0;
404 CCP5_CMD_EOM(&desc) = 1;
405 CCP5_CMD_PROT(&desc) = 0;
406
407 function.raw = 0;
408 CCP_RSA_SIZE(&function) = op->u.rsa.mod_size >> 3;
409 CCP5_CMD_FUNCTION(&desc) = function.raw;
410
411 CCP5_CMD_LEN(&desc) = op->u.rsa.input_len;
412
413 /* Source is from external memory */
414 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
415 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
416 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
417
418 /* Destination is in external memory */
419 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
420 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
421 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
422
423 /* Exponent is in LSB memory */
424 CCP5_CMD_KEY_LO(&desc) = op->sb_key * LSB_ITEM_SIZE;
425 CCP5_CMD_KEY_HI(&desc) = 0;
426 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
427
428 return ccp5_do_cmd(&desc, op->cmd_q);
429 }
430
431 static int ccp5_perform_passthru(struct ccp_op *op)
432 {
433 struct ccp5_desc desc;
434 union ccp_function function;
435 struct ccp_dma_info *saddr = &op->src.u.dma;
436 struct ccp_dma_info *daddr = &op->dst.u.dma;
437
438 memset(&desc, 0, Q_DESC_SIZE);
439
440 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_PASSTHRU;
441
442 CCP5_CMD_SOC(&desc) = 0;
443 CCP5_CMD_IOC(&desc) = 1;
444 CCP5_CMD_INIT(&desc) = 0;
445 CCP5_CMD_EOM(&desc) = op->eom;
446 CCP5_CMD_PROT(&desc) = 0;
447
448 function.raw = 0;
449 CCP_PT_BYTESWAP(&function) = op->u.passthru.byte_swap;
450 CCP_PT_BITWISE(&function) = op->u.passthru.bit_mod;
451 CCP5_CMD_FUNCTION(&desc) = function.raw;
452
453 /* Length of source data is always 256 bytes */
454 if (op->src.type == CCP_MEMTYPE_SYSTEM)
455 CCP5_CMD_LEN(&desc) = saddr->length;
456 else
457 CCP5_CMD_LEN(&desc) = daddr->length;
458
459 if (op->src.type == CCP_MEMTYPE_SYSTEM) {
460 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
461 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
462 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
463
464 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
465 CCP5_CMD_LSB_ID(&desc) = op->sb_key;
466 } else {
467 u32 key_addr = op->src.u.sb * CCP_SB_BYTES;
468
469 CCP5_CMD_SRC_LO(&desc) = lower_32_bits(key_addr);
470 CCP5_CMD_SRC_HI(&desc) = 0;
471 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SB;
472 }
473
474 if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
475 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
476 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
477 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
478 } else {
479 u32 key_addr = op->dst.u.sb * CCP_SB_BYTES;
480
481 CCP5_CMD_DST_LO(&desc) = lower_32_bits(key_addr);
482 CCP5_CMD_DST_HI(&desc) = 0;
483 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SB;
484 }
485
486 return ccp5_do_cmd(&desc, op->cmd_q);
487 }
488
489 static int ccp5_perform_ecc(struct ccp_op *op)
490 {
491 struct ccp5_desc desc;
492 union ccp_function function;
493
494 /* Zero out all the fields of the command desc */
495 memset(&desc, 0, Q_DESC_SIZE);
496
497 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_ECC;
498
499 CCP5_CMD_SOC(&desc) = 0;
500 CCP5_CMD_IOC(&desc) = 1;
501 CCP5_CMD_INIT(&desc) = 0;
502 CCP5_CMD_EOM(&desc) = 1;
503 CCP5_CMD_PROT(&desc) = 0;
504
505 function.raw = 0;
506 function.ecc.mode = op->u.ecc.function;
507 CCP5_CMD_FUNCTION(&desc) = function.raw;
508
509 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
510
511 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
512 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
513 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
514
515 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
516 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
517 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
518
519 return ccp5_do_cmd(&desc, op->cmd_q);
520 }
521
522 static int ccp_find_lsb_regions(struct ccp_cmd_queue *cmd_q, u64 status)
523 {
524 int q_mask = 1 << cmd_q->id;
525 int queues = 0;
526 int j;
527
528 /* Build a bit mask to know which LSBs this queue has access to.
529 * Don't bother with segment 0 as it has special privileges.
530 */
531 for (j = 1; j < MAX_LSB_CNT; j++) {
532 if (status & q_mask)
533 bitmap_set(cmd_q->lsbmask, j, 1);
534 status >>= LSB_REGION_WIDTH;
535 }
536 queues = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);
537 dev_dbg(cmd_q->ccp->dev, "Queue %d can access %d LSB regions\n",
538 cmd_q->id, queues);
539
540 return queues ? 0 : -EINVAL;
541 }
542
543
544 static int ccp_find_and_assign_lsb_to_q(struct ccp_device *ccp,
545 int lsb_cnt, int n_lsbs,
546 unsigned long *lsb_pub)
547 {
548 DECLARE_BITMAP(qlsb, MAX_LSB_CNT);
549 int bitno;
550 int qlsb_wgt;
551 int i;
552
553 /* For each queue:
554 * If the count of potential LSBs available to a queue matches the
555 * ordinal given to us in lsb_cnt:
556 * Copy the mask of possible LSBs for this queue into "qlsb";
557 * For each bit in qlsb, see if the corresponding bit in the
558 * aggregation mask is set; if so, we have a match.
559 * If we have a match, clear the bit in the aggregation to
560 * mark it as no longer available.
561 * If there is no match, clear the bit in qlsb and keep looking.
562 */
563 for (i = 0; i < ccp->cmd_q_count; i++) {
564 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];
565
566 qlsb_wgt = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);
567
568 if (qlsb_wgt == lsb_cnt) {
569 bitmap_copy(qlsb, cmd_q->lsbmask, MAX_LSB_CNT);
570
571 bitno = find_first_bit(qlsb, MAX_LSB_CNT);
572 while (bitno < MAX_LSB_CNT) {
573 if (test_bit(bitno, lsb_pub)) {
574 /* We found an available LSB
575 * that this queue can access
576 */
577 cmd_q->lsb = bitno;
578 bitmap_clear(lsb_pub, bitno, 1);
579 dev_dbg(ccp->dev,
580 "Queue %d gets LSB %d\n",
581 i, bitno);
582 break;
583 }
584 bitmap_clear(qlsb, bitno, 1);
585 bitno = find_first_bit(qlsb, MAX_LSB_CNT);
586 }
587 if (bitno >= MAX_LSB_CNT)
588 return -EINVAL;
589 n_lsbs--;
590 }
591 }
592 return n_lsbs;
593 }
594
595 /* For each queue, from the most- to least-constrained:
596 * find an LSB that can be assigned to the queue. If there are N queues that
597 * can only use M LSBs, where N > M, fail; otherwise, every queue will get a
598 * dedicated LSB. Remaining LSB regions become a shared resource.
599 * If we have fewer LSBs than queues, all LSB regions become shared resources.
600 */
601 static int ccp_assign_lsbs(struct ccp_device *ccp)
602 {
603 DECLARE_BITMAP(lsb_pub, MAX_LSB_CNT);
604 DECLARE_BITMAP(qlsb, MAX_LSB_CNT);
605 int n_lsbs = 0;
606 int bitno;
607 int i, lsb_cnt;
608 int rc = 0;
609
610 bitmap_zero(lsb_pub, MAX_LSB_CNT);
611
612 /* Create an aggregate bitmap to get a total count of available LSBs */
613 for (i = 0; i < ccp->cmd_q_count; i++)
614 bitmap_or(lsb_pub,
615 lsb_pub, ccp->cmd_q[i].lsbmask,
616 MAX_LSB_CNT);
617
618 n_lsbs = bitmap_weight(lsb_pub, MAX_LSB_CNT);
619
620 if (n_lsbs >= ccp->cmd_q_count) {
621 /* We have enough LSBS to give every queue a private LSB.
622 * Brute force search to start with the queues that are more
623 * constrained in LSB choice. When an LSB is privately
624 * assigned, it is removed from the public mask.
625 * This is an ugly N squared algorithm with some optimization.
626 */
627 for (lsb_cnt = 1;
628 n_lsbs && (lsb_cnt <= MAX_LSB_CNT);
629 lsb_cnt++) {
630 rc = ccp_find_and_assign_lsb_to_q(ccp, lsb_cnt, n_lsbs,
631 lsb_pub);
632 if (rc < 0)
633 return -EINVAL;
634 n_lsbs = rc;
635 }
636 }
637
638 rc = 0;
639 /* What's left of the LSBs, according to the public mask, now become
640 * shared. Any zero bits in the lsb_pub mask represent an LSB region
641 * that can't be used as a shared resource, so mark the LSB slots for
642 * them as "in use".
643 */
644 bitmap_copy(qlsb, lsb_pub, MAX_LSB_CNT);
645
646 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT);
647 while (bitno < MAX_LSB_CNT) {
648 bitmap_set(ccp->lsbmap, bitno * LSB_SIZE, LSB_SIZE);
649 bitmap_set(qlsb, bitno, 1);
650 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT);
651 }
652
653 return rc;
654 }
655
656 static int ccp5_init(struct ccp_device *ccp)
657 {
658 struct device *dev = ccp->dev;
659 struct ccp_cmd_queue *cmd_q;
660 struct dma_pool *dma_pool;
661 char dma_pool_name[MAX_DMAPOOL_NAME_LEN];
662 unsigned int qmr, qim, i;
663 u64 status;
664 u32 status_lo, status_hi;
665 int ret;
666
667 /* Find available queues */
668 qim = 0;
669 qmr = ioread32(ccp->io_regs + Q_MASK_REG);
670 for (i = 0; i < MAX_HW_QUEUES; i++) {
671
672 if (!(qmr & (1 << i)))
673 continue;
674
675 /* Allocate a dma pool for this queue */
676 snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d",
677 ccp->name, i);
678 dma_pool = dma_pool_create(dma_pool_name, dev,
679 CCP_DMAPOOL_MAX_SIZE,
680 CCP_DMAPOOL_ALIGN, 0);
681 if (!dma_pool) {
682 dev_err(dev, "unable to allocate dma pool\n");
683 ret = -ENOMEM;
684 }
685
686 cmd_q = &ccp->cmd_q[ccp->cmd_q_count];
687 ccp->cmd_q_count++;
688
689 cmd_q->ccp = ccp;
690 cmd_q->id = i;
691 cmd_q->dma_pool = dma_pool;
692 mutex_init(&cmd_q->q_mutex);
693
694 /* Page alignment satisfies our needs for N <= 128 */
695 BUILD_BUG_ON(COMMANDS_PER_QUEUE > 128);
696 cmd_q->qsize = Q_SIZE(Q_DESC_SIZE);
697 cmd_q->qbase = dma_zalloc_coherent(dev, cmd_q->qsize,
698 &cmd_q->qbase_dma,
699 GFP_KERNEL);
700 if (!cmd_q->qbase) {
701 dev_err(dev, "unable to allocate command queue\n");
702 ret = -ENOMEM;
703 goto e_pool;
704 }
705
706 cmd_q->qidx = 0;
707 /* Preset some register values and masks that are queue
708 * number dependent
709 */
710 cmd_q->reg_control = ccp->io_regs +
711 CMD5_Q_STATUS_INCR * (i + 1);
712 cmd_q->reg_tail_lo = cmd_q->reg_control + CMD5_Q_TAIL_LO_BASE;
713 cmd_q->reg_head_lo = cmd_q->reg_control + CMD5_Q_HEAD_LO_BASE;
714 cmd_q->reg_int_enable = cmd_q->reg_control +
715 CMD5_Q_INT_ENABLE_BASE;
716 cmd_q->reg_interrupt_status = cmd_q->reg_control +
717 CMD5_Q_INTERRUPT_STATUS_BASE;
718 cmd_q->reg_status = cmd_q->reg_control + CMD5_Q_STATUS_BASE;
719 cmd_q->reg_int_status = cmd_q->reg_control +
720 CMD5_Q_INT_STATUS_BASE;
721 cmd_q->reg_dma_status = cmd_q->reg_control +
722 CMD5_Q_DMA_STATUS_BASE;
723 cmd_q->reg_dma_read_status = cmd_q->reg_control +
724 CMD5_Q_DMA_READ_STATUS_BASE;
725 cmd_q->reg_dma_write_status = cmd_q->reg_control +
726 CMD5_Q_DMA_WRITE_STATUS_BASE;
727
728 init_waitqueue_head(&cmd_q->int_queue);
729
730 dev_dbg(dev, "queue #%u available\n", i);
731 }
732 if (ccp->cmd_q_count == 0) {
733 dev_notice(dev, "no command queues available\n");
734 ret = -EIO;
735 goto e_pool;
736 }
737
738 /* Turn off the queues and disable interrupts until ready */
739 for (i = 0; i < ccp->cmd_q_count; i++) {
740 cmd_q = &ccp->cmd_q[i];
741
742 cmd_q->qcontrol = 0; /* Start with nothing */
743 iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
744
745 /* Disable the interrupts */
746 iowrite32(0x00, cmd_q->reg_int_enable);
747 ioread32(cmd_q->reg_int_status);
748 ioread32(cmd_q->reg_status);
749
750 /* Clear the interrupts */
751 iowrite32(ALL_INTERRUPTS, cmd_q->reg_interrupt_status);
752 }
753
754 dev_dbg(dev, "Requesting an IRQ...\n");
755 /* Request an irq */
756 ret = ccp->get_irq(ccp);
757 if (ret) {
758 dev_err(dev, "unable to allocate an IRQ\n");
759 goto e_pool;
760 }
761
762 dev_dbg(dev, "Loading LSB map...\n");
763 /* Copy the private LSB mask to the public registers */
764 status_lo = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);
765 status_hi = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);
766 iowrite32(status_lo, ccp->io_regs + LSB_PUBLIC_MASK_LO_OFFSET);
767 iowrite32(status_hi, ccp->io_regs + LSB_PUBLIC_MASK_HI_OFFSET);
768 status = ((u64)status_hi<<30) | (u64)status_lo;
769
770 dev_dbg(dev, "Configuring virtual queues...\n");
771 /* Configure size of each virtual queue accessible to host */
772 for (i = 0; i < ccp->cmd_q_count; i++) {
773 u32 dma_addr_lo;
774 u32 dma_addr_hi;
775
776 cmd_q = &ccp->cmd_q[i];
777
778 cmd_q->qcontrol &= ~(CMD5_Q_SIZE << CMD5_Q_SHIFT);
779 cmd_q->qcontrol |= QUEUE_SIZE_VAL << CMD5_Q_SHIFT;
780
781 cmd_q->qdma_tail = cmd_q->qbase_dma;
782 dma_addr_lo = low_address(cmd_q->qdma_tail);
783 iowrite32((u32)dma_addr_lo, cmd_q->reg_tail_lo);
784 iowrite32((u32)dma_addr_lo, cmd_q->reg_head_lo);
785
786 dma_addr_hi = high_address(cmd_q->qdma_tail);
787 cmd_q->qcontrol |= (dma_addr_hi << 16);
788 iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
789
790 /* Find the LSB regions accessible to the queue */
791 ccp_find_lsb_regions(cmd_q, status);
792 cmd_q->lsb = -1; /* Unassigned value */
793 }
794
795 dev_dbg(dev, "Assigning LSBs...\n");
796 ret = ccp_assign_lsbs(ccp);
797 if (ret) {
798 dev_err(dev, "Unable to assign LSBs (%d)\n", ret);
799 goto e_irq;
800 }
801
802 /* Optimization: pre-allocate LSB slots for each queue */
803 for (i = 0; i < ccp->cmd_q_count; i++) {
804 ccp->cmd_q[i].sb_key = ccp_lsb_alloc(&ccp->cmd_q[i], 2);
805 ccp->cmd_q[i].sb_ctx = ccp_lsb_alloc(&ccp->cmd_q[i], 2);
806 }
807
808 dev_dbg(dev, "Starting threads...\n");
809 /* Create a kthread for each queue */
810 for (i = 0; i < ccp->cmd_q_count; i++) {
811 struct task_struct *kthread;
812
813 cmd_q = &ccp->cmd_q[i];
814
815 kthread = kthread_create(ccp_cmd_queue_thread, cmd_q,
816 "%s-q%u", ccp->name, cmd_q->id);
817 if (IS_ERR(kthread)) {
818 dev_err(dev, "error creating queue thread (%ld)\n",
819 PTR_ERR(kthread));
820 ret = PTR_ERR(kthread);
821 goto e_kthread;
822 }
823
824 cmd_q->kthread = kthread;
825 wake_up_process(kthread);
826 }
827
828 dev_dbg(dev, "Enabling interrupts...\n");
829 /* Enable interrupts */
830 for (i = 0; i < ccp->cmd_q_count; i++) {
831 cmd_q = &ccp->cmd_q[i];
832 iowrite32(ALL_INTERRUPTS, cmd_q->reg_int_enable);
833 }
834
835 dev_dbg(dev, "Registering device...\n");
836 /* Put this on the unit list to make it available */
837 ccp_add_device(ccp);
838
839 ret = ccp_register_rng(ccp);
840 if (ret)
841 goto e_kthread;
842
843 /* Register the DMA engine support */
844 ret = ccp_dmaengine_register(ccp);
845 if (ret)
846 goto e_hwrng;
847
848 return 0;
849
850 e_hwrng:
851 ccp_unregister_rng(ccp);
852
853 e_kthread:
854 for (i = 0; i < ccp->cmd_q_count; i++)
855 if (ccp->cmd_q[i].kthread)
856 kthread_stop(ccp->cmd_q[i].kthread);
857
858 e_irq:
859 ccp->free_irq(ccp);
860
861 e_pool:
862 for (i = 0; i < ccp->cmd_q_count; i++)
863 dma_pool_destroy(ccp->cmd_q[i].dma_pool);
864
865 return ret;
866 }
867
868 static void ccp5_destroy(struct ccp_device *ccp)
869 {
870 struct device *dev = ccp->dev;
871 struct ccp_cmd_queue *cmd_q;
872 struct ccp_cmd *cmd;
873 unsigned int i;
874
875 /* Unregister the DMA engine */
876 ccp_dmaengine_unregister(ccp);
877
878 /* Unregister the RNG */
879 ccp_unregister_rng(ccp);
880
881 /* Remove this device from the list of available units first */
882 ccp_del_device(ccp);
883
884 /* Disable and clear interrupts */
885 for (i = 0; i < ccp->cmd_q_count; i++) {
886 cmd_q = &ccp->cmd_q[i];
887
888 /* Turn off the run bit */
889 iowrite32(cmd_q->qcontrol & ~CMD5_Q_RUN, cmd_q->reg_control);
890
891 /* Disable the interrupts */
892 iowrite32(ALL_INTERRUPTS, cmd_q->reg_interrupt_status);
893
894 /* Clear the interrupt status */
895 iowrite32(0x00, cmd_q->reg_int_enable);
896 ioread32(cmd_q->reg_int_status);
897 ioread32(cmd_q->reg_status);
898 }
899
900 /* Stop the queue kthreads */
901 for (i = 0; i < ccp->cmd_q_count; i++)
902 if (ccp->cmd_q[i].kthread)
903 kthread_stop(ccp->cmd_q[i].kthread);
904
905 ccp->free_irq(ccp);
906
907 for (i = 0; i < ccp->cmd_q_count; i++) {
908 cmd_q = &ccp->cmd_q[i];
909 dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase,
910 cmd_q->qbase_dma);
911 }
912
913 /* Flush the cmd and backlog queue */
914 while (!list_empty(&ccp->cmd)) {
915 /* Invoke the callback directly with an error code */
916 cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);
917 list_del(&cmd->entry);
918 cmd->callback(cmd->data, -ENODEV);
919 }
920 while (!list_empty(&ccp->backlog)) {
921 /* Invoke the callback directly with an error code */
922 cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry);
923 list_del(&cmd->entry);
924 cmd->callback(cmd->data, -ENODEV);
925 }
926 }
927
928 static irqreturn_t ccp5_irq_handler(int irq, void *data)
929 {
930 struct device *dev = data;
931 struct ccp_device *ccp = dev_get_drvdata(dev);
932 u32 status;
933 unsigned int i;
934
935 for (i = 0; i < ccp->cmd_q_count; i++) {
936 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];
937
938 status = ioread32(cmd_q->reg_interrupt_status);
939
940 if (status) {
941 cmd_q->int_status = status;
942 cmd_q->q_status = ioread32(cmd_q->reg_status);
943 cmd_q->q_int_status = ioread32(cmd_q->reg_int_status);
944
945 /* On error, only save the first error value */
946 if ((status & INT_ERROR) && !cmd_q->cmd_error)
947 cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);
948
949 cmd_q->int_rcvd = 1;
950
951 /* Acknowledge the interrupt and wake the kthread */
952 iowrite32(ALL_INTERRUPTS, cmd_q->reg_interrupt_status);
953 wake_up_interruptible(&cmd_q->int_queue);
954 }
955 }
956
957 return IRQ_HANDLED;
958 }
959
960 static void ccp5_config(struct ccp_device *ccp)
961 {
962 /* Public side */
963 iowrite32(0x0, ccp->io_regs + CMD5_REQID_CONFIG_OFFSET);
964 }
965
966 static void ccp5other_config(struct ccp_device *ccp)
967 {
968 int i;
969 u32 rnd;
970
971 /* We own all of the queues on the NTB CCP */
972
973 iowrite32(0x00012D57, ccp->io_regs + CMD5_TRNG_CTL_OFFSET);
974 iowrite32(0x00000003, ccp->io_regs + CMD5_CONFIG_0_OFFSET);
975 for (i = 0; i < 12; i++) {
976 rnd = ioread32(ccp->io_regs + TRNG_OUT_REG);
977 iowrite32(rnd, ccp->io_regs + CMD5_AES_MASK_OFFSET);
978 }
979
980 iowrite32(0x0000001F, ccp->io_regs + CMD5_QUEUE_MASK_OFFSET);
981 iowrite32(0x00005B6D, ccp->io_regs + CMD5_QUEUE_PRIO_OFFSET);
982 iowrite32(0x00000000, ccp->io_regs + CMD5_CMD_TIMEOUT_OFFSET);
983
984 iowrite32(0x3FFFFFFF, ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);
985 iowrite32(0x000003FF, ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);
986
987 iowrite32(0x00108823, ccp->io_regs + CMD5_CLK_GATE_CTL_OFFSET);
988
989 ccp5_config(ccp);
990 }
991
992 /* Version 5 adds some function, but is essentially the same as v5 */
993 static const struct ccp_actions ccp5_actions = {
994 .aes = ccp5_perform_aes,
995 .xts_aes = ccp5_perform_xts_aes,
996 .sha = ccp5_perform_sha,
997 .rsa = ccp5_perform_rsa,
998 .passthru = ccp5_perform_passthru,
999 .ecc = ccp5_perform_ecc,
1000 .sballoc = ccp_lsb_alloc,
1001 .sbfree = ccp_lsb_free,
1002 .init = ccp5_init,
1003 .destroy = ccp5_destroy,
1004 .get_free_slots = ccp5_get_free_slots,
1005 .irqhandler = ccp5_irq_handler,
1006 };
1007
1008 const struct ccp_vdata ccpv5a = {
1009 .version = CCP_VERSION(5, 0),
1010 .setup = ccp5_config,
1011 .perform = &ccp5_actions,
1012 .bar = 2,
1013 .offset = 0x0,
1014 };
1015
1016 const struct ccp_vdata ccpv5b = {
1017 .version = CCP_VERSION(5, 0),
1018 .dma_chan_attr = DMA_PRIVATE,
1019 .setup = ccp5other_config,
1020 .perform = &ccp5_actions,
1021 .bar = 2,
1022 .offset = 0x0,
1023 };
ubuntu-bionic-kernel mirror