2 * Driver for IBM PowerNV 842 compression accelerator
4 * Copyright (C) 2015 Dan Streetman, IBM Corp
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
21 #include <linux/timer.h>
24 #include <asm/icswx.h>
27 MODULE_LICENSE("GPL");
28 MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");
29 MODULE_DESCRIPTION("842 H/W Compression driver for IBM PowerNV processors");
30 MODULE_ALIAS_CRYPTO("842");
31 MODULE_ALIAS_CRYPTO("842-nx");
33 #define WORKMEM_ALIGN (CRB_ALIGN)
34 #define CSB_WAIT_MAX (5000) /* ms */
36 struct nx842_workmem
{
37 /* Below fields must be properly aligned */
38 struct coprocessor_request_block crb
; /* CRB_ALIGN align */
39 struct data_descriptor_entry ddl_in
[DDL_LEN_MAX
]; /* DDE_ALIGN align */
40 struct data_descriptor_entry ddl_out
[DDL_LEN_MAX
]; /* DDE_ALIGN align */
41 /* Above fields must be properly aligned */
45 char padding
[WORKMEM_ALIGN
]; /* unused, to allow alignment */
46 } __packed
__aligned(WORKMEM_ALIGN
);
52 struct list_head list
;
55 /* no cpu hotplug on powernv, so this list never changes after init */
56 static LIST_HEAD(nx842_coprocs
);
57 static unsigned int nx842_ct
; /* used in icswx function */
59 static int (*nx842_powernv_exec
)(const unsigned char *in
,
60 unsigned int inlen
, unsigned char *out
,
61 unsigned int *outlenp
, void *workmem
, int fc
);
64 * setup_indirect_dde - Setup an indirect DDE
66 * The DDE is setup with the the DDE count, byte count, and address of
67 * first direct DDE in the list.
69 static void setup_indirect_dde(struct data_descriptor_entry
*dde
,
70 struct data_descriptor_entry
*ddl
,
71 unsigned int dde_count
, unsigned int byte_count
)
74 dde
->count
= dde_count
;
76 dde
->length
= cpu_to_be32(byte_count
);
77 dde
->address
= cpu_to_be64(nx842_get_pa(ddl
));
81 * setup_direct_dde - Setup single DDE from buffer
83 * The DDE is setup with the buffer and length. The buffer must be properly
84 * aligned. The used length is returned.
86 * N Successfully set up DDE with N bytes
88 static unsigned int setup_direct_dde(struct data_descriptor_entry
*dde
,
89 unsigned long pa
, unsigned int len
)
91 unsigned int l
= min_t(unsigned int, len
, LEN_ON_PAGE(pa
));
96 dde
->length
= cpu_to_be32(l
);
97 dde
->address
= cpu_to_be64(pa
);
103 * setup_ddl - Setup DDL from buffer
106 * 0 Successfully set up DDL
108 static int setup_ddl(struct data_descriptor_entry
*dde
,
109 struct data_descriptor_entry
*ddl
,
110 unsigned char *buf
, unsigned int len
,
113 unsigned long pa
= nx842_get_pa(buf
);
114 int i
, ret
, total_len
= len
;
116 if (!IS_ALIGNED(pa
, DDE_BUFFER_ALIGN
)) {
117 pr_debug("%s buffer pa 0x%lx not 0x%x-byte aligned\n",
118 in
? "input" : "output", pa
, DDE_BUFFER_ALIGN
);
122 /* only need to check last mult; since buffer must be
123 * DDE_BUFFER_ALIGN aligned, and that is a multiple of
124 * DDE_BUFFER_SIZE_MULT, and pre-last page DDE buffers
125 * are guaranteed a multiple of DDE_BUFFER_SIZE_MULT.
127 if (len
% DDE_BUFFER_LAST_MULT
) {
128 pr_debug("%s buffer len 0x%x not a multiple of 0x%x\n",
129 in
? "input" : "output", len
, DDE_BUFFER_LAST_MULT
);
132 len
= round_down(len
, DDE_BUFFER_LAST_MULT
);
135 /* use a single direct DDE */
136 if (len
<= LEN_ON_PAGE(pa
)) {
137 ret
= setup_direct_dde(dde
, pa
, len
);
143 for (i
= 0; i
< DDL_LEN_MAX
&& len
> 0; i
++) {
144 ret
= setup_direct_dde(&ddl
[i
], pa
, len
);
147 pa
= nx842_get_pa(buf
);
151 pr_debug("0x%x total %s bytes 0x%x too many for DDL.\n",
152 total_len
, in
? "input" : "output", len
);
157 setup_indirect_dde(dde
, ddl
, i
, total_len
);
162 #define CSB_ERR(csb, msg, ...) \
163 pr_err("ERROR: " msg " : %02x %02x %02x %02x %08x\n", \
164 ##__VA_ARGS__, (csb)->flags, \
165 (csb)->cs, (csb)->cc, (csb)->ce, \
166 be32_to_cpu((csb)->count))
168 #define CSB_ERR_ADDR(csb, msg, ...) \
169 CSB_ERR(csb, msg " at %lx", ##__VA_ARGS__, \
170 (unsigned long)be64_to_cpu((csb)->address))
175 static int wait_for_csb(struct nx842_workmem
*wmem
,
176 struct coprocessor_status_block
*csb
)
178 ktime_t start
= wmem
->start
, now
= ktime_get();
179 ktime_t timeout
= ktime_add_ms(start
, CSB_WAIT_MAX
);
181 while (!(ACCESS_ONCE(csb
->flags
) & CSB_V
)) {
184 if (ktime_after(now
, timeout
))
188 /* hw has updated csb and output buffer */
191 /* check CSB flags */
192 if (!(csb
->flags
& CSB_V
)) {
193 CSB_ERR(csb
, "CSB still not valid after %ld us, giving up",
194 (long)ktime_us_delta(now
, start
));
197 if (csb
->flags
& CSB_F
) {
198 CSB_ERR(csb
, "Invalid CSB format");
201 if (csb
->flags
& CSB_CH
) {
202 CSB_ERR(csb
, "Invalid CSB chaining state");
206 /* verify CSB completion sequence is 0 */
208 CSB_ERR(csb
, "Invalid CSB completion sequence");
212 /* check CSB Completion Code */
217 case CSB_CC_TPBC_GT_SPBC
:
218 /* not an error, but the compressed data is
219 * larger than the uncompressed data :(
223 /* input data errors */
224 case CSB_CC_OPERAND_OVERLAP
:
225 /* input and output buffers overlap */
226 CSB_ERR(csb
, "Operand Overlap error");
228 case CSB_CC_INVALID_OPERAND
:
229 CSB_ERR(csb
, "Invalid operand");
232 /* output buffer too small */
235 CSB_ERR(csb
, "Function aborted");
237 case CSB_CC_CRC_MISMATCH
:
238 CSB_ERR(csb
, "CRC mismatch");
240 case CSB_CC_TEMPL_INVALID
:
241 CSB_ERR(csb
, "Compressed data template invalid");
243 case CSB_CC_TEMPL_OVERFLOW
:
244 CSB_ERR(csb
, "Compressed data template shows data past end");
247 /* these should not happen */
248 case CSB_CC_INVALID_ALIGN
:
249 /* setup_ddl should have detected this */
250 CSB_ERR_ADDR(csb
, "Invalid alignment");
252 case CSB_CC_DATA_LENGTH
:
253 /* setup_ddl should have detected this */
254 CSB_ERR(csb
, "Invalid data length");
256 case CSB_CC_WR_TRANSLATION
:
257 case CSB_CC_TRANSLATION
:
258 case CSB_CC_TRANSLATION_DUP1
:
259 case CSB_CC_TRANSLATION_DUP2
:
260 case CSB_CC_TRANSLATION_DUP3
:
261 case CSB_CC_TRANSLATION_DUP4
:
262 case CSB_CC_TRANSLATION_DUP5
:
263 case CSB_CC_TRANSLATION_DUP6
:
264 /* should not happen, we use physical addrs */
265 CSB_ERR_ADDR(csb
, "Translation error");
267 case CSB_CC_WR_PROTECTION
:
268 case CSB_CC_PROTECTION
:
269 case CSB_CC_PROTECTION_DUP1
:
270 case CSB_CC_PROTECTION_DUP2
:
271 case CSB_CC_PROTECTION_DUP3
:
272 case CSB_CC_PROTECTION_DUP4
:
273 case CSB_CC_PROTECTION_DUP5
:
274 case CSB_CC_PROTECTION_DUP6
:
275 /* should not happen, we use physical addrs */
276 CSB_ERR_ADDR(csb
, "Protection error");
278 case CSB_CC_PRIVILEGE
:
279 /* shouldn't happen, we're in HYP mode */
280 CSB_ERR(csb
, "Insufficient Privilege error");
282 case CSB_CC_EXCESSIVE_DDE
:
283 /* shouldn't happen, setup_ddl doesn't use many dde's */
284 CSB_ERR(csb
, "Too many DDEs in DDL");
286 case CSB_CC_TRANSPORT
:
287 /* shouldn't happen, we setup CRB correctly */
288 CSB_ERR(csb
, "Invalid CRB");
290 case CSB_CC_SEGMENTED_DDL
:
291 /* shouldn't happen, setup_ddl creates DDL right */
292 CSB_ERR(csb
, "Segmented DDL error");
294 case CSB_CC_DDE_OVERFLOW
:
295 /* shouldn't happen, setup_ddl creates DDL right */
296 CSB_ERR(csb
, "DDE overflow error");
299 /* should not happen with ICSWX */
300 CSB_ERR(csb
, "Session violation error");
303 /* should not happen, we don't use chained CRBs */
304 CSB_ERR(csb
, "Chained CRB error");
306 case CSB_CC_SEQUENCE
:
307 /* should not happen, we don't use chained CRBs */
308 CSB_ERR(csb
, "CRB seqeunce number error");
310 case CSB_CC_UNKNOWN_CODE
:
311 CSB_ERR(csb
, "Unknown subfunction code");
314 /* hardware errors */
315 case CSB_CC_RD_EXTERNAL
:
316 case CSB_CC_RD_EXTERNAL_DUP1
:
317 case CSB_CC_RD_EXTERNAL_DUP2
:
318 case CSB_CC_RD_EXTERNAL_DUP3
:
319 CSB_ERR_ADDR(csb
, "Read error outside coprocessor");
321 case CSB_CC_WR_EXTERNAL
:
322 CSB_ERR_ADDR(csb
, "Write error outside coprocessor");
324 case CSB_CC_INTERNAL
:
325 CSB_ERR(csb
, "Internal error in coprocessor");
327 case CSB_CC_PROVISION
:
328 CSB_ERR(csb
, "Storage provision error");
331 CSB_ERR(csb
, "Correctable hardware error");
335 CSB_ERR(csb
, "Invalid CC %d", csb
->cc
);
339 /* check Completion Extension state */
340 if (csb
->ce
& CSB_CE_TERMINATION
) {
341 CSB_ERR(csb
, "CSB request was terminated");
344 if (csb
->ce
& CSB_CE_INCOMPLETE
) {
345 CSB_ERR(csb
, "CSB request not complete");
348 if (!(csb
->ce
& CSB_CE_TPBC
)) {
349 CSB_ERR(csb
, "TPBC not provided, unknown target length");
353 /* successful completion */
354 pr_debug_ratelimited("Processed %u bytes in %lu us\n",
355 be32_to_cpu(csb
->count
),
356 (unsigned long)ktime_us_delta(now
, start
));
361 static int nx842_config_crb(const unsigned char *in
, unsigned int inlen
,
362 unsigned char *out
, unsigned int outlen
,
363 struct nx842_workmem
*wmem
)
365 struct coprocessor_request_block
*crb
;
366 struct coprocessor_status_block
*csb
;
373 /* Clear any previous values */
374 memset(crb
, 0, sizeof(*crb
));
377 ret
= setup_ddl(&crb
->source
, wmem
->ddl_in
,
378 (unsigned char *)in
, inlen
, true);
382 ret
= setup_ddl(&crb
->target
, wmem
->ddl_out
,
387 /* set up CRB's CSB addr */
388 csb_addr
= nx842_get_pa(csb
) & CRB_CSB_ADDRESS
;
389 csb_addr
|= CRB_CSB_AT
; /* Addrs are phys */
390 crb
->csb_addr
= cpu_to_be64(csb_addr
);
396 * nx842_exec_icswx - compress/decompress data using the 842 algorithm
398 * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems.
399 * This compresses or decompresses the provided input buffer into the provided
402 * Upon return from this function @outlen contains the length of the
403 * output data. If there is an error then @outlen will be 0 and an
404 * error will be specified by the return code from this function.
406 * The @workmem buffer should only be used by one function call at a time.
408 * @in: input buffer pointer
409 * @inlen: input buffer size
410 * @out: output buffer pointer
411 * @outlenp: output buffer size pointer
412 * @workmem: working memory buffer pointer, size determined by
413 * nx842_powernv_driver.workmem_size
414 * @fc: function code, see CCW Function Codes in nx-842.h
417 * 0 Success, output of length @outlenp stored in the buffer at @out
418 * -ENODEV Hardware unavailable
419 * -ENOSPC Output buffer is to small
420 * -EMSGSIZE Input buffer too large
421 * -EINVAL buffer constraints do not fix nx842_constraints
422 * -EPROTO hardware error during operation
423 * -ETIMEDOUT hardware did not complete operation in reasonable time
424 * -EINTR operation was aborted
426 static int nx842_exec_icswx(const unsigned char *in
, unsigned int inlen
,
427 unsigned char *out
, unsigned int *outlenp
,
428 void *workmem
, int fc
)
430 struct coprocessor_request_block
*crb
;
431 struct coprocessor_status_block
*csb
;
432 struct nx842_workmem
*wmem
;
435 unsigned int outlen
= *outlenp
;
437 wmem
= PTR_ALIGN(workmem
, WORKMEM_ALIGN
);
441 /* shoudn't happen, we don't load without a coproc */
443 pr_err_ratelimited("coprocessor CT is 0");
447 ret
= nx842_config_crb(in
, inlen
, out
, outlen
, wmem
);
456 ccw
= SET_FIELD(CCW_CT
, ccw
, nx842_ct
);
457 ccw
= SET_FIELD(CCW_CI_842
, ccw
, 0); /* use 0 for hw auto-selection */
458 ccw
= SET_FIELD(CCW_FC_842
, ccw
, fc
);
460 wmem
->start
= ktime_get();
463 ret
= icswx(cpu_to_be32(ccw
), crb
);
465 pr_debug_ratelimited("icswx CR %x ccw %x crb->ccw %x\n", ret
,
467 (unsigned int)be32_to_cpu(crb
->ccw
));
470 * NX842 coprocessor sets 3rd bit in CR register with XER[S0].
471 * XER[S0] is the integer summary overflow bit which is nothing
472 * to do NX. Since this bit can be set with other return values,
478 case ICSWX_INITIATED
:
479 ret
= wait_for_csb(wmem
, csb
);
482 pr_debug_ratelimited("842 Coprocessor busy\n");
486 pr_err_ratelimited("ICSWX rejected\n");
492 *outlenp
= be32_to_cpu(csb
->count
);
498 * nx842_powernv_compress - Compress data using the 842 algorithm
500 * Compression provided by the NX842 coprocessor on IBM PowerNV systems.
501 * The input buffer is compressed and the result is stored in the
502 * provided output buffer.
504 * Upon return from this function @outlen contains the length of the
505 * compressed data. If there is an error then @outlen will be 0 and an
506 * error will be specified by the return code from this function.
508 * @in: input buffer pointer
509 * @inlen: input buffer size
510 * @out: output buffer pointer
511 * @outlenp: output buffer size pointer
512 * @workmem: working memory buffer pointer, size determined by
513 * nx842_powernv_driver.workmem_size
515 * Returns: see @nx842_powernv_exec()
517 static int nx842_powernv_compress(const unsigned char *in
, unsigned int inlen
,
518 unsigned char *out
, unsigned int *outlenp
,
521 return nx842_powernv_exec(in
, inlen
, out
, outlenp
,
522 wmem
, CCW_FC_842_COMP_CRC
);
526 * nx842_powernv_decompress - Decompress data using the 842 algorithm
528 * Decompression provided by the NX842 coprocessor on IBM PowerNV systems.
529 * The input buffer is decompressed and the result is stored in the
530 * provided output buffer.
532 * Upon return from this function @outlen contains the length of the
533 * decompressed data. If there is an error then @outlen will be 0 and an
534 * error will be specified by the return code from this function.
536 * @in: input buffer pointer
537 * @inlen: input buffer size
538 * @out: output buffer pointer
539 * @outlenp: output buffer size pointer
540 * @workmem: working memory buffer pointer, size determined by
541 * nx842_powernv_driver.workmem_size
543 * Returns: see @nx842_powernv_exec()
545 static int nx842_powernv_decompress(const unsigned char *in
, unsigned int inlen
,
546 unsigned char *out
, unsigned int *outlenp
,
549 return nx842_powernv_exec(in
, inlen
, out
, outlenp
,
550 wmem
, CCW_FC_842_DECOMP_CRC
);
553 static inline void nx842_add_coprocs_list(struct nx842_coproc
*coproc
,
556 coproc
->chip_id
= chipid
;
557 INIT_LIST_HEAD(&coproc
->list
);
558 list_add(&coproc
->list
, &nx842_coprocs
);
561 static int __init
nx842_powernv_probe(struct device_node
*dn
)
563 struct nx842_coproc
*coproc
;
567 chip_id
= of_get_ibm_chip_id(dn
);
569 pr_err("ibm,chip-id missing\n");
573 if (of_property_read_u32(dn
, "ibm,842-coprocessor-type", &ct
)) {
574 pr_err("ibm,842-coprocessor-type missing\n");
578 if (of_property_read_u32(dn
, "ibm,842-coprocessor-instance", &ci
)) {
579 pr_err("ibm,842-coprocessor-instance missing\n");
583 coproc
= kmalloc(sizeof(*coproc
), GFP_KERNEL
);
589 nx842_add_coprocs_list(coproc
, chip_id
);
591 pr_info("coprocessor found on chip %d, CT %d CI %d\n", chip_id
, ct
, ci
);
595 else if (nx842_ct
!= ct
)
596 pr_err("NX842 chip %d, CT %d != first found CT %d\n",
597 chip_id
, ct
, nx842_ct
);
602 static void nx842_delete_coprocs(void)
604 struct nx842_coproc
*coproc
, *n
;
606 list_for_each_entry_safe(coproc
, n
, &nx842_coprocs
, list
) {
607 list_del(&coproc
->list
);
612 static struct nx842_constraints nx842_powernv_constraints
= {
613 .alignment
= DDE_BUFFER_ALIGN
,
614 .multiple
= DDE_BUFFER_LAST_MULT
,
615 .minimum
= DDE_BUFFER_LAST_MULT
,
616 .maximum
= (DDL_LEN_MAX
- 1) * PAGE_SIZE
,
619 static struct nx842_driver nx842_powernv_driver
= {
620 .name
= KBUILD_MODNAME
,
621 .owner
= THIS_MODULE
,
622 .workmem_size
= sizeof(struct nx842_workmem
),
623 .constraints
= &nx842_powernv_constraints
,
624 .compress
= nx842_powernv_compress
,
625 .decompress
= nx842_powernv_decompress
,
628 static int nx842_powernv_crypto_init(struct crypto_tfm
*tfm
)
630 return nx842_crypto_init(tfm
, &nx842_powernv_driver
);
633 static struct crypto_alg nx842_powernv_alg
= {
635 .cra_driver_name
= "842-nx",
637 .cra_flags
= CRYPTO_ALG_TYPE_COMPRESS
,
638 .cra_ctxsize
= sizeof(struct nx842_crypto_ctx
),
639 .cra_module
= THIS_MODULE
,
640 .cra_init
= nx842_powernv_crypto_init
,
641 .cra_exit
= nx842_crypto_exit
,
642 .cra_u
= { .compress
= {
643 .coa_compress
= nx842_crypto_compress
,
644 .coa_decompress
= nx842_crypto_decompress
} }
647 static __init
int nx842_powernv_init(void)
649 struct device_node
*dn
;
652 /* verify workmem size/align restrictions */
653 BUILD_BUG_ON(WORKMEM_ALIGN
% CRB_ALIGN
);
654 BUILD_BUG_ON(CRB_ALIGN
% DDE_ALIGN
);
655 BUILD_BUG_ON(CRB_SIZE
% DDE_ALIGN
);
656 /* verify buffer size/align restrictions */
657 BUILD_BUG_ON(PAGE_SIZE
% DDE_BUFFER_ALIGN
);
658 BUILD_BUG_ON(DDE_BUFFER_ALIGN
% DDE_BUFFER_SIZE_MULT
);
659 BUILD_BUG_ON(DDE_BUFFER_SIZE_MULT
% DDE_BUFFER_LAST_MULT
);
661 for_each_compatible_node(dn
, NULL
, "ibm,power-nx")
662 nx842_powernv_probe(dn
);
667 nx842_powernv_exec
= nx842_exec_icswx
;
669 ret
= crypto_register_alg(&nx842_powernv_alg
);
671 nx842_delete_coprocs();
677 module_init(nx842_powernv_init
);
679 static void __exit
nx842_powernv_exit(void)
681 crypto_unregister_alg(&nx842_powernv_alg
);
683 nx842_delete_coprocs();
685 module_exit(nx842_powernv_exit
);