2 * Driver for IBM PowerNV 842 compression accelerator
4 * Copyright (C) 2015 Dan Streetman, IBM Corp
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
21 #include <linux/timer.h>
24 #include <asm/icswx.h>
28 MODULE_LICENSE("GPL");
29 MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");
30 MODULE_DESCRIPTION("842 H/W Compression driver for IBM PowerNV processors");
31 MODULE_ALIAS_CRYPTO("842");
32 MODULE_ALIAS_CRYPTO("842-nx");
34 #define WORKMEM_ALIGN (CRB_ALIGN)
35 #define CSB_WAIT_MAX (5000) /* ms */
36 #define VAS_RETRIES (10)
37 /* # of requests allowed per RxFIFO at a time. 0 for unlimited */
38 #define MAX_CREDITS_PER_RXFIFO (1024)
40 struct nx842_workmem
{
41 /* Below fields must be properly aligned */
42 struct coprocessor_request_block crb
; /* CRB_ALIGN align */
43 struct data_descriptor_entry ddl_in
[DDL_LEN_MAX
]; /* DDE_ALIGN align */
44 struct data_descriptor_entry ddl_out
[DDL_LEN_MAX
]; /* DDE_ALIGN align */
45 /* Above fields must be properly aligned */
49 struct vas_window
*txwin
; /* Used with VAS function */
50 char padding
[WORKMEM_ALIGN
]; /* unused, to allow alignment */
51 } __packed
__aligned(WORKMEM_ALIGN
);
56 unsigned int ci
; /* Coprocessor instance, used with icswx */
58 struct vas_window
*rxwin
;
61 struct list_head list
;
65 * Send the request to NX engine on the chip for the corresponding CPU
66 * where the process is executing. Use with VAS function.
68 static DEFINE_PER_CPU(struct nx842_coproc
*, coproc_inst
);
70 /* no cpu hotplug on powernv, so this list never changes after init */
71 static LIST_HEAD(nx842_coprocs
);
72 static unsigned int nx842_ct
; /* used in icswx function */
74 static int (*nx842_powernv_exec
)(const unsigned char *in
,
75 unsigned int inlen
, unsigned char *out
,
76 unsigned int *outlenp
, void *workmem
, int fc
);
79 * setup_indirect_dde - Setup an indirect DDE
81 * The DDE is setup with the the DDE count, byte count, and address of
82 * first direct DDE in the list.
84 static void setup_indirect_dde(struct data_descriptor_entry
*dde
,
85 struct data_descriptor_entry
*ddl
,
86 unsigned int dde_count
, unsigned int byte_count
)
89 dde
->count
= dde_count
;
91 dde
->length
= cpu_to_be32(byte_count
);
92 dde
->address
= cpu_to_be64(nx842_get_pa(ddl
));
96 * setup_direct_dde - Setup single DDE from buffer
98 * The DDE is setup with the buffer and length. The buffer must be properly
99 * aligned. The used length is returned.
101 * N Successfully set up DDE with N bytes
103 static unsigned int setup_direct_dde(struct data_descriptor_entry
*dde
,
104 unsigned long pa
, unsigned int len
)
106 unsigned int l
= min_t(unsigned int, len
, LEN_ON_PAGE(pa
));
111 dde
->length
= cpu_to_be32(l
);
112 dde
->address
= cpu_to_be64(pa
);
118 * setup_ddl - Setup DDL from buffer
121 * 0 Successfully set up DDL
123 static int setup_ddl(struct data_descriptor_entry
*dde
,
124 struct data_descriptor_entry
*ddl
,
125 unsigned char *buf
, unsigned int len
,
128 unsigned long pa
= nx842_get_pa(buf
);
129 int i
, ret
, total_len
= len
;
131 if (!IS_ALIGNED(pa
, DDE_BUFFER_ALIGN
)) {
132 pr_debug("%s buffer pa 0x%lx not 0x%x-byte aligned\n",
133 in
? "input" : "output", pa
, DDE_BUFFER_ALIGN
);
137 /* only need to check last mult; since buffer must be
138 * DDE_BUFFER_ALIGN aligned, and that is a multiple of
139 * DDE_BUFFER_SIZE_MULT, and pre-last page DDE buffers
140 * are guaranteed a multiple of DDE_BUFFER_SIZE_MULT.
142 if (len
% DDE_BUFFER_LAST_MULT
) {
143 pr_debug("%s buffer len 0x%x not a multiple of 0x%x\n",
144 in
? "input" : "output", len
, DDE_BUFFER_LAST_MULT
);
147 len
= round_down(len
, DDE_BUFFER_LAST_MULT
);
150 /* use a single direct DDE */
151 if (len
<= LEN_ON_PAGE(pa
)) {
152 ret
= setup_direct_dde(dde
, pa
, len
);
158 for (i
= 0; i
< DDL_LEN_MAX
&& len
> 0; i
++) {
159 ret
= setup_direct_dde(&ddl
[i
], pa
, len
);
162 pa
= nx842_get_pa(buf
);
166 pr_debug("0x%x total %s bytes 0x%x too many for DDL.\n",
167 total_len
, in
? "input" : "output", len
);
172 setup_indirect_dde(dde
, ddl
, i
, total_len
);
177 #define CSB_ERR(csb, msg, ...) \
178 pr_err("ERROR: " msg " : %02x %02x %02x %02x %08x\n", \
179 ##__VA_ARGS__, (csb)->flags, \
180 (csb)->cs, (csb)->cc, (csb)->ce, \
181 be32_to_cpu((csb)->count))
183 #define CSB_ERR_ADDR(csb, msg, ...) \
184 CSB_ERR(csb, msg " at %lx", ##__VA_ARGS__, \
185 (unsigned long)be64_to_cpu((csb)->address))
190 static int wait_for_csb(struct nx842_workmem
*wmem
,
191 struct coprocessor_status_block
*csb
)
193 ktime_t start
= wmem
->start
, now
= ktime_get();
194 ktime_t timeout
= ktime_add_ms(start
, CSB_WAIT_MAX
);
196 while (!(ACCESS_ONCE(csb
->flags
) & CSB_V
)) {
199 if (ktime_after(now
, timeout
))
203 /* hw has updated csb and output buffer */
206 /* check CSB flags */
207 if (!(csb
->flags
& CSB_V
)) {
208 CSB_ERR(csb
, "CSB still not valid after %ld us, giving up",
209 (long)ktime_us_delta(now
, start
));
212 if (csb
->flags
& CSB_F
) {
213 CSB_ERR(csb
, "Invalid CSB format");
216 if (csb
->flags
& CSB_CH
) {
217 CSB_ERR(csb
, "Invalid CSB chaining state");
221 /* verify CSB completion sequence is 0 */
223 CSB_ERR(csb
, "Invalid CSB completion sequence");
227 /* check CSB Completion Code */
232 case CSB_CC_TPBC_GT_SPBC
:
233 /* not an error, but the compressed data is
234 * larger than the uncompressed data :(
238 /* input data errors */
239 case CSB_CC_OPERAND_OVERLAP
:
240 /* input and output buffers overlap */
241 CSB_ERR(csb
, "Operand Overlap error");
243 case CSB_CC_INVALID_OPERAND
:
244 CSB_ERR(csb
, "Invalid operand");
247 /* output buffer too small */
250 CSB_ERR(csb
, "Function aborted");
252 case CSB_CC_CRC_MISMATCH
:
253 CSB_ERR(csb
, "CRC mismatch");
255 case CSB_CC_TEMPL_INVALID
:
256 CSB_ERR(csb
, "Compressed data template invalid");
258 case CSB_CC_TEMPL_OVERFLOW
:
259 CSB_ERR(csb
, "Compressed data template shows data past end");
261 case CSB_CC_EXCEED_BYTE_COUNT
: /* P9 or later */
263 * DDE byte count exceeds the limit specified in Maximum
264 * byte count register.
266 CSB_ERR(csb
, "DDE byte count exceeds the limit");
269 /* these should not happen */
270 case CSB_CC_INVALID_ALIGN
:
271 /* setup_ddl should have detected this */
272 CSB_ERR_ADDR(csb
, "Invalid alignment");
274 case CSB_CC_DATA_LENGTH
:
275 /* setup_ddl should have detected this */
276 CSB_ERR(csb
, "Invalid data length");
278 case CSB_CC_WR_TRANSLATION
:
279 case CSB_CC_TRANSLATION
:
280 case CSB_CC_TRANSLATION_DUP1
:
281 case CSB_CC_TRANSLATION_DUP2
:
282 case CSB_CC_TRANSLATION_DUP3
:
283 case CSB_CC_TRANSLATION_DUP4
:
284 case CSB_CC_TRANSLATION_DUP5
:
285 case CSB_CC_TRANSLATION_DUP6
:
286 /* should not happen, we use physical addrs */
287 CSB_ERR_ADDR(csb
, "Translation error");
289 case CSB_CC_WR_PROTECTION
:
290 case CSB_CC_PROTECTION
:
291 case CSB_CC_PROTECTION_DUP1
:
292 case CSB_CC_PROTECTION_DUP2
:
293 case CSB_CC_PROTECTION_DUP3
:
294 case CSB_CC_PROTECTION_DUP4
:
295 case CSB_CC_PROTECTION_DUP5
:
296 case CSB_CC_PROTECTION_DUP6
:
297 /* should not happen, we use physical addrs */
298 CSB_ERR_ADDR(csb
, "Protection error");
300 case CSB_CC_PRIVILEGE
:
301 /* shouldn't happen, we're in HYP mode */
302 CSB_ERR(csb
, "Insufficient Privilege error");
304 case CSB_CC_EXCESSIVE_DDE
:
305 /* shouldn't happen, setup_ddl doesn't use many dde's */
306 CSB_ERR(csb
, "Too many DDEs in DDL");
308 case CSB_CC_TRANSPORT
:
309 case CSB_CC_INVALID_CRB
: /* P9 or later */
310 /* shouldn't happen, we setup CRB correctly */
311 CSB_ERR(csb
, "Invalid CRB");
313 case CSB_CC_INVALID_DDE
: /* P9 or later */
315 * shouldn't happen, setup_direct/indirect_dde creates
318 CSB_ERR(csb
, "Invalid DDE");
320 case CSB_CC_SEGMENTED_DDL
:
321 /* shouldn't happen, setup_ddl creates DDL right */
322 CSB_ERR(csb
, "Segmented DDL error");
324 case CSB_CC_DDE_OVERFLOW
:
325 /* shouldn't happen, setup_ddl creates DDL right */
326 CSB_ERR(csb
, "DDE overflow error");
329 /* should not happen with ICSWX */
330 CSB_ERR(csb
, "Session violation error");
333 /* should not happen, we don't use chained CRBs */
334 CSB_ERR(csb
, "Chained CRB error");
336 case CSB_CC_SEQUENCE
:
337 /* should not happen, we don't use chained CRBs */
338 CSB_ERR(csb
, "CRB seqeunce number error");
340 case CSB_CC_UNKNOWN_CODE
:
341 CSB_ERR(csb
, "Unknown subfunction code");
344 /* hardware errors */
345 case CSB_CC_RD_EXTERNAL
:
346 case CSB_CC_RD_EXTERNAL_DUP1
:
347 case CSB_CC_RD_EXTERNAL_DUP2
:
348 case CSB_CC_RD_EXTERNAL_DUP3
:
349 CSB_ERR_ADDR(csb
, "Read error outside coprocessor");
351 case CSB_CC_WR_EXTERNAL
:
352 CSB_ERR_ADDR(csb
, "Write error outside coprocessor");
354 case CSB_CC_INTERNAL
:
355 CSB_ERR(csb
, "Internal error in coprocessor");
357 case CSB_CC_PROVISION
:
358 CSB_ERR(csb
, "Storage provision error");
361 CSB_ERR(csb
, "Correctable hardware error");
363 case CSB_CC_HW_EXPIRED_TIMER
: /* P9 or later */
364 CSB_ERR(csb
, "Job did not finish within allowed time");
368 CSB_ERR(csb
, "Invalid CC %d", csb
->cc
);
372 /* check Completion Extension state */
373 if (csb
->ce
& CSB_CE_TERMINATION
) {
374 CSB_ERR(csb
, "CSB request was terminated");
377 if (csb
->ce
& CSB_CE_INCOMPLETE
) {
378 CSB_ERR(csb
, "CSB request not complete");
381 if (!(csb
->ce
& CSB_CE_TPBC
)) {
382 CSB_ERR(csb
, "TPBC not provided, unknown target length");
386 /* successful completion */
387 pr_debug_ratelimited("Processed %u bytes in %lu us\n",
388 be32_to_cpu(csb
->count
),
389 (unsigned long)ktime_us_delta(now
, start
));
394 static int nx842_config_crb(const unsigned char *in
, unsigned int inlen
,
395 unsigned char *out
, unsigned int outlen
,
396 struct nx842_workmem
*wmem
)
398 struct coprocessor_request_block
*crb
;
399 struct coprocessor_status_block
*csb
;
406 /* Clear any previous values */
407 memset(crb
, 0, sizeof(*crb
));
410 ret
= setup_ddl(&crb
->source
, wmem
->ddl_in
,
411 (unsigned char *)in
, inlen
, true);
415 ret
= setup_ddl(&crb
->target
, wmem
->ddl_out
,
420 /* set up CRB's CSB addr */
421 csb_addr
= nx842_get_pa(csb
) & CRB_CSB_ADDRESS
;
422 csb_addr
|= CRB_CSB_AT
; /* Addrs are phys */
423 crb
->csb_addr
= cpu_to_be64(csb_addr
);
429 * nx842_exec_icswx - compress/decompress data using the 842 algorithm
431 * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems.
432 * This compresses or decompresses the provided input buffer into the provided
435 * Upon return from this function @outlen contains the length of the
436 * output data. If there is an error then @outlen will be 0 and an
437 * error will be specified by the return code from this function.
439 * The @workmem buffer should only be used by one function call at a time.
441 * @in: input buffer pointer
442 * @inlen: input buffer size
443 * @out: output buffer pointer
444 * @outlenp: output buffer size pointer
445 * @workmem: working memory buffer pointer, size determined by
446 * nx842_powernv_driver.workmem_size
447 * @fc: function code, see CCW Function Codes in nx-842.h
450 * 0 Success, output of length @outlenp stored in the buffer at @out
451 * -ENODEV Hardware unavailable
452 * -ENOSPC Output buffer is to small
453 * -EMSGSIZE Input buffer too large
454 * -EINVAL buffer constraints do not fix nx842_constraints
455 * -EPROTO hardware error during operation
456 * -ETIMEDOUT hardware did not complete operation in reasonable time
457 * -EINTR operation was aborted
459 static int nx842_exec_icswx(const unsigned char *in
, unsigned int inlen
,
460 unsigned char *out
, unsigned int *outlenp
,
461 void *workmem
, int fc
)
463 struct coprocessor_request_block
*crb
;
464 struct coprocessor_status_block
*csb
;
465 struct nx842_workmem
*wmem
;
468 unsigned int outlen
= *outlenp
;
470 wmem
= PTR_ALIGN(workmem
, WORKMEM_ALIGN
);
474 /* shoudn't happen, we don't load without a coproc */
476 pr_err_ratelimited("coprocessor CT is 0");
480 ret
= nx842_config_crb(in
, inlen
, out
, outlen
, wmem
);
489 ccw
= SET_FIELD(CCW_CT
, ccw
, nx842_ct
);
490 ccw
= SET_FIELD(CCW_CI_842
, ccw
, 0); /* use 0 for hw auto-selection */
491 ccw
= SET_FIELD(CCW_FC_842
, ccw
, fc
);
493 wmem
->start
= ktime_get();
496 ret
= icswx(cpu_to_be32(ccw
), crb
);
498 pr_debug_ratelimited("icswx CR %x ccw %x crb->ccw %x\n", ret
,
500 (unsigned int)be32_to_cpu(crb
->ccw
));
503 * NX842 coprocessor sets 3rd bit in CR register with XER[S0].
504 * XER[S0] is the integer summary overflow bit which is nothing
505 * to do NX. Since this bit can be set with other return values,
511 case ICSWX_INITIATED
:
512 ret
= wait_for_csb(wmem
, csb
);
515 pr_debug_ratelimited("842 Coprocessor busy\n");
519 pr_err_ratelimited("ICSWX rejected\n");
525 *outlenp
= be32_to_cpu(csb
->count
);
531 * nx842_exec_vas - compress/decompress data using the 842 algorithm
533 * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems.
534 * This compresses or decompresses the provided input buffer into the provided
537 * Upon return from this function @outlen contains the length of the
538 * output data. If there is an error then @outlen will be 0 and an
539 * error will be specified by the return code from this function.
541 * The @workmem buffer should only be used by one function call at a time.
543 * @in: input buffer pointer
544 * @inlen: input buffer size
545 * @out: output buffer pointer
546 * @outlenp: output buffer size pointer
547 * @workmem: working memory buffer pointer, size determined by
548 * nx842_powernv_driver.workmem_size
549 * @fc: function code, see CCW Function Codes in nx-842.h
552 * 0 Success, output of length @outlenp stored in the buffer
554 * -ENODEV Hardware unavailable
555 * -ENOSPC Output buffer is to small
556 * -EMSGSIZE Input buffer too large
557 * -EINVAL buffer constraints do not fix nx842_constraints
558 * -EPROTO hardware error during operation
559 * -ETIMEDOUT hardware did not complete operation in reasonable time
560 * -EINTR operation was aborted
562 static int nx842_exec_vas(const unsigned char *in
, unsigned int inlen
,
563 unsigned char *out
, unsigned int *outlenp
,
564 void *workmem
, int fc
)
566 struct coprocessor_request_block
*crb
;
567 struct coprocessor_status_block
*csb
;
568 struct nx842_workmem
*wmem
;
569 struct vas_window
*txwin
;
572 unsigned int outlen
= *outlenp
;
574 wmem
= PTR_ALIGN(workmem
, WORKMEM_ALIGN
);
581 ret
= nx842_config_crb(in
, inlen
, out
, outlen
, wmem
);
586 ccw
= SET_FIELD(CCW_FC_842
, ccw
, fc
);
587 crb
->ccw
= cpu_to_be32(ccw
);
590 /* shoudn't happen, we don't load without a coproc */
592 pr_err_ratelimited("NX-842 coprocessor is not available");
597 wmem
->start
= ktime_get();
600 * VAS copy CRB into L2 cache. Refer <asm/vas.h>.
603 vas_copy_crb(crb
, 0);
606 * VAS paste previously copied CRB to NX.
607 * @txwin, @offset and @last (must be true).
609 ret
= vas_paste_crb(txwin
, 0, 1);
612 * Retry copy/paste function for VAS failures.
614 } while (ret
&& (i
++ < VAS_RETRIES
));
617 pr_err_ratelimited("VAS copy/paste failed\n");
621 ret
= wait_for_csb(wmem
, csb
);
623 *outlenp
= be32_to_cpu(csb
->count
);
629 * nx842_powernv_compress - Compress data using the 842 algorithm
631 * Compression provided by the NX842 coprocessor on IBM PowerNV systems.
632 * The input buffer is compressed and the result is stored in the
633 * provided output buffer.
635 * Upon return from this function @outlen contains the length of the
636 * compressed data. If there is an error then @outlen will be 0 and an
637 * error will be specified by the return code from this function.
639 * @in: input buffer pointer
640 * @inlen: input buffer size
641 * @out: output buffer pointer
642 * @outlenp: output buffer size pointer
643 * @workmem: working memory buffer pointer, size determined by
644 * nx842_powernv_driver.workmem_size
646 * Returns: see @nx842_powernv_exec()
648 static int nx842_powernv_compress(const unsigned char *in
, unsigned int inlen
,
649 unsigned char *out
, unsigned int *outlenp
,
652 return nx842_powernv_exec(in
, inlen
, out
, outlenp
,
653 wmem
, CCW_FC_842_COMP_CRC
);
657 * nx842_powernv_decompress - Decompress data using the 842 algorithm
659 * Decompression provided by the NX842 coprocessor on IBM PowerNV systems.
660 * The input buffer is decompressed and the result is stored in the
661 * provided output buffer.
663 * Upon return from this function @outlen contains the length of the
664 * decompressed data. If there is an error then @outlen will be 0 and an
665 * error will be specified by the return code from this function.
667 * @in: input buffer pointer
668 * @inlen: input buffer size
669 * @out: output buffer pointer
670 * @outlenp: output buffer size pointer
671 * @workmem: working memory buffer pointer, size determined by
672 * nx842_powernv_driver.workmem_size
674 * Returns: see @nx842_powernv_exec()
676 static int nx842_powernv_decompress(const unsigned char *in
, unsigned int inlen
,
677 unsigned char *out
, unsigned int *outlenp
,
680 return nx842_powernv_exec(in
, inlen
, out
, outlenp
,
681 wmem
, CCW_FC_842_DECOMP_CRC
);
684 static inline void nx842_add_coprocs_list(struct nx842_coproc
*coproc
,
687 coproc
->chip_id
= chipid
;
688 INIT_LIST_HEAD(&coproc
->list
);
689 list_add(&coproc
->list
, &nx842_coprocs
);
693 * Identify chip ID for each CPU and save coprocesor adddress for the
694 * corresponding NX engine in percpu coproc_inst.
695 * coproc_inst is used in crypto_init to open send window on the NX instance
696 * for the corresponding CPU / chip where the open request is executed.
698 static void nx842_set_per_cpu_coproc(struct nx842_coproc
*coproc
)
700 unsigned int i
, chip_id
;
702 for_each_possible_cpu(i
) {
703 chip_id
= cpu_to_chip_id(i
);
705 if (coproc
->chip_id
== chip_id
)
706 per_cpu(coproc_inst
, i
) = coproc
;
711 static struct vas_window
*nx842_alloc_txwin(struct nx842_coproc
*coproc
)
713 struct vas_window
*txwin
= NULL
;
714 struct vas_tx_win_attr txattr
;
717 * Kernel requests will be high priority. So open send
718 * windows only for high priority RxFIFO entries.
720 vas_init_tx_win_attr(&txattr
, coproc
->ct
);
721 txattr
.lpid
= 0; /* lpid is 0 for kernel requests */
722 txattr
.pid
= 0; /* pid is 0 for kernel requests */
725 * Open a VAS send window which is used to send request to NX.
727 txwin
= vas_tx_win_open(coproc
->vas
.id
, coproc
->ct
, &txattr
);
729 pr_err("ibm,nx-842: Can not open TX window: %ld\n",
737 static int __init
vas_cfg_coproc_info(struct device_node
*dn
, int chip_id
,
740 struct vas_window
*rxwin
= NULL
;
741 struct vas_rx_win_attr rxattr
;
742 struct nx842_coproc
*coproc
;
743 u32 lpid
, pid
, tid
, fifo_size
;
745 const char *priority
;
748 ret
= of_property_read_u64(dn
, "rx-fifo-address", &rx_fifo
);
750 pr_err("Missing rx-fifo-address property\n");
754 ret
= of_property_read_u32(dn
, "rx-fifo-size", &fifo_size
);
756 pr_err("Missing rx-fifo-size property\n");
760 ret
= of_property_read_u32(dn
, "lpid", &lpid
);
762 pr_err("Missing lpid property\n");
766 ret
= of_property_read_u32(dn
, "pid", &pid
);
768 pr_err("Missing pid property\n");
772 ret
= of_property_read_u32(dn
, "tid", &tid
);
774 pr_err("Missing tid property\n");
778 ret
= of_property_read_string(dn
, "priority", &priority
);
780 pr_err("Missing priority property\n");
784 coproc
= kzalloc(sizeof(*coproc
), GFP_KERNEL
);
788 if (!strcmp(priority
, "High"))
789 coproc
->ct
= VAS_COP_TYPE_842_HIPRI
;
790 else if (!strcmp(priority
, "Normal"))
791 coproc
->ct
= VAS_COP_TYPE_842
;
793 pr_err("Invalid RxFIFO priority value\n");
798 vas_init_rx_win_attr(&rxattr
, coproc
->ct
);
799 rxattr
.rx_fifo
= (void *)rx_fifo
;
800 rxattr
.rx_fifo_size
= fifo_size
;
801 rxattr
.lnotify_lpid
= lpid
;
802 rxattr
.lnotify_pid
= pid
;
803 rxattr
.lnotify_tid
= tid
;
804 rxattr
.wcreds_max
= MAX_CREDITS_PER_RXFIFO
;
807 * Open a VAS receice window which is used to configure RxFIFO
810 rxwin
= vas_rx_win_open(vasid
, coproc
->ct
, &rxattr
);
812 ret
= PTR_ERR(rxwin
);
813 pr_err("setting RxFIFO with VAS failed: %d\n",
818 coproc
->vas
.rxwin
= rxwin
;
819 coproc
->vas
.id
= vasid
;
820 nx842_add_coprocs_list(coproc
, chip_id
);
823 * Kernel requests use only high priority FIFOs. So save coproc
824 * info in percpu coproc_inst which will be used to open send
825 * windows for crypto open requests later.
827 if (coproc
->ct
== VAS_COP_TYPE_842_HIPRI
)
828 nx842_set_per_cpu_coproc(coproc
);
838 static int __init
nx842_powernv_probe_vas(struct device_node
*pn
)
840 struct device_node
*dn
;
841 int chip_id
, vasid
, ret
= 0;
842 int nx_fifo_found
= 0;
844 chip_id
= of_get_ibm_chip_id(pn
);
846 pr_err("ibm,chip-id missing\n");
850 for_each_compatible_node(dn
, NULL
, "ibm,power9-vas-x") {
851 if (of_get_ibm_chip_id(dn
) == chip_id
)
856 pr_err("Missing VAS device node\n");
860 if (of_property_read_u32(dn
, "ibm,vas-id", &vasid
)) {
861 pr_err("Missing ibm,vas-id device property\n");
868 for_each_child_of_node(pn
, dn
) {
869 if (of_device_is_compatible(dn
, "ibm,p9-nx-842")) {
870 ret
= vas_cfg_coproc_info(dn
, chip_id
, vasid
);
879 if (!nx_fifo_found
) {
880 pr_err("NX842 FIFO nodes are missing\n");
887 static int __init
nx842_powernv_probe(struct device_node
*dn
)
889 struct nx842_coproc
*coproc
;
893 chip_id
= of_get_ibm_chip_id(dn
);
895 pr_err("ibm,chip-id missing\n");
899 if (of_property_read_u32(dn
, "ibm,842-coprocessor-type", &ct
)) {
900 pr_err("ibm,842-coprocessor-type missing\n");
904 if (of_property_read_u32(dn
, "ibm,842-coprocessor-instance", &ci
)) {
905 pr_err("ibm,842-coprocessor-instance missing\n");
909 coproc
= kmalloc(sizeof(*coproc
), GFP_KERNEL
);
915 nx842_add_coprocs_list(coproc
, chip_id
);
917 pr_info("coprocessor found on chip %d, CT %d CI %d\n", chip_id
, ct
, ci
);
921 else if (nx842_ct
!= ct
)
922 pr_err("NX842 chip %d, CT %d != first found CT %d\n",
923 chip_id
, ct
, nx842_ct
);
928 static void nx842_delete_coprocs(void)
930 struct nx842_coproc
*coproc
, *n
;
932 list_for_each_entry_safe(coproc
, n
, &nx842_coprocs
, list
) {
933 if (coproc
->vas
.rxwin
)
934 vas_win_close(coproc
->vas
.rxwin
);
936 list_del(&coproc
->list
);
941 static struct nx842_constraints nx842_powernv_constraints
= {
942 .alignment
= DDE_BUFFER_ALIGN
,
943 .multiple
= DDE_BUFFER_LAST_MULT
,
944 .minimum
= DDE_BUFFER_LAST_MULT
,
945 .maximum
= (DDL_LEN_MAX
- 1) * PAGE_SIZE
,
948 static struct nx842_driver nx842_powernv_driver
= {
949 .name
= KBUILD_MODNAME
,
950 .owner
= THIS_MODULE
,
951 .workmem_size
= sizeof(struct nx842_workmem
),
952 .constraints
= &nx842_powernv_constraints
,
953 .compress
= nx842_powernv_compress
,
954 .decompress
= nx842_powernv_decompress
,
957 static int nx842_powernv_crypto_init_vas(struct crypto_tfm
*tfm
)
959 struct nx842_crypto_ctx
*ctx
= crypto_tfm_ctx(tfm
);
960 struct nx842_workmem
*wmem
;
961 struct nx842_coproc
*coproc
;
964 ret
= nx842_crypto_init(tfm
, &nx842_powernv_driver
);
969 wmem
= PTR_ALIGN((struct nx842_workmem
*)ctx
->wmem
, WORKMEM_ALIGN
);
970 coproc
= per_cpu(coproc_inst
, smp_processor_id());
973 if (coproc
&& coproc
->vas
.rxwin
) {
974 wmem
->txwin
= nx842_alloc_txwin(coproc
);
975 if (!IS_ERR(wmem
->txwin
))
978 ret
= PTR_ERR(wmem
->txwin
);
984 void nx842_powernv_crypto_exit_vas(struct crypto_tfm
*tfm
)
986 struct nx842_crypto_ctx
*ctx
= crypto_tfm_ctx(tfm
);
987 struct nx842_workmem
*wmem
;
989 wmem
= PTR_ALIGN((struct nx842_workmem
*)ctx
->wmem
, WORKMEM_ALIGN
);
991 if (wmem
&& wmem
->txwin
)
992 vas_win_close(wmem
->txwin
);
994 nx842_crypto_exit(tfm
);
997 static int nx842_powernv_crypto_init(struct crypto_tfm
*tfm
)
999 return nx842_crypto_init(tfm
, &nx842_powernv_driver
);
1002 static struct crypto_alg nx842_powernv_alg
= {
1004 .cra_driver_name
= "842-nx",
1005 .cra_priority
= 300,
1006 .cra_flags
= CRYPTO_ALG_TYPE_COMPRESS
,
1007 .cra_ctxsize
= sizeof(struct nx842_crypto_ctx
),
1008 .cra_module
= THIS_MODULE
,
1009 .cra_init
= nx842_powernv_crypto_init
,
1010 .cra_exit
= nx842_crypto_exit
,
1011 .cra_u
= { .compress
= {
1012 .coa_compress
= nx842_crypto_compress
,
1013 .coa_decompress
= nx842_crypto_decompress
} }
1016 static __init
int nx842_powernv_init(void)
1018 struct device_node
*dn
;
1021 /* verify workmem size/align restrictions */
1022 BUILD_BUG_ON(WORKMEM_ALIGN
% CRB_ALIGN
);
1023 BUILD_BUG_ON(CRB_ALIGN
% DDE_ALIGN
);
1024 BUILD_BUG_ON(CRB_SIZE
% DDE_ALIGN
);
1025 /* verify buffer size/align restrictions */
1026 BUILD_BUG_ON(PAGE_SIZE
% DDE_BUFFER_ALIGN
);
1027 BUILD_BUG_ON(DDE_BUFFER_ALIGN
% DDE_BUFFER_SIZE_MULT
);
1028 BUILD_BUG_ON(DDE_BUFFER_SIZE_MULT
% DDE_BUFFER_LAST_MULT
);
1030 for_each_compatible_node(dn
, NULL
, "ibm,power9-nx") {
1031 ret
= nx842_powernv_probe_vas(dn
);
1033 nx842_delete_coprocs();
1038 if (list_empty(&nx842_coprocs
)) {
1039 for_each_compatible_node(dn
, NULL
, "ibm,power-nx")
1040 nx842_powernv_probe(dn
);
1045 nx842_powernv_exec
= nx842_exec_icswx
;
1047 nx842_powernv_exec
= nx842_exec_vas
;
1048 nx842_powernv_alg
.cra_init
= nx842_powernv_crypto_init_vas
;
1049 nx842_powernv_alg
.cra_exit
= nx842_powernv_crypto_exit_vas
;
1052 ret
= crypto_register_alg(&nx842_powernv_alg
);
1054 nx842_delete_coprocs();
1060 module_init(nx842_powernv_init
);
1062 static void __exit
nx842_powernv_exit(void)
1064 crypto_unregister_alg(&nx842_powernv_alg
);
1066 nx842_delete_coprocs();
1068 module_exit(nx842_powernv_exit
);