4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 #if defined(_KERNEL) && defined(HAVE_QAT)
23 #include <linux/slab.h>
24 #include <linux/vmalloc.h>
25 #include <linux/pagemap.h>
26 #include <linux/completion.h>
27 #include <linux/mod_compat.h>
28 #include <sys/zfs_context.h>
29 #include <sys/byteorder.h>
34 * Max instances in a QAT device, each instance is a channel to submit
35 * jobs to QAT hardware, this is only for pre-allocating instance and
36 * session arrays; the actual number of instances are defined in the
37 * QAT driver's configuration file.
39 #define QAT_DC_MAX_INSTANCES 48
42 * ZLIB head and foot size
44 #define ZLIB_HEAD_SZ 2
45 #define ZLIB_FOOT_SZ 4
47 static CpaInstanceHandle dc_inst_handles
[QAT_DC_MAX_INSTANCES
];
48 static CpaDcSessionHandle session_handles
[QAT_DC_MAX_INSTANCES
];
49 static CpaBufferList
**buffer_array
[QAT_DC_MAX_INSTANCES
];
50 static Cpa16U num_inst
= 0;
51 static Cpa32U inst_num
= 0;
52 static boolean_t qat_dc_init_done
= B_FALSE
;
53 int zfs_qat_compress_disable
= 0;
56 qat_dc_use_accel(size_t s_len
)
58 return (!zfs_qat_compress_disable
&&
60 s_len
>= QAT_MIN_BUF_SIZE
&&
61 s_len
<= QAT_MAX_BUF_SIZE
);
65 qat_dc_callback(void *p_callback
, CpaStatus status
)
67 if (p_callback
!= NULL
)
68 complete((struct completion
*)p_callback
);
75 Cpa16U num_inter_buff_lists
= 0;
77 for (Cpa16U i
= 0; i
< num_inst
; i
++) {
78 cpaDcStopInstance(dc_inst_handles
[i
]);
79 QAT_PHYS_CONTIG_FREE(session_handles
[i
]);
80 /* free intermediate buffers */
81 if (buffer_array
[i
] != NULL
) {
82 cpaDcGetNumIntermediateBuffers(
83 dc_inst_handles
[i
], &num_inter_buff_lists
);
84 for (buff_num
= 0; buff_num
< num_inter_buff_lists
;
86 CpaBufferList
*buffer_inter
=
87 buffer_array
[i
][buff_num
];
88 if (buffer_inter
->pBuffers
) {
90 buffer_inter
->pBuffers
->pData
);
92 buffer_inter
->pBuffers
);
95 buffer_inter
->pPrivateMetaData
);
96 QAT_PHYS_CONTIG_FREE(buffer_inter
);
102 qat_dc_init_done
= B_FALSE
;
108 CpaStatus status
= CPA_STATUS_SUCCESS
;
109 Cpa32U sess_size
= 0;
111 Cpa16U num_inter_buff_lists
= 0;
113 Cpa32U buff_meta_size
= 0;
114 CpaDcSessionSetupData sd
= {0};
116 if (qat_dc_init_done
)
119 status
= cpaDcGetNumInstances(&num_inst
);
120 if (status
!= CPA_STATUS_SUCCESS
)
123 /* if the user has configured no QAT compression units just return */
127 if (num_inst
> QAT_DC_MAX_INSTANCES
)
128 num_inst
= QAT_DC_MAX_INSTANCES
;
130 status
= cpaDcGetInstances(num_inst
, &dc_inst_handles
[0]);
131 if (status
!= CPA_STATUS_SUCCESS
)
134 for (Cpa16U i
= 0; i
< num_inst
; i
++) {
135 cpaDcSetAddressTranslation(dc_inst_handles
[i
],
136 (void*)virt_to_phys
);
138 status
= cpaDcBufferListGetMetaSize(dc_inst_handles
[i
],
141 if (status
== CPA_STATUS_SUCCESS
)
142 status
= cpaDcGetNumIntermediateBuffers(
143 dc_inst_handles
[i
], &num_inter_buff_lists
);
145 if (status
== CPA_STATUS_SUCCESS
&& num_inter_buff_lists
!= 0)
146 status
= QAT_PHYS_CONTIG_ALLOC(&buffer_array
[i
],
147 num_inter_buff_lists
*
148 sizeof (CpaBufferList
*));
150 for (buff_num
= 0; buff_num
< num_inter_buff_lists
;
152 if (status
== CPA_STATUS_SUCCESS
)
153 status
= QAT_PHYS_CONTIG_ALLOC(
154 &buffer_array
[i
][buff_num
],
155 sizeof (CpaBufferList
));
157 if (status
== CPA_STATUS_SUCCESS
)
158 status
= QAT_PHYS_CONTIG_ALLOC(
159 &buffer_array
[i
][buff_num
]->
163 if (status
== CPA_STATUS_SUCCESS
)
164 status
= QAT_PHYS_CONTIG_ALLOC(
165 &buffer_array
[i
][buff_num
]->pBuffers
,
166 sizeof (CpaFlatBuffer
));
168 if (status
== CPA_STATUS_SUCCESS
) {
170 * implementation requires an intermediate
171 * buffer approximately twice the size of
172 * output buffer, which is 2x max buffer
175 status
= QAT_PHYS_CONTIG_ALLOC(
176 &buffer_array
[i
][buff_num
]->pBuffers
->
177 pData
, 2 * QAT_MAX_BUF_SIZE
);
178 if (status
!= CPA_STATUS_SUCCESS
)
181 buffer_array
[i
][buff_num
]->numBuffers
= 1;
182 buffer_array
[i
][buff_num
]->pBuffers
->
183 dataLenInBytes
= 2 * QAT_MAX_BUF_SIZE
;
187 status
= cpaDcStartInstance(dc_inst_handles
[i
],
188 num_inter_buff_lists
, buffer_array
[i
]);
189 if (status
!= CPA_STATUS_SUCCESS
)
192 sd
.compLevel
= CPA_DC_L1
;
193 sd
.compType
= CPA_DC_DEFLATE
;
194 sd
.huffType
= CPA_DC_HT_FULL_DYNAMIC
;
195 sd
.sessDirection
= CPA_DC_DIR_COMBINED
;
196 sd
.sessState
= CPA_DC_STATELESS
;
197 sd
.deflateWindowSize
= 7;
198 sd
.checksum
= CPA_DC_ADLER32
;
199 status
= cpaDcGetSessionSize(dc_inst_handles
[i
],
200 &sd
, &sess_size
, &ctx_size
);
201 if (status
!= CPA_STATUS_SUCCESS
)
204 QAT_PHYS_CONTIG_ALLOC(&session_handles
[i
], sess_size
);
205 if (session_handles
[i
] == NULL
)
208 status
= cpaDcInitSession(dc_inst_handles
[i
],
210 &sd
, NULL
, qat_dc_callback
);
211 if (status
!= CPA_STATUS_SUCCESS
)
215 qat_dc_init_done
= B_TRUE
;
225 if (!qat_dc_init_done
)
232 * The "add" parameter is an additional buffer which is passed
233 * to QAT as a scratch buffer alongside the destination buffer
234 * in case the "compressed" data ends up being larger than the
235 * original source data. This is necessary to prevent QAT from
236 * generating buffer overflow warnings for incompressible data.
239 qat_compress_impl(qat_compress_dir_t dir
, char *src
, int src_len
,
240 char *dst
, int dst_len
, char *add
, int add_len
, size_t *c_len
)
242 CpaInstanceHandle dc_inst_handle
;
243 CpaDcSessionHandle session_handle
;
244 CpaBufferList
*buf_list_src
= NULL
;
245 CpaBufferList
*buf_list_dst
= NULL
;
246 CpaFlatBuffer
*flat_buf_src
= NULL
;
247 CpaFlatBuffer
*flat_buf_dst
= NULL
;
248 Cpa8U
*buffer_meta_src
= NULL
;
249 Cpa8U
*buffer_meta_dst
= NULL
;
250 Cpa32U buffer_meta_size
= 0;
251 CpaDcRqResults dc_results
;
252 CpaStatus status
= CPA_STATUS_SUCCESS
;
254 Cpa32U compressed_sz
;
255 Cpa32U num_src_buf
= (src_len
>> PAGE_SHIFT
) + 2;
256 Cpa32U num_dst_buf
= (dst_len
>> PAGE_SHIFT
) + 2;
257 Cpa32U num_add_buf
= (add_len
>> PAGE_SHIFT
) + 2;
259 Cpa32U dst_pages
= 0;
263 struct page
**in_pages
= NULL
;
264 struct page
**out_pages
= NULL
;
265 struct page
**add_pages
= NULL
;
267 struct completion complete
;
272 * We increment num_src_buf and num_dst_buf by 2 to allow
273 * us to handle non page-aligned buffer addresses and buffers
274 * whose sizes are not divisible by PAGE_SIZE.
276 Cpa32U src_buffer_list_mem_size
= sizeof (CpaBufferList
) +
277 (num_src_buf
* sizeof (CpaFlatBuffer
));
278 Cpa32U dst_buffer_list_mem_size
= sizeof (CpaBufferList
) +
279 ((num_dst_buf
+ num_add_buf
) * sizeof (CpaFlatBuffer
));
281 if (QAT_PHYS_CONTIG_ALLOC(&in_pages
,
282 num_src_buf
* sizeof (struct page
*)) != CPA_STATUS_SUCCESS
)
285 if (QAT_PHYS_CONTIG_ALLOC(&out_pages
,
286 num_dst_buf
* sizeof (struct page
*)) != CPA_STATUS_SUCCESS
)
289 if (QAT_PHYS_CONTIG_ALLOC(&add_pages
,
290 num_add_buf
* sizeof (struct page
*)) != CPA_STATUS_SUCCESS
)
293 i
= (Cpa32U
)atomic_inc_32_nv(&inst_num
) % num_inst
;
294 dc_inst_handle
= dc_inst_handles
[i
];
295 session_handle
= session_handles
[i
];
297 cpaDcBufferListGetMetaSize(dc_inst_handle
, num_src_buf
,
299 if (QAT_PHYS_CONTIG_ALLOC(&buffer_meta_src
, buffer_meta_size
) !=
303 cpaDcBufferListGetMetaSize(dc_inst_handle
, num_dst_buf
+ num_add_buf
,
305 if (QAT_PHYS_CONTIG_ALLOC(&buffer_meta_dst
, buffer_meta_size
) !=
309 /* build source buffer list */
310 if (QAT_PHYS_CONTIG_ALLOC(&buf_list_src
, src_buffer_list_mem_size
) !=
314 flat_buf_src
= (CpaFlatBuffer
*)(buf_list_src
+ 1);
316 buf_list_src
->pBuffers
= flat_buf_src
; /* always point to first one */
318 /* build destination buffer list */
319 if (QAT_PHYS_CONTIG_ALLOC(&buf_list_dst
, dst_buffer_list_mem_size
) !=
323 flat_buf_dst
= (CpaFlatBuffer
*)(buf_list_dst
+ 1);
325 buf_list_dst
->pBuffers
= flat_buf_dst
; /* always point to first one */
327 buf_list_src
->numBuffers
= 0;
328 buf_list_src
->pPrivateMetaData
= buffer_meta_src
;
329 bytes_left
= src_len
;
332 while (bytes_left
> 0) {
333 page_off
= ((long)data
& ~PAGE_MASK
);
334 page
= qat_mem_to_page(data
);
335 in_pages
[page_num
] = page
;
336 flat_buf_src
->pData
= kmap(page
) + page_off
;
337 flat_buf_src
->dataLenInBytes
=
338 min((long)PAGE_SIZE
- page_off
, (long)bytes_left
);
340 bytes_left
-= flat_buf_src
->dataLenInBytes
;
341 data
+= flat_buf_src
->dataLenInBytes
;
343 buf_list_src
->numBuffers
++;
347 buf_list_dst
->numBuffers
= 0;
348 buf_list_dst
->pPrivateMetaData
= buffer_meta_dst
;
349 bytes_left
= dst_len
;
352 while (bytes_left
> 0) {
353 page_off
= ((long)data
& ~PAGE_MASK
);
354 page
= qat_mem_to_page(data
);
355 flat_buf_dst
->pData
= kmap(page
) + page_off
;
356 out_pages
[page_num
] = page
;
357 flat_buf_dst
->dataLenInBytes
=
358 min((long)PAGE_SIZE
- page_off
, (long)bytes_left
);
360 bytes_left
-= flat_buf_dst
->dataLenInBytes
;
361 data
+= flat_buf_dst
->dataLenInBytes
;
363 buf_list_dst
->numBuffers
++;
368 /* map additional scratch pages into the destination buffer list */
369 bytes_left
= add_len
;
372 while (bytes_left
> 0) {
373 page_off
= ((long)data
& ~PAGE_MASK
);
374 page
= qat_mem_to_page(data
);
375 flat_buf_dst
->pData
= kmap(page
) + page_off
;
376 add_pages
[page_num
] = page
;
377 flat_buf_dst
->dataLenInBytes
=
378 min((long)PAGE_SIZE
- page_off
, (long)bytes_left
);
380 bytes_left
-= flat_buf_dst
->dataLenInBytes
;
381 data
+= flat_buf_dst
->dataLenInBytes
;
383 buf_list_dst
->numBuffers
++;
387 init_completion(&complete
);
389 if (dir
== QAT_COMPRESS
) {
390 QAT_STAT_BUMP(comp_requests
);
391 QAT_STAT_INCR(comp_total_in_bytes
, src_len
);
393 cpaDcGenerateHeader(session_handle
,
394 buf_list_dst
->pBuffers
, &hdr_sz
);
395 buf_list_dst
->pBuffers
->pData
+= hdr_sz
;
396 buf_list_dst
->pBuffers
->dataLenInBytes
-= hdr_sz
;
397 status
= cpaDcCompressData(
398 dc_inst_handle
, session_handle
,
399 buf_list_src
, buf_list_dst
,
400 &dc_results
, CPA_DC_FLUSH_FINAL
,
402 if (status
!= CPA_STATUS_SUCCESS
) {
406 /* we now wait until the completion of the operation. */
407 if (!wait_for_completion_interruptible_timeout(&complete
,
409 status
= CPA_STATUS_FAIL
;
413 if (dc_results
.status
!= CPA_STATUS_SUCCESS
) {
414 status
= CPA_STATUS_FAIL
;
418 compressed_sz
= dc_results
.produced
;
419 if (compressed_sz
+ hdr_sz
+ ZLIB_FOOT_SZ
> dst_len
) {
420 status
= CPA_STATUS_INCOMPRESSIBLE
;
424 flat_buf_dst
= (CpaFlatBuffer
*)(buf_list_dst
+ 1);
425 /* move to the last page */
426 flat_buf_dst
+= (compressed_sz
+ hdr_sz
) >> PAGE_SHIFT
;
428 /* no space for gzip footer in the last page */
429 if (((compressed_sz
+ hdr_sz
) % PAGE_SIZE
)
430 + ZLIB_FOOT_SZ
> PAGE_SIZE
) {
431 status
= CPA_STATUS_INCOMPRESSIBLE
;
435 /* jump to the end of the buffer and append footer */
436 flat_buf_dst
->pData
=
437 (char *)((unsigned long)flat_buf_dst
->pData
& PAGE_MASK
)
438 + ((compressed_sz
+ hdr_sz
) % PAGE_SIZE
);
439 flat_buf_dst
->dataLenInBytes
= ZLIB_FOOT_SZ
;
441 dc_results
.produced
= 0;
442 status
= cpaDcGenerateFooter(session_handle
,
443 flat_buf_dst
, &dc_results
);
444 if (status
!= CPA_STATUS_SUCCESS
)
447 *c_len
= compressed_sz
+ dc_results
.produced
+ hdr_sz
;
448 QAT_STAT_INCR(comp_total_out_bytes
, *c_len
);
450 ASSERT3U(dir
, ==, QAT_DECOMPRESS
);
451 QAT_STAT_BUMP(decomp_requests
);
452 QAT_STAT_INCR(decomp_total_in_bytes
, src_len
);
454 buf_list_src
->pBuffers
->pData
+= ZLIB_HEAD_SZ
;
455 buf_list_src
->pBuffers
->dataLenInBytes
-= ZLIB_HEAD_SZ
;
456 status
= cpaDcDecompressData(dc_inst_handle
, session_handle
,
457 buf_list_src
, buf_list_dst
, &dc_results
, CPA_DC_FLUSH_FINAL
,
460 if (CPA_STATUS_SUCCESS
!= status
) {
461 status
= CPA_STATUS_FAIL
;
465 /* we now wait until the completion of the operation. */
466 if (!wait_for_completion_interruptible_timeout(&complete
,
468 status
= CPA_STATUS_FAIL
;
472 if (dc_results
.status
!= CPA_STATUS_SUCCESS
) {
473 status
= CPA_STATUS_FAIL
;
477 /* verify adler checksum */
478 adler32
= *(Cpa32U
*)(src
+ dc_results
.consumed
+ ZLIB_HEAD_SZ
);
479 if (adler32
!= BSWAP_32(dc_results
.checksum
)) {
480 status
= CPA_STATUS_FAIL
;
483 *c_len
= dc_results
.produced
;
484 QAT_STAT_INCR(decomp_total_out_bytes
, *c_len
);
488 if (status
!= CPA_STATUS_SUCCESS
&& status
!= CPA_STATUS_INCOMPRESSIBLE
)
489 QAT_STAT_BUMP(dc_fails
);
493 page_num
< buf_list_src
->numBuffers
;
495 kunmap(in_pages
[page_num
]);
497 QAT_PHYS_CONTIG_FREE(in_pages
);
501 for (page_num
= 0; page_num
< dst_pages
; page_num
++) {
502 kunmap(out_pages
[page_num
]);
504 QAT_PHYS_CONTIG_FREE(out_pages
);
509 page_num
< buf_list_dst
->numBuffers
- dst_pages
;
511 kunmap(add_pages
[page_num
]);
513 QAT_PHYS_CONTIG_FREE(add_pages
);
516 QAT_PHYS_CONTIG_FREE(buffer_meta_src
);
517 QAT_PHYS_CONTIG_FREE(buffer_meta_dst
);
518 QAT_PHYS_CONTIG_FREE(buf_list_src
);
519 QAT_PHYS_CONTIG_FREE(buf_list_dst
);
525 * Entry point for QAT accelerated compression / decompression.
528 qat_compress(qat_compress_dir_t dir
, char *src
, int src_len
,
529 char *dst
, int dst_len
, size_t *c_len
)
535 if (dir
== QAT_COMPRESS
) {
537 add
= zio_data_buf_alloc(add_len
);
540 ret
= qat_compress_impl(dir
, src
, src_len
, dst
,
541 dst_len
, add
, add_len
, c_len
);
543 if (dir
== QAT_COMPRESS
)
544 zio_data_buf_free(add
, add_len
);
550 param_set_qat_compress(const char *val
, struct kernel_param
*kp
)
553 int *pvalue
= kp
->arg
;
554 ret
= param_set_int(val
, kp
);
558 * zfs_qat_compress_disable = 0: enable qat compress
559 * try to initialize qat instance if it has not been done
561 if (*pvalue
== 0 && !qat_dc_init_done
) {
564 zfs_qat_compress_disable
= 1;
571 module_param_call(zfs_qat_compress_disable
, param_set_qat_compress
,
572 param_get_int
, &zfs_qat_compress_disable
, 0644);
573 MODULE_PARM_DESC(zfs_qat_compress_disable
, "Enable/Disable QAT compression");