]> git.proxmox.com Git - mirror_zfs-debian.git/blob - module/zfs/qat_compress.c
projects / mirror_zfs-debian.git / blob
? search:


4d17d7ac9a18361613da2339812d96803fdc17ec
[mirror_zfs-debian.git] / module / zfs / qat_compress.c
1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 #if defined(_KERNEL) && defined(HAVE_QAT)
23 #include <linux/slab.h>
24 #include <linux/vmalloc.h>
25 #include <linux/pagemap.h>
26 #include <linux/completion.h>
27 #include <sys/zfs_context.h>
28 #include "qat_compress.h"
29
30 /*
31 * Timeout - no response from hardware after 0.5 seconds
32 */
33 #define TIMEOUT_MS 500
34
35 /*
36 * Max instances in QAT device, each instance is a channel to submit
37 * jobs to QAT hardware, this is only for pre-allocating instance,
38 * and session arrays, the actual number of instances are defined in
39 * the QAT driver's configure file.
40 */
41 #define MAX_INSTANCES 48
42
43 /*
44 * ZLIB head and foot size
45 */
46 #define ZLIB_HEAD_SZ 2
47 #define ZLIB_FOOT_SZ 4
48
49 /*
50 * The minimal and maximal buffer size, which are not restricted
51 * in the QAT hardware, but with the input buffer size between 4KB
52 * and 128KB, the hardware can provide the optimal performance.
53 */
54 #define QAT_MIN_BUF_SIZE (4*1024)
55 #define QAT_MAX_BUF_SIZE (128*1024)
56
57 /*
58 * Used for qat kstat.
59 */
60 typedef struct qat_stats {
61 /*
62 * Number of jobs submitted to qat compression engine.
63 */
64 kstat_named_t comp_requests;
65 /*
66 * Total bytes sent to qat compression engine.
67 */
68 kstat_named_t comp_total_in_bytes;
69 /*
70 * Total bytes output from qat compression engine.
71 */
72 kstat_named_t comp_total_out_bytes;
73 /*
74 * Number of jobs submitted to qat de-compression engine.
75 */
76 kstat_named_t decomp_requests;
77 /*
78 * Total bytes sent to qat de-compression engine.
79 */
80 kstat_named_t decomp_total_in_bytes;
81 /*
82 * Total bytes output from qat de-compression engine.
83 */
84 kstat_named_t decomp_total_out_bytes;
85 /*
86 * Number of fails in qat engine.
87 * Note: when qat fail happens, it doesn't mean a critical hardware
88 * issue, sometimes it is because the output buffer is not big enough,
89 * and the compression job will be transfered to gzip software again,
90 * so the functionality of ZFS is not impacted.
91 */
92 kstat_named_t dc_fails;
93 } qat_stats_t;
94
95 qat_stats_t qat_stats = {
96 { "comp_reqests", KSTAT_DATA_UINT64 },
97 { "comp_total_in_bytes", KSTAT_DATA_UINT64 },
98 { "comp_total_out_bytes", KSTAT_DATA_UINT64 },
99 { "decomp_reqests", KSTAT_DATA_UINT64 },
100 { "decomp_total_in_bytes", KSTAT_DATA_UINT64 },
101 { "decomp_total_out_bytes", KSTAT_DATA_UINT64 },
102 { "dc_fails", KSTAT_DATA_UINT64 },
103 };
104
105 static kstat_t *qat_ksp;
106 static CpaInstanceHandle dc_inst_handles[MAX_INSTANCES];
107 static CpaDcSessionHandle session_handles[MAX_INSTANCES];
108 static CpaBufferList **buffer_array[MAX_INSTANCES];
109 static Cpa16U num_inst = 0;
110 static Cpa32U inst_num = 0;
111 static boolean_t qat_init_done = B_FALSE;
112 int zfs_qat_disable = 0;
113
114 #define QAT_STAT_INCR(stat, val) \
115 atomic_add_64(&qat_stats.stat.value.ui64, (val));
116 #define QAT_STAT_BUMP(stat) \
117 QAT_STAT_INCR(stat, 1);
118
119 #define PHYS_CONTIG_ALLOC(pp_mem_addr, size_bytes) \
120 mem_alloc_contig((void *)(pp_mem_addr), (size_bytes))
121
122 #define PHYS_CONTIG_FREE(p_mem_addr) \
123 mem_free_contig((void *)&(p_mem_addr))
124
125 static inline struct page *
126 mem_to_page(void *addr)
127 {
128 if (!is_vmalloc_addr(addr))
129 return (virt_to_page(addr));
130
131 return (vmalloc_to_page(addr));
132 }
133
134 static void
135 qat_dc_callback(void *p_callback, CpaStatus status)
136 {
137 if (p_callback != NULL)
138 complete((struct completion *)p_callback);
139 }
140
141 static inline CpaStatus
142 mem_alloc_contig(void **pp_mem_addr, Cpa32U size_bytes)
143 {
144 *pp_mem_addr = kmalloc(size_bytes, GFP_KERNEL);
145 if (*pp_mem_addr == NULL)
146 return (CPA_STATUS_RESOURCE);
147 return (CPA_STATUS_SUCCESS);
148 }
149
150 static inline void
151 mem_free_contig(void **pp_mem_addr)
152 {
153 if (*pp_mem_addr != NULL) {
154 kfree(*pp_mem_addr);
155 *pp_mem_addr = NULL;
156 }
157 }
158
159 static void
160 qat_clean(void)
161 {
162 Cpa16U buff_num = 0;
163 Cpa16U num_inter_buff_lists = 0;
164 Cpa16U i = 0;
165
166 for (i = 0; i < num_inst; i++) {
167 cpaDcStopInstance(dc_inst_handles[i]);
168 PHYS_CONTIG_FREE(session_handles[i]);
169 /* free intermediate buffers */
170 if (buffer_array[i] != NULL) {
171 cpaDcGetNumIntermediateBuffers(
172 dc_inst_handles[i], &num_inter_buff_lists);
173 for (buff_num = 0; buff_num < num_inter_buff_lists;
174 buff_num++) {
175 CpaBufferList *buffer_inter =
176 buffer_array[i][buff_num];
177 if (buffer_inter->pBuffers) {
178 PHYS_CONTIG_FREE(
179 buffer_inter->pBuffers->pData);
180 PHYS_CONTIG_FREE(
181 buffer_inter->pBuffers);
182 }
183 PHYS_CONTIG_FREE(
184 buffer_inter->pPrivateMetaData);
185 PHYS_CONTIG_FREE(buffer_inter);
186 }
187 }
188 }
189
190 num_inst = 0;
191 qat_init_done = B_FALSE;
192 }
193
194 int
195 qat_init(void)
196 {
197 CpaStatus status = CPA_STATUS_SUCCESS;
198 Cpa32U sess_size = 0;
199 Cpa32U ctx_size = 0;
200 Cpa16U num_inter_buff_lists = 0;
201 Cpa16U buff_num = 0;
202 Cpa32U buff_meta_size = 0;
203 CpaDcSessionSetupData sd = {0};
204 Cpa16U i;
205
206 status = cpaDcGetNumInstances(&num_inst);
207 if (status != CPA_STATUS_SUCCESS || num_inst == 0)
208 return (-1);
209
210 if (num_inst > MAX_INSTANCES)
211 num_inst = MAX_INSTANCES;
212
213 status = cpaDcGetInstances(num_inst, &dc_inst_handles[0]);
214 if (status != CPA_STATUS_SUCCESS)
215 return (-1);
216
217 for (i = 0; i < num_inst; i++) {
218 cpaDcSetAddressTranslation(dc_inst_handles[i],
219 (void*)virt_to_phys);
220
221 status = cpaDcBufferListGetMetaSize(dc_inst_handles[i],
222 1, &buff_meta_size);
223
224 if (status == CPA_STATUS_SUCCESS)
225 status = cpaDcGetNumIntermediateBuffers(
226 dc_inst_handles[i], &num_inter_buff_lists);
227
228 if (status == CPA_STATUS_SUCCESS && num_inter_buff_lists != 0)
229 status = PHYS_CONTIG_ALLOC(&buffer_array[i],
230 num_inter_buff_lists *
231 sizeof (CpaBufferList *));
232
233 for (buff_num = 0; buff_num < num_inter_buff_lists;
234 buff_num++) {
235 if (status == CPA_STATUS_SUCCESS)
236 status = PHYS_CONTIG_ALLOC(
237 &buffer_array[i][buff_num],
238 sizeof (CpaBufferList));
239
240 if (status == CPA_STATUS_SUCCESS)
241 status = PHYS_CONTIG_ALLOC(
242 &buffer_array[i][buff_num]->
243 pPrivateMetaData,
244 buff_meta_size);
245
246 if (status == CPA_STATUS_SUCCESS)
247 status = PHYS_CONTIG_ALLOC(
248 &buffer_array[i][buff_num]->pBuffers,
249 sizeof (CpaFlatBuffer));
250
251 if (status == CPA_STATUS_SUCCESS) {
252 /*
253 * implementation requires an intermediate
254 * buffer approximately twice the size of
255 * output buffer, which is 2x max buffer
256 * size here.
257 */
258 status = PHYS_CONTIG_ALLOC(
259 &buffer_array[i][buff_num]->pBuffers->
260 pData, 2 * QAT_MAX_BUF_SIZE);
261 if (status != CPA_STATUS_SUCCESS)
262 goto fail;
263
264 buffer_array[i][buff_num]->numBuffers = 1;
265 buffer_array[i][buff_num]->pBuffers->
266 dataLenInBytes = 2 * QAT_MAX_BUF_SIZE;
267 }
268 }
269
270 status = cpaDcStartInstance(dc_inst_handles[i],
271 num_inter_buff_lists, buffer_array[i]);
272 if (status != CPA_STATUS_SUCCESS)
273 goto fail;
274
275 sd.compLevel = CPA_DC_L1;
276 sd.compType = CPA_DC_DEFLATE;
277 sd.huffType = CPA_DC_HT_FULL_DYNAMIC;
278 sd.sessDirection = CPA_DC_DIR_COMBINED;
279 sd.sessState = CPA_DC_STATELESS;
280 sd.deflateWindowSize = 7;
281 sd.checksum = CPA_DC_ADLER32;
282 status = cpaDcGetSessionSize(dc_inst_handles[i],
283 &sd, &sess_size, &ctx_size);
284 if (status != CPA_STATUS_SUCCESS)
285 goto fail;
286
287 PHYS_CONTIG_ALLOC(&session_handles[i], sess_size);
288 if (session_handles[i] == NULL)
289 goto fail;
290
291 status = cpaDcInitSession(dc_inst_handles[i],
292 session_handles[i],
293 &sd, NULL, qat_dc_callback);
294 if (status != CPA_STATUS_SUCCESS)
295 goto fail;
296 }
297
298 qat_ksp = kstat_create("zfs", 0, "qat", "misc",
299 KSTAT_TYPE_NAMED, sizeof (qat_stats) / sizeof (kstat_named_t),
300 KSTAT_FLAG_VIRTUAL);
301 if (qat_ksp != NULL) {
302 qat_ksp->ks_data = &qat_stats;
303 kstat_install(qat_ksp);
304 }
305
306 qat_init_done = B_TRUE;
307 return (0);
308 fail:
309 qat_clean();
310 return (-1);
311 }
312
313 void
314 qat_fini(void)
315 {
316 qat_clean();
317
318 if (qat_ksp != NULL) {
319 kstat_delete(qat_ksp);
320 qat_ksp = NULL;
321 }
322 }
323
324 boolean_t
325 qat_use_accel(size_t s_len)
326 {
327 return (!zfs_qat_disable &&
328 qat_init_done &&
329 s_len >= QAT_MIN_BUF_SIZE &&
330 s_len <= QAT_MAX_BUF_SIZE);
331 }
332
333 int
334 qat_compress(qat_compress_dir_t dir, char *src, int src_len,
335 char *dst, int dst_len, size_t *c_len)
336 {
337 CpaInstanceHandle dc_inst_handle;
338 CpaDcSessionHandle session_handle;
339 CpaBufferList *buf_list_src = NULL;
340 CpaBufferList *buf_list_dst = NULL;
341 CpaFlatBuffer *flat_buf_src = NULL;
342 CpaFlatBuffer *flat_buf_dst = NULL;
343 Cpa8U *buffer_meta_src = NULL;
344 Cpa8U *buffer_meta_dst = NULL;
345 Cpa32U buffer_meta_size = 0;
346 CpaDcRqResults dc_results;
347 CpaStatus status = CPA_STATUS_SUCCESS;
348 Cpa32U hdr_sz = 0;
349 Cpa32U compressed_sz;
350 Cpa32U num_src_buf = (src_len >> PAGE_SHIFT) + 1;
351 Cpa32U num_dst_buf = (dst_len >> PAGE_SHIFT) + 1;
352 Cpa32U bytes_left;
353 char *data;
354 struct page *in_page, *out_page;
355 struct page **in_pages = NULL;
356 struct page **out_pages = NULL;
357 struct completion complete;
358 size_t ret = -1;
359 Cpa16U page_num = 0;
360 Cpa16U i;
361
362 Cpa32U src_buffer_list_mem_size = sizeof (CpaBufferList) +
363 (num_src_buf * sizeof (CpaFlatBuffer));
364 Cpa32U dst_buffer_list_mem_size = sizeof (CpaBufferList) +
365 (num_dst_buf * sizeof (CpaFlatBuffer));
366
367 if (!is_vmalloc_addr(src) || !is_vmalloc_addr(src + src_len - 1) ||
368 !is_vmalloc_addr(dst) || !is_vmalloc_addr(dst + dst_len - 1))
369 return (-1);
370
371 if (PHYS_CONTIG_ALLOC(&in_pages,
372 num_src_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS)
373 goto fail;
374
375 if (PHYS_CONTIG_ALLOC(&out_pages,
376 num_dst_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS)
377 goto fail;
378
379 i = atomic_inc_32_nv(&inst_num) % num_inst;
380 dc_inst_handle = dc_inst_handles[i];
381 session_handle = session_handles[i];
382
383 cpaDcBufferListGetMetaSize(dc_inst_handle, num_src_buf,
384 &buffer_meta_size);
385 if (PHYS_CONTIG_ALLOC(&buffer_meta_src, buffer_meta_size) !=
386 CPA_STATUS_SUCCESS)
387 goto fail;
388
389 cpaDcBufferListGetMetaSize(dc_inst_handle, num_dst_buf,
390 &buffer_meta_size);
391 if (PHYS_CONTIG_ALLOC(&buffer_meta_dst, buffer_meta_size) !=
392 CPA_STATUS_SUCCESS)
393 goto fail;
394
395 /* build source buffer list */
396 if (PHYS_CONTIG_ALLOC(&buf_list_src, src_buffer_list_mem_size) !=
397 CPA_STATUS_SUCCESS)
398 goto fail;
399
400 flat_buf_src = (CpaFlatBuffer *)(buf_list_src + 1);
401
402 buf_list_src->pBuffers = flat_buf_src; /* always point to first one */
403
404 /* build destination buffer list */
405 if (PHYS_CONTIG_ALLOC(&buf_list_dst, dst_buffer_list_mem_size) !=
406 CPA_STATUS_SUCCESS)
407 goto fail;
408
409 flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1);
410
411 buf_list_dst->pBuffers = flat_buf_dst; /* always point to first one */
412
413 buf_list_src->numBuffers = 0;
414 buf_list_src->pPrivateMetaData = buffer_meta_src;
415 bytes_left = src_len;
416 data = src;
417 page_num = 0;
418 while (bytes_left > 0) {
419 in_page = mem_to_page(data);
420 in_pages[page_num] = in_page;
421 flat_buf_src->pData = kmap(in_page);
422 flat_buf_src->dataLenInBytes =
423 min((long)bytes_left, (long)PAGE_SIZE);
424
425 bytes_left -= flat_buf_src->dataLenInBytes;
426 data += flat_buf_src->dataLenInBytes;
427 flat_buf_src++;
428 buf_list_src->numBuffers++;
429 page_num++;
430 }
431
432 buf_list_dst->numBuffers = 0;
433 buf_list_dst->pPrivateMetaData = buffer_meta_dst;
434 bytes_left = dst_len;
435 data = dst;
436 page_num = 0;
437 while (bytes_left > 0) {
438 out_page = mem_to_page(data);
439 flat_buf_dst->pData = kmap(out_page);
440 out_pages[page_num] = out_page;
441 flat_buf_dst->dataLenInBytes =
442 min((long)bytes_left, (long)PAGE_SIZE);
443
444 bytes_left -= flat_buf_dst->dataLenInBytes;
445 data += flat_buf_dst->dataLenInBytes;
446 flat_buf_dst++;
447 buf_list_dst->numBuffers++;
448 page_num++;
449 }
450
451 init_completion(&complete);
452
453 if (dir == QAT_COMPRESS) {
454 QAT_STAT_BUMP(comp_requests);
455 QAT_STAT_INCR(comp_total_in_bytes, src_len);
456
457 cpaDcGenerateHeader(session_handle,
458 buf_list_dst->pBuffers, &hdr_sz);
459 buf_list_dst->pBuffers->pData += hdr_sz;
460 buf_list_dst->pBuffers->dataLenInBytes -= hdr_sz;
461 status = cpaDcCompressData(
462 dc_inst_handle, session_handle,
463 buf_list_src, buf_list_dst,
464 &dc_results, CPA_DC_FLUSH_FINAL,
465 &complete);
466 if (status != CPA_STATUS_SUCCESS) {
467 goto fail;
468 }
469
470 /* we now wait until the completion of the operation. */
471 if (!wait_for_completion_interruptible_timeout(&complete,
472 TIMEOUT_MS)) {
473 status = CPA_STATUS_FAIL;
474 goto fail;
475 }
476
477 if (dc_results.status != CPA_STATUS_SUCCESS) {
478 status = CPA_STATUS_FAIL;
479 goto fail;
480 }
481
482 compressed_sz = dc_results.produced;
483 if (compressed_sz + hdr_sz + ZLIB_FOOT_SZ > dst_len) {
484 goto fail;
485 }
486
487 flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1);
488 /* move to the last page */
489 flat_buf_dst += (compressed_sz + hdr_sz) >> PAGE_SHIFT;
490
491 /* no space for gzip foot in the last page */
492 if (((compressed_sz + hdr_sz) % PAGE_SIZE)
493 + ZLIB_FOOT_SZ > PAGE_SIZE)
494 goto fail;
495
496 /* jump to the end of the buffer and append footer */
497 flat_buf_dst->pData =
498 (char *)((unsigned long)flat_buf_dst->pData & PAGE_MASK)
499 + ((compressed_sz + hdr_sz) % PAGE_SIZE);
500 flat_buf_dst->dataLenInBytes = ZLIB_FOOT_SZ;
501
502 dc_results.produced = 0;
503 status = cpaDcGenerateFooter(session_handle,
504 flat_buf_dst, &dc_results);
505 if (status != CPA_STATUS_SUCCESS) {
506 goto fail;
507 }
508
509 *c_len = compressed_sz + dc_results.produced + hdr_sz;
510
511 QAT_STAT_INCR(comp_total_out_bytes, *c_len);
512
513 ret = 0;
514
515 } else if (dir == QAT_DECOMPRESS) {
516 QAT_STAT_BUMP(decomp_requests);
517 QAT_STAT_INCR(decomp_total_in_bytes, src_len);
518
519 buf_list_src->pBuffers->pData += ZLIB_HEAD_SZ;
520 buf_list_src->pBuffers->dataLenInBytes -= ZLIB_HEAD_SZ;
521 status = cpaDcDecompressData(dc_inst_handle,
522 session_handle,
523 buf_list_src,
524 buf_list_dst,
525 &dc_results,
526 CPA_DC_FLUSH_FINAL,
527 &complete);
528
529 if (CPA_STATUS_SUCCESS != status) {
530 status = CPA_STATUS_FAIL;
531 goto fail;
532 }
533
534 /* we now wait until the completion of the operation. */
535 if (!wait_for_completion_interruptible_timeout(&complete,
536 TIMEOUT_MS)) {
537 status = CPA_STATUS_FAIL;
538 goto fail;
539 }
540
541 if (dc_results.status != CPA_STATUS_SUCCESS) {
542 status = CPA_STATUS_FAIL;
543 goto fail;
544 }
545
546 *c_len = dc_results.produced;
547
548 QAT_STAT_INCR(decomp_total_out_bytes, *c_len);
549
550 ret = 0;
551 }
552
553 fail:
554 if (status != CPA_STATUS_SUCCESS) {
555 QAT_STAT_BUMP(dc_fails);
556 }
557
558 if (in_pages) {
559 for (page_num = 0;
560 page_num < buf_list_src->numBuffers;
561 page_num++) {
562 kunmap(in_pages[page_num]);
563 }
564 PHYS_CONTIG_FREE(in_pages);
565 }
566
567 if (out_pages) {
568 for (page_num = 0;
569 page_num < buf_list_dst->numBuffers;
570 page_num++) {
571 kunmap(out_pages[page_num]);
572 }
573 PHYS_CONTIG_FREE(out_pages);
574 }
575
576 PHYS_CONTIG_FREE(buffer_meta_src);
577 PHYS_CONTIG_FREE(buffer_meta_dst);
578 PHYS_CONTIG_FREE(buf_list_src);
579 PHYS_CONTIG_FREE(buf_list_dst);
580
581 return (ret);
582 }
583
584 module_param(zfs_qat_disable, int, 0644);
585 MODULE_PARM_DESC(zfs_qat_disable, "Disable QAT compression");
586
587 #endif
Mirror of ZFSOnLinux for Debian