]>
Commit | Line | Data |
---|---|---|
6a9d6359 | 1 | /* |
2 | * CDDL HEADER START | |
3 | * | |
4 | * The contents of this file are subject to the terms of the | |
5 | * Common Development and Distribution License (the "License"). | |
6 | * You may not use this file except in compliance with the License. | |
7 | * | |
8 | * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE | |
1d3ba0bf | 9 | * or https://opensource.org/licenses/CDDL-1.0. |
6a9d6359 | 10 | * See the License for the specific language governing permissions |
11 | * and limitations under the License. | |
12 | * | |
13 | * When distributing Covered Code, include this CDDL HEADER in each | |
14 | * file and include the License file at usr/src/OPENSOLARIS.LICENSE. | |
15 | * If applicable, add the following below this CDDL HEADER, with the | |
16 | * fields enclosed by brackets "[]" replaced with your own identifying | |
17 | * information: Portions Copyright [yyyy] [name of copyright owner] | |
18 | * | |
19 | * CDDL HEADER END | |
20 | */ | |
21 | ||
22 | #if defined(_KERNEL) && defined(HAVE_QAT) | |
23 | #include <linux/slab.h> | |
24 | #include <linux/vmalloc.h> | |
25 | #include <linux/pagemap.h> | |
26 | #include <linux/completion.h> | |
27 | #include <sys/zfs_context.h> | |
5090f727 | 28 | #include <sys/byteorder.h> |
32dce2da | 29 | #include <sys/zio.h> |
7238cbd4 | 30 | #include <sys/qat.h> |
6a9d6359 | 31 | |
32 | /* | |
8d9e7c8f TC |
33 | * Max instances in a QAT device, each instance is a channel to submit |
34 | * jobs to QAT hardware, this is only for pre-allocating instance and | |
35 | * session arrays; the actual number of instances are defined in the | |
36 | * QAT driver's configuration file. | |
6a9d6359 | 37 | */ |
cf637391 | 38 | #define QAT_DC_MAX_INSTANCES 48 |
6a9d6359 | 39 | |
40 | /* | |
41 | * ZLIB head and foot size | |
42 | */ | |
43 | #define ZLIB_HEAD_SZ 2 | |
44 | #define ZLIB_FOOT_SZ 4 | |
45 | ||
cf637391 TC |
46 | static CpaInstanceHandle dc_inst_handles[QAT_DC_MAX_INSTANCES]; |
47 | static CpaDcSessionHandle session_handles[QAT_DC_MAX_INSTANCES]; | |
48 | static CpaBufferList **buffer_array[QAT_DC_MAX_INSTANCES]; | |
1cfdb0e6 | 49 | static Cpa16U num_inst = 0; |
7a25f089 | 50 | static Cpa32U inst_num = 0; |
cf637391 | 51 | static boolean_t qat_dc_init_done = B_FALSE; |
38742209 | 52 | int zfs_qat_compress_disable = 0; |
6a9d6359 | 53 | |
cf637391 TC |
54 | boolean_t |
55 | qat_dc_use_accel(size_t s_len) | |
6a9d6359 | 56 | { |
38742209 | 57 | return (!zfs_qat_compress_disable && |
cf637391 TC |
58 | qat_dc_init_done && |
59 | s_len >= QAT_MIN_BUF_SIZE && | |
60 | s_len <= QAT_MAX_BUF_SIZE); | |
6a9d6359 | 61 | } |
62 | ||
63 | static void | |
64 | qat_dc_callback(void *p_callback, CpaStatus status) | |
65 | { | |
66 | if (p_callback != NULL) | |
67 | complete((struct completion *)p_callback); | |
68 | } | |
69 | ||
6a9d6359 | 70 | static void |
cf637391 | 71 | qat_dc_clean(void) |
6a9d6359 | 72 | { |
73 | Cpa16U buff_num = 0; | |
74 | Cpa16U num_inter_buff_lists = 0; | |
6a9d6359 | 75 | |
bc8a6a60 | 76 | for (Cpa16U i = 0; i < num_inst; i++) { |
6a9d6359 | 77 | cpaDcStopInstance(dc_inst_handles[i]); |
cf637391 | 78 | QAT_PHYS_CONTIG_FREE(session_handles[i]); |
6a9d6359 | 79 | /* free intermediate buffers */ |
80 | if (buffer_array[i] != NULL) { | |
81 | cpaDcGetNumIntermediateBuffers( | |
82 | dc_inst_handles[i], &num_inter_buff_lists); | |
83 | for (buff_num = 0; buff_num < num_inter_buff_lists; | |
84 | buff_num++) { | |
85 | CpaBufferList *buffer_inter = | |
86 | buffer_array[i][buff_num]; | |
87 | if (buffer_inter->pBuffers) { | |
cf637391 | 88 | QAT_PHYS_CONTIG_FREE( |
6a9d6359 | 89 | buffer_inter->pBuffers->pData); |
cf637391 | 90 | QAT_PHYS_CONTIG_FREE( |
6a9d6359 | 91 | buffer_inter->pBuffers); |
92 | } | |
cf637391 | 93 | QAT_PHYS_CONTIG_FREE( |
6a9d6359 | 94 | buffer_inter->pPrivateMetaData); |
cf637391 | 95 | QAT_PHYS_CONTIG_FREE(buffer_inter); |
6a9d6359 | 96 | } |
97 | } | |
98 | } | |
99 | ||
100 | num_inst = 0; | |
cf637391 | 101 | qat_dc_init_done = B_FALSE; |
6a9d6359 | 102 | } |
103 | ||
104 | int | |
cf637391 | 105 | qat_dc_init(void) |
6a9d6359 | 106 | { |
107 | CpaStatus status = CPA_STATUS_SUCCESS; | |
108 | Cpa32U sess_size = 0; | |
109 | Cpa32U ctx_size = 0; | |
110 | Cpa16U num_inter_buff_lists = 0; | |
111 | Cpa16U buff_num = 0; | |
112 | Cpa32U buff_meta_size = 0; | |
113 | CpaDcSessionSetupData sd = {0}; | |
6a9d6359 | 114 | |
5090f727 CZ |
115 | if (qat_dc_init_done) |
116 | return (0); | |
117 | ||
6a9d6359 | 118 | status = cpaDcGetNumInstances(&num_inst); |
cf637391 | 119 | if (status != CPA_STATUS_SUCCESS) |
6a9d6359 | 120 | return (-1); |
121 | ||
cf637391 TC |
122 | /* if the user has configured no QAT compression units just return */ |
123 | if (num_inst == 0) | |
124 | return (0); | |
125 | ||
126 | if (num_inst > QAT_DC_MAX_INSTANCES) | |
127 | num_inst = QAT_DC_MAX_INSTANCES; | |
6a9d6359 | 128 | |
129 | status = cpaDcGetInstances(num_inst, &dc_inst_handles[0]); | |
130 | if (status != CPA_STATUS_SUCCESS) | |
131 | return (-1); | |
132 | ||
bc8a6a60 | 133 | for (Cpa16U i = 0; i < num_inst; i++) { |
6a9d6359 | 134 | cpaDcSetAddressTranslation(dc_inst_handles[i], |
135 | (void*)virt_to_phys); | |
136 | ||
137 | status = cpaDcBufferListGetMetaSize(dc_inst_handles[i], | |
138 | 1, &buff_meta_size); | |
139 | ||
140 | if (status == CPA_STATUS_SUCCESS) | |
141 | status = cpaDcGetNumIntermediateBuffers( | |
142 | dc_inst_handles[i], &num_inter_buff_lists); | |
143 | ||
144 | if (status == CPA_STATUS_SUCCESS && num_inter_buff_lists != 0) | |
cf637391 | 145 | status = QAT_PHYS_CONTIG_ALLOC(&buffer_array[i], |
6a9d6359 | 146 | num_inter_buff_lists * |
147 | sizeof (CpaBufferList *)); | |
148 | ||
149 | for (buff_num = 0; buff_num < num_inter_buff_lists; | |
150 | buff_num++) { | |
151 | if (status == CPA_STATUS_SUCCESS) | |
cf637391 | 152 | status = QAT_PHYS_CONTIG_ALLOC( |
6a9d6359 | 153 | &buffer_array[i][buff_num], |
154 | sizeof (CpaBufferList)); | |
155 | ||
156 | if (status == CPA_STATUS_SUCCESS) | |
cf637391 | 157 | status = QAT_PHYS_CONTIG_ALLOC( |
6a9d6359 | 158 | &buffer_array[i][buff_num]-> |
159 | pPrivateMetaData, | |
160 | buff_meta_size); | |
161 | ||
162 | if (status == CPA_STATUS_SUCCESS) | |
cf637391 | 163 | status = QAT_PHYS_CONTIG_ALLOC( |
6a9d6359 | 164 | &buffer_array[i][buff_num]->pBuffers, |
165 | sizeof (CpaFlatBuffer)); | |
166 | ||
167 | if (status == CPA_STATUS_SUCCESS) { | |
168 | /* | |
169 | * implementation requires an intermediate | |
170 | * buffer approximately twice the size of | |
171 | * output buffer, which is 2x max buffer | |
172 | * size here. | |
173 | */ | |
cf637391 | 174 | status = QAT_PHYS_CONTIG_ALLOC( |
6a9d6359 | 175 | &buffer_array[i][buff_num]->pBuffers-> |
176 | pData, 2 * QAT_MAX_BUF_SIZE); | |
177 | if (status != CPA_STATUS_SUCCESS) | |
178 | goto fail; | |
179 | ||
180 | buffer_array[i][buff_num]->numBuffers = 1; | |
181 | buffer_array[i][buff_num]->pBuffers-> | |
182 | dataLenInBytes = 2 * QAT_MAX_BUF_SIZE; | |
183 | } | |
184 | } | |
185 | ||
186 | status = cpaDcStartInstance(dc_inst_handles[i], | |
187 | num_inter_buff_lists, buffer_array[i]); | |
188 | if (status != CPA_STATUS_SUCCESS) | |
189 | goto fail; | |
190 | ||
191 | sd.compLevel = CPA_DC_L1; | |
192 | sd.compType = CPA_DC_DEFLATE; | |
193 | sd.huffType = CPA_DC_HT_FULL_DYNAMIC; | |
194 | sd.sessDirection = CPA_DC_DIR_COMBINED; | |
195 | sd.sessState = CPA_DC_STATELESS; | |
4da8c7d1 | 196 | #if (CPA_DC_API_VERSION_NUM_MAJOR == 1 && CPA_DC_API_VERSION_NUM_MINOR < 6) |
6a9d6359 | 197 | sd.deflateWindowSize = 7; |
4da8c7d1 | 198 | #endif |
6a9d6359 | 199 | sd.checksum = CPA_DC_ADLER32; |
200 | status = cpaDcGetSessionSize(dc_inst_handles[i], | |
201 | &sd, &sess_size, &ctx_size); | |
202 | if (status != CPA_STATUS_SUCCESS) | |
203 | goto fail; | |
204 | ||
cf637391 | 205 | QAT_PHYS_CONTIG_ALLOC(&session_handles[i], sess_size); |
6a9d6359 | 206 | if (session_handles[i] == NULL) |
207 | goto fail; | |
208 | ||
209 | status = cpaDcInitSession(dc_inst_handles[i], | |
210 | session_handles[i], | |
211 | &sd, NULL, qat_dc_callback); | |
212 | if (status != CPA_STATUS_SUCCESS) | |
213 | goto fail; | |
214 | } | |
215 | ||
cf637391 | 216 | qat_dc_init_done = B_TRUE; |
6a9d6359 | 217 | return (0); |
218 | fail: | |
cf637391 | 219 | qat_dc_clean(); |
6a9d6359 | 220 | return (-1); |
221 | } | |
222 | ||
223 | void | |
cf637391 | 224 | qat_dc_fini(void) |
6a9d6359 | 225 | { |
cf637391 TC |
226 | if (!qat_dc_init_done) |
227 | return; | |
6a9d6359 | 228 | |
cf637391 | 229 | qat_dc_clean(); |
6a9d6359 | 230 | } |
231 | ||
32dce2da TC |
232 | /* |
233 | * The "add" parameter is an additional buffer which is passed | |
234 | * to QAT as a scratch buffer alongside the destination buffer | |
235 | * in case the "compressed" data ends up being larger than the | |
236 | * original source data. This is necessary to prevent QAT from | |
237 | * generating buffer overflow warnings for incompressible data. | |
238 | */ | |
239 | static int | |
240 | qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len, | |
241 | char *dst, int dst_len, char *add, int add_len, size_t *c_len) | |
6a9d6359 | 242 | { |
243 | CpaInstanceHandle dc_inst_handle; | |
244 | CpaDcSessionHandle session_handle; | |
245 | CpaBufferList *buf_list_src = NULL; | |
246 | CpaBufferList *buf_list_dst = NULL; | |
247 | CpaFlatBuffer *flat_buf_src = NULL; | |
248 | CpaFlatBuffer *flat_buf_dst = NULL; | |
249 | Cpa8U *buffer_meta_src = NULL; | |
250 | Cpa8U *buffer_meta_dst = NULL; | |
251 | Cpa32U buffer_meta_size = 0; | |
60cfd3bb | 252 | CpaDcRqResults dc_results = {.checksum = 1}; |
bc9cef11 | 253 | CpaStatus status = CPA_STATUS_FAIL; |
6a9d6359 | 254 | Cpa32U hdr_sz = 0; |
255 | Cpa32U compressed_sz; | |
17dd8835 TC |
256 | Cpa32U num_src_buf = (src_len >> PAGE_SHIFT) + 2; |
257 | Cpa32U num_dst_buf = (dst_len >> PAGE_SHIFT) + 2; | |
32dce2da | 258 | Cpa32U num_add_buf = (add_len >> PAGE_SHIFT) + 2; |
6a9d6359 | 259 | Cpa32U bytes_left; |
32dce2da | 260 | Cpa32U dst_pages = 0; |
5090f727 | 261 | Cpa32U adler32 = 0; |
6a9d6359 | 262 | char *data; |
32dce2da | 263 | struct page *page; |
6a9d6359 | 264 | struct page **in_pages = NULL; |
265 | struct page **out_pages = NULL; | |
32dce2da | 266 | struct page **add_pages = NULL; |
17dd8835 | 267 | Cpa32U page_off = 0; |
6a9d6359 | 268 | struct completion complete; |
17dd8835 | 269 | Cpa32U page_num = 0; |
6a9d6359 | 270 | Cpa16U i; |
271 | ||
17dd8835 TC |
272 | /* |
273 | * We increment num_src_buf and num_dst_buf by 2 to allow | |
274 | * us to handle non page-aligned buffer addresses and buffers | |
275 | * whose sizes are not divisible by PAGE_SIZE. | |
276 | */ | |
6a9d6359 | 277 | Cpa32U src_buffer_list_mem_size = sizeof (CpaBufferList) + |
278 | (num_src_buf * sizeof (CpaFlatBuffer)); | |
279 | Cpa32U dst_buffer_list_mem_size = sizeof (CpaBufferList) + | |
32dce2da | 280 | ((num_dst_buf + num_add_buf) * sizeof (CpaFlatBuffer)); |
6a9d6359 | 281 | |
bc9cef11 BB |
282 | status = QAT_PHYS_CONTIG_ALLOC(&in_pages, |
283 | num_src_buf * sizeof (struct page *)); | |
284 | if (status != CPA_STATUS_SUCCESS) | |
6a9d6359 | 285 | goto fail; |
286 | ||
bc9cef11 BB |
287 | status = QAT_PHYS_CONTIG_ALLOC(&out_pages, |
288 | num_dst_buf * sizeof (struct page *)); | |
289 | if (status != CPA_STATUS_SUCCESS) | |
6a9d6359 | 290 | goto fail; |
291 | ||
bc9cef11 BB |
292 | status = QAT_PHYS_CONTIG_ALLOC(&add_pages, |
293 | num_add_buf * sizeof (struct page *)); | |
294 | if (status != CPA_STATUS_SUCCESS) | |
32dce2da TC |
295 | goto fail; |
296 | ||
bc8a6a60 | 297 | i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst; |
6a9d6359 | 298 | dc_inst_handle = dc_inst_handles[i]; |
299 | session_handle = session_handles[i]; | |
300 | ||
301 | cpaDcBufferListGetMetaSize(dc_inst_handle, num_src_buf, | |
302 | &buffer_meta_size); | |
bc9cef11 BB |
303 | status = QAT_PHYS_CONTIG_ALLOC(&buffer_meta_src, buffer_meta_size); |
304 | if (status != CPA_STATUS_SUCCESS) | |
6a9d6359 | 305 | goto fail; |
306 | ||
32dce2da | 307 | cpaDcBufferListGetMetaSize(dc_inst_handle, num_dst_buf + num_add_buf, |
6a9d6359 | 308 | &buffer_meta_size); |
bc9cef11 BB |
309 | status = QAT_PHYS_CONTIG_ALLOC(&buffer_meta_dst, buffer_meta_size); |
310 | if (status != CPA_STATUS_SUCCESS) | |
6a9d6359 | 311 | goto fail; |
312 | ||
313 | /* build source buffer list */ | |
bc9cef11 BB |
314 | status = QAT_PHYS_CONTIG_ALLOC(&buf_list_src, src_buffer_list_mem_size); |
315 | if (status != CPA_STATUS_SUCCESS) | |
6a9d6359 | 316 | goto fail; |
317 | ||
318 | flat_buf_src = (CpaFlatBuffer *)(buf_list_src + 1); | |
319 | ||
320 | buf_list_src->pBuffers = flat_buf_src; /* always point to first one */ | |
321 | ||
322 | /* build destination buffer list */ | |
bc9cef11 BB |
323 | status = QAT_PHYS_CONTIG_ALLOC(&buf_list_dst, dst_buffer_list_mem_size); |
324 | if (status != CPA_STATUS_SUCCESS) | |
6a9d6359 | 325 | goto fail; |
326 | ||
327 | flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1); | |
328 | ||
329 | buf_list_dst->pBuffers = flat_buf_dst; /* always point to first one */ | |
330 | ||
331 | buf_list_src->numBuffers = 0; | |
332 | buf_list_src->pPrivateMetaData = buffer_meta_src; | |
333 | bytes_left = src_len; | |
334 | data = src; | |
335 | page_num = 0; | |
336 | while (bytes_left > 0) { | |
17dd8835 | 337 | page_off = ((long)data & ~PAGE_MASK); |
32dce2da TC |
338 | page = qat_mem_to_page(data); |
339 | in_pages[page_num] = page; | |
340 | flat_buf_src->pData = kmap(page) + page_off; | |
6a9d6359 | 341 | flat_buf_src->dataLenInBytes = |
17dd8835 | 342 | min((long)PAGE_SIZE - page_off, (long)bytes_left); |
6a9d6359 | 343 | |
344 | bytes_left -= flat_buf_src->dataLenInBytes; | |
345 | data += flat_buf_src->dataLenInBytes; | |
346 | flat_buf_src++; | |
347 | buf_list_src->numBuffers++; | |
348 | page_num++; | |
349 | } | |
350 | ||
351 | buf_list_dst->numBuffers = 0; | |
352 | buf_list_dst->pPrivateMetaData = buffer_meta_dst; | |
353 | bytes_left = dst_len; | |
354 | data = dst; | |
355 | page_num = 0; | |
356 | while (bytes_left > 0) { | |
17dd8835 | 357 | page_off = ((long)data & ~PAGE_MASK); |
32dce2da TC |
358 | page = qat_mem_to_page(data); |
359 | flat_buf_dst->pData = kmap(page) + page_off; | |
360 | out_pages[page_num] = page; | |
361 | flat_buf_dst->dataLenInBytes = | |
362 | min((long)PAGE_SIZE - page_off, (long)bytes_left); | |
363 | ||
364 | bytes_left -= flat_buf_dst->dataLenInBytes; | |
365 | data += flat_buf_dst->dataLenInBytes; | |
366 | flat_buf_dst++; | |
367 | buf_list_dst->numBuffers++; | |
368 | page_num++; | |
369 | dst_pages++; | |
370 | } | |
371 | ||
372 | /* map additional scratch pages into the destination buffer list */ | |
373 | bytes_left = add_len; | |
374 | data = add; | |
375 | page_num = 0; | |
376 | while (bytes_left > 0) { | |
377 | page_off = ((long)data & ~PAGE_MASK); | |
378 | page = qat_mem_to_page(data); | |
379 | flat_buf_dst->pData = kmap(page) + page_off; | |
380 | add_pages[page_num] = page; | |
6a9d6359 | 381 | flat_buf_dst->dataLenInBytes = |
17dd8835 | 382 | min((long)PAGE_SIZE - page_off, (long)bytes_left); |
6a9d6359 | 383 | |
384 | bytes_left -= flat_buf_dst->dataLenInBytes; | |
385 | data += flat_buf_dst->dataLenInBytes; | |
386 | flat_buf_dst++; | |
387 | buf_list_dst->numBuffers++; | |
388 | page_num++; | |
389 | } | |
390 | ||
391 | init_completion(&complete); | |
392 | ||
393 | if (dir == QAT_COMPRESS) { | |
394 | QAT_STAT_BUMP(comp_requests); | |
395 | QAT_STAT_INCR(comp_total_in_bytes, src_len); | |
396 | ||
397 | cpaDcGenerateHeader(session_handle, | |
398 | buf_list_dst->pBuffers, &hdr_sz); | |
399 | buf_list_dst->pBuffers->pData += hdr_sz; | |
400 | buf_list_dst->pBuffers->dataLenInBytes -= hdr_sz; | |
401 | status = cpaDcCompressData( | |
402 | dc_inst_handle, session_handle, | |
403 | buf_list_src, buf_list_dst, | |
404 | &dc_results, CPA_DC_FLUSH_FINAL, | |
405 | &complete); | |
406 | if (status != CPA_STATUS_SUCCESS) { | |
407 | goto fail; | |
408 | } | |
409 | ||
410 | /* we now wait until the completion of the operation. */ | |
7238cbd4 | 411 | wait_for_completion(&complete); |
6a9d6359 | 412 | |
413 | if (dc_results.status != CPA_STATUS_SUCCESS) { | |
414 | status = CPA_STATUS_FAIL; | |
415 | goto fail; | |
416 | } | |
417 | ||
418 | compressed_sz = dc_results.produced; | |
419 | if (compressed_sz + hdr_sz + ZLIB_FOOT_SZ > dst_len) { | |
32dce2da | 420 | status = CPA_STATUS_INCOMPRESSIBLE; |
6a9d6359 | 421 | goto fail; |
422 | } | |
423 | ||
43575527 | 424 | /* get adler32 checksum and append footer */ |
425 | *(Cpa32U*)(dst + hdr_sz + compressed_sz) = | |
426 | BSWAP_32(dc_results.checksum); | |
6a9d6359 | 427 | |
43575527 | 428 | *c_len = hdr_sz + compressed_sz + ZLIB_FOOT_SZ; |
6a9d6359 | 429 | QAT_STAT_INCR(comp_total_out_bytes, *c_len); |
cf637391 TC |
430 | } else { |
431 | ASSERT3U(dir, ==, QAT_DECOMPRESS); | |
6a9d6359 | 432 | QAT_STAT_BUMP(decomp_requests); |
433 | QAT_STAT_INCR(decomp_total_in_bytes, src_len); | |
434 | ||
435 | buf_list_src->pBuffers->pData += ZLIB_HEAD_SZ; | |
436 | buf_list_src->pBuffers->dataLenInBytes -= ZLIB_HEAD_SZ; | |
32dce2da TC |
437 | status = cpaDcDecompressData(dc_inst_handle, session_handle, |
438 | buf_list_src, buf_list_dst, &dc_results, CPA_DC_FLUSH_FINAL, | |
6a9d6359 | 439 | &complete); |
440 | ||
441 | if (CPA_STATUS_SUCCESS != status) { | |
442 | status = CPA_STATUS_FAIL; | |
443 | goto fail; | |
444 | } | |
445 | ||
446 | /* we now wait until the completion of the operation. */ | |
7238cbd4 | 447 | wait_for_completion(&complete); |
6a9d6359 | 448 | |
449 | if (dc_results.status != CPA_STATUS_SUCCESS) { | |
450 | status = CPA_STATUS_FAIL; | |
451 | goto fail; | |
452 | } | |
453 | ||
5090f727 CZ |
454 | /* verify adler checksum */ |
455 | adler32 = *(Cpa32U *)(src + dc_results.consumed + ZLIB_HEAD_SZ); | |
456 | if (adler32 != BSWAP_32(dc_results.checksum)) { | |
457 | status = CPA_STATUS_FAIL; | |
458 | goto fail; | |
459 | } | |
6a9d6359 | 460 | *c_len = dc_results.produced; |
6a9d6359 | 461 | QAT_STAT_INCR(decomp_total_out_bytes, *c_len); |
6a9d6359 | 462 | } |
463 | ||
464 | fail: | |
32dce2da | 465 | if (status != CPA_STATUS_SUCCESS && status != CPA_STATUS_INCOMPRESSIBLE) |
6a9d6359 | 466 | QAT_STAT_BUMP(dc_fails); |
6a9d6359 | 467 | |
468 | if (in_pages) { | |
469 | for (page_num = 0; | |
470 | page_num < buf_list_src->numBuffers; | |
471 | page_num++) { | |
472 | kunmap(in_pages[page_num]); | |
473 | } | |
cf637391 | 474 | QAT_PHYS_CONTIG_FREE(in_pages); |
6a9d6359 | 475 | } |
476 | ||
477 | if (out_pages) { | |
32dce2da | 478 | for (page_num = 0; page_num < dst_pages; page_num++) { |
6a9d6359 | 479 | kunmap(out_pages[page_num]); |
480 | } | |
cf637391 | 481 | QAT_PHYS_CONTIG_FREE(out_pages); |
6a9d6359 | 482 | } |
483 | ||
32dce2da TC |
484 | if (add_pages) { |
485 | for (page_num = 0; | |
486 | page_num < buf_list_dst->numBuffers - dst_pages; | |
487 | page_num++) { | |
488 | kunmap(add_pages[page_num]); | |
489 | } | |
490 | QAT_PHYS_CONTIG_FREE(add_pages); | |
491 | } | |
492 | ||
cf637391 TC |
493 | QAT_PHYS_CONTIG_FREE(buffer_meta_src); |
494 | QAT_PHYS_CONTIG_FREE(buffer_meta_dst); | |
495 | QAT_PHYS_CONTIG_FREE(buf_list_src); | |
496 | QAT_PHYS_CONTIG_FREE(buf_list_dst); | |
6a9d6359 | 497 | |
32dce2da TC |
498 | return (status); |
499 | } | |
500 | ||
501 | /* | |
502 | * Entry point for QAT accelerated compression / decompression. | |
503 | */ | |
504 | int | |
505 | qat_compress(qat_compress_dir_t dir, char *src, int src_len, | |
506 | char *dst, int dst_len, size_t *c_len) | |
507 | { | |
508 | int ret; | |
509 | size_t add_len = 0; | |
510 | void *add = NULL; | |
511 | ||
512 | if (dir == QAT_COMPRESS) { | |
513 | add_len = dst_len; | |
514 | add = zio_data_buf_alloc(add_len); | |
515 | } | |
516 | ||
517 | ret = qat_compress_impl(dir, src, src_len, dst, | |
518 | dst_len, add, add_len, c_len); | |
519 | ||
520 | if (dir == QAT_COMPRESS) | |
521 | zio_data_buf_free(add, add_len); | |
522 | ||
6a9d6359 | 523 | return (ret); |
524 | } | |
525 | ||
5090f727 | 526 | static int |
6988f3ed | 527 | param_set_qat_compress(const char *val, zfs_kernel_param_t *kp) |
5090f727 CZ |
528 | { |
529 | int ret; | |
530 | int *pvalue = kp->arg; | |
531 | ret = param_set_int(val, kp); | |
532 | if (ret) | |
533 | return (ret); | |
534 | /* | |
535 | * zfs_qat_compress_disable = 0: enable qat compress | |
536 | * try to initialize qat instance if it has not been done | |
537 | */ | |
538 | if (*pvalue == 0 && !qat_dc_init_done) { | |
539 | ret = qat_dc_init(); | |
540 | if (ret != 0) { | |
541 | zfs_qat_compress_disable = 1; | |
542 | return (ret); | |
543 | } | |
544 | } | |
545 | return (ret); | |
546 | } | |
547 | ||
548 | module_param_call(zfs_qat_compress_disable, param_set_qat_compress, | |
549 | param_get_int, &zfs_qat_compress_disable, 0644); | |
550 | MODULE_PARM_DESC(zfs_qat_compress_disable, "Enable/Disable QAT compression"); | |
38742209 | 551 | |
6a9d6359 | 552 | #endif |