]>
Commit | Line | Data |
---|---|---|
6a9d6359 | 1 | /* |
2 | * CDDL HEADER START | |
3 | * | |
4 | * The contents of this file are subject to the terms of the | |
5 | * Common Development and Distribution License (the "License"). | |
6 | * You may not use this file except in compliance with the License. | |
7 | * | |
8 | * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE | |
9 | * or http://www.opensolaris.org/os/licensing. | |
10 | * See the License for the specific language governing permissions | |
11 | * and limitations under the License. | |
12 | * | |
13 | * When distributing Covered Code, include this CDDL HEADER in each | |
14 | * file and include the License file at usr/src/OPENSOLARIS.LICENSE. | |
15 | * If applicable, add the following below this CDDL HEADER, with the | |
16 | * fields enclosed by brackets "[]" replaced with your own identifying | |
17 | * information: Portions Copyright [yyyy] [name of copyright owner] | |
18 | * | |
19 | * CDDL HEADER END | |
20 | */ | |
21 | ||
22 | #if defined(_KERNEL) && defined(HAVE_QAT) | |
23 | #include <linux/slab.h> | |
24 | #include <linux/vmalloc.h> | |
25 | #include <linux/pagemap.h> | |
26 | #include <linux/completion.h> | |
27 | #include <sys/zfs_context.h> | |
32dce2da | 28 | #include <sys/zio.h> |
cf637391 | 29 | #include "qat.h" |
6a9d6359 | 30 | |
31 | /* | |
8d9e7c8f TC |
32 | * Max instances in a QAT device, each instance is a channel to submit |
33 | * jobs to QAT hardware, this is only for pre-allocating instance and | |
34 | * session arrays; the actual number of instances are defined in the | |
35 | * QAT driver's configuration file. | |
6a9d6359 | 36 | */ |
cf637391 | 37 | #define QAT_DC_MAX_INSTANCES 48 |
6a9d6359 | 38 | |
39 | /* | |
40 | * ZLIB head and foot size | |
41 | */ | |
42 | #define ZLIB_HEAD_SZ 2 | |
43 | #define ZLIB_FOOT_SZ 4 | |
44 | ||
cf637391 TC |
45 | static CpaInstanceHandle dc_inst_handles[QAT_DC_MAX_INSTANCES]; |
46 | static CpaDcSessionHandle session_handles[QAT_DC_MAX_INSTANCES]; | |
47 | static CpaBufferList **buffer_array[QAT_DC_MAX_INSTANCES]; | |
1cfdb0e6 | 48 | static Cpa16U num_inst = 0; |
7a25f089 | 49 | static Cpa32U inst_num = 0; |
cf637391 | 50 | static boolean_t qat_dc_init_done = B_FALSE; |
38742209 | 51 | int zfs_qat_compress_disable = 0; |
6a9d6359 | 52 | |
cf637391 TC |
53 | boolean_t |
54 | qat_dc_use_accel(size_t s_len) | |
6a9d6359 | 55 | { |
38742209 | 56 | return (!zfs_qat_compress_disable && |
cf637391 TC |
57 | qat_dc_init_done && |
58 | s_len >= QAT_MIN_BUF_SIZE && | |
59 | s_len <= QAT_MAX_BUF_SIZE); | |
6a9d6359 | 60 | } |
61 | ||
62 | static void | |
63 | qat_dc_callback(void *p_callback, CpaStatus status) | |
64 | { | |
65 | if (p_callback != NULL) | |
66 | complete((struct completion *)p_callback); | |
67 | } | |
68 | ||
6a9d6359 | 69 | static void |
cf637391 | 70 | qat_dc_clean(void) |
6a9d6359 | 71 | { |
72 | Cpa16U buff_num = 0; | |
73 | Cpa16U num_inter_buff_lists = 0; | |
74 | Cpa16U i = 0; | |
75 | ||
76 | for (i = 0; i < num_inst; i++) { | |
77 | cpaDcStopInstance(dc_inst_handles[i]); | |
cf637391 | 78 | QAT_PHYS_CONTIG_FREE(session_handles[i]); |
6a9d6359 | 79 | /* free intermediate buffers */ |
80 | if (buffer_array[i] != NULL) { | |
81 | cpaDcGetNumIntermediateBuffers( | |
82 | dc_inst_handles[i], &num_inter_buff_lists); | |
83 | for (buff_num = 0; buff_num < num_inter_buff_lists; | |
84 | buff_num++) { | |
85 | CpaBufferList *buffer_inter = | |
86 | buffer_array[i][buff_num]; | |
87 | if (buffer_inter->pBuffers) { | |
cf637391 | 88 | QAT_PHYS_CONTIG_FREE( |
6a9d6359 | 89 | buffer_inter->pBuffers->pData); |
cf637391 | 90 | QAT_PHYS_CONTIG_FREE( |
6a9d6359 | 91 | buffer_inter->pBuffers); |
92 | } | |
cf637391 | 93 | QAT_PHYS_CONTIG_FREE( |
6a9d6359 | 94 | buffer_inter->pPrivateMetaData); |
cf637391 | 95 | QAT_PHYS_CONTIG_FREE(buffer_inter); |
6a9d6359 | 96 | } |
97 | } | |
98 | } | |
99 | ||
100 | num_inst = 0; | |
cf637391 | 101 | qat_dc_init_done = B_FALSE; |
6a9d6359 | 102 | } |
103 | ||
104 | int | |
cf637391 | 105 | qat_dc_init(void) |
6a9d6359 | 106 | { |
107 | CpaStatus status = CPA_STATUS_SUCCESS; | |
108 | Cpa32U sess_size = 0; | |
109 | Cpa32U ctx_size = 0; | |
110 | Cpa16U num_inter_buff_lists = 0; | |
111 | Cpa16U buff_num = 0; | |
112 | Cpa32U buff_meta_size = 0; | |
113 | CpaDcSessionSetupData sd = {0}; | |
114 | Cpa16U i; | |
115 | ||
116 | status = cpaDcGetNumInstances(&num_inst); | |
cf637391 | 117 | if (status != CPA_STATUS_SUCCESS) |
6a9d6359 | 118 | return (-1); |
119 | ||
cf637391 TC |
120 | /* if the user has configured no QAT compression units just return */ |
121 | if (num_inst == 0) | |
122 | return (0); | |
123 | ||
124 | if (num_inst > QAT_DC_MAX_INSTANCES) | |
125 | num_inst = QAT_DC_MAX_INSTANCES; | |
6a9d6359 | 126 | |
127 | status = cpaDcGetInstances(num_inst, &dc_inst_handles[0]); | |
128 | if (status != CPA_STATUS_SUCCESS) | |
129 | return (-1); | |
130 | ||
131 | for (i = 0; i < num_inst; i++) { | |
132 | cpaDcSetAddressTranslation(dc_inst_handles[i], | |
133 | (void*)virt_to_phys); | |
134 | ||
135 | status = cpaDcBufferListGetMetaSize(dc_inst_handles[i], | |
136 | 1, &buff_meta_size); | |
137 | ||
138 | if (status == CPA_STATUS_SUCCESS) | |
139 | status = cpaDcGetNumIntermediateBuffers( | |
140 | dc_inst_handles[i], &num_inter_buff_lists); | |
141 | ||
142 | if (status == CPA_STATUS_SUCCESS && num_inter_buff_lists != 0) | |
cf637391 | 143 | status = QAT_PHYS_CONTIG_ALLOC(&buffer_array[i], |
6a9d6359 | 144 | num_inter_buff_lists * |
145 | sizeof (CpaBufferList *)); | |
146 | ||
147 | for (buff_num = 0; buff_num < num_inter_buff_lists; | |
148 | buff_num++) { | |
149 | if (status == CPA_STATUS_SUCCESS) | |
cf637391 | 150 | status = QAT_PHYS_CONTIG_ALLOC( |
6a9d6359 | 151 | &buffer_array[i][buff_num], |
152 | sizeof (CpaBufferList)); | |
153 | ||
154 | if (status == CPA_STATUS_SUCCESS) | |
cf637391 | 155 | status = QAT_PHYS_CONTIG_ALLOC( |
6a9d6359 | 156 | &buffer_array[i][buff_num]-> |
157 | pPrivateMetaData, | |
158 | buff_meta_size); | |
159 | ||
160 | if (status == CPA_STATUS_SUCCESS) | |
cf637391 | 161 | status = QAT_PHYS_CONTIG_ALLOC( |
6a9d6359 | 162 | &buffer_array[i][buff_num]->pBuffers, |
163 | sizeof (CpaFlatBuffer)); | |
164 | ||
165 | if (status == CPA_STATUS_SUCCESS) { | |
166 | /* | |
167 | * implementation requires an intermediate | |
168 | * buffer approximately twice the size of | |
169 | * output buffer, which is 2x max buffer | |
170 | * size here. | |
171 | */ | |
cf637391 | 172 | status = QAT_PHYS_CONTIG_ALLOC( |
6a9d6359 | 173 | &buffer_array[i][buff_num]->pBuffers-> |
174 | pData, 2 * QAT_MAX_BUF_SIZE); | |
175 | if (status != CPA_STATUS_SUCCESS) | |
176 | goto fail; | |
177 | ||
178 | buffer_array[i][buff_num]->numBuffers = 1; | |
179 | buffer_array[i][buff_num]->pBuffers-> | |
180 | dataLenInBytes = 2 * QAT_MAX_BUF_SIZE; | |
181 | } | |
182 | } | |
183 | ||
184 | status = cpaDcStartInstance(dc_inst_handles[i], | |
185 | num_inter_buff_lists, buffer_array[i]); | |
186 | if (status != CPA_STATUS_SUCCESS) | |
187 | goto fail; | |
188 | ||
189 | sd.compLevel = CPA_DC_L1; | |
190 | sd.compType = CPA_DC_DEFLATE; | |
191 | sd.huffType = CPA_DC_HT_FULL_DYNAMIC; | |
192 | sd.sessDirection = CPA_DC_DIR_COMBINED; | |
193 | sd.sessState = CPA_DC_STATELESS; | |
194 | sd.deflateWindowSize = 7; | |
195 | sd.checksum = CPA_DC_ADLER32; | |
196 | status = cpaDcGetSessionSize(dc_inst_handles[i], | |
197 | &sd, &sess_size, &ctx_size); | |
198 | if (status != CPA_STATUS_SUCCESS) | |
199 | goto fail; | |
200 | ||
cf637391 | 201 | QAT_PHYS_CONTIG_ALLOC(&session_handles[i], sess_size); |
6a9d6359 | 202 | if (session_handles[i] == NULL) |
203 | goto fail; | |
204 | ||
205 | status = cpaDcInitSession(dc_inst_handles[i], | |
206 | session_handles[i], | |
207 | &sd, NULL, qat_dc_callback); | |
208 | if (status != CPA_STATUS_SUCCESS) | |
209 | goto fail; | |
210 | } | |
211 | ||
cf637391 | 212 | qat_dc_init_done = B_TRUE; |
6a9d6359 | 213 | return (0); |
214 | fail: | |
cf637391 | 215 | qat_dc_clean(); |
6a9d6359 | 216 | return (-1); |
217 | } | |
218 | ||
219 | void | |
cf637391 | 220 | qat_dc_fini(void) |
6a9d6359 | 221 | { |
cf637391 TC |
222 | if (!qat_dc_init_done) |
223 | return; | |
6a9d6359 | 224 | |
cf637391 | 225 | qat_dc_clean(); |
6a9d6359 | 226 | } |
227 | ||
32dce2da TC |
228 | /* |
229 | * The "add" parameter is an additional buffer which is passed | |
230 | * to QAT as a scratch buffer alongside the destination buffer | |
231 | * in case the "compressed" data ends up being larger than the | |
232 | * original source data. This is necessary to prevent QAT from | |
233 | * generating buffer overflow warnings for incompressible data. | |
234 | */ | |
235 | static int | |
236 | qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len, | |
237 | char *dst, int dst_len, char *add, int add_len, size_t *c_len) | |
6a9d6359 | 238 | { |
239 | CpaInstanceHandle dc_inst_handle; | |
240 | CpaDcSessionHandle session_handle; | |
241 | CpaBufferList *buf_list_src = NULL; | |
242 | CpaBufferList *buf_list_dst = NULL; | |
243 | CpaFlatBuffer *flat_buf_src = NULL; | |
244 | CpaFlatBuffer *flat_buf_dst = NULL; | |
245 | Cpa8U *buffer_meta_src = NULL; | |
246 | Cpa8U *buffer_meta_dst = NULL; | |
247 | Cpa32U buffer_meta_size = 0; | |
248 | CpaDcRqResults dc_results; | |
249 | CpaStatus status = CPA_STATUS_SUCCESS; | |
250 | Cpa32U hdr_sz = 0; | |
251 | Cpa32U compressed_sz; | |
17dd8835 TC |
252 | Cpa32U num_src_buf = (src_len >> PAGE_SHIFT) + 2; |
253 | Cpa32U num_dst_buf = (dst_len >> PAGE_SHIFT) + 2; | |
32dce2da | 254 | Cpa32U num_add_buf = (add_len >> PAGE_SHIFT) + 2; |
6a9d6359 | 255 | Cpa32U bytes_left; |
32dce2da | 256 | Cpa32U dst_pages = 0; |
6a9d6359 | 257 | char *data; |
32dce2da | 258 | struct page *page; |
6a9d6359 | 259 | struct page **in_pages = NULL; |
260 | struct page **out_pages = NULL; | |
32dce2da | 261 | struct page **add_pages = NULL; |
17dd8835 | 262 | Cpa32U page_off = 0; |
6a9d6359 | 263 | struct completion complete; |
17dd8835 | 264 | Cpa32U page_num = 0; |
6a9d6359 | 265 | Cpa16U i; |
266 | ||
17dd8835 TC |
267 | /* |
268 | * We increment num_src_buf and num_dst_buf by 2 to allow | |
269 | * us to handle non page-aligned buffer addresses and buffers | |
270 | * whose sizes are not divisible by PAGE_SIZE. | |
271 | */ | |
6a9d6359 | 272 | Cpa32U src_buffer_list_mem_size = sizeof (CpaBufferList) + |
273 | (num_src_buf * sizeof (CpaFlatBuffer)); | |
274 | Cpa32U dst_buffer_list_mem_size = sizeof (CpaBufferList) + | |
32dce2da | 275 | ((num_dst_buf + num_add_buf) * sizeof (CpaFlatBuffer)); |
6a9d6359 | 276 | |
cf637391 | 277 | if (QAT_PHYS_CONTIG_ALLOC(&in_pages, |
6a9d6359 | 278 | num_src_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS) |
279 | goto fail; | |
280 | ||
cf637391 | 281 | if (QAT_PHYS_CONTIG_ALLOC(&out_pages, |
6a9d6359 | 282 | num_dst_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS) |
283 | goto fail; | |
284 | ||
32dce2da TC |
285 | if (QAT_PHYS_CONTIG_ALLOC(&add_pages, |
286 | num_add_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS) | |
287 | goto fail; | |
288 | ||
6a9d6359 | 289 | i = atomic_inc_32_nv(&inst_num) % num_inst; |
290 | dc_inst_handle = dc_inst_handles[i]; | |
291 | session_handle = session_handles[i]; | |
292 | ||
293 | cpaDcBufferListGetMetaSize(dc_inst_handle, num_src_buf, | |
294 | &buffer_meta_size); | |
cf637391 | 295 | if (QAT_PHYS_CONTIG_ALLOC(&buffer_meta_src, buffer_meta_size) != |
6a9d6359 | 296 | CPA_STATUS_SUCCESS) |
297 | goto fail; | |
298 | ||
32dce2da | 299 | cpaDcBufferListGetMetaSize(dc_inst_handle, num_dst_buf + num_add_buf, |
6a9d6359 | 300 | &buffer_meta_size); |
cf637391 | 301 | if (QAT_PHYS_CONTIG_ALLOC(&buffer_meta_dst, buffer_meta_size) != |
6a9d6359 | 302 | CPA_STATUS_SUCCESS) |
303 | goto fail; | |
304 | ||
305 | /* build source buffer list */ | |
cf637391 | 306 | if (QAT_PHYS_CONTIG_ALLOC(&buf_list_src, src_buffer_list_mem_size) != |
6a9d6359 | 307 | CPA_STATUS_SUCCESS) |
308 | goto fail; | |
309 | ||
310 | flat_buf_src = (CpaFlatBuffer *)(buf_list_src + 1); | |
311 | ||
312 | buf_list_src->pBuffers = flat_buf_src; /* always point to first one */ | |
313 | ||
314 | /* build destination buffer list */ | |
cf637391 | 315 | if (QAT_PHYS_CONTIG_ALLOC(&buf_list_dst, dst_buffer_list_mem_size) != |
6a9d6359 | 316 | CPA_STATUS_SUCCESS) |
317 | goto fail; | |
318 | ||
319 | flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1); | |
320 | ||
321 | buf_list_dst->pBuffers = flat_buf_dst; /* always point to first one */ | |
322 | ||
323 | buf_list_src->numBuffers = 0; | |
324 | buf_list_src->pPrivateMetaData = buffer_meta_src; | |
325 | bytes_left = src_len; | |
326 | data = src; | |
327 | page_num = 0; | |
328 | while (bytes_left > 0) { | |
17dd8835 | 329 | page_off = ((long)data & ~PAGE_MASK); |
32dce2da TC |
330 | page = qat_mem_to_page(data); |
331 | in_pages[page_num] = page; | |
332 | flat_buf_src->pData = kmap(page) + page_off; | |
6a9d6359 | 333 | flat_buf_src->dataLenInBytes = |
17dd8835 | 334 | min((long)PAGE_SIZE - page_off, (long)bytes_left); |
6a9d6359 | 335 | |
336 | bytes_left -= flat_buf_src->dataLenInBytes; | |
337 | data += flat_buf_src->dataLenInBytes; | |
338 | flat_buf_src++; | |
339 | buf_list_src->numBuffers++; | |
340 | page_num++; | |
341 | } | |
342 | ||
343 | buf_list_dst->numBuffers = 0; | |
344 | buf_list_dst->pPrivateMetaData = buffer_meta_dst; | |
345 | bytes_left = dst_len; | |
346 | data = dst; | |
347 | page_num = 0; | |
348 | while (bytes_left > 0) { | |
17dd8835 | 349 | page_off = ((long)data & ~PAGE_MASK); |
32dce2da TC |
350 | page = qat_mem_to_page(data); |
351 | flat_buf_dst->pData = kmap(page) + page_off; | |
352 | out_pages[page_num] = page; | |
353 | flat_buf_dst->dataLenInBytes = | |
354 | min((long)PAGE_SIZE - page_off, (long)bytes_left); | |
355 | ||
356 | bytes_left -= flat_buf_dst->dataLenInBytes; | |
357 | data += flat_buf_dst->dataLenInBytes; | |
358 | flat_buf_dst++; | |
359 | buf_list_dst->numBuffers++; | |
360 | page_num++; | |
361 | dst_pages++; | |
362 | } | |
363 | ||
364 | /* map additional scratch pages into the destination buffer list */ | |
365 | bytes_left = add_len; | |
366 | data = add; | |
367 | page_num = 0; | |
368 | while (bytes_left > 0) { | |
369 | page_off = ((long)data & ~PAGE_MASK); | |
370 | page = qat_mem_to_page(data); | |
371 | flat_buf_dst->pData = kmap(page) + page_off; | |
372 | add_pages[page_num] = page; | |
6a9d6359 | 373 | flat_buf_dst->dataLenInBytes = |
17dd8835 | 374 | min((long)PAGE_SIZE - page_off, (long)bytes_left); |
6a9d6359 | 375 | |
376 | bytes_left -= flat_buf_dst->dataLenInBytes; | |
377 | data += flat_buf_dst->dataLenInBytes; | |
378 | flat_buf_dst++; | |
379 | buf_list_dst->numBuffers++; | |
380 | page_num++; | |
381 | } | |
382 | ||
383 | init_completion(&complete); | |
384 | ||
385 | if (dir == QAT_COMPRESS) { | |
386 | QAT_STAT_BUMP(comp_requests); | |
387 | QAT_STAT_INCR(comp_total_in_bytes, src_len); | |
388 | ||
389 | cpaDcGenerateHeader(session_handle, | |
390 | buf_list_dst->pBuffers, &hdr_sz); | |
391 | buf_list_dst->pBuffers->pData += hdr_sz; | |
392 | buf_list_dst->pBuffers->dataLenInBytes -= hdr_sz; | |
393 | status = cpaDcCompressData( | |
394 | dc_inst_handle, session_handle, | |
395 | buf_list_src, buf_list_dst, | |
396 | &dc_results, CPA_DC_FLUSH_FINAL, | |
397 | &complete); | |
398 | if (status != CPA_STATUS_SUCCESS) { | |
399 | goto fail; | |
400 | } | |
401 | ||
402 | /* we now wait until the completion of the operation. */ | |
403 | if (!wait_for_completion_interruptible_timeout(&complete, | |
cf637391 | 404 | QAT_TIMEOUT_MS)) { |
6a9d6359 | 405 | status = CPA_STATUS_FAIL; |
406 | goto fail; | |
407 | } | |
408 | ||
409 | if (dc_results.status != CPA_STATUS_SUCCESS) { | |
410 | status = CPA_STATUS_FAIL; | |
411 | goto fail; | |
412 | } | |
413 | ||
414 | compressed_sz = dc_results.produced; | |
415 | if (compressed_sz + hdr_sz + ZLIB_FOOT_SZ > dst_len) { | |
32dce2da | 416 | status = CPA_STATUS_INCOMPRESSIBLE; |
6a9d6359 | 417 | goto fail; |
418 | } | |
419 | ||
420 | flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1); | |
421 | /* move to the last page */ | |
422 | flat_buf_dst += (compressed_sz + hdr_sz) >> PAGE_SHIFT; | |
423 | ||
8d9e7c8f | 424 | /* no space for gzip footer in the last page */ |
6a9d6359 | 425 | if (((compressed_sz + hdr_sz) % PAGE_SIZE) |
32dce2da TC |
426 | + ZLIB_FOOT_SZ > PAGE_SIZE) { |
427 | status = CPA_STATUS_INCOMPRESSIBLE; | |
6a9d6359 | 428 | goto fail; |
32dce2da | 429 | } |
6a9d6359 | 430 | |
a3df7fa7 | 431 | /* jump to the end of the buffer and append footer */ |
432 | flat_buf_dst->pData = | |
433 | (char *)((unsigned long)flat_buf_dst->pData & PAGE_MASK) | |
434 | + ((compressed_sz + hdr_sz) % PAGE_SIZE); | |
6a9d6359 | 435 | flat_buf_dst->dataLenInBytes = ZLIB_FOOT_SZ; |
436 | ||
437 | dc_results.produced = 0; | |
438 | status = cpaDcGenerateFooter(session_handle, | |
439 | flat_buf_dst, &dc_results); | |
32dce2da | 440 | if (status != CPA_STATUS_SUCCESS) |
6a9d6359 | 441 | goto fail; |
6a9d6359 | 442 | |
443 | *c_len = compressed_sz + dc_results.produced + hdr_sz; | |
6a9d6359 | 444 | QAT_STAT_INCR(comp_total_out_bytes, *c_len); |
cf637391 TC |
445 | } else { |
446 | ASSERT3U(dir, ==, QAT_DECOMPRESS); | |
6a9d6359 | 447 | QAT_STAT_BUMP(decomp_requests); |
448 | QAT_STAT_INCR(decomp_total_in_bytes, src_len); | |
449 | ||
450 | buf_list_src->pBuffers->pData += ZLIB_HEAD_SZ; | |
451 | buf_list_src->pBuffers->dataLenInBytes -= ZLIB_HEAD_SZ; | |
32dce2da TC |
452 | status = cpaDcDecompressData(dc_inst_handle, session_handle, |
453 | buf_list_src, buf_list_dst, &dc_results, CPA_DC_FLUSH_FINAL, | |
6a9d6359 | 454 | &complete); |
455 | ||
456 | if (CPA_STATUS_SUCCESS != status) { | |
457 | status = CPA_STATUS_FAIL; | |
458 | goto fail; | |
459 | } | |
460 | ||
461 | /* we now wait until the completion of the operation. */ | |
462 | if (!wait_for_completion_interruptible_timeout(&complete, | |
cf637391 | 463 | QAT_TIMEOUT_MS)) { |
6a9d6359 | 464 | status = CPA_STATUS_FAIL; |
465 | goto fail; | |
466 | } | |
467 | ||
468 | if (dc_results.status != CPA_STATUS_SUCCESS) { | |
469 | status = CPA_STATUS_FAIL; | |
470 | goto fail; | |
471 | } | |
472 | ||
473 | *c_len = dc_results.produced; | |
6a9d6359 | 474 | QAT_STAT_INCR(decomp_total_out_bytes, *c_len); |
6a9d6359 | 475 | } |
476 | ||
477 | fail: | |
32dce2da | 478 | if (status != CPA_STATUS_SUCCESS && status != CPA_STATUS_INCOMPRESSIBLE) |
6a9d6359 | 479 | QAT_STAT_BUMP(dc_fails); |
6a9d6359 | 480 | |
481 | if (in_pages) { | |
482 | for (page_num = 0; | |
483 | page_num < buf_list_src->numBuffers; | |
484 | page_num++) { | |
485 | kunmap(in_pages[page_num]); | |
486 | } | |
cf637391 | 487 | QAT_PHYS_CONTIG_FREE(in_pages); |
6a9d6359 | 488 | } |
489 | ||
490 | if (out_pages) { | |
32dce2da | 491 | for (page_num = 0; page_num < dst_pages; page_num++) { |
6a9d6359 | 492 | kunmap(out_pages[page_num]); |
493 | } | |
cf637391 | 494 | QAT_PHYS_CONTIG_FREE(out_pages); |
6a9d6359 | 495 | } |
496 | ||
32dce2da TC |
497 | if (add_pages) { |
498 | for (page_num = 0; | |
499 | page_num < buf_list_dst->numBuffers - dst_pages; | |
500 | page_num++) { | |
501 | kunmap(add_pages[page_num]); | |
502 | } | |
503 | QAT_PHYS_CONTIG_FREE(add_pages); | |
504 | } | |
505 | ||
cf637391 TC |
506 | QAT_PHYS_CONTIG_FREE(buffer_meta_src); |
507 | QAT_PHYS_CONTIG_FREE(buffer_meta_dst); | |
508 | QAT_PHYS_CONTIG_FREE(buf_list_src); | |
509 | QAT_PHYS_CONTIG_FREE(buf_list_dst); | |
6a9d6359 | 510 | |
32dce2da TC |
511 | return (status); |
512 | } | |
513 | ||
514 | /* | |
515 | * Entry point for QAT accelerated compression / decompression. | |
516 | */ | |
517 | int | |
518 | qat_compress(qat_compress_dir_t dir, char *src, int src_len, | |
519 | char *dst, int dst_len, size_t *c_len) | |
520 | { | |
521 | int ret; | |
522 | size_t add_len = 0; | |
523 | void *add = NULL; | |
524 | ||
525 | if (dir == QAT_COMPRESS) { | |
526 | add_len = dst_len; | |
527 | add = zio_data_buf_alloc(add_len); | |
528 | } | |
529 | ||
530 | ret = qat_compress_impl(dir, src, src_len, dst, | |
531 | dst_len, add, add_len, c_len); | |
532 | ||
533 | if (dir == QAT_COMPRESS) | |
534 | zio_data_buf_free(add, add_len); | |
535 | ||
6a9d6359 | 536 | return (ret); |
537 | } | |
538 | ||
38742209 TC |
539 | module_param(zfs_qat_compress_disable, int, 0644); |
540 | MODULE_PARM_DESC(zfs_qat_compress_disable, "Disable QAT compression"); | |
541 | ||
6a9d6359 | 542 | #endif |