]>
Commit | Line | Data |
---|---|---|
6a9d6359 | 1 | /* |
2 | * CDDL HEADER START | |
3 | * | |
4 | * The contents of this file are subject to the terms of the | |
5 | * Common Development and Distribution License (the "License"). | |
6 | * You may not use this file except in compliance with the License. | |
7 | * | |
8 | * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE | |
9 | * or http://www.opensolaris.org/os/licensing. | |
10 | * See the License for the specific language governing permissions | |
11 | * and limitations under the License. | |
12 | * | |
13 | * When distributing Covered Code, include this CDDL HEADER in each | |
14 | * file and include the License file at usr/src/OPENSOLARIS.LICENSE. | |
15 | * If applicable, add the following below this CDDL HEADER, with the | |
16 | * fields enclosed by brackets "[]" replaced with your own identifying | |
17 | * information: Portions Copyright [yyyy] [name of copyright owner] | |
18 | * | |
19 | * CDDL HEADER END | |
20 | */ | |
21 | ||
22 | #if defined(_KERNEL) && defined(HAVE_QAT) | |
23 | #include <linux/slab.h> | |
24 | #include <linux/vmalloc.h> | |
25 | #include <linux/pagemap.h> | |
26 | #include <linux/completion.h> | |
27 | #include <sys/zfs_context.h> | |
32dce2da | 28 | #include <sys/zio.h> |
cf637391 | 29 | #include "qat.h" |
6a9d6359 | 30 | |
31 | /* | |
8d9e7c8f TC |
32 | * Max instances in a QAT device, each instance is a channel to submit |
33 | * jobs to QAT hardware, this is only for pre-allocating instance and | |
34 | * session arrays; the actual number of instances are defined in the | |
35 | * QAT driver's configuration file. | |
6a9d6359 | 36 | */ |
cf637391 | 37 | #define QAT_DC_MAX_INSTANCES 48 |
6a9d6359 | 38 | |
39 | /* | |
40 | * ZLIB head and foot size | |
41 | */ | |
42 | #define ZLIB_HEAD_SZ 2 | |
43 | #define ZLIB_FOOT_SZ 4 | |
44 | ||
cf637391 TC |
45 | static CpaInstanceHandle dc_inst_handles[QAT_DC_MAX_INSTANCES]; |
46 | static CpaDcSessionHandle session_handles[QAT_DC_MAX_INSTANCES]; | |
47 | static CpaBufferList **buffer_array[QAT_DC_MAX_INSTANCES]; | |
1cfdb0e6 | 48 | static Cpa16U num_inst = 0; |
7a25f089 | 49 | static Cpa32U inst_num = 0; |
cf637391 | 50 | static boolean_t qat_dc_init_done = B_FALSE; |
38742209 | 51 | int zfs_qat_compress_disable = 0; |
6a9d6359 | 52 | |
cf637391 TC |
53 | boolean_t |
54 | qat_dc_use_accel(size_t s_len) | |
6a9d6359 | 55 | { |
38742209 | 56 | return (!zfs_qat_compress_disable && |
cf637391 TC |
57 | qat_dc_init_done && |
58 | s_len >= QAT_MIN_BUF_SIZE && | |
59 | s_len <= QAT_MAX_BUF_SIZE); | |
6a9d6359 | 60 | } |
61 | ||
62 | static void | |
63 | qat_dc_callback(void *p_callback, CpaStatus status) | |
64 | { | |
65 | if (p_callback != NULL) | |
66 | complete((struct completion *)p_callback); | |
67 | } | |
68 | ||
6a9d6359 | 69 | static void |
cf637391 | 70 | qat_dc_clean(void) |
6a9d6359 | 71 | { |
72 | Cpa16U buff_num = 0; | |
73 | Cpa16U num_inter_buff_lists = 0; | |
6a9d6359 | 74 | |
bc8a6a60 | 75 | for (Cpa16U i = 0; i < num_inst; i++) { |
6a9d6359 | 76 | cpaDcStopInstance(dc_inst_handles[i]); |
cf637391 | 77 | QAT_PHYS_CONTIG_FREE(session_handles[i]); |
6a9d6359 | 78 | /* free intermediate buffers */ |
79 | if (buffer_array[i] != NULL) { | |
80 | cpaDcGetNumIntermediateBuffers( | |
81 | dc_inst_handles[i], &num_inter_buff_lists); | |
82 | for (buff_num = 0; buff_num < num_inter_buff_lists; | |
83 | buff_num++) { | |
84 | CpaBufferList *buffer_inter = | |
85 | buffer_array[i][buff_num]; | |
86 | if (buffer_inter->pBuffers) { | |
cf637391 | 87 | QAT_PHYS_CONTIG_FREE( |
6a9d6359 | 88 | buffer_inter->pBuffers->pData); |
cf637391 | 89 | QAT_PHYS_CONTIG_FREE( |
6a9d6359 | 90 | buffer_inter->pBuffers); |
91 | } | |
cf637391 | 92 | QAT_PHYS_CONTIG_FREE( |
6a9d6359 | 93 | buffer_inter->pPrivateMetaData); |
cf637391 | 94 | QAT_PHYS_CONTIG_FREE(buffer_inter); |
6a9d6359 | 95 | } |
96 | } | |
97 | } | |
98 | ||
99 | num_inst = 0; | |
cf637391 | 100 | qat_dc_init_done = B_FALSE; |
6a9d6359 | 101 | } |
102 | ||
103 | int | |
cf637391 | 104 | qat_dc_init(void) |
6a9d6359 | 105 | { |
106 | CpaStatus status = CPA_STATUS_SUCCESS; | |
107 | Cpa32U sess_size = 0; | |
108 | Cpa32U ctx_size = 0; | |
109 | Cpa16U num_inter_buff_lists = 0; | |
110 | Cpa16U buff_num = 0; | |
111 | Cpa32U buff_meta_size = 0; | |
112 | CpaDcSessionSetupData sd = {0}; | |
6a9d6359 | 113 | |
114 | status = cpaDcGetNumInstances(&num_inst); | |
cf637391 | 115 | if (status != CPA_STATUS_SUCCESS) |
6a9d6359 | 116 | return (-1); |
117 | ||
cf637391 TC |
118 | /* if the user has configured no QAT compression units just return */ |
119 | if (num_inst == 0) | |
120 | return (0); | |
121 | ||
122 | if (num_inst > QAT_DC_MAX_INSTANCES) | |
123 | num_inst = QAT_DC_MAX_INSTANCES; | |
6a9d6359 | 124 | |
125 | status = cpaDcGetInstances(num_inst, &dc_inst_handles[0]); | |
126 | if (status != CPA_STATUS_SUCCESS) | |
127 | return (-1); | |
128 | ||
bc8a6a60 | 129 | for (Cpa16U i = 0; i < num_inst; i++) { |
6a9d6359 | 130 | cpaDcSetAddressTranslation(dc_inst_handles[i], |
131 | (void*)virt_to_phys); | |
132 | ||
133 | status = cpaDcBufferListGetMetaSize(dc_inst_handles[i], | |
134 | 1, &buff_meta_size); | |
135 | ||
136 | if (status == CPA_STATUS_SUCCESS) | |
137 | status = cpaDcGetNumIntermediateBuffers( | |
138 | dc_inst_handles[i], &num_inter_buff_lists); | |
139 | ||
140 | if (status == CPA_STATUS_SUCCESS && num_inter_buff_lists != 0) | |
cf637391 | 141 | status = QAT_PHYS_CONTIG_ALLOC(&buffer_array[i], |
6a9d6359 | 142 | num_inter_buff_lists * |
143 | sizeof (CpaBufferList *)); | |
144 | ||
145 | for (buff_num = 0; buff_num < num_inter_buff_lists; | |
146 | buff_num++) { | |
147 | if (status == CPA_STATUS_SUCCESS) | |
cf637391 | 148 | status = QAT_PHYS_CONTIG_ALLOC( |
6a9d6359 | 149 | &buffer_array[i][buff_num], |
150 | sizeof (CpaBufferList)); | |
151 | ||
152 | if (status == CPA_STATUS_SUCCESS) | |
cf637391 | 153 | status = QAT_PHYS_CONTIG_ALLOC( |
6a9d6359 | 154 | &buffer_array[i][buff_num]-> |
155 | pPrivateMetaData, | |
156 | buff_meta_size); | |
157 | ||
158 | if (status == CPA_STATUS_SUCCESS) | |
cf637391 | 159 | status = QAT_PHYS_CONTIG_ALLOC( |
6a9d6359 | 160 | &buffer_array[i][buff_num]->pBuffers, |
161 | sizeof (CpaFlatBuffer)); | |
162 | ||
163 | if (status == CPA_STATUS_SUCCESS) { | |
164 | /* | |
165 | * implementation requires an intermediate | |
166 | * buffer approximately twice the size of | |
167 | * output buffer, which is 2x max buffer | |
168 | * size here. | |
169 | */ | |
cf637391 | 170 | status = QAT_PHYS_CONTIG_ALLOC( |
6a9d6359 | 171 | &buffer_array[i][buff_num]->pBuffers-> |
172 | pData, 2 * QAT_MAX_BUF_SIZE); | |
173 | if (status != CPA_STATUS_SUCCESS) | |
174 | goto fail; | |
175 | ||
176 | buffer_array[i][buff_num]->numBuffers = 1; | |
177 | buffer_array[i][buff_num]->pBuffers-> | |
178 | dataLenInBytes = 2 * QAT_MAX_BUF_SIZE; | |
179 | } | |
180 | } | |
181 | ||
182 | status = cpaDcStartInstance(dc_inst_handles[i], | |
183 | num_inter_buff_lists, buffer_array[i]); | |
184 | if (status != CPA_STATUS_SUCCESS) | |
185 | goto fail; | |
186 | ||
187 | sd.compLevel = CPA_DC_L1; | |
188 | sd.compType = CPA_DC_DEFLATE; | |
189 | sd.huffType = CPA_DC_HT_FULL_DYNAMIC; | |
190 | sd.sessDirection = CPA_DC_DIR_COMBINED; | |
191 | sd.sessState = CPA_DC_STATELESS; | |
192 | sd.deflateWindowSize = 7; | |
193 | sd.checksum = CPA_DC_ADLER32; | |
194 | status = cpaDcGetSessionSize(dc_inst_handles[i], | |
195 | &sd, &sess_size, &ctx_size); | |
196 | if (status != CPA_STATUS_SUCCESS) | |
197 | goto fail; | |
198 | ||
cf637391 | 199 | QAT_PHYS_CONTIG_ALLOC(&session_handles[i], sess_size); |
6a9d6359 | 200 | if (session_handles[i] == NULL) |
201 | goto fail; | |
202 | ||
203 | status = cpaDcInitSession(dc_inst_handles[i], | |
204 | session_handles[i], | |
205 | &sd, NULL, qat_dc_callback); | |
206 | if (status != CPA_STATUS_SUCCESS) | |
207 | goto fail; | |
208 | } | |
209 | ||
cf637391 | 210 | qat_dc_init_done = B_TRUE; |
6a9d6359 | 211 | return (0); |
212 | fail: | |
cf637391 | 213 | qat_dc_clean(); |
6a9d6359 | 214 | return (-1); |
215 | } | |
216 | ||
217 | void | |
cf637391 | 218 | qat_dc_fini(void) |
6a9d6359 | 219 | { |
cf637391 TC |
220 | if (!qat_dc_init_done) |
221 | return; | |
6a9d6359 | 222 | |
cf637391 | 223 | qat_dc_clean(); |
6a9d6359 | 224 | } |
225 | ||
32dce2da TC |
226 | /* |
227 | * The "add" parameter is an additional buffer which is passed | |
228 | * to QAT as a scratch buffer alongside the destination buffer | |
229 | * in case the "compressed" data ends up being larger than the | |
230 | * original source data. This is necessary to prevent QAT from | |
231 | * generating buffer overflow warnings for incompressible data. | |
232 | */ | |
233 | static int | |
234 | qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len, | |
235 | char *dst, int dst_len, char *add, int add_len, size_t *c_len) | |
6a9d6359 | 236 | { |
237 | CpaInstanceHandle dc_inst_handle; | |
238 | CpaDcSessionHandle session_handle; | |
239 | CpaBufferList *buf_list_src = NULL; | |
240 | CpaBufferList *buf_list_dst = NULL; | |
241 | CpaFlatBuffer *flat_buf_src = NULL; | |
242 | CpaFlatBuffer *flat_buf_dst = NULL; | |
243 | Cpa8U *buffer_meta_src = NULL; | |
244 | Cpa8U *buffer_meta_dst = NULL; | |
245 | Cpa32U buffer_meta_size = 0; | |
246 | CpaDcRqResults dc_results; | |
247 | CpaStatus status = CPA_STATUS_SUCCESS; | |
248 | Cpa32U hdr_sz = 0; | |
249 | Cpa32U compressed_sz; | |
17dd8835 TC |
250 | Cpa32U num_src_buf = (src_len >> PAGE_SHIFT) + 2; |
251 | Cpa32U num_dst_buf = (dst_len >> PAGE_SHIFT) + 2; | |
32dce2da | 252 | Cpa32U num_add_buf = (add_len >> PAGE_SHIFT) + 2; |
6a9d6359 | 253 | Cpa32U bytes_left; |
32dce2da | 254 | Cpa32U dst_pages = 0; |
6a9d6359 | 255 | char *data; |
32dce2da | 256 | struct page *page; |
6a9d6359 | 257 | struct page **in_pages = NULL; |
258 | struct page **out_pages = NULL; | |
32dce2da | 259 | struct page **add_pages = NULL; |
17dd8835 | 260 | Cpa32U page_off = 0; |
6a9d6359 | 261 | struct completion complete; |
17dd8835 | 262 | Cpa32U page_num = 0; |
6a9d6359 | 263 | Cpa16U i; |
264 | ||
17dd8835 TC |
265 | /* |
266 | * We increment num_src_buf and num_dst_buf by 2 to allow | |
267 | * us to handle non page-aligned buffer addresses and buffers | |
268 | * whose sizes are not divisible by PAGE_SIZE. | |
269 | */ | |
6a9d6359 | 270 | Cpa32U src_buffer_list_mem_size = sizeof (CpaBufferList) + |
271 | (num_src_buf * sizeof (CpaFlatBuffer)); | |
272 | Cpa32U dst_buffer_list_mem_size = sizeof (CpaBufferList) + | |
32dce2da | 273 | ((num_dst_buf + num_add_buf) * sizeof (CpaFlatBuffer)); |
6a9d6359 | 274 | |
cf637391 | 275 | if (QAT_PHYS_CONTIG_ALLOC(&in_pages, |
6a9d6359 | 276 | num_src_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS) |
277 | goto fail; | |
278 | ||
cf637391 | 279 | if (QAT_PHYS_CONTIG_ALLOC(&out_pages, |
6a9d6359 | 280 | num_dst_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS) |
281 | goto fail; | |
282 | ||
32dce2da TC |
283 | if (QAT_PHYS_CONTIG_ALLOC(&add_pages, |
284 | num_add_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS) | |
285 | goto fail; | |
286 | ||
bc8a6a60 | 287 | i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst; |
6a9d6359 | 288 | dc_inst_handle = dc_inst_handles[i]; |
289 | session_handle = session_handles[i]; | |
290 | ||
291 | cpaDcBufferListGetMetaSize(dc_inst_handle, num_src_buf, | |
292 | &buffer_meta_size); | |
cf637391 | 293 | if (QAT_PHYS_CONTIG_ALLOC(&buffer_meta_src, buffer_meta_size) != |
6a9d6359 | 294 | CPA_STATUS_SUCCESS) |
295 | goto fail; | |
296 | ||
32dce2da | 297 | cpaDcBufferListGetMetaSize(dc_inst_handle, num_dst_buf + num_add_buf, |
6a9d6359 | 298 | &buffer_meta_size); |
cf637391 | 299 | if (QAT_PHYS_CONTIG_ALLOC(&buffer_meta_dst, buffer_meta_size) != |
6a9d6359 | 300 | CPA_STATUS_SUCCESS) |
301 | goto fail; | |
302 | ||
303 | /* build source buffer list */ | |
cf637391 | 304 | if (QAT_PHYS_CONTIG_ALLOC(&buf_list_src, src_buffer_list_mem_size) != |
6a9d6359 | 305 | CPA_STATUS_SUCCESS) |
306 | goto fail; | |
307 | ||
308 | flat_buf_src = (CpaFlatBuffer *)(buf_list_src + 1); | |
309 | ||
310 | buf_list_src->pBuffers = flat_buf_src; /* always point to first one */ | |
311 | ||
312 | /* build destination buffer list */ | |
cf637391 | 313 | if (QAT_PHYS_CONTIG_ALLOC(&buf_list_dst, dst_buffer_list_mem_size) != |
6a9d6359 | 314 | CPA_STATUS_SUCCESS) |
315 | goto fail; | |
316 | ||
317 | flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1); | |
318 | ||
319 | buf_list_dst->pBuffers = flat_buf_dst; /* always point to first one */ | |
320 | ||
321 | buf_list_src->numBuffers = 0; | |
322 | buf_list_src->pPrivateMetaData = buffer_meta_src; | |
323 | bytes_left = src_len; | |
324 | data = src; | |
325 | page_num = 0; | |
326 | while (bytes_left > 0) { | |
17dd8835 | 327 | page_off = ((long)data & ~PAGE_MASK); |
32dce2da TC |
328 | page = qat_mem_to_page(data); |
329 | in_pages[page_num] = page; | |
330 | flat_buf_src->pData = kmap(page) + page_off; | |
6a9d6359 | 331 | flat_buf_src->dataLenInBytes = |
17dd8835 | 332 | min((long)PAGE_SIZE - page_off, (long)bytes_left); |
6a9d6359 | 333 | |
334 | bytes_left -= flat_buf_src->dataLenInBytes; | |
335 | data += flat_buf_src->dataLenInBytes; | |
336 | flat_buf_src++; | |
337 | buf_list_src->numBuffers++; | |
338 | page_num++; | |
339 | } | |
340 | ||
341 | buf_list_dst->numBuffers = 0; | |
342 | buf_list_dst->pPrivateMetaData = buffer_meta_dst; | |
343 | bytes_left = dst_len; | |
344 | data = dst; | |
345 | page_num = 0; | |
346 | while (bytes_left > 0) { | |
17dd8835 | 347 | page_off = ((long)data & ~PAGE_MASK); |
32dce2da TC |
348 | page = qat_mem_to_page(data); |
349 | flat_buf_dst->pData = kmap(page) + page_off; | |
350 | out_pages[page_num] = page; | |
351 | flat_buf_dst->dataLenInBytes = | |
352 | min((long)PAGE_SIZE - page_off, (long)bytes_left); | |
353 | ||
354 | bytes_left -= flat_buf_dst->dataLenInBytes; | |
355 | data += flat_buf_dst->dataLenInBytes; | |
356 | flat_buf_dst++; | |
357 | buf_list_dst->numBuffers++; | |
358 | page_num++; | |
359 | dst_pages++; | |
360 | } | |
361 | ||
362 | /* map additional scratch pages into the destination buffer list */ | |
363 | bytes_left = add_len; | |
364 | data = add; | |
365 | page_num = 0; | |
366 | while (bytes_left > 0) { | |
367 | page_off = ((long)data & ~PAGE_MASK); | |
368 | page = qat_mem_to_page(data); | |
369 | flat_buf_dst->pData = kmap(page) + page_off; | |
370 | add_pages[page_num] = page; | |
6a9d6359 | 371 | flat_buf_dst->dataLenInBytes = |
17dd8835 | 372 | min((long)PAGE_SIZE - page_off, (long)bytes_left); |
6a9d6359 | 373 | |
374 | bytes_left -= flat_buf_dst->dataLenInBytes; | |
375 | data += flat_buf_dst->dataLenInBytes; | |
376 | flat_buf_dst++; | |
377 | buf_list_dst->numBuffers++; | |
378 | page_num++; | |
379 | } | |
380 | ||
381 | init_completion(&complete); | |
382 | ||
383 | if (dir == QAT_COMPRESS) { | |
384 | QAT_STAT_BUMP(comp_requests); | |
385 | QAT_STAT_INCR(comp_total_in_bytes, src_len); | |
386 | ||
387 | cpaDcGenerateHeader(session_handle, | |
388 | buf_list_dst->pBuffers, &hdr_sz); | |
389 | buf_list_dst->pBuffers->pData += hdr_sz; | |
390 | buf_list_dst->pBuffers->dataLenInBytes -= hdr_sz; | |
391 | status = cpaDcCompressData( | |
392 | dc_inst_handle, session_handle, | |
393 | buf_list_src, buf_list_dst, | |
394 | &dc_results, CPA_DC_FLUSH_FINAL, | |
395 | &complete); | |
396 | if (status != CPA_STATUS_SUCCESS) { | |
397 | goto fail; | |
398 | } | |
399 | ||
400 | /* we now wait until the completion of the operation. */ | |
401 | if (!wait_for_completion_interruptible_timeout(&complete, | |
cf637391 | 402 | QAT_TIMEOUT_MS)) { |
6a9d6359 | 403 | status = CPA_STATUS_FAIL; |
404 | goto fail; | |
405 | } | |
406 | ||
407 | if (dc_results.status != CPA_STATUS_SUCCESS) { | |
408 | status = CPA_STATUS_FAIL; | |
409 | goto fail; | |
410 | } | |
411 | ||
412 | compressed_sz = dc_results.produced; | |
413 | if (compressed_sz + hdr_sz + ZLIB_FOOT_SZ > dst_len) { | |
32dce2da | 414 | status = CPA_STATUS_INCOMPRESSIBLE; |
6a9d6359 | 415 | goto fail; |
416 | } | |
417 | ||
418 | flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1); | |
419 | /* move to the last page */ | |
420 | flat_buf_dst += (compressed_sz + hdr_sz) >> PAGE_SHIFT; | |
421 | ||
8d9e7c8f | 422 | /* no space for gzip footer in the last page */ |
6a9d6359 | 423 | if (((compressed_sz + hdr_sz) % PAGE_SIZE) |
32dce2da TC |
424 | + ZLIB_FOOT_SZ > PAGE_SIZE) { |
425 | status = CPA_STATUS_INCOMPRESSIBLE; | |
6a9d6359 | 426 | goto fail; |
32dce2da | 427 | } |
6a9d6359 | 428 | |
a3df7fa7 | 429 | /* jump to the end of the buffer and append footer */ |
430 | flat_buf_dst->pData = | |
431 | (char *)((unsigned long)flat_buf_dst->pData & PAGE_MASK) | |
432 | + ((compressed_sz + hdr_sz) % PAGE_SIZE); | |
6a9d6359 | 433 | flat_buf_dst->dataLenInBytes = ZLIB_FOOT_SZ; |
434 | ||
435 | dc_results.produced = 0; | |
436 | status = cpaDcGenerateFooter(session_handle, | |
437 | flat_buf_dst, &dc_results); | |
32dce2da | 438 | if (status != CPA_STATUS_SUCCESS) |
6a9d6359 | 439 | goto fail; |
6a9d6359 | 440 | |
441 | *c_len = compressed_sz + dc_results.produced + hdr_sz; | |
6a9d6359 | 442 | QAT_STAT_INCR(comp_total_out_bytes, *c_len); |
cf637391 TC |
443 | } else { |
444 | ASSERT3U(dir, ==, QAT_DECOMPRESS); | |
6a9d6359 | 445 | QAT_STAT_BUMP(decomp_requests); |
446 | QAT_STAT_INCR(decomp_total_in_bytes, src_len); | |
447 | ||
448 | buf_list_src->pBuffers->pData += ZLIB_HEAD_SZ; | |
449 | buf_list_src->pBuffers->dataLenInBytes -= ZLIB_HEAD_SZ; | |
32dce2da TC |
450 | status = cpaDcDecompressData(dc_inst_handle, session_handle, |
451 | buf_list_src, buf_list_dst, &dc_results, CPA_DC_FLUSH_FINAL, | |
6a9d6359 | 452 | &complete); |
453 | ||
454 | if (CPA_STATUS_SUCCESS != status) { | |
455 | status = CPA_STATUS_FAIL; | |
456 | goto fail; | |
457 | } | |
458 | ||
459 | /* we now wait until the completion of the operation. */ | |
460 | if (!wait_for_completion_interruptible_timeout(&complete, | |
cf637391 | 461 | QAT_TIMEOUT_MS)) { |
6a9d6359 | 462 | status = CPA_STATUS_FAIL; |
463 | goto fail; | |
464 | } | |
465 | ||
466 | if (dc_results.status != CPA_STATUS_SUCCESS) { | |
467 | status = CPA_STATUS_FAIL; | |
468 | goto fail; | |
469 | } | |
470 | ||
471 | *c_len = dc_results.produced; | |
6a9d6359 | 472 | QAT_STAT_INCR(decomp_total_out_bytes, *c_len); |
6a9d6359 | 473 | } |
474 | ||
475 | fail: | |
32dce2da | 476 | if (status != CPA_STATUS_SUCCESS && status != CPA_STATUS_INCOMPRESSIBLE) |
6a9d6359 | 477 | QAT_STAT_BUMP(dc_fails); |
6a9d6359 | 478 | |
479 | if (in_pages) { | |
480 | for (page_num = 0; | |
481 | page_num < buf_list_src->numBuffers; | |
482 | page_num++) { | |
483 | kunmap(in_pages[page_num]); | |
484 | } | |
cf637391 | 485 | QAT_PHYS_CONTIG_FREE(in_pages); |
6a9d6359 | 486 | } |
487 | ||
488 | if (out_pages) { | |
32dce2da | 489 | for (page_num = 0; page_num < dst_pages; page_num++) { |
6a9d6359 | 490 | kunmap(out_pages[page_num]); |
491 | } | |
cf637391 | 492 | QAT_PHYS_CONTIG_FREE(out_pages); |
6a9d6359 | 493 | } |
494 | ||
32dce2da TC |
495 | if (add_pages) { |
496 | for (page_num = 0; | |
497 | page_num < buf_list_dst->numBuffers - dst_pages; | |
498 | page_num++) { | |
499 | kunmap(add_pages[page_num]); | |
500 | } | |
501 | QAT_PHYS_CONTIG_FREE(add_pages); | |
502 | } | |
503 | ||
cf637391 TC |
504 | QAT_PHYS_CONTIG_FREE(buffer_meta_src); |
505 | QAT_PHYS_CONTIG_FREE(buffer_meta_dst); | |
506 | QAT_PHYS_CONTIG_FREE(buf_list_src); | |
507 | QAT_PHYS_CONTIG_FREE(buf_list_dst); | |
6a9d6359 | 508 | |
32dce2da TC |
509 | return (status); |
510 | } | |
511 | ||
512 | /* | |
513 | * Entry point for QAT accelerated compression / decompression. | |
514 | */ | |
515 | int | |
516 | qat_compress(qat_compress_dir_t dir, char *src, int src_len, | |
517 | char *dst, int dst_len, size_t *c_len) | |
518 | { | |
519 | int ret; | |
520 | size_t add_len = 0; | |
521 | void *add = NULL; | |
522 | ||
523 | if (dir == QAT_COMPRESS) { | |
524 | add_len = dst_len; | |
525 | add = zio_data_buf_alloc(add_len); | |
526 | } | |
527 | ||
528 | ret = qat_compress_impl(dir, src, src_len, dst, | |
529 | dst_len, add, add_len, c_len); | |
530 | ||
531 | if (dir == QAT_COMPRESS) | |
532 | zio_data_buf_free(add, add_len); | |
533 | ||
6a9d6359 | 534 | return (ret); |
535 | } | |
536 | ||
38742209 TC |
537 | module_param(zfs_qat_compress_disable, int, 0644); |
538 | MODULE_PARM_DESC(zfs_qat_compress_disable, "Disable QAT compression"); | |
539 | ||
6a9d6359 | 540 | #endif |