module/zfs/qat_compress.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 #if defined(_KERNEL) && defined(HAVE_QAT)
  23 #include <linux/slab.h>
  24 #include <linux/vmalloc.h>
  25 #include <linux/pagemap.h>
  26 #include <linux/completion.h>
  27 #include <sys/zfs_context.h>
  28 #include <sys/zio.h>
  29 #include "qat.h"
  30
  31 /*
  32  * Max instances in a QAT device, each instance is a channel to submit
  33  * jobs to QAT hardware, this is only for pre-allocating instance and
  34  * session arrays; the actual number of instances are defined in the
  35  * QAT driver's configuration file.
  36  */
  37 #define QAT_DC_MAX_INSTANCES    48
  38
  39 /*
  40  * ZLIB head and foot size
  41  */
  42 #define ZLIB_HEAD_SZ            2
  43 #define ZLIB_FOOT_SZ            4
  44
  45 static CpaInstanceHandle dc_inst_handles[QAT_DC_MAX_INSTANCES];
  46 static CpaDcSessionHandle session_handles[QAT_DC_MAX_INSTANCES];
  47 static CpaBufferList **buffer_array[QAT_DC_MAX_INSTANCES];
  48 static Cpa16U num_inst = 0;
  49 static Cpa32U inst_num = 0;
  50 static boolean_t qat_dc_init_done = B_FALSE;
  51 int zfs_qat_compress_disable = 0;
  52
  53 boolean_t
  54 qat_dc_use_accel(size_t s_len)
  55 {
  56         return (!zfs_qat_compress_disable &&
  57             qat_dc_init_done &&
  58             s_len >= QAT_MIN_BUF_SIZE &&
  59             s_len <= QAT_MAX_BUF_SIZE);
  60 }
  61
  62 static void
  63 qat_dc_callback(void *p_callback, CpaStatus status)
  64 {
  65         if (p_callback != NULL)
  66                 complete((struct completion *)p_callback);
  67 }
  68
  69 static void
  70 qat_dc_clean(void)
  71 {
  72         Cpa16U buff_num = 0;
  73         Cpa16U num_inter_buff_lists = 0;
  74         Cpa16U i = 0;
  75
  76         for (i = 0; i < num_inst; i++) {
  77                 cpaDcStopInstance(dc_inst_handles[i]);
  78                 QAT_PHYS_CONTIG_FREE(session_handles[i]);
  79                 /* free intermediate buffers  */
  80                 if (buffer_array[i] != NULL) {
  81                         cpaDcGetNumIntermediateBuffers(
  82                             dc_inst_handles[i], &num_inter_buff_lists);
  83                         for (buff_num = 0; buff_num < num_inter_buff_lists;
  84                             buff_num++) {
  85                                 CpaBufferList *buffer_inter =
  86                                     buffer_array[i][buff_num];
  87                                 if (buffer_inter->pBuffers) {
  88                                         QAT_PHYS_CONTIG_FREE(
  89                                             buffer_inter->pBuffers->pData);
  90                                         QAT_PHYS_CONTIG_FREE(
  91                                             buffer_inter->pBuffers);
  92                                 }
  93                                 QAT_PHYS_CONTIG_FREE(
  94                                     buffer_inter->pPrivateMetaData);
  95                                 QAT_PHYS_CONTIG_FREE(buffer_inter);
  96                         }
  97                 }
  98         }
  99
 100         num_inst = 0;
 101         qat_dc_init_done = B_FALSE;
 102 }
 103
 104 int
 105 qat_dc_init(void)
 106 {
 107         CpaStatus status = CPA_STATUS_SUCCESS;
 108         Cpa32U sess_size = 0;
 109         Cpa32U ctx_size = 0;
 110         Cpa16U num_inter_buff_lists = 0;
 111         Cpa16U buff_num = 0;
 112         Cpa32U buff_meta_size = 0;
 113         CpaDcSessionSetupData sd = {0};
 114         Cpa16U i;
 115
 116         status = cpaDcGetNumInstances(&num_inst);
 117         if (status != CPA_STATUS_SUCCESS)
 118                 return (-1);
 119
 120         /* if the user has configured no QAT compression units just return */
 121         if (num_inst == 0)
 122                 return (0);
 123
 124         if (num_inst > QAT_DC_MAX_INSTANCES)
 125                 num_inst = QAT_DC_MAX_INSTANCES;
 126
 127         status = cpaDcGetInstances(num_inst, &dc_inst_handles[0]);
 128         if (status != CPA_STATUS_SUCCESS)
 129                 return (-1);
 130
 131         for (i = 0; i < num_inst; i++) {
 132                 cpaDcSetAddressTranslation(dc_inst_handles[i],
 133                     (void*)virt_to_phys);
 134
 135                 status = cpaDcBufferListGetMetaSize(dc_inst_handles[i],
 136                     1, &buff_meta_size);
 137
 138                 if (status == CPA_STATUS_SUCCESS)
 139                         status = cpaDcGetNumIntermediateBuffers(
 140                             dc_inst_handles[i], &num_inter_buff_lists);
 141
 142                 if (status == CPA_STATUS_SUCCESS && num_inter_buff_lists != 0)
 143                         status = QAT_PHYS_CONTIG_ALLOC(&buffer_array[i],
 144                             num_inter_buff_lists *
 145                             sizeof (CpaBufferList *));
 146
 147                 for (buff_num = 0; buff_num < num_inter_buff_lists;
 148                     buff_num++) {
 149                         if (status == CPA_STATUS_SUCCESS)
 150                                 status = QAT_PHYS_CONTIG_ALLOC(
 151                                     &buffer_array[i][buff_num],
 152                                     sizeof (CpaBufferList));
 153
 154                         if (status == CPA_STATUS_SUCCESS)
 155                                 status = QAT_PHYS_CONTIG_ALLOC(
 156                                     &buffer_array[i][buff_num]->
 157                                     pPrivateMetaData,
 158                                     buff_meta_size);
 159
 160                         if (status == CPA_STATUS_SUCCESS)
 161                                 status = QAT_PHYS_CONTIG_ALLOC(
 162                                     &buffer_array[i][buff_num]->pBuffers,
 163                                     sizeof (CpaFlatBuffer));
 164
 165                         if (status == CPA_STATUS_SUCCESS) {
 166                                 /*
 167                                  *  implementation requires an intermediate
 168                                  *  buffer approximately twice the size of
 169                                  *  output buffer, which is 2x max buffer
 170                                  *  size here.
 171                                  */
 172                                 status = QAT_PHYS_CONTIG_ALLOC(
 173                                     &buffer_array[i][buff_num]->pBuffers->
 174                                     pData, 2 * QAT_MAX_BUF_SIZE);
 175                                 if (status != CPA_STATUS_SUCCESS)
 176                                         goto fail;
 177
 178                                 buffer_array[i][buff_num]->numBuffers = 1;
 179                                 buffer_array[i][buff_num]->pBuffers->
 180                                     dataLenInBytes = 2 * QAT_MAX_BUF_SIZE;
 181                         }
 182                 }
 183
 184                 status = cpaDcStartInstance(dc_inst_handles[i],
 185                     num_inter_buff_lists, buffer_array[i]);
 186                 if (status != CPA_STATUS_SUCCESS)
 187                         goto fail;
 188
 189                 sd.compLevel = CPA_DC_L1;
 190                 sd.compType = CPA_DC_DEFLATE;
 191                 sd.huffType = CPA_DC_HT_FULL_DYNAMIC;
 192                 sd.sessDirection = CPA_DC_DIR_COMBINED;
 193                 sd.sessState = CPA_DC_STATELESS;
 194                 sd.deflateWindowSize = 7;
 195                 sd.checksum = CPA_DC_ADLER32;
 196                 status = cpaDcGetSessionSize(dc_inst_handles[i],
 197                     &sd, &sess_size, &ctx_size);
 198                 if (status != CPA_STATUS_SUCCESS)
 199                         goto fail;
 200
 201                 QAT_PHYS_CONTIG_ALLOC(&session_handles[i], sess_size);
 202                 if (session_handles[i] == NULL)
 203                         goto fail;
 204
 205                 status = cpaDcInitSession(dc_inst_handles[i],
 206                     session_handles[i],
 207                     &sd, NULL, qat_dc_callback);
 208                 if (status != CPA_STATUS_SUCCESS)
 209                         goto fail;
 210         }
 211
 212         qat_dc_init_done = B_TRUE;
 213         return (0);
 214 fail:
 215         qat_dc_clean();
 216         return (-1);
 217 }
 218
 219 void
 220 qat_dc_fini(void)
 221 {
 222         if (!qat_dc_init_done)
 223                 return;
 224
 225         qat_dc_clean();
 226 }
 227
 228 /*
 229  * The "add" parameter is an additional buffer which is passed
 230  * to QAT as a scratch buffer alongside the destination buffer
 231  * in case the "compressed" data ends up being larger than the
 232  * original source data. This is necessary to prevent QAT from
 233  * generating buffer overflow warnings for incompressible data.
 234  */
 235 static int
 236 qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len,
 237     char *dst, int dst_len, char *add, int add_len, size_t *c_len)
 238 {
 239         CpaInstanceHandle dc_inst_handle;
 240         CpaDcSessionHandle session_handle;
 241         CpaBufferList *buf_list_src = NULL;
 242         CpaBufferList *buf_list_dst = NULL;
 243         CpaFlatBuffer *flat_buf_src = NULL;
 244         CpaFlatBuffer *flat_buf_dst = NULL;
 245         Cpa8U *buffer_meta_src = NULL;
 246         Cpa8U *buffer_meta_dst = NULL;
 247         Cpa32U buffer_meta_size = 0;
 248         CpaDcRqResults dc_results;
 249         CpaStatus status = CPA_STATUS_SUCCESS;
 250         Cpa32U hdr_sz = 0;
 251         Cpa32U compressed_sz;
 252         Cpa32U num_src_buf = (src_len >> PAGE_SHIFT) + 2;
 253         Cpa32U num_dst_buf = (dst_len >> PAGE_SHIFT) + 2;
 254         Cpa32U num_add_buf = (add_len >> PAGE_SHIFT) + 2;
 255         Cpa32U bytes_left;
 256         Cpa32U dst_pages = 0;
 257         char *data;
 258         struct page *page;
 259         struct page **in_pages = NULL;
 260         struct page **out_pages = NULL;
 261         struct page **add_pages = NULL;
 262         Cpa32U page_off = 0;
 263         struct completion complete;
 264         Cpa32U page_num = 0;
 265         Cpa16U i;
 266
 267         /*
 268          * We increment num_src_buf and num_dst_buf by 2 to allow
 269          * us to handle non page-aligned buffer addresses and buffers
 270          * whose sizes are not divisible by PAGE_SIZE.
 271          */
 272         Cpa32U src_buffer_list_mem_size = sizeof (CpaBufferList) +
 273             (num_src_buf * sizeof (CpaFlatBuffer));
 274         Cpa32U dst_buffer_list_mem_size = sizeof (CpaBufferList) +
 275             ((num_dst_buf + num_add_buf) * sizeof (CpaFlatBuffer));
 276
 277         if (QAT_PHYS_CONTIG_ALLOC(&in_pages,
 278             num_src_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS)
 279                 goto fail;
 280
 281         if (QAT_PHYS_CONTIG_ALLOC(&out_pages,
 282             num_dst_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS)
 283                 goto fail;
 284
 285         if (QAT_PHYS_CONTIG_ALLOC(&add_pages,
 286             num_add_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS)
 287                 goto fail;
 288
 289         i = atomic_inc_32_nv(&inst_num) % num_inst;
 290         dc_inst_handle = dc_inst_handles[i];
 291         session_handle = session_handles[i];
 292
 293         cpaDcBufferListGetMetaSize(dc_inst_handle, num_src_buf,
 294             &buffer_meta_size);
 295         if (QAT_PHYS_CONTIG_ALLOC(&buffer_meta_src, buffer_meta_size) !=
 296             CPA_STATUS_SUCCESS)
 297                 goto fail;
 298
 299         cpaDcBufferListGetMetaSize(dc_inst_handle, num_dst_buf + num_add_buf,
 300             &buffer_meta_size);
 301         if (QAT_PHYS_CONTIG_ALLOC(&buffer_meta_dst, buffer_meta_size) !=
 302             CPA_STATUS_SUCCESS)
 303                 goto fail;
 304
 305         /* build source buffer list */
 306         if (QAT_PHYS_CONTIG_ALLOC(&buf_list_src, src_buffer_list_mem_size) !=
 307             CPA_STATUS_SUCCESS)
 308                 goto fail;
 309
 310         flat_buf_src = (CpaFlatBuffer *)(buf_list_src + 1);
 311
 312         buf_list_src->pBuffers = flat_buf_src; /* always point to first one */
 313
 314         /* build destination buffer list */
 315         if (QAT_PHYS_CONTIG_ALLOC(&buf_list_dst, dst_buffer_list_mem_size) !=
 316             CPA_STATUS_SUCCESS)
 317                 goto fail;
 318
 319         flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1);
 320
 321         buf_list_dst->pBuffers = flat_buf_dst; /* always point to first one */
 322
 323         buf_list_src->numBuffers = 0;
 324         buf_list_src->pPrivateMetaData = buffer_meta_src;
 325         bytes_left = src_len;
 326         data = src;
 327         page_num = 0;
 328         while (bytes_left > 0) {
 329                 page_off = ((long)data & ~PAGE_MASK);
 330                 page = qat_mem_to_page(data);
 331                 in_pages[page_num] = page;
 332                 flat_buf_src->pData = kmap(page) + page_off;
 333                 flat_buf_src->dataLenInBytes =
 334                     min((long)PAGE_SIZE - page_off, (long)bytes_left);
 335
 336                 bytes_left -= flat_buf_src->dataLenInBytes;
 337                 data += flat_buf_src->dataLenInBytes;
 338                 flat_buf_src++;
 339                 buf_list_src->numBuffers++;
 340                 page_num++;
 341         }
 342
 343         buf_list_dst->numBuffers = 0;
 344         buf_list_dst->pPrivateMetaData = buffer_meta_dst;
 345         bytes_left = dst_len;
 346         data = dst;
 347         page_num = 0;
 348         while (bytes_left > 0) {
 349                 page_off = ((long)data & ~PAGE_MASK);
 350                 page = qat_mem_to_page(data);
 351                 flat_buf_dst->pData = kmap(page) + page_off;
 352                 out_pages[page_num] = page;
 353                 flat_buf_dst->dataLenInBytes =
 354                     min((long)PAGE_SIZE - page_off, (long)bytes_left);
 355
 356                 bytes_left -= flat_buf_dst->dataLenInBytes;
 357                 data += flat_buf_dst->dataLenInBytes;
 358                 flat_buf_dst++;
 359                 buf_list_dst->numBuffers++;
 360                 page_num++;
 361                 dst_pages++;
 362         }
 363
 364         /* map additional scratch pages into the destination buffer list */
 365         bytes_left = add_len;
 366         data = add;
 367         page_num = 0;
 368         while (bytes_left > 0) {
 369                 page_off = ((long)data & ~PAGE_MASK);
 370                 page = qat_mem_to_page(data);
 371                 flat_buf_dst->pData = kmap(page) + page_off;
 372                 add_pages[page_num] = page;
 373                 flat_buf_dst->dataLenInBytes =
 374                     min((long)PAGE_SIZE - page_off, (long)bytes_left);
 375
 376                 bytes_left -= flat_buf_dst->dataLenInBytes;
 377                 data += flat_buf_dst->dataLenInBytes;
 378                 flat_buf_dst++;
 379                 buf_list_dst->numBuffers++;
 380                 page_num++;
 381         }
 382
 383         init_completion(&complete);
 384
 385         if (dir == QAT_COMPRESS) {
 386                 QAT_STAT_BUMP(comp_requests);
 387                 QAT_STAT_INCR(comp_total_in_bytes, src_len);
 388
 389                 cpaDcGenerateHeader(session_handle,
 390                     buf_list_dst->pBuffers, &hdr_sz);
 391                 buf_list_dst->pBuffers->pData += hdr_sz;
 392                 buf_list_dst->pBuffers->dataLenInBytes -= hdr_sz;
 393                 status = cpaDcCompressData(
 394                     dc_inst_handle, session_handle,
 395                     buf_list_src, buf_list_dst,
 396                     &dc_results, CPA_DC_FLUSH_FINAL,
 397                     &complete);
 398                 if (status != CPA_STATUS_SUCCESS) {
 399                         goto fail;
 400                 }
 401
 402                 /* we now wait until the completion of the operation. */
 403                 if (!wait_for_completion_interruptible_timeout(&complete,
 404                     QAT_TIMEOUT_MS)) {
 405                         status = CPA_STATUS_FAIL;
 406                         goto fail;
 407                 }
 408
 409                 if (dc_results.status != CPA_STATUS_SUCCESS) {
 410                         status = CPA_STATUS_FAIL;
 411                         goto fail;
 412                 }
 413
 414                 compressed_sz = dc_results.produced;
 415                 if (compressed_sz + hdr_sz + ZLIB_FOOT_SZ > dst_len) {
 416                         status = CPA_STATUS_INCOMPRESSIBLE;
 417                         goto fail;
 418                 }
 419
 420                 flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1);
 421                 /* move to the last page */
 422                 flat_buf_dst += (compressed_sz + hdr_sz) >> PAGE_SHIFT;
 423
 424                 /* no space for gzip footer in the last page */
 425                 if (((compressed_sz + hdr_sz) % PAGE_SIZE)
 426                     + ZLIB_FOOT_SZ > PAGE_SIZE) {
 427                         status = CPA_STATUS_INCOMPRESSIBLE;
 428                         goto fail;
 429                 }
 430
 431                 /* jump to the end of the buffer and append footer */
 432                 flat_buf_dst->pData =
 433                     (char *)((unsigned long)flat_buf_dst->pData & PAGE_MASK)
 434                     + ((compressed_sz + hdr_sz) % PAGE_SIZE);
 435                 flat_buf_dst->dataLenInBytes = ZLIB_FOOT_SZ;
 436
 437                 dc_results.produced = 0;
 438                 status = cpaDcGenerateFooter(session_handle,
 439                     flat_buf_dst, &dc_results);
 440                 if (status != CPA_STATUS_SUCCESS)
 441                         goto fail;
 442
 443                 *c_len = compressed_sz + dc_results.produced + hdr_sz;
 444                 QAT_STAT_INCR(comp_total_out_bytes, *c_len);
 445         } else {
 446                 ASSERT3U(dir, ==, QAT_DECOMPRESS);
 447                 QAT_STAT_BUMP(decomp_requests);
 448                 QAT_STAT_INCR(decomp_total_in_bytes, src_len);
 449
 450                 buf_list_src->pBuffers->pData += ZLIB_HEAD_SZ;
 451                 buf_list_src->pBuffers->dataLenInBytes -= ZLIB_HEAD_SZ;
 452                 status = cpaDcDecompressData(dc_inst_handle, session_handle,
 453                     buf_list_src, buf_list_dst, &dc_results, CPA_DC_FLUSH_FINAL,
 454                     &complete);
 455
 456                 if (CPA_STATUS_SUCCESS != status) {
 457                         status = CPA_STATUS_FAIL;
 458                         goto fail;
 459                 }
 460
 461                 /* we now wait until the completion of the operation. */
 462                 if (!wait_for_completion_interruptible_timeout(&complete,
 463                     QAT_TIMEOUT_MS)) {
 464                         status = CPA_STATUS_FAIL;
 465                         goto fail;
 466                 }
 467
 468                 if (dc_results.status != CPA_STATUS_SUCCESS) {
 469                         status = CPA_STATUS_FAIL;
 470                         goto fail;
 471                 }
 472
 473                 *c_len = dc_results.produced;
 474                 QAT_STAT_INCR(decomp_total_out_bytes, *c_len);
 475         }
 476
 477 fail:
 478         if (status != CPA_STATUS_SUCCESS && status != CPA_STATUS_INCOMPRESSIBLE)
 479                 QAT_STAT_BUMP(dc_fails);
 480
 481         if (in_pages) {
 482                 for (page_num = 0;
 483                     page_num < buf_list_src->numBuffers;
 484                     page_num++) {
 485                         kunmap(in_pages[page_num]);
 486                 }
 487                 QAT_PHYS_CONTIG_FREE(in_pages);
 488         }
 489
 490         if (out_pages) {
 491                 for (page_num = 0; page_num < dst_pages; page_num++) {
 492                         kunmap(out_pages[page_num]);
 493                 }
 494                 QAT_PHYS_CONTIG_FREE(out_pages);
 495         }
 496
 497         if (add_pages) {
 498                 for (page_num = 0;
 499                     page_num < buf_list_dst->numBuffers - dst_pages;
 500                     page_num++) {
 501                         kunmap(add_pages[page_num]);
 502                 }
 503                 QAT_PHYS_CONTIG_FREE(add_pages);
 504         }
 505
 506         QAT_PHYS_CONTIG_FREE(buffer_meta_src);
 507         QAT_PHYS_CONTIG_FREE(buffer_meta_dst);
 508         QAT_PHYS_CONTIG_FREE(buf_list_src);
 509         QAT_PHYS_CONTIG_FREE(buf_list_dst);
 510
 511         return (status);
 512 }
 513
 514 /*
 515  * Entry point for QAT accelerated compression / decompression.
 516  */
 517 int
 518 qat_compress(qat_compress_dir_t dir, char *src, int src_len,
 519     char *dst, int dst_len, size_t *c_len)
 520 {
 521         int ret;
 522         size_t add_len = 0;
 523         void *add = NULL;
 524
 525         if (dir == QAT_COMPRESS) {
 526                 add_len = dst_len;
 527                 add = zio_data_buf_alloc(add_len);
 528         }
 529
 530         ret = qat_compress_impl(dir, src, src_len, dst,
 531             dst_len, add, add_len, c_len);
 532
 533         if (dir == QAT_COMPRESS)
 534                 zio_data_buf_free(add, add_len);
 535
 536         return (ret);
 537 }
 538
 539 module_param(zfs_qat_compress_disable, int, 0644);
 540 MODULE_PARM_DESC(zfs_qat_compress_disable, "Disable QAT compression");
 541
 542 #endif