compress: change zio_compress API to use ABDs

This commit changes the frontend zio_compress_data and
zio_decompress_data APIs to take ABD points instead of buffer pointers.

All callers are updated to match. Any that already have an appropriate
ABD nearby now use it directly, while at the rest we create an one.

Internally, the ABDs are passed through to the provider directly.

Sponsored-by: Klara, Inc.
Sponsored-by: Wasabi Technology, Inc.
Signed-off-by: Rob Norris <rob.norris@klarasystems.com>

diff --git a/cmd/zdb/zdb.c b/cmd/zdb/zdb.c
index c72df3909356a53d7c5275f93a22eaf8e45506d2..41c2b67655852ca18cda4d060446e2f090a755a1 100644 (file)
--- a/cmd/zdb/zdb.c
+++ b/cmd/zdb/zdb.c
@@ -4657,7 +4657,6 @@ dump_l2arc_log_blocks(int fd, const l2arc_dev_hdr_phys_t *l2dhdr,
        l2arc_log_blk_phys_t this_lb;
        uint64_t asize;
        l2arc_log_blkptr_t lbps[2];
-       abd_t *abd;
        zio_cksum_t cksum;
        int failed = 0;
        l2arc_dev_t dev;
@@ -4711,20 +4710,25 @@ dump_l2arc_log_blocks(int fd, const l2arc_dev_hdr_phys_t *l2dhdr,
                switch (L2BLK_GET_COMPRESS((&lbps[0])->lbp_prop)) {
                case ZIO_COMPRESS_OFF:
                        break;
-               default:
-                       abd = abd_alloc_for_io(asize, B_TRUE);
+               default: {
+                       abd_t *abd = abd_alloc_linear(asize, B_TRUE);
                        abd_copy_from_buf_off(abd, &this_lb, 0, asize);
-                       if (zio_decompress_data(L2BLK_GET_COMPRESS(
-                           (&lbps[0])->lbp_prop), abd, &this_lb,
-                           asize, sizeof (this_lb), NULL) != 0) {
+                       abd_t dabd;
+                       abd_get_from_buf_struct(&dabd, &this_lb,
+                           sizeof (this_lb));
+                       int err = zio_decompress_data(L2BLK_GET_COMPRESS(
+                           (&lbps[0])->lbp_prop), abd, &dabd,
+                           asize, sizeof (this_lb), NULL);
+                       abd_free(&dabd);
+                       abd_free(abd);
+                       if (err != 0) {
                                (void) printf("L2ARC block decompression "
                                    "failed\n");
-                               abd_free(abd);
                                goto out;
                        }
-                       abd_free(abd);
                        break;
                }
+               }
 
                if (this_lb.lb_magic == BSWAP_64(L2ARC_LOG_BLK_MAGIC))
                        byteswap_uint64_array(&this_lb, sizeof (this_lb));
@@ -8618,13 +8622,22 @@ try_decompress_block(abd_t *pabd, uint64_t lsize, uint64_t psize,
        memset(lbuf, 0x00, lsize);
        memset(lbuf2, 0xff, lsize);
 
+       abd_t labd, labd2;
+       abd_get_from_buf_struct(&labd, lbuf, lsize);
+       abd_get_from_buf_struct(&labd2, lbuf2, lsize);
+
+       boolean_t ret = B_FALSE;
        if (zio_decompress_data(cfunc, pabd,
-           lbuf, psize, lsize, NULL) == 0 &&
+           &labd, psize, lsize, NULL) == 0 &&
            zio_decompress_data(cfunc, pabd,
-           lbuf2, psize, lsize, NULL) == 0 &&
+           &labd2, psize, lsize, NULL) == 0 &&
            memcmp(lbuf, lbuf2, lsize) == 0)
-               return (B_TRUE);
-       return (B_FALSE);
+               ret = B_TRUE;
+
+       abd_free(&labd2);
+       abd_free(&labd);
+
+       return (ret);
 }
 
 static uint64_t

diff --git a/cmd/zstream/zstream_recompress.c b/cmd/zstream/zstream_recompress.c
index 0e5cc9cd8158f6e92c91bc52db5a587a5af53c17..32ef6fa54433a08675dfc43c12cfe6dc8394e4a4 100644 (file)
--- a/cmd/zstream/zstream_recompress.c
+++ b/cmd/zstream/zstream_recompress.c
@@ -259,12 +259,13 @@ zstream_do_recompress(int argc, char *argv[])
                        /* Read and decompress the payload */
                        (void) sfread(cbuf, payload_size, stdin);
                        if (dtype != ZIO_COMPRESS_OFF) {
-                               abd_t cabd;
+                               abd_t cabd, dabd;
                                abd_get_from_buf_struct(&cabd,
                                    cbuf, payload_size);
-                               if (zio_decompress_data(dtype, &cabd, dbuf,
-                                   payload_size,
-                                   MIN(bufsz, drrw->drr_logical_size),
+                               abd_get_from_buf_struct(&dabd, dbuf,
+                                   MIN(bufsz, drrw->drr_logical_size));
+                               if (zio_decompress_data(dtype, &cabd, &dabd,
+                                   payload_size, abd_get_size(&dabd),
                                    NULL) != 0) {
                                        warnx("decompression type %d failed "
                                            "for ino %llu offset %llu",
@@ -274,17 +275,20 @@ zstream_do_recompress(int argc, char *argv[])
                                        exit(4);
                                }
                                payload_size = drrw->drr_logical_size;
+                               abd_free(&dabd);
                                abd_free(&cabd);
                                free(cbuf);
                        }
 
                        /* Recompress the payload */
                        if (ctype != ZIO_COMPRESS_OFF) {
-                               abd_t dabd;
+                               abd_t dabd, abd;
                                abd_get_from_buf_struct(&dabd,
                                    dbuf, drrw->drr_logical_size);
+                               abd_t *pabd =
+                                   abd_get_from_buf_struct(&abd, buf, bufsz);
                                payload_size = P2ROUNDUP(zio_compress_data(
-                                   ctype, &dabd, (void **)&buf,
+                                   ctype, &dabd, &pabd,
                                    drrw->drr_logical_size, level),
                                    SPA_MINBLOCKSIZE);
                                if (payload_size != drrw->drr_logical_size) {
@@ -296,6 +300,7 @@ zstream_do_recompress(int argc, char *argv[])
                                        drrw->drr_compressiontype = 0;
                                        drrw->drr_compressed_size = 0;
                                }
+                               abd_free(&abd);
                                abd_free(&dabd);
                                free(dbuf);
                        } else {

diff --git a/include/sys/zio_compress.h b/include/sys/zio_compress.h
index d41b5dfd447f8c7c4df99611763a0dc8c2bbc021..31602039a150dd713d26335df346f418c340dd2c 100644 (file)
--- a/include/sys/zio_compress.h
+++ b/include/sys/zio_compress.h
@@ -173,9 +173,9 @@ extern int zfs_lz4_decompress(abd_t *src, abd_t *dst, size_t s_len,
 /*
  * Compress and decompress data if necessary.
  */
-extern size_t zio_compress_data(enum zio_compress c, abd_t *src, void **dst,
+extern size_t zio_compress_data(enum zio_compress c, abd_t *src, abd_t **dst,
     size_t s_len, uint8_t level);
-extern int zio_decompress_data(enum zio_compress c, abd_t *src, void *dst,
+extern int zio_decompress_data(enum zio_compress c, abd_t *src, abd_t *abd,
     size_t s_len, size_t d_len, uint8_t *level);
 extern int zio_compress_to_feature(enum zio_compress comp);
 

diff --git a/module/zfs/arc.c b/module/zfs/arc.c
index 3c657c979cdcde2ad9a5dc2a024b598f6d86f19e..714a30e863a7fbff4d291e77dedacc1032c1c895 100644 (file)
--- a/module/zfs/arc.c
+++ b/module/zfs/arc.c
@@ -1767,12 +1767,12 @@ arc_hdr_authenticate(arc_buf_hdr_t *hdr, spa_t *spa, uint64_t dsobj)
        uint64_t csize;
        uint64_t lsize = HDR_GET_LSIZE(hdr);
        uint64_t psize = HDR_GET_PSIZE(hdr);
-       void *tmpbuf = NULL;
        abd_t *abd = hdr->b_l1hdr.b_pabd;
+       boolean_t free_abd = B_FALSE;
 
        ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
        ASSERT(HDR_AUTHENTICATED(hdr));
-       ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
+       ASSERT3P(abd, !=, NULL);
 
        /*
         * The MAC is calculated on the compressed data that is stored on disk.
@@ -1784,14 +1784,13 @@ arc_hdr_authenticate(arc_buf_hdr_t *hdr, spa_t *spa, uint64_t dsobj)
         */
        if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
            !HDR_COMPRESSION_ENABLED(hdr)) {
-
+               abd = NULL;
                csize = zio_compress_data(HDR_GET_COMPRESS(hdr),
-                   hdr->b_l1hdr.b_pabd, &tmpbuf, lsize, hdr->b_complevel);
-               ASSERT3P(tmpbuf, !=, NULL);
+                   hdr->b_l1hdr.b_pabd, &abd, lsize, hdr->b_complevel);
+               ASSERT3P(abd, !=, NULL);
                ASSERT3U(csize, <=, psize);
-               abd = abd_get_from_buf(tmpbuf, lsize);
-               abd_take_ownership_of_buf(abd, B_TRUE);
                abd_zero_off(abd, csize, psize - csize);
+               free_abd = B_TRUE;
        }
 
        /*
@@ -1810,16 +1809,10 @@ arc_hdr_authenticate(arc_buf_hdr_t *hdr, spa_t *spa, uint64_t dsobj)
 
        if (ret == 0)
                arc_hdr_clear_flags(hdr, ARC_FLAG_NOAUTH);
-       else if (ret != ENOENT)
-               goto error;
+       else if (ret == ENOENT)
+               ret = 0;
 
-       if (tmpbuf != NULL)
-               abd_free(abd);
-
-       return (0);
-
-error:
-       if (tmpbuf != NULL)
+       if (free_abd)
                abd_free(abd);
 
        return (ret);
@@ -1836,7 +1829,6 @@ arc_hdr_decrypt(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb)
 {
        int ret;
        abd_t *cabd = NULL;
-       void *tmp = NULL;
        boolean_t no_crypt = B_FALSE;
        boolean_t bswap = (hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);
 
@@ -1871,17 +1863,14 @@ arc_hdr_decrypt(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb)
                 * linear buffer and wrapping it in an abd later.
                 */
                cabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr, 0);
-               tmp = abd_borrow_buf(cabd, arc_hdr_size(hdr));
 
                ret = zio_decompress_data(HDR_GET_COMPRESS(hdr),
-                   hdr->b_l1hdr.b_pabd, tmp, HDR_GET_PSIZE(hdr),
+                   hdr->b_l1hdr.b_pabd, cabd, HDR_GET_PSIZE(hdr),
                    HDR_GET_LSIZE(hdr), &hdr->b_complevel);
                if (ret != 0) {
-                       abd_return_buf(cabd, tmp, arc_hdr_size(hdr));
                        goto error;
                }
 
-               abd_return_buf_copy(cabd, tmp, arc_hdr_size(hdr));
                arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
                    arc_hdr_size(hdr), hdr);
                hdr->b_l1hdr.b_pabd = cabd;
@@ -2123,10 +2112,14 @@ arc_buf_fill(arc_buf_t *buf, spa_t *spa, const zbookmark_phys_t *zb,
                        /* Skip byteswapping and checksumming (already done) */
                        return (0);
                } else {
+                       abd_t dabd;
+                       abd_get_from_buf_struct(&dabd, buf->b_data,
+                           HDR_GET_LSIZE(hdr));
                        error = zio_decompress_data(HDR_GET_COMPRESS(hdr),
-                           hdr->b_l1hdr.b_pabd, buf->b_data,
+                           hdr->b_l1hdr.b_pabd, &dabd,
                            HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr),
                            &hdr->b_complevel);
+                       abd_free(&dabd);
 
                        /*
                         * Absent hardware errors or software bugs, this should
@@ -8531,18 +8524,15 @@ l2arc_untransform(zio_t *zio, l2arc_read_callback_t *cb)
            !HDR_COMPRESSION_ENABLED(hdr)) {
                abd_t *cabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr,
                    ARC_HDR_USE_RESERVE);
-               void *tmp = abd_borrow_buf(cabd, arc_hdr_size(hdr));
 
                ret = zio_decompress_data(HDR_GET_COMPRESS(hdr),
-                   hdr->b_l1hdr.b_pabd, tmp, HDR_GET_PSIZE(hdr),
+                   hdr->b_l1hdr.b_pabd, cabd, HDR_GET_PSIZE(hdr),
                    HDR_GET_LSIZE(hdr), &hdr->b_complevel);
                if (ret != 0) {
-                       abd_return_buf_copy(cabd, tmp, arc_hdr_size(hdr));
                        arc_free_data_abd(hdr, cabd, arc_hdr_size(hdr), hdr);
                        goto error;
                }
 
-               abd_return_buf_copy(cabd, tmp, arc_hdr_size(hdr));
                arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
                    arc_hdr_size(hdr), hdr);
                hdr->b_l1hdr.b_pabd = cabd;
@@ -9037,9 +9027,8 @@ l2arc_apply_transforms(spa_t *spa, arc_buf_hdr_t *hdr, uint64_t asize,
        }
 
        if (compress != ZIO_COMPRESS_OFF && !HDR_COMPRESSION_ENABLED(hdr)) {
-               size_t bufsize = MAX(size, asize);
-               void *buf = zio_buf_alloc(bufsize);
-               uint64_t csize = zio_compress_data(compress, to_write, &buf,
+               cabd = abd_alloc_for_io(MAX(size, asize), ismd);
+               uint64_t csize = zio_compress_data(compress, to_write, &cabd,
                    size, hdr->b_complevel);
                if (csize > psize) {
                        /*
@@ -9047,13 +9036,12 @@ l2arc_apply_transforms(spa_t *spa, arc_buf_hdr_t *hdr, uint64_t asize,
                         * psize.  Even if it fits into asize, it does not
                         * matter, since checksum will never match on read.
                         */
-                       zio_buf_free(buf, bufsize);
+                       abd_free(cabd);
                        return (SET_ERROR(EIO));
                }
                if (asize > csize)
-                       memset((char *)buf + csize, 0, asize - csize);
-               to_write = cabd = abd_get_from_buf(buf, bufsize);
-               abd_take_ownership_of_buf(cabd, B_TRUE);
+                       abd_zero_off(cabd, csize, asize - csize);
+               to_write = cabd;
        }
 
        if (HDR_ENCRYPTED(hdr)) {
@@ -10184,7 +10172,6 @@ l2arc_log_blk_read(l2arc_dev_t *dev,
 {
        int             err = 0;
        zio_cksum_t     cksum;
-       abd_t           *abd = NULL;
        uint64_t        asize;
 
        ASSERT(this_lbp != NULL && next_lbp != NULL);
@@ -10246,16 +10233,22 @@ l2arc_log_blk_read(l2arc_dev_t *dev,
        switch (L2BLK_GET_COMPRESS((this_lbp)->lbp_prop)) {
        case ZIO_COMPRESS_OFF:
                break;
-       case ZIO_COMPRESS_LZ4:
-               abd = abd_alloc_for_io(asize, B_TRUE);
+       case ZIO_COMPRESS_LZ4: {
+               abd_t *abd = abd_alloc_linear(asize, B_TRUE);
                abd_copy_from_buf_off(abd, this_lb, 0, asize);
-               if ((err = zio_decompress_data(
+               abd_t dabd;
+               abd_get_from_buf_struct(&dabd, this_lb, sizeof (*this_lb));
+               err = zio_decompress_data(
                    L2BLK_GET_COMPRESS((this_lbp)->lbp_prop),
-                   abd, this_lb, asize, sizeof (*this_lb), NULL)) != 0) {
+                   abd, &dabd, asize, sizeof (*this_lb), NULL);
+               abd_free(&dabd);
+               abd_free(abd);
+               if (err != 0) {
                        err = SET_ERROR(EINVAL);
                        goto cleanup;
                }
                break;
+       }
        default:
                err = SET_ERROR(EINVAL);
                goto cleanup;
@@ -10272,8 +10265,6 @@ cleanup:
                l2arc_log_blk_fetch_abort(*next_io);
                *next_io = NULL;
        }
-       if (abd != NULL)
-               abd_free(abd);
        return (err);
 }
 
@@ -10509,7 +10500,7 @@ l2arc_log_blk_commit(l2arc_dev_t *dev, zio_t *pio, l2arc_write_callback_t *cb)
        uint64_t                psize, asize;
        zio_t                   *wzio;
        l2arc_lb_abd_buf_t      *abd_buf;
-       uint8_t                 *tmpbuf = NULL;
+       abd_t                   *abd = NULL;
        l2arc_lb_ptr_buf_t      *lb_ptr_buf;
 
        VERIFY3S(dev->l2ad_log_ent_idx, ==, dev->l2ad_log_entries);
@@ -10532,7 +10523,7 @@ l2arc_log_blk_commit(l2arc_dev_t *dev, zio_t *pio, l2arc_write_callback_t *cb)
 
        /* try to compress the buffer */
        psize = zio_compress_data(ZIO_COMPRESS_LZ4,
-           abd_buf->abd, (void **) &tmpbuf, sizeof (*lb), 0);
+           abd_buf->abd, &abd, sizeof (*lb), 0);
 
        /* a log block is never entirely zero */
        ASSERT(psize != 0);
@@ -10558,27 +10549,26 @@ l2arc_log_blk_commit(l2arc_dev_t *dev, zio_t *pio, l2arc_write_callback_t *cb)
            ZIO_CHECKSUM_FLETCHER_4);
        if (asize < sizeof (*lb)) {
                /* compression succeeded */
-               memset(tmpbuf + psize, 0, asize - psize);
+               abd_zero_off(abd, psize, asize - psize);
                L2BLK_SET_COMPRESS(
                    (&l2dhdr->dh_start_lbps[0])->lbp_prop,
                    ZIO_COMPRESS_LZ4);
        } else {
                /* compression failed */
-               memcpy(tmpbuf, lb, sizeof (*lb));
+               abd_copy_from_buf_off(abd, lb, 0, sizeof (*lb));
                L2BLK_SET_COMPRESS(
                    (&l2dhdr->dh_start_lbps[0])->lbp_prop,
                    ZIO_COMPRESS_OFF);
        }
 
        /* checksum what we're about to write */
-       fletcher_4_native(tmpbuf, asize, NULL,
+       abd_fletcher_4_native(abd, asize, NULL,
            &l2dhdr->dh_start_lbps[0].lbp_cksum);
 
        abd_free(abd_buf->abd);
 
        /* perform the write itself */
-       abd_buf->abd = abd_get_from_buf(tmpbuf, sizeof (*lb));
-       abd_take_ownership_of_buf(abd_buf->abd, B_TRUE);
+       abd_buf->abd = abd;
        wzio = zio_write_phys(pio, dev->l2ad_vdev, dev->l2ad_hand,
            asize, abd_buf->abd, ZIO_CHECKSUM_OFF, NULL, NULL,
            ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_CANFAIL, B_FALSE);

diff --git a/module/zfs/blkptr.c b/module/zfs/blkptr.c
index 6a6f06c735785ec18e18e2d2a7897656b5de9c94..ac801c2bcf3feb68c7d291a5741af9579fe7d489 100644 (file)
--- a/module/zfs/blkptr.c
+++ b/module/zfs/blkptr.c
@@ -142,11 +142,13 @@ decode_embedded_bp(const blkptr_t *bp, void *buf, int buflen)
        if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF) {
                uint8_t dstbuf[BPE_PAYLOAD_SIZE];
                decode_embedded_bp_compressed(bp, dstbuf);
-               abd_t dstabd;
-               abd_get_from_buf_struct(&dstabd, dstbuf, psize);
-               VERIFY0(zio_decompress_data(BP_GET_COMPRESS(bp), &dstabd,
-                   buf, psize, buflen, NULL));
-               abd_free(&dstabd);
+               abd_t cabd, dabd;
+               abd_get_from_buf_struct(&cabd, dstbuf, psize);
+               abd_get_from_buf_struct(&dabd, buf, buflen);
+               VERIFY0(zio_decompress_data(BP_GET_COMPRESS(bp), &cabd,
+                   &dabd, psize, buflen, NULL));
+               abd_free(&dabd);
+               abd_free(&cabd);
        } else {
                ASSERT3U(lsize, ==, psize);
                decode_embedded_bp_compressed(bp, buf);

diff --git a/module/zfs/ddt_zap.c b/module/zfs/ddt_zap.c
index e96984b86f0c97c35df5760ea96a4ea56230b407..d96dc505cdeadd14aabf0003e8599c1c2e5354c0 100644 (file)
--- a/module/zfs/ddt_zap.c
+++ b/module/zfs/ddt_zap.c
@@ -83,9 +83,11 @@ ddt_zap_decompress(uchar_t *src, void *dst, size_t s_len, size_t d_len)
                return;
        }
 
-       abd_t sabd;
+       abd_t sabd, dabd;
        abd_get_from_buf_struct(&sabd, src, s_len);
-       VERIFY0(zio_decompress_data(cpfunc, &sabd, dst, s_len, d_len, NULL));
+       abd_get_from_buf_struct(&dabd, dst, d_len);
+       VERIFY0(zio_decompress_data(cpfunc, &sabd, &dabd, s_len, d_len, NULL));
+       abd_free(&dabd);
        abd_free(&sabd);
 
        if (((version & DDT_ZAP_COMPRESS_BYTEORDER_MASK) != 0) !=

diff --git a/module/zfs/dmu_recv.c b/module/zfs/dmu_recv.c
index 0119191d79208244a84027349f02c149f8357d0f..a1752650f3bae0595f6a039b28befd7068698af9 100644 (file)
--- a/module/zfs/dmu_recv.c
+++ b/module/zfs/dmu_recv.c
@@ -1391,7 +1391,7 @@ do_corrective_recv(struct receive_writer_arg *rwa, struct drr_write *drrw,
                abd_t *dabd = abd_alloc_linear(
                    drrw->drr_logical_size, B_FALSE);
                err = zio_decompress_data(drrw->drr_compressiontype,
-                   abd, abd_to_buf(dabd), abd_get_size(abd),
+                   abd, dabd, abd_get_size(abd),
                    abd_get_size(dabd), NULL);
 
                if (err != 0) {
@@ -1407,9 +1407,8 @@ do_corrective_recv(struct receive_writer_arg *rwa, struct drr_write *drrw,
                /* Recompress the data */
                abd_t *cabd = abd_alloc_linear(BP_GET_PSIZE(bp),
                    B_FALSE);
-               void *buf = abd_to_buf(cabd);
                uint64_t csize = zio_compress_data(BP_GET_COMPRESS(bp),
-                   abd, &buf, abd_get_size(abd),
+                   abd, &cabd, abd_get_size(abd),
                    rwa->os->os_complevel);
                abd_zero_off(cabd, csize, BP_GET_PSIZE(bp) - csize);
                /* Swap in newly compressed data into the abd */
@@ -2221,7 +2220,7 @@ flush_write_batch_impl(struct receive_writer_arg *rwa)
 
                                err = zio_decompress_data(
                                    drrw->drr_compressiontype,
-                                   abd, abd_to_buf(decomp_abd),
+                                   abd, decomp_abd,
                                    abd_get_size(abd),
                                    abd_get_size(decomp_abd), NULL);
 

diff --git a/module/zfs/zio.c b/module/zfs/zio.c
index 73252c2da9703f7534efaa3aeafa6bad5d17eda6..a841e0a79107c2cba6cec16f5bdeec70edc62b2e 100644 (file)
--- a/module/zfs/zio.c
+++ b/module/zfs/zio.c
@@ -487,11 +487,9 @@ static void
 zio_decompress(zio_t *zio, abd_t *data, uint64_t size)
 {
        if (zio->io_error == 0) {
-               void *tmp = abd_borrow_buf(data, size);
                int ret = zio_decompress_data(BP_GET_COMPRESS(zio->io_bp),
-                   zio->io_abd, tmp, zio->io_size, size,
+                   zio->io_abd, data, zio->io_size, size,
                    &zio->io_prop.zp_complevel);
-               abd_return_buf_copy(data, tmp, size);
 
                if (zio_injection_enabled && ret == 0)
                        ret = zio_handle_fault_injection(zio, EINVAL);
@@ -538,17 +536,18 @@ zio_decrypt(zio_t *zio, abd_t *data, uint64_t size)
                         * from the indirect block. We decompress it now and
                         * throw away the result after we are finished.
                         */
-                       tmp = zio_buf_alloc(lsize);
+                       abd_t *abd = abd_alloc_linear(lsize, B_TRUE);
                        ret = zio_decompress_data(BP_GET_COMPRESS(bp),
-                           zio->io_abd, tmp, zio->io_size, lsize,
+                           zio->io_abd, abd, zio->io_size, lsize,
                            &zio->io_prop.zp_complevel);
                        if (ret != 0) {
+                               abd_free(abd);
                                ret = SET_ERROR(EIO);
                                goto error;
                        }
-                       ret = zio_crypt_do_indirect_mac_checksum(B_FALSE,
-                           tmp, lsize, BP_SHOULD_BYTESWAP(bp), mac);
-                       zio_buf_free(tmp, lsize);
+                       ret = zio_crypt_do_indirect_mac_checksum_abd(B_FALSE,
+                           abd, lsize, BP_SHOULD_BYTESWAP(bp), mac);
+                       abd_free(abd);
                } else {
                        ret = zio_crypt_do_indirect_mac_checksum_abd(B_FALSE,
                            zio->io_abd, size, BP_SHOULD_BYTESWAP(bp), mac);
@@ -1866,30 +1865,32 @@ zio_write_compress(zio_t *zio)
        /* If it's a compressed write that is not raw, compress the buffer. */
        if (compress != ZIO_COMPRESS_OFF &&
            !(zio->io_flags & ZIO_FLAG_RAW_COMPRESS)) {
-               void *cbuf = NULL;
+               abd_t *cabd = NULL;
                if (abd_cmp_zero(zio->io_abd, lsize) == 0)
                        psize = 0;
                else if (compress == ZIO_COMPRESS_EMPTY)
                        psize = lsize;
                else
-                       psize = zio_compress_data(compress, zio->io_abd, &cbuf,
+                       psize = zio_compress_data(compress, zio->io_abd, &cabd,
                            lsize, zp->zp_complevel);
                if (psize == 0) {
                        compress = ZIO_COMPRESS_OFF;
                } else if (psize >= lsize) {
                        compress = ZIO_COMPRESS_OFF;
-                       if (cbuf != NULL)
-                               zio_buf_free(cbuf, lsize);
+                       if (cabd != NULL)
+                               abd_free(cabd);
                } else if (!zp->zp_dedup && !zp->zp_encrypt &&
                    psize <= BPE_PAYLOAD_SIZE &&
                    zp->zp_level == 0 && !DMU_OT_HAS_FILL(zp->zp_type) &&
                    spa_feature_is_enabled(spa, SPA_FEATURE_EMBEDDED_DATA)) {
+                       void *cbuf = abd_borrow_buf_copy(cabd, lsize);
                        encode_embedded_bp_compressed(bp,
                            cbuf, compress, lsize, psize);
                        BPE_SET_ETYPE(bp, BP_EMBEDDED_TYPE_DATA);
                        BP_SET_TYPE(bp, zio->io_prop.zp_type);
                        BP_SET_LEVEL(bp, zio->io_prop.zp_level);
-                       zio_buf_free(cbuf, lsize);
+                       abd_return_buf(cabd, cbuf, lsize);
+                       abd_free(cabd);
                        BP_SET_LOGICAL_BIRTH(bp, zio->io_txg);
                        zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
                        ASSERT(spa_feature_is_active(spa,
@@ -1908,14 +1909,12 @@ zio_write_compress(zio_t *zio)
                            psize);
                        if (rounded >= lsize) {
                                compress = ZIO_COMPRESS_OFF;
-                               zio_buf_free(cbuf, lsize);
+                               abd_free(cabd);
                                psize = lsize;
                        } else {
-                               abd_t *cdata = abd_get_from_buf(cbuf, lsize);
-                               abd_take_ownership_of_buf(cdata, B_TRUE);
-                               abd_zero_off(cdata, psize, rounded - psize);
+                               abd_zero_off(cabd, psize, rounded - psize);
                                psize = rounded;
-                               zio_push_transform(zio, cdata,
+                               zio_push_transform(zio, cabd,
                                    psize, lsize, NULL);
                        }
                }

diff --git a/module/zfs/zio_compress.c b/module/zfs/zio_compress.c
index 118003bd295a33656ac530a134d926e939cc13e6..faf430972078fec426fb38b676b8fdaceb3e1a34 100644 (file)
--- a/module/zfs/zio_compress.c
+++ b/module/zfs/zio_compress.c
@@ -128,7 +128,7 @@ zio_compress_select(spa_t *spa, enum zio_compress child,
 }
 
 size_t
-zio_compress_data(enum zio_compress c, abd_t *src, void **dst, size_t s_len,
+zio_compress_data(enum zio_compress c, abd_t *src, abd_t **dst, size_t s_len,
     uint8_t level)
 {
        size_t c_len, d_len;
@@ -158,12 +158,9 @@ zio_compress_data(enum zio_compress c, abd_t *src, void **dst, size_t s_len,
        }
 
        if (*dst == NULL)
-               *dst = zio_buf_alloc(s_len);
+               *dst = abd_alloc_sametype(src, s_len);
 
-       abd_t dabd;
-       abd_get_from_buf_struct(&dabd, dst, d_len);
-       c_len = ci->ci_compress(src, &dabd, s_len, d_len, complevel);
-       abd_free(&dabd);
+       c_len = ci->ci_compress(src, *dst, s_len, d_len, complevel);
 
        if (c_len > d_len)
                return (s_len);
@@ -173,23 +170,18 @@ zio_compress_data(enum zio_compress c, abd_t *src, void **dst, size_t s_len,
 }
 
 int
-zio_decompress_data(enum zio_compress c, abd_t *src, void *dst,
+zio_decompress_data(enum zio_compress c, abd_t *src, abd_t *dst,
     size_t s_len, size_t d_len, uint8_t *level)
 {
        zio_compress_info_t *ci = &zio_compress_table[c];
        if ((uint_t)c >= ZIO_COMPRESS_FUNCTIONS || ci->ci_decompress == NULL)
                return (SET_ERROR(EINVAL));
 
-       abd_t dabd;
-       abd_get_from_buf_struct(&dabd, dst, d_len);
-
        int err;
        if (ci->ci_decompress_level != NULL && level != NULL)
-               err = ci->ci_decompress_level(src, &dabd, s_len, d_len, level);
+               err = ci->ci_decompress_level(src, dst, s_len, d_len, level);
        else
-               err = ci->ci_decompress(src, &dabd, s_len, d_len, ci->ci_level);
-
-       abd_free(&dabd);
+               err = ci->ci_decompress(src, dst, s_len, d_len, ci->ci_level);
 
        /*
         * Decompression shouldn't fail, because we've already verified

diff --git a/module/zstd/zfs_zstd.c b/module/zstd/zfs_zstd.c
index 8d1d53d234b46831bbfcc9b7a23f4876034528d2..e113962f65b6272731a09b844b581ecd3732eaca 100644 (file)
--- a/module/zstd/zfs_zstd.c
+++ b/module/zstd/zfs_zstd.c
@@ -569,9 +569,11 @@ zfs_zstd_compress_buf(void *s_start, void *d_start, size_t s_len, size_t d_len,
        if (zstd_earlyabort_pass > 0 && zstd_level >= zstd_cutoff_level &&
            s_len >= actual_abort_size) {
                int pass_len = 1;
-               abd_t sabd;
+               abd_t sabd, dabd;
                abd_get_from_buf_struct(&sabd, s_start, s_len);
-               pass_len = zfs_lz4_compress(&sabd, d_start, s_len, d_len, 0);
+               abd_get_from_buf_struct(&dabd, d_start, d_len);
+               pass_len = zfs_lz4_compress(&sabd, &dabd, s_len, d_len, 0);
+               abd_free(&dabd);
                abd_free(&sabd);
                if (pass_len < d_len) {
                        ZSTDSTAT_BUMP(zstd_stat_lz4pass_allowed);