Refactor arc_hdr_realloc_crypt()

The arc_hdr_realloc_crypt() function is responsible for converting
a "full" arc header to an extended "crypt" header and visa versa.
This code was originally written with a bcopy() so that any new
members added to arc headers would automatically be included
without requiring a code change. However, in practice this (along
with small differences in kmem_cache implementations between
various platforms) has caused a number of hard-to-find problems in
ports to other operating systems. This patch solves this problem
by making all member copies explicit and adding ASSERTs for fields
that cannot be set during the transfer. It also manually resets the
old header after the reallocation is finished so it can be properly
reallocated and reused.

Reviewed-by: Jorgen Lundman <lundman@lundman.net>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Signed-off-by: Tom Caputi <tcaputi@datto.com>
Closes #7711

diff --git a/module/zfs/arc.c b/module/zfs/arc.c
index ab256484119fa1567f85cf576faddbe7ee059ccf..e49f8a547e5527d0b98164b590f0a9145b46c6b1 100644 (file)
--- a/module/zfs/arc.c
+++ b/module/zfs/arc.c
@@ -3512,22 +3512,47 @@ arc_hdr_realloc_crypt(arc_buf_hdr_t *hdr, boolean_t need_crypt)
        arc_buf_hdr_t *nhdr;
        arc_buf_t *buf;
        kmem_cache_t *ncache, *ocache;
+       unsigned nsize, osize;
 
+       /*
+        * This function requires that hdr is in the arc_anon state.
+        * Therefore it won't have any L2ARC data for us to worry
+        * about copying.
+        */
        ASSERT(HDR_HAS_L1HDR(hdr));
+       ASSERT(!HDR_HAS_L2HDR(hdr));
        ASSERT3U(!!HDR_PROTECTED(hdr), !=, need_crypt);
        ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
        ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
+       ASSERT(!list_link_active(&hdr->b_l2hdr.b_l2node));
+       ASSERT3P(hdr->b_hash_next, ==, NULL);
 
        if (need_crypt) {
                ncache = hdr_full_crypt_cache;
+               nsize = sizeof (hdr->b_crypt_hdr);
                ocache = hdr_full_cache;
+               osize = HDR_FULL_SIZE;
        } else {
                ncache = hdr_full_cache;
+               nsize = HDR_FULL_SIZE;
                ocache = hdr_full_crypt_cache;
+               osize = sizeof (hdr->b_crypt_hdr);
        }
 
        nhdr = kmem_cache_alloc(ncache, KM_PUSHPAGE);
-       bcopy(hdr, nhdr, HDR_L2ONLY_SIZE);
+
+       /*
+        * Copy all members that aren't locks or condvars to the new header.
+        * No lists are pointing to us (as we asserted above), so we don't
+        * need to worry about the list nodes.
+        */
+       nhdr->b_dva = hdr->b_dva;
+       nhdr->b_birth = hdr->b_birth;
+       nhdr->b_type = hdr->b_type;
+       nhdr->b_flags = hdr->b_flags;
+       nhdr->b_psize = hdr->b_psize;
+       nhdr->b_lsize = hdr->b_lsize;
+       nhdr->b_spa = hdr->b_spa;
        nhdr->b_l1hdr.b_freeze_cksum = hdr->b_l1hdr.b_freeze_cksum;
        nhdr->b_l1hdr.b_bufcnt = hdr->b_l1hdr.b_bufcnt;
        nhdr->b_l1hdr.b_byteswap = hdr->b_l1hdr.b_byteswap;
@@ -3540,7 +3565,6 @@ arc_hdr_realloc_crypt(arc_buf_hdr_t *hdr, boolean_t need_crypt)
        nhdr->b_l1hdr.b_l2_hits = hdr->b_l1hdr.b_l2_hits;
        nhdr->b_l1hdr.b_acb = hdr->b_l1hdr.b_acb;
        nhdr->b_l1hdr.b_pabd = hdr->b_l1hdr.b_pabd;
-       nhdr->b_l1hdr.b_buf = hdr->b_l1hdr.b_buf;
 
        /*
         * This refcount_add() exists only to ensure that the individual
@@ -3548,7 +3572,7 @@ arc_hdr_realloc_crypt(arc_buf_hdr_t *hdr, boolean_t need_crypt)
         * a small race condition that could trigger ASSERTs.
         */
        (void) refcount_add(&nhdr->b_l1hdr.b_refcnt, FTAG);
-
+       nhdr->b_l1hdr.b_buf = hdr->b_l1hdr.b_buf;
        for (buf = nhdr->b_l1hdr.b_buf; buf != NULL; buf = buf->b_next) {
                mutex_enter(&buf->b_evict_lock);
                buf->b_hdr = nhdr;
@@ -3557,6 +3581,7 @@ arc_hdr_realloc_crypt(arc_buf_hdr_t *hdr, boolean_t need_crypt)
 
        refcount_transfer(&nhdr->b_l1hdr.b_refcnt, &hdr->b_l1hdr.b_refcnt);
        (void) refcount_remove(&nhdr->b_l1hdr.b_refcnt, FTAG);
+       ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
 
        if (need_crypt) {
                arc_hdr_set_flags(nhdr, ARC_FLAG_PROTECTED);
@@ -3564,6 +3589,38 @@ arc_hdr_realloc_crypt(arc_buf_hdr_t *hdr, boolean_t need_crypt)
                arc_hdr_clear_flags(nhdr, ARC_FLAG_PROTECTED);
        }
 
+       /* unset all members of the original hdr */
+       bzero(&hdr->b_dva, sizeof (dva_t));
+       hdr->b_birth = 0;
+       hdr->b_type = ARC_BUFC_INVALID;
+       hdr->b_flags = 0;
+       hdr->b_psize = 0;
+       hdr->b_lsize = 0;
+       hdr->b_spa = 0;
+       hdr->b_l1hdr.b_freeze_cksum = NULL;
+       hdr->b_l1hdr.b_buf = NULL;
+       hdr->b_l1hdr.b_bufcnt = 0;
+       hdr->b_l1hdr.b_byteswap = 0;
+       hdr->b_l1hdr.b_state = NULL;
+       hdr->b_l1hdr.b_arc_access = 0;
+       hdr->b_l1hdr.b_mru_hits = 0;
+       hdr->b_l1hdr.b_mru_ghost_hits = 0;
+       hdr->b_l1hdr.b_mfu_hits = 0;
+       hdr->b_l1hdr.b_mfu_ghost_hits = 0;
+       hdr->b_l1hdr.b_l2_hits = 0;
+       hdr->b_l1hdr.b_acb = NULL;
+       hdr->b_l1hdr.b_pabd = NULL;
+
+       if (ocache == hdr_full_crypt_cache) {
+               ASSERT(!HDR_HAS_RABD(hdr));
+               hdr->b_crypt_hdr.b_ot = DMU_OT_NONE;
+               hdr->b_crypt_hdr.b_ebufcnt = 0;
+               hdr->b_crypt_hdr.b_dsobj = 0;
+               bzero(hdr->b_crypt_hdr.b_salt, ZIO_DATA_SALT_LEN);
+               bzero(hdr->b_crypt_hdr.b_iv, ZIO_DATA_IV_LEN);
+               bzero(hdr->b_crypt_hdr.b_mac, ZIO_DATA_MAC_LEN);
+       }
+
        buf_discard_identity(hdr);
        kmem_cache_free(ocache, hdr);