]> git.proxmox.com Git - mirror_zfs.git/commitdiff
OpenZFS 7247 - zfs receive of deduplicated stream fails
authorGeorge Melikov <mail@gmelikov.ru>
Sat, 4 Feb 2017 17:10:24 +0000 (20:10 +0300)
committerBrian Behlendorf <behlendorf1@llnl.gov>
Sat, 4 Feb 2017 17:10:24 +0000 (09:10 -0800)
Authored by: Chris Williamson <chris.williamson@delphix.com>
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Dan Kimmel <dan.kimmel@delphix.com>
Approved by: Robert Mustacchi <rm@joyent.com>
Reviewed-by: loli10K <ezomori.nozomu@gmail.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Ported-by: George Melikov <mail@gmelikov.ru>
OpenZFS-issue: https://www.illumos.org/issues/7247
OpenZFS-commit: https://github.com/openzfs/openzfs/commit/2ad25b4
Closes #5689

Porting notes:
- tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_013_pos.ksh
  renamed as zfs_receive_015_pos.ksh, zfs_receive_013_pos.ksh is now
  used for OpenZFS test.
- libzfs_sendrecv.c: SMALLEST_POSSIBLE_MAX_DDT_MB is always used
  for all 32-bit builds.

lib/libzfs/libzfs_sendrecv.c
module/zfs/dmu_send.c
tests/runfiles/linux.run
tests/zfs-tests/tests/functional/cli_root/zfs_receive/Makefile.am
tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_013_pos.ksh
tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_015_pos.ksh [new file with mode: 0755]

index b08cd2b631616c23b07e06f71e592bfdf03a5964..c1a09e3a9c846fdf5f292df064490dc95c2b0690 100644 (file)
@@ -245,12 +245,16 @@ cksummer(void *arg)
        int outfd;
        dedup_table_t ddt;
        zio_cksum_t stream_cksum;
-       uint64_t physmem = sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE);
        uint64_t numbuckets;
 
+#ifdef _ILP32
+       ddt.max_ddt_size = SMALLEST_POSSIBLE_MAX_DDT_MB << 20;
+#else
+       uint64_t physmem = sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE);
        ddt.max_ddt_size =
            MAX((physmem * MAX_DDT_PHYSMEM_PERCENT) / 100,
            SMALLEST_POSSIBLE_MAX_DDT_MB << 20);
+#endif
 
        numbuckets = ddt.max_ddt_size / (sizeof (dedup_entry_t));
 
index 6e117651ba0ce92bc982d36c81013af9f1e16b34..6f17671ee6a3884666ae1ca78b54c1db7ee4f0ab 100644 (file)
@@ -3230,6 +3230,9 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx)
                dsl_dataset_phys(origin_head)->ds_flags &=
                    ~DS_FLAG_INCONSISTENT;
 
+               drc->drc_newsnapobj =
+                   dsl_dataset_phys(origin_head)->ds_prev_snap_obj;
+
                dsl_dataset_rele(origin_head, FTAG);
                dsl_destroy_head_sync_impl(drc->drc_ds, tx);
 
@@ -3265,8 +3268,9 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx)
                        (void) zap_remove(dp->dp_meta_objset, ds->ds_object,
                            DS_FIELD_RESUME_TONAME, tx);
                }
+               drc->drc_newsnapobj =
+                   dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj;
        }
-       drc->drc_newsnapobj = dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj;
        zvol_create_minors(dp->dp_spa, drc->drc_tofs, B_TRUE);
        /*
         * Release the hold from dmu_recv_begin.  This must be done before
@@ -3310,8 +3314,6 @@ static int dmu_recv_end_modified_blocks = 3;
 static int
 dmu_recv_existing_end(dmu_recv_cookie_t *drc)
 {
-       int error;
-
 #ifdef _KERNEL
        /*
         * We will be destroying the ds; make sure its origin is unmounted if
@@ -3322,23 +3324,30 @@ dmu_recv_existing_end(dmu_recv_cookie_t *drc)
        zfs_destroy_unmount_origin(name);
 #endif
 
-       error = dsl_sync_task(drc->drc_tofs,
+       return (dsl_sync_task(drc->drc_tofs,
            dmu_recv_end_check, dmu_recv_end_sync, drc,
-           dmu_recv_end_modified_blocks, ZFS_SPACE_CHECK_NORMAL);
-
-       if (error != 0)
-               dmu_recv_cleanup_ds(drc);
-       return (error);
+           dmu_recv_end_modified_blocks, ZFS_SPACE_CHECK_NORMAL));
 }
 
 static int
 dmu_recv_new_end(dmu_recv_cookie_t *drc)
+{
+       return (dsl_sync_task(drc->drc_tofs,
+           dmu_recv_end_check, dmu_recv_end_sync, drc,
+           dmu_recv_end_modified_blocks, ZFS_SPACE_CHECK_NORMAL));
+}
+
+int
+dmu_recv_end(dmu_recv_cookie_t *drc, void *owner)
 {
        int error;
 
-       error = dsl_sync_task(drc->drc_tofs,
-           dmu_recv_end_check, dmu_recv_end_sync, drc,
-           dmu_recv_end_modified_blocks, ZFS_SPACE_CHECK_NORMAL);
+       drc->drc_owner = owner;
+
+       if (drc->drc_newfs)
+               error = dmu_recv_new_end(drc);
+       else
+               error = dmu_recv_existing_end(drc);
 
        if (error != 0) {
                dmu_recv_cleanup_ds(drc);
@@ -3350,17 +3359,6 @@ dmu_recv_new_end(dmu_recv_cookie_t *drc)
        return (error);
 }
 
-int
-dmu_recv_end(dmu_recv_cookie_t *drc, void *owner)
-{
-       drc->drc_owner = owner;
-
-       if (drc->drc_newfs)
-               return (dmu_recv_new_end(drc));
-       else
-               return (dmu_recv_existing_end(drc));
-}
-
 /*
  * Return TRUE if this objset is currently being received into.
  */
index 0a8198236fbfbcbe09adeb544df894688f604e52..117117661574fc5ec1889379163980a3c07c6c09 100644 (file)
@@ -150,7 +150,7 @@ tests = ['zfs_receive_001_pos', 'zfs_receive_002_pos', 'zfs_receive_003_pos',
     'zfs_receive_005_neg', 'zfs_receive_006_pos',
     'zfs_receive_007_neg', 'zfs_receive_008_pos', 'zfs_receive_009_neg',
     'zfs_receive_010_pos', 'zfs_receive_011_pos', 'zfs_receive_012_pos',
-    'zfs_receive_013_pos', 'zfs_receive_014_pos']
+    'zfs_receive_013_pos', 'zfs_receive_014_pos', 'zfs_receive_015_pos']
 
 # DISABLED:
 # zfs_rename_006_pos - https://github.com/zfsonlinux/zfs/issues/5647
index 564aab75698f64009ac13a2586a2c0d6d7d3b6d1..3c87a992706d561cee29bdcd071cc0365cc4ed8f 100644 (file)
@@ -15,4 +15,5 @@ dist_pkgdata_SCRIPTS = \
        zfs_receive_011_pos.ksh \
        zfs_receive_012_pos.ksh \
        zfs_receive_013_pos.ksh \
-       zfs_receive_014_pos.ksh
+       zfs_receive_014_pos.ksh \
+       zfs_receive_015_pos.ksh
index 11081bb7f1e106a97d423dae0c0198a90c37650c..f70d0f4bfa45c71aefc332d4a85c85e1e3ee4c96 100755 (executable)
 #
 
 #
-# Copyright 2016, loli10K. All rights reserved.
+# Copyright (c) 2015 by Delphix. All rights reserved.
 #
 
-. $STF_SUITE/include/libtest.shlib
-. $STF_SUITE/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib
+. $STF_SUITE/tests/functional/cli_root/cli_common.kshlib
 
 #
 # DESCRIPTION:
-#      Verify ZFS can receive custom properties on both filesystems and
-#      snapshots from full and incremental streams.
+#   Verifying 'zfs receive' works correctly on deduplicated streams
 #
 # STRATEGY:
-#      1. Create a filesystem.
-#      2. Snapshot the filesystem.
-#      3. Set custom properties on both the fs and snapshots.
-#      4. Create different send streams with the properties.
-#      5. Receive the send streams and verify the properties.
+#   1. Create some snapshots with duplicated data
+#   2. Send a deduplicated stream of the last snapshot
+#   3. Attempt to receive the deduplicated stream
 #
 
-verify_runnable "both"
-
-typeset streamfile_full=/var/tmp/streamfile_full.$$
-typeset streamfile_incr=/var/tmp/streamfile_incr.$$
-orig=$TESTPOOL/$TESTFS1
-dest=$TESTPOOL/$TESTFS2
-typeset user_prop=$(valid_user_property 8)
-typeset value=$(user_property_value 8)
+src_fs=$TESTPOOL/drecvsrc
+temppool=recvtank
+dst_fs=$temppool/drecvdest
+streamfile=/var/tmp/drecvstream.$$
+tpoolfile=/temptank.$$
 
 function cleanup
 {
-       log_must $RM $streamfile_full
-       log_must $RM $streamfile_incr
-       log_must $ZFS destroy -rf $TESTPOOL/$TESTFS1
-       log_must $ZFS destroy -rf $TESTPOOL/$TESTFS2
+    for fs in $src_fs $dst_fs; do
+        datasetexists $fs && log_must $ZFS destroy -rf $fs
+    done
+    $ZPOOL destroy $temppool
+    [[ -f $streamfile ]] && log_must $RM -f $streamfile
+    [[ -f $tpoolfile ]] && log_must $RM -f $tpoolfile
 }
 
-log_assert "ZFS can receive custom properties."
+log_assert "Verifying 'zfs receive' works correctly on deduplicated streams"
 log_onexit cleanup
 
-#      1. Create a filesystem.
-log_must $ZFS create $orig
+truncate -s 100M $tpoolfile
+log_must $ZPOOL create $temppool $tpoolfile
+log_must $ZFS create $src_fs
+src_mnt=$(get_prop mountpoint $src_fs) || log_fail "get_prop mountpoint $src_fs"
+
+echo blah > $src_mnt/blah
+$ZFS snapshot $src_fs@base
 
-#      2. Snapshot the filesystem.
-log_must $ZFS snapshot $orig@snap1
-log_must $ZFS snapshot $orig@snap2
-log_must $ZFS snapshot $orig@snap3
+echo grumble > $src_mnt/grumble
+echo blah > $src_mnt/blah2
+$ZFS snapshot $src_fs@snap2
 
-#      3. Set custom properties on both the fs and snapshots.
-log_must eval "$ZFS set '$user_prop'='$value' $orig"
-log_must eval "$ZFS set '$user_prop:snap1'='$value:snap1' $orig@snap1"
-log_must eval "$ZFS set '$user_prop:snap2'='$value:snap2' $orig@snap2"
-log_must eval "$ZFS set '$user_prop:snap3'='$value:snap3' $orig@snap3"
+echo grumble > $src_mnt/mumble
+echo blah > $src_mnt/blah3
+$ZFS snapshot $src_fs@snap3
 
-#      4. Create different send streams with the properties.
-log_must eval "$ZFS send -p $orig@snap1 > $streamfile_full"
-log_must eval "$ZFS send -p -I $orig@snap1 $orig@snap3 > $streamfile_incr"
+log_must eval "$ZFS send -D -R $src_fs@snap3 > $streamfile"
+log_must eval "$ZFS receive -v $dst_fs < $streamfile"
 
-#      5. Receive the send streams and verify the properties.
-log_must eval "$ZFS recv $dest < $streamfile_full"
-log_must eval "check_user_prop $dest $user_prop '$value'"
-log_must eval "check_user_prop $dest@snap1 '$user_prop:snap1' '$value:snap1'"
-log_must eval "$ZFS recv $dest < $streamfile_incr"
-log_must eval "check_user_prop $dest@snap2 '$user_prop:snap2' '$value:snap2'"
-log_must eval "check_user_prop $dest@snap3 '$user_prop:snap3' '$value:snap3'"
+cleanup
 
-log_pass "ZFS can receive custom properties passed."
+log_pass "Verifying 'zfs receive' works correctly on deduplicated streams"
diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_015_pos.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_015_pos.ksh
new file mode 100755 (executable)
index 0000000..11081bb
--- /dev/null
@@ -0,0 +1,83 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2016, loli10K. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib
+
+#
+# DESCRIPTION:
+#      Verify ZFS can receive custom properties on both filesystems and
+#      snapshots from full and incremental streams.
+#
+# STRATEGY:
+#      1. Create a filesystem.
+#      2. Snapshot the filesystem.
+#      3. Set custom properties on both the fs and snapshots.
+#      4. Create different send streams with the properties.
+#      5. Receive the send streams and verify the properties.
+#
+
+verify_runnable "both"
+
+typeset streamfile_full=/var/tmp/streamfile_full.$$
+typeset streamfile_incr=/var/tmp/streamfile_incr.$$
+orig=$TESTPOOL/$TESTFS1
+dest=$TESTPOOL/$TESTFS2
+typeset user_prop=$(valid_user_property 8)
+typeset value=$(user_property_value 8)
+
+function cleanup
+{
+       log_must $RM $streamfile_full
+       log_must $RM $streamfile_incr
+       log_must $ZFS destroy -rf $TESTPOOL/$TESTFS1
+       log_must $ZFS destroy -rf $TESTPOOL/$TESTFS2
+}
+
+log_assert "ZFS can receive custom properties."
+log_onexit cleanup
+
+#      1. Create a filesystem.
+log_must $ZFS create $orig
+
+#      2. Snapshot the filesystem.
+log_must $ZFS snapshot $orig@snap1
+log_must $ZFS snapshot $orig@snap2
+log_must $ZFS snapshot $orig@snap3
+
+#      3. Set custom properties on both the fs and snapshots.
+log_must eval "$ZFS set '$user_prop'='$value' $orig"
+log_must eval "$ZFS set '$user_prop:snap1'='$value:snap1' $orig@snap1"
+log_must eval "$ZFS set '$user_prop:snap2'='$value:snap2' $orig@snap2"
+log_must eval "$ZFS set '$user_prop:snap3'='$value:snap3' $orig@snap3"
+
+#      4. Create different send streams with the properties.
+log_must eval "$ZFS send -p $orig@snap1 > $streamfile_full"
+log_must eval "$ZFS send -p -I $orig@snap1 $orig@snap3 > $streamfile_incr"
+
+#      5. Receive the send streams and verify the properties.
+log_must eval "$ZFS recv $dest < $streamfile_full"
+log_must eval "check_user_prop $dest $user_prop '$value'"
+log_must eval "check_user_prop $dest@snap1 '$user_prop:snap1' '$value:snap1'"
+log_must eval "$ZFS recv $dest < $streamfile_incr"
+log_must eval "check_user_prop $dest@snap2 '$user_prop:snap2' '$value:snap2'"
+log_must eval "check_user_prop $dest@snap3 '$user_prop:snap3' '$value:snap3'"
+
+log_pass "ZFS can receive custom properties passed."