1ULL << drro->drr_indblkshift : 0;
int nblkptr = deduce_nblkptr(drro->drr_bonustype,
drro->drr_bonuslen);
+ boolean_t did_free = B_FALSE;
object = drro->drr_object;
drro->drr_object, 0, DMU_OBJECT_END);
if (err != 0)
return (SET_ERROR(EINVAL));
+ else
+ did_free = B_TRUE;
}
/*
* processed. However, for raw receives we manually set the
* maxblkid from the drr_maxblkid and so we must first free
* everything above that blkid to ensure the DMU is always
- * consistent with itself.
+ * consistent with itself. We will never free the first block
+ * of the object here because a maxblkid of 0 could indicate
+ * an object with a single block or one with no blocks. This
+ * free may be skipped when dmu_free_long_range() was called
+ * above since it covers the entire object's contents.
*/
- if (rwa->raw) {
+ if (rwa->raw && object != DMU_NEW_OBJECT && !did_free) {
err = dmu_free_long_range(rwa->os, drro->drr_object,
- (drro->drr_maxblkid + 1) * drro->drr_blksz,
+ (drro->drr_maxblkid + 1) * doi.doi_data_block_size,
DMU_OBJECT_END);
if (err != 0)
return (SET_ERROR(EINVAL));
drro->drr_nlevels, tx));
/*
- * Set the maxblkid. We will never free the first block of
- * an object here because a maxblkid of 0 could indicate
- * an object with a single block or one with no blocks.
- * This will always succeed because we freed all blocks
- * beyond the new maxblkid above.
+ * Set the maxblkid. This will always succeed because
+ * we freed all blocks beyond the new maxblkid above.
*/
VERIFY0(dmu_object_set_maxblkid(rwa->os, drro->drr_object,
drro->drr_maxblkid, tx));
'send-c_mixed_compression', 'send-c_stream_size_estimate', 'send-cD',
'send-c_embedded_blocks', 'send-c_resume', 'send-cpL_varied_recsize',
'send-c_recv_dedup', 'send_encrypted_files', 'send_encrypted_hierarchy',
- 'send_encrypted_props', 'send_freeobjects', 'send_realloc_dnode_size',
- 'send_realloc_files', 'send_holds', 'send_hole_birth', 'send_mixed_raw',
- 'send-wDR_encrypted_zvol']
+ 'send_encrypted_props', 'send_encrypted_truncated_files',
+ 'send_freeobjects', 'send_realloc_dnode_size', 'send_realloc_files',
+ 'send_realloc_encrypted_files', 'send_holds', 'send_hole_birth',
+ 'send_mixed_raw', 'send-wDR_encrypted_zvol']
tags = ['functional', 'rsend']
[tests/functional/scrub_mirror]
send_encrypted_files.ksh \
send_encrypted_hierarchy.ksh \
send_encrypted_props.ksh \
+ send_encrypted_truncated_files.ksh \
send-cD.ksh \
send-c_embedded_blocks.ksh \
send-c_incremental.ksh \
send_freeobjects.ksh \
send_realloc_dnode_size.ksh \
send_realloc_files.ksh \
+ send_realloc_encrypted_files.ksh \
send_holds.ksh \
send_hole_birth.ksh \
send_mixed_raw.ksh \
# $1 Number of files to modify
# $2 Maximum file size
# $3 File system to modify the file on
+# $4 Enabled xattrs (optional)
#
function churn_files
{
nfiles=$1
maxsize=$2
fs=$3
+ xattrs=${4:-1}
#
# Remove roughly half of the files in order to make it more
#
if [[ -e $file_name ]]; then
value=$((RANDOM % 5))
- if [ $value -eq 0 ]; then
+ if [ $value -eq 0 -a $xattrs -ne 0 ]; then
attrname="testattr$((RANDOM % 3))"
attr -qr $attrname $file_name || \
log_fail "Failed to remove $attrname"
bs=$file_size count=1 >/dev/null 2>&1 || \
log_fail "Failed to create $file_name"
- for j in {0..2}; do
- attrname="testattr$j"
- attr -qs $attrname -V TestValue $file_name || \
- log_fail "Failed to set $attrname"
- done
+ if [ $xattrs -ne 0 ]; then
+ for j in {0..2}; do
+ attrname="testattr$j"
+ attr -qs $attrname -V TestValue \
+ $file_name || log_fail \
+ "Failed to set $attrname"
+ done
+ fi
fi
done
#
# DESCRIPTION:
-#
+# Verify that a raw zfs send and receive can deal with several different
+# types of file layouts.
#
# STRATEGY:
# 1. Create a new encrypted filesystem
# 3. Add a small 512 byte file to the filesystem
# 4. Add a larger 32M file to the filesystem
# 5. Add a large sparse file to the filesystem
-# 6. Add a 3 files that are to be truncated later
-# 7. Add 1000 empty files to the filesystem
-# 8. Add a file with a large xattr value
-# 9. Use xattrtest to create files with random xattrs (with and without xattrs=on)
-# 10. Take a snapshot of the filesystem
-# 11. Truncate one of the files from 32M to 128k
-# 12. Truncate one of the files from 512k to 384k
-# 13. Truncate one of the files from 512k to 0 to 384k
-# 14. Remove the 1000 empty files to the filesystem
-# 15. Take another snapshot of the filesystem
-# 16. Send and receive both snapshots
-# 17. Mount the filesystem and check the contents
+# 6. Add 1000 empty files to the filesystem
+# 7. Add a file with a large xattr value
+# 8. Use xattrtest to create files with random xattrs (with and without xattrs=on)
+# 9. Take a snapshot of the filesystem
+# 10. Remove the 1000 empty files to the filesystem
+# 11. Take another snapshot of the filesystem
+# 12. Send and receive both snapshots
+# 13. Mount the filesystem and check the contents
#
verify_runnable "both"
log_must zfs create -o encryption=on -o keyformat=passphrase \
-o keylocation=file://$keyfile $TESTPOOL/$TESTFS2
-# Create files with vaired layouts on disk
+# Create files with varied layouts on disk
log_must touch /$TESTPOOL/$TESTFS2/empty
log_must mkfile 512 /$TESTPOOL/$TESTFS2/small
log_must mkfile 32M /$TESTPOOL/$TESTFS2/full
log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/sparse \
bs=512 count=1 seek=1048576 >/dev/null 2>&1
-log_must mkfile 32M /$TESTPOOL/$TESTFS2/truncated
-log_must mkfile 524288 /$TESTPOOL/$TESTFS2/truncated2
-log_must mkfile 524288 /$TESTPOOL/$TESTFS2/truncated3
log_must mkdir -p /$TESTPOOL/$TESTFS2/dir
for i in {1..1000}; do
log_must touch /$TESTPOOL/$TESTFS2/attrs
log_must eval "python -c 'print \"a\" * 4096' | \
attr -s bigval /$TESTPOOL/$TESTFS2/attrs"
+log_must zfs set compression=off xattr=on $TESTPOOL/$TESTFS2
log_must zfs snapshot $TESTPOOL/$TESTFS2@snap1
-#
-# Truncate files created in the first snapshot. The first tests
-# truncating a large file to a single block. The second tests
-# truncating one block off the end of a file without changing
-# the required nlevels to hold it. The last tests handling
-# of a maxblkid that is dropped and then raised again.
-#
-log_must truncate -s 131072 /$TESTPOOL/$TESTFS2/truncated
-log_must truncate -s 393216 /$TESTPOOL/$TESTFS2/truncated2
-log_must truncate -s 0 /$TESTPOOL/$TESTFS2/truncated3
-log_must zpool sync $TESTPOOL
-log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/truncated3 \
- bs=128k count=3 iflag=fullblock
-
# Remove the empty files created in the first snapshot
for i in {1..1000}; do
log_must rm /$TESTPOOL/$TESTFS2/dir/file-$i
--- /dev/null
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source. A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2018 by Datto Inc. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+#
+# DESCRIPTION:
+#
+#
+# STRATEGY:
+# 1. Create a new encrypted filesystem
+# 2. Add a 4 files that are to be truncated later
+# 3. Take a snapshot of the filesystem
+# 4. Truncate one of the files from 32M to 128k
+# 5. Truncate one of the files from 512k to 384k
+# 6. Truncate one of the files from 512k to 0 to 384k via reallocation
+# 7. Truncate one of the files from 1k to 0 to 512b via reallocation
+# 8. Take another snapshot of the filesystem
+# 9. Send and receive both snapshots
+# 10. Mount the filesystem and check the contents
+#
+
+verify_runnable "both"
+
+function cleanup
+{
+ datasetexists $TESTPOOL/$TESTFS2 && \
+ log_must zfs destroy -r $TESTPOOL/$TESTFS2
+ datasetexists $TESTPOOL/recv && \
+ log_must zfs destroy -r $TESTPOOL/recv
+ [[ -f $keyfile ]] && log_must rm $keyfile
+ [[ -f $sendfile ]] && log_must rm $sendfile
+}
+log_onexit cleanup
+
+function recursive_cksum
+{
+ find $1 -type f -exec sha256sum {} \; | \
+ sort -k 2 | awk '{ print $1 }' | sha256sum
+}
+
+log_assert "Verify 'zfs send -w' works with many different file layouts"
+
+typeset keyfile=/$TESTPOOL/pkey
+typeset sendfile=/$TESTPOOL/sendfile
+typeset sendfile2=/$TESTPOOL/sendfile2
+
+# Create an encrypted dataset
+log_must eval "echo 'password' > $keyfile"
+log_must zfs create -o encryption=on -o keyformat=passphrase \
+ -o keylocation=file://$keyfile $TESTPOOL/$TESTFS2
+
+# Explicitly set the recordsize since the truncation sizes below depend on
+# this value being 128k. This is currently same as the default recordsize.
+log_must zfs set recordsize=128k $TESTPOOL/$TESTFS2
+
+# Create files with varied layouts on disk
+log_must mkfile 32M /$TESTPOOL/$TESTFS2/truncated
+log_must mkfile 524288 /$TESTPOOL/$TESTFS2/truncated2
+log_must mkfile 524288 /$TESTPOOL/$TESTFS2/truncated3
+log_must mkfile 1024 /$TESTPOOL/$TESTFS2/truncated4
+
+log_must zfs snapshot $TESTPOOL/$TESTFS2@snap1
+
+#
+# Truncate files created in the first snapshot. The first tests
+# truncating a large file to a single block. The second tests
+# truncating one block off the end of a file without changing
+# the required nlevels to hold it. The third tests handling
+# of a maxblkid that is dropped and then raised again. The
+# fourth tests an object that is truncated from a single block
+# to a smaller single block.
+#
+log_must truncate -s 131072 /$TESTPOOL/$TESTFS2/truncated
+log_must truncate -s 393216 /$TESTPOOL/$TESTFS2/truncated2
+log_must rm -f /$TESTPOOL/$TESTFS2/truncated3
+log_must rm -f /$TESTPOOL/$TESTFS2/truncated4
+log_must zpool sync $TESTPOOL
+log_must zfs umount $TESTPOOL/$TESTFS2
+log_must zfs mount $TESTPOOL/$TESTFS2
+log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/truncated3 \
+ bs=128k count=3 iflag=fullblock
+log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/truncated4 \
+ bs=512 count=1 iflag=fullblock
+
+log_must zfs snapshot $TESTPOOL/$TESTFS2@snap2
+expected_cksum=$(recursive_cksum /$TESTPOOL/$TESTFS2)
+
+log_must eval "zfs send -wp $TESTPOOL/$TESTFS2@snap1 > $sendfile"
+log_must eval "zfs send -wp -i @snap1 $TESTPOOL/$TESTFS2@snap2 > $sendfile2"
+
+log_must eval "zfs recv -F $TESTPOOL/recv < $sendfile"
+log_must eval "zfs recv -F $TESTPOOL/recv < $sendfile2"
+log_must zfs load-key $TESTPOOL/recv
+
+log_must zfs mount -a
+actual_cksum=$(recursive_cksum /$TESTPOOL/recv)
+[[ "$expected_cksum" != "$actual_cksum" ]] && \
+ log_fail "Recursive checksums differ ($expected_cksum != $actual_cksum)"
+
+log_pass "Verified 'zfs send -w' works with many different file layouts"
--- /dev/null
+#!/bin/ksh
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source. A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2019 by Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+#
+# Description:
+# Verify encrypted raw incremental receives handle dnode reallocation.
+
+# Strategy:
+# 1. Create a pool containing an encrypted filesystem.
+# 2. Use 'zfs send -wp' to perform a raw send of the initial filesystem.
+# 3. Repeat the followings steps N times to verify raw incremental receives.
+# a) Randomly change several key dataset properties.
+# b) Modify the contents of the filesystem such that dnode reallocation
+# is likely during the 'zfs receive', and receive_object() exercises
+# as much of its functionality as possible.
+# c) Create a new snapshot and generate an raw incremental stream.
+# d) Receive the raw incremental stream and verify the received contents.
+# e) Destroy the incremental stream and old snapshot.
+#
+
+verify_runnable "both"
+
+log_assert "Verify encrypted raw incremental receive handles reallocation"
+
+function cleanup
+{
+ rm -f $BACKDIR/fs@*
+ rm -f $keyfile
+ destroy_dataset $POOL/fs "-rR"
+ destroy_dataset $POOL/newfs "-rR"
+}
+
+log_onexit cleanup
+
+typeset keyfile=/$TESTPOOL/pkey
+
+# Create an encrypted dataset
+log_must eval "echo 'password' > $keyfile"
+log_must zfs create -o encryption=on -o keyformat=passphrase \
+ -o keylocation=file://$keyfile $POOL/fs
+
+last_snap=1
+log_must zfs snapshot $POOL/fs@snap${last_snap}
+log_must eval "zfs send -wp $POOL/fs@snap${last_snap} \
+ >$BACKDIR/fs@snap${last_snap}"
+log_must eval "zfs recv $POOL/newfs < $BACKDIR/fs@snap${last_snap}"
+
+# Set atime=off to prevent the recursive_cksum from modifying newfs.
+log_must zfs set atime=off $POOL/newfs
+
+for i in {1..5}; do
+ # Randomly modify several dataset properties in order to generate
+ # more interesting incremental send streams.
+ rand_set_prop $POOL/fs checksum "off" "fletcher4" "sha256"
+ rand_set_prop $POOL/fs compression "off" "lzjb" "gzip" "lz4"
+ rand_set_prop $POOL/fs recordsize "32K" "128K"
+ rand_set_prop $POOL/fs dnodesize "legacy" "auto" "4k"
+ rand_set_prop $POOL/fs xattr "on" "sa"
+
+ # Churn the filesystem in such a way that we're likely to be both
+ # allocating and reallocating objects in the incremental stream.
+ #
+ # Disable xattrs until the following spill block issue is resolved:
+ # https://github.com/openzfs/openzfs/pull/705
+ #
+ log_must churn_files 1000 524288 $POOL/fs 0
+ expected_cksum=$(recursive_cksum /$fs)
+
+ # Create a snapshot and use it to send an incremental stream.
+ this_snap=$((last_snap + 1))
+ log_must zfs snapshot $POOL/fs@snap${this_snap}
+ log_must eval "zfs send -wp -i $POOL/fs@snap${last_snap} \
+ $POOL/fs@snap${this_snap} > $BACKDIR/fs@snap${this_snap}"
+
+ # Receive the incremental stream and verify the received contents.
+ log_must eval "zfs recv -Fu $POOL/newfs < $BACKDIR/fs@snap${this_snap}"
+
+ log_must zfs load-key $POOL/newfs
+ log_must zfs mount $POOL/newfs
+ actual_cksum=$(recursive_cksum /$POOL/newfs)
+ log_must zfs umount $POOL/newfs
+ log_must zfs unload-key $POOL/newfs
+
+ if [[ "$expected_cksum" != "$actual_cksum" ]]; then
+ log_fail "Checksums differ ($expected_cksum != $actual_cksum)"
+ fi
+
+ # Destroy the incremental stream and old snapshot.
+ rm -f $BACKDIR/fs@snap${last_snap}
+ log_must zfs destroy $POOL/fs@snap${last_snap}
+ log_must zfs destroy $POOL/newfs@snap${last_snap}
+ last_snap=$this_snap
+done
+
+log_pass "Verify encrypted raw incremental receive handles reallocation"
projects / mirror_zfs.git / commitdiff