tests/zfs-tests/tests/functional/rsend/rsend.kshlib

   1 #
   2 # CDDL HEADER START
   3 #
   4 # The contents of this file are subject to the terms of the
   5 # Common Development and Distribution License (the "License").
   6 # You may not use this file except in compliance with the License.
   7 #
   8 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9 # or http://www.opensolaris.org/os/licensing.
  10 # See the License for the specific language governing permissions
  11 # and limitations under the License.
  12 #
  13 # When distributing Covered Code, include this CDDL HEADER in each
  14 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15 # If applicable, add the following below this CDDL HEADER, with the
  16 # fields enclosed by brackets "[]" replaced with your own identifying
  17 # information: Portions Copyright [yyyy] [name of copyright owner]
  18 #
  19 # CDDL HEADER END
  20 #
  21
  22 #
  23 # Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  24 # Use is subject to license terms.
  25 #
  26
  27 #
  28 # Copyright (c) 2013, 2016 by Delphix. All rights reserved.
  29 #
  30
  31 . $STF_SUITE/include/libtest.shlib
  32 . $STF_SUITE/include/math.shlib
  33 . $STF_SUITE/tests/functional/rsend/rsend.cfg
  34
  35 #
  36 # Set up test model which includes various datasets
  37 #
  38 #               @final
  39 #               @snapB
  40 #               @init
  41 #               |
  42 #   ______ pclone
  43 #  |      /
  44 #  |@psnap
  45 #  ||                         @final
  46 #  ||@final       @final      @snapC
  47 #  ||@snapC       @snapC      @snapB
  48 #  ||@snapA       @snapB      @snapA
  49 #  ||@init        @init       @init
  50 #  |||            |           |
  51 # $pool -------- $FS ------- fs1 ------- fs2
  52 #    \             \\_____     \          |
  53 #     vol           vol   \____ \         @fsnap
  54 #      |              |        \ \              \
  55 #      @init          @vsnap   |  ------------ fclone
  56 #      @snapA         @init \  |                    |
  57 #      @final         @snapB \ |                    @init
  58 #                     @snapC  vclone                @snapA
  59 #                     @final       |                @final
  60 #                                 @init
  61 #                                 @snapC
  62 #                                 @final
  63 #
  64 # $1 pool name
  65 #
  66 function setup_test_model
  67 {
  68         typeset pool=$1
  69
  70         log_must zfs create -p $pool/$FS/fs1/fs2
  71
  72         log_must zfs snapshot $pool@psnap
  73         log_must zfs clone $pool@psnap $pool/pclone
  74
  75         if is_global_zone ; then
  76                 log_must zfs create -V 16M $pool/vol
  77                 log_must zfs create -V 16M $pool/$FS/vol
  78                 block_device_wait
  79
  80                 log_must zfs snapshot $pool/$FS/vol@vsnap
  81                 log_must zfs clone $pool/$FS/vol@vsnap $pool/$FS/vclone
  82                 block_device_wait
  83         fi
  84
  85         log_must snapshot_tree $pool/$FS/fs1/fs2@fsnap
  86         log_must zfs clone $pool/$FS/fs1/fs2@fsnap $pool/$FS/fs1/fclone
  87         log_must zfs snapshot -r $pool@init
  88
  89         log_must snapshot_tree $pool@snapA
  90         log_must snapshot_tree $pool@snapC
  91         log_must snapshot_tree $pool/pclone@snapB
  92         log_must snapshot_tree $pool/$FS@snapB
  93         log_must snapshot_tree $pool/$FS@snapC
  94         log_must snapshot_tree $pool/$FS/fs1@snapA
  95         log_must snapshot_tree $pool/$FS/fs1@snapB
  96         log_must snapshot_tree $pool/$FS/fs1@snapC
  97         log_must snapshot_tree $pool/$FS/fs1/fclone@snapA
  98
  99         if is_global_zone ; then
 100                 log_must zfs snapshot $pool/vol@snapA
 101                 log_must zfs snapshot $pool/$FS/vol@snapB
 102                 log_must zfs snapshot $pool/$FS/vol@snapC
 103                 log_must zfs snapshot $pool/$FS/vclone@snapC
 104         fi
 105
 106         log_must zfs snapshot -r $pool@final
 107
 108         return 0
 109 }
 110
 111 #
 112 # Cleanup the BACKDIR and given pool content and all the sub datasets
 113 #
 114 # $1 pool name
 115 #
 116 function cleanup_pool
 117 {
 118         typeset pool=$1
 119         log_must rm -rf $BACKDIR/*
 120
 121         if is_global_zone ; then
 122                 log_must_busy zfs destroy -Rf $pool
 123         else
 124                 typeset list=$(zfs list -H -r -t all -o name $pool)
 125                 for ds in $list ; do
 126                         if [[ $ds != $pool ]] ; then
 127                                 if datasetexists $ds ; then
 128                                         log_must_busy zfs destroy -Rf $ds
 129                                 fi
 130                         fi
 131                 done
 132         fi
 133
 134         typeset mntpnt=$(get_prop mountpoint $pool)
 135         if ! ismounted $pool ; then
 136                 # Make sure mountpoint directory is empty
 137                 if [[ -d $mntpnt ]]; then
 138                         log_must rm -rf $mntpnt/*
 139                 fi
 140
 141                 log_must zfs mount $pool
 142         fi
 143         if [[ -d $mntpnt ]]; then
 144                 rm -rf $mntpnt/*
 145         fi
 146
 147         return 0
 148 }
 149
 150 function cleanup_pools
 151 {
 152         cleanup_pool $POOL2
 153         destroy_pool $POOL3
 154 }
 155
 156 function cmp_md5s {
 157         typeset file1=$1
 158         typeset file2=$2
 159
 160         eval md5sum $file1 | awk '{ print $1 }' > $BACKDIR/md5_file1
 161         eval md5sum $file2 | awk '{ print $1 }' > $BACKDIR/md5_file2
 162         diff $BACKDIR/md5_file1 $BACKDIR/md5_file2
 163         typeset -i ret=$?
 164
 165         rm -f $BACKDIR/md5_file1 $BACKDIR/md5_file2
 166
 167         return $ret
 168 }
 169
 170 #
 171 # Detect if the given two filesystems have same sub-datasets
 172 #
 173 # $1 source filesystem
 174 # $2 destination filesystem
 175 #
 176 function cmp_ds_subs
 177 {
 178         typeset src_fs=$1
 179         typeset dst_fs=$2
 180
 181         zfs list -r -H -t all -o name $src_fs > $BACKDIR/src1
 182         zfs list -r -H -t all -o name $dst_fs > $BACKDIR/dst1
 183
 184         eval sed -e 's:^$src_fs:PREFIX:g' < $BACKDIR/src1 > $BACKDIR/src
 185         eval sed -e 's:^$dst_fs:PREFIX:g' < $BACKDIR/dst1 > $BACKDIR/dst
 186
 187         diff $BACKDIR/src $BACKDIR/dst
 188         typeset -i ret=$?
 189
 190         rm -f $BACKDIR/src $BACKDIR/dst $BACKDIR/src1 $BACKDIR/dst1
 191
 192         return $ret
 193 }
 194
 195 #
 196 # Compare all the directories and files in two filesystems
 197 #
 198 # $1 source filesystem
 199 # $2 destination filesystem
 200 #
 201 function cmp_ds_cont
 202 {
 203         typeset src_fs=$1
 204         typeset dst_fs=$2
 205
 206         typeset srcdir dstdir
 207         srcdir=$(get_prop mountpoint $src_fs)
 208         dstdir=$(get_prop mountpoint $dst_fs)
 209
 210         diff -r $srcdir $dstdir > /dev/null 2>&1
 211         return $?
 212 }
 213
 214 #
 215 # Compare the given two dataset properties
 216 #
 217 # $1 dataset 1
 218 # $2 dataset 2
 219 #
 220 function cmp_ds_prop
 221 {
 222         typeset dtst1=$1
 223         typeset dtst2=$2
 224
 225         for item in "type" "origin" "volblocksize" "acltype" "dnodesize" \
 226             "atime" "canmount" "checksum" "compression" "copies" "devices" \
 227             "exec" "quota" "readonly" "recordsize" "reservation" "setuid" \
 228             "snapdir" "version" "volsize" "xattr" "zoned" "mountpoint";
 229         do
 230                 zfs get -H -o property,value,source $item $dtst1 >> \
 231                     $BACKDIR/dtst1
 232                 zfs get -H -o property,value,source $item $dtst2 >> \
 233                     $BACKDIR/dtst2
 234         done
 235
 236         eval sed -e 's:$dtst1:PREFIX:g' < $BACKDIR/dtst1 > $BACKDIR/dtst1
 237         eval sed -e 's:$dtst2:PREFIX:g' < $BACKDIR/dtst2 > $BACKDIR/dtst2
 238
 239         diff $BACKDIR/dtst1 $BACKDIR/dtst2
 240         typeset -i ret=$?
 241
 242         rm -f $BACKDIR/dtst1 $BACKDIR/dtst2
 243
 244         return $ret
 245
 246 }
 247
 248 #
 249 # Random create directories and files
 250 #
 251 # $1 directory
 252 #
 253 function random_tree
 254 {
 255         typeset dir=$1
 256
 257         if [[ -d $dir ]]; then
 258                 rm -rf $dir
 259         fi
 260         mkdir -p $dir
 261         typeset -i ret=$?
 262
 263         typeset -i nl nd nf
 264         ((nl = RANDOM % 6 + 1))
 265         ((nd = RANDOM % 3 ))
 266         ((nf = RANDOM % 5 ))
 267         mktree -b $dir -l $nl -d $nd -f $nf
 268         ((ret |= $?))
 269
 270         return $ret
 271 }
 272
 273 #
 274 # Put data in filesystem and take snapshot
 275 #
 276 # $1 snapshot name
 277 #
 278 function snapshot_tree
 279 {
 280         typeset snap=$1
 281         typeset ds=${snap%%@*}
 282         typeset type=$(get_prop "type" $ds)
 283
 284         typeset -i ret=0
 285         if [[ $type == "filesystem" ]]; then
 286                 typeset mntpnt=$(get_prop mountpoint $ds)
 287                 ((ret |= $?))
 288
 289                 if ((ret == 0)) ; then
 290                         eval random_tree $mntpnt/${snap##$ds}
 291                         ((ret |= $?))
 292                 fi
 293         fi
 294
 295         if ((ret == 0)) ; then
 296                 zfs snapshot $snap
 297                 ((ret |= $?))
 298         fi
 299
 300         return $ret
 301 }
 302
 303 #
 304 # Destroy the given snapshot and stuff
 305 #
 306 # $1 snapshot
 307 #
 308 function destroy_tree
 309 {
 310         typeset -i ret=0
 311         typeset snap
 312         for snap in "$@" ; do
 313                 log_must_busy zfs destroy $snap
 314
 315                 typeset ds=${snap%%@*}
 316                 typeset type=$(get_prop "type" $ds)
 317                 if [[ $type == "filesystem" ]]; then
 318                         typeset mntpnt=$(get_prop mountpoint $ds)
 319                         if [[ -n $mntpnt ]]; then
 320                                 rm -rf $mntpnt/$snap
 321                         fi
 322                 fi
 323         done
 324
 325         return 0
 326 }
 327
 328 #
 329 # Get all the sub-datasets of give dataset with specific suffix
 330 #
 331 # $1 Given dataset
 332 # $2 Suffix
 333 #
 334 function getds_with_suffix
 335 {
 336         typeset ds=$1
 337         typeset suffix=$2
 338
 339         typeset list=$(zfs list -r -H -t all -o name $ds | grep "$suffix$")
 340
 341         echo $list
 342 }
 343
 344 #
 345 # Output inherited properties whitch is edited for file system
 346 #
 347 function fs_inherit_prop
 348 {
 349         typeset fs_prop
 350         if is_global_zone ; then
 351                 fs_prop=$(zfs inherit 2>&1 | \
 352                     awk '$2=="YES" && $3=="YES" {print $1}')
 353                 if ! is_te_enabled ; then
 354                         fs_prop=$(echo $fs_prop | grep -v "mlslabel")
 355                 fi
 356         else
 357                 fs_prop=$(zfs inherit 2>&1 | \
 358                     awk '$2=="YES" && $3=="YES" {print $1}'|
 359                     egrep -v "devices|mlslabel|sharenfs|sharesmb|zoned")
 360         fi
 361
 362         echo $fs_prop
 363 }
 364
 365 #
 366 # Output inherited properties for volume
 367 #
 368 function vol_inherit_prop
 369 {
 370         echo "checksum readonly"
 371 }
 372
 373 #
 374 # Get the destination dataset to compare
 375 #
 376 function get_dst_ds
 377 {
 378         typeset srcfs=$1
 379         typeset dstfs=$2
 380
 381         #
 382         # If the srcfs is not pool
 383         #
 384         if ! zpool list $srcfs > /dev/null 2>&1 ; then
 385                 eval dstfs="$dstfs/${srcfs#*/}"
 386         fi
 387
 388         echo $dstfs
 389 }
 390
 391 #
 392 # Make test files
 393 #
 394 # $1 Number of files to create
 395 # $2 Maximum file size
 396 # $3 File ID offset
 397 # $4 File system to create the files on
 398 #
 399 function mk_files
 400 {
 401         nfiles=$1
 402         maxsize=$2
 403         file_id_offset=$3
 404         fs=$4
 405         bs=512
 406
 407         for ((i=0; i<$nfiles; i=i+1)); do
 408                 file_name="/$fs/file-$maxsize-$((i+$file_id_offset))"
 409                 file_size=$((($RANDOM * $RANDOM % ($maxsize - 1)) + 1))
 410
 411                 #
 412                 # Create an interesting mix of files which contain both
 413                 # data blocks and holes for more realistic test coverage.
 414                 # Half the files are created as sparse then partially filled,
 415                 # the other half is dense then a hole is punched in the file.
 416                 #
 417                 if [ $((RANDOM % 2)) -eq 0 ]; then
 418                         truncate -s $file_size $file_name || \
 419                             log_fail "Failed to create $file_name"
 420                         dd if=/dev/urandom of=$file_name \
 421                             bs=$bs count=$(($file_size / 2 / $bs)) \
 422                             seek=$(($RANDOM % (($file_size / 2 / $bs) + 1))) \
 423                             conv=notrunc >/dev/null 2>&1 || \
 424                             log_fail "Failed to create $file_name"
 425                 else
 426                         dd if=/dev/urandom of=$file_name \
 427                             bs=$file_size count=1 >/dev/null 2>&1 || \
 428                             log_fail "Failed to create $file_name"
 429                         dd if=/dev/zero of=$file_name \
 430                             bs=$bs count=$(($file_size / 2 / $bs)) \
 431                             seek=$(($RANDOM % (($file_size / 2 / $bs) + 1))) \
 432                             conv=notrunc >/dev/null 2>&1 || \
 433                             log_fail "Failed to create $file_name"
 434                 fi
 435         done
 436         echo Created $nfiles files of random sizes up to $maxsize bytes
 437 }
 438
 439 #
 440 # Remove test files
 441 #
 442 # $1 Number of files to remove
 443 # $2 Maximum file size
 444 # $3 File ID offset
 445 # $4 File system to remove the files from
 446 #
 447 function rm_files
 448 {
 449         nfiles=$1
 450         maxsize=$2
 451         file_id_offset=$3
 452         fs=$4
 453
 454         for ((i=0; i<$nfiles; i=i+1)); do
 455                 rm -f /$fs/file-$maxsize-$((i+$file_id_offset))
 456         done
 457         echo Removed $nfiles files of random sizes up to $maxsize bytes
 458 }
 459
 460 #
 461 # Simulate a random set of operations which could be reasonably expected
 462 # to occur on an average filesystem.
 463 #
 464 # $1 Number of files to modify
 465 # $2 Maximum file size
 466 # $3 File system to modify the file on
 467 # $4 Enabled xattrs (optional)
 468 #
 469 function churn_files
 470 {
 471         nfiles=$1
 472         maxsize=$2
 473         fs=$3
 474         xattrs=${4:-1}
 475
 476         #
 477         # Remove roughly half of the files in order to make it more
 478         # likely that a dnode will be reallocated.
 479         #
 480         for ((i=0; i<$nfiles; i=i+1)); do
 481                 file_name="/$fs/file-$i"
 482
 483                 if [[ -e $file_name ]]; then
 484                         if [ $((RANDOM % 2)) -eq 0 ]; then
 485                                 rm $file_name || \
 486                                     log_fail "Failed to remove $file_name"
 487                         fi
 488                 fi
 489         done
 490
 491         #
 492         # Remount the filesystem to simulate normal usage.  This resets
 493         # the last allocated object id allowing for new objects to be
 494         # reallocated in the locations of previously freed objects.
 495         #
 496         log_must zfs unmount $fs
 497         log_must zfs mount $fs
 498
 499         for i in {0..$nfiles}; do
 500                 file_name="/$fs/file-$i"
 501                 file_size=$((($RANDOM * $RANDOM % ($maxsize - 1)) + 1))
 502
 503                 #
 504                 # When the file exists modify it in one of five ways to
 505                 # simulate normal usage:
 506                 # - (20%) Remove and set and extended attribute on the file
 507                 # - (20%) Overwrite the existing file
 508                 # - (20%) Truncate the existing file to a random length
 509                 # - (20%) Truncate the existing file to zero length
 510                 # - (20%) Remove the file
 511                 #
 512                 # Otherwise create the missing file.  20% of the created
 513                 # files will be small and use embedded block pointers, the
 514                 # remainder with have random sizes up to the maximum size.
 515                 # Three extended attributes are attached to all of the files.
 516                 #
 517                 if [[ -e $file_name ]]; then
 518                         value=$((RANDOM % 5))
 519                         if [ $value -eq 0 -a $xattrs -ne 0 ]; then
 520                                 attrname="testattr$((RANDOM % 3))"
 521                                 attr -qr $attrname $file_name || \
 522                                     log_fail "Failed to remove $attrname"
 523                                 attr -qs $attrname -V TestValue $file_name || \
 524                                     log_fail "Failed to set $attrname"
 525                         elif [ $value -eq 1 ]; then
 526                                 dd if=/dev/urandom of=$file_name \
 527                                     bs=$file_size count=1 >/dev/null 2>&1 || \
 528                                     log_fail "Failed to overwrite $file_name"
 529                         elif [ $value -eq 2 ]; then
 530                                 truncate -s $file_size $file_name || \
 531                                     log_fail "Failed to truncate $file_name"
 532                         elif [ $value -eq 3 ]; then
 533                                 truncate -s 0 $file_name || \
 534                                     log_fail "Failed to truncate $file_name"
 535                         else
 536                                 rm $file_name || \
 537                                     log_fail "Failed to remove $file_name"
 538                         fi
 539                 else
 540                         if [ $((RANDOM % 5)) -eq 0 ]; then
 541                                 file_size=$((($RANDOM % 64) + 1))
 542                         fi
 543
 544                         dd if=/dev/urandom of=$file_name \
 545                             bs=$file_size count=1 >/dev/null 2>&1 || \
 546                             log_fail "Failed to create $file_name"
 547
 548                         if [ $xattrs -ne 0 ]; then
 549                                 for j in {0..2}; do
 550                                         attrname="testattr$j"
 551                                         attr -qs $attrname -V TestValue \
 552                                             $file_name || log_fail \
 553                                             "Failed to set $attrname"
 554                                 done
 555                         fi
 556                 fi
 557         done
 558
 559         return 0
 560 }
 561
 562 #
 563 # Mess up file contents
 564 #
 565 # $1 The file path
 566 #
 567 function mess_file
 568 {
 569         file=$1
 570
 571         filesize=$(stat -c '%s' $file)
 572         offset=$(($RANDOM * $RANDOM % $filesize))
 573         if (($RANDOM % 7 <= 1)); then
 574                 #
 575                 # We corrupt 2 bytes to minimize the chance that we
 576                 # write the same value that's already there.
 577                 #
 578                 log_must eval "dd if=/dev/urandom of=$file conv=notrunc " \
 579                     "bs=1 count=2 seek=$offset >/dev/null 2>&1"
 580         else
 581                 log_must truncate -s $offset $file
 582         fi
 583 }
 584
 585 #
 586 # Diff the send/receive filesystems
 587 #
 588 # $1 The sent filesystem
 589 # $2 The received filesystem
 590 #
 591 function file_check
 592 {
 593         sendfs=$1
 594         recvfs=$2
 595
 596         if [[ -d /$recvfs/.zfs/snapshot/a && -d \
 597             /$sendfs/.zfs/snapshot/a ]]; then
 598                 diff -r /$recvfs/.zfs/snapshot/a /$sendfs/.zfs/snapshot/a
 599                 [[ $? -eq 0 ]] || log_fail "Differences found in snap a"
 600         fi
 601         if [[ -d /$recvfs/.zfs/snapshot/b && -d \
 602             /$sendfs/.zfs/snapshot/b ]]; then
 603                 diff -r /$recvfs/.zfs/snapshot/b /$sendfs/.zfs/snapshot/b
 604                 [[ $? -eq 0 ]] || log_fail "Differences found in snap b"
 605         fi
 606 }
 607
 608 #
 609 # Resume test helper
 610 #
 611 # $1 The ZFS send command
 612 # $2 The filesystem where the streams are sent
 613 # $3 The receive filesystem
 614 #
 615 function resume_test
 616 {
 617         sendcmd=$1
 618         streamfs=$2
 619         recvfs=$3
 620
 621         stream_num=1
 622         log_must eval "$sendcmd >/$streamfs/$stream_num"
 623
 624         for ((i=0; i<2; i=i+1)); do
 625                 mess_file /$streamfs/$stream_num
 626                 log_mustnot zfs recv -suv $recvfs </$streamfs/$stream_num
 627                 stream_num=$((stream_num+1))
 628
 629                 token=$(zfs get -Hp -o value receive_resume_token $recvfs)
 630                 log_must eval "zfs send -v -t $token >/$streamfs/$stream_num"
 631                 [[ -f /$streamfs/$stream_num ]] || \
 632                     log_fail "NO FILE /$streamfs/$stream_num"
 633         done
 634         log_must zfs recv -suv $recvfs </$streamfs/$stream_num
 635 }
 636
 637 #
 638 # Setup filesystems for the resumable send/receive tests
 639 #
 640 # $1 The "send" filesystem
 641 # $2 The "recv" filesystem
 642 #
 643 function test_fs_setup
 644 {
 645         typeset sendfs=$1
 646         typeset recvfs=$2
 647         typeset streamfs=$3
 648         typeset sendpool=${sendfs%%/*}
 649         typeset recvpool=${recvfs%%/*}
 650
 651         datasetexists $sendfs && log_must_busy zfs destroy -r $sendpool
 652         datasetexists $recvfs && log_must_busy zfs destroy -r $recvpool
 653         datasetexists $streamfs && log_must_busy zfs destroy -r $streamfs
 654
 655         if $(datasetexists $sendfs || zfs create -o compress=lz4 $sendfs); then
 656                 mk_files 1000 256 0 $sendfs &
 657                 mk_files 1000 131072 0 $sendfs &
 658                 mk_files 100 1048576 0 $sendfs &
 659                 mk_files 10 10485760 0 $sendfs &
 660                 mk_files 1 104857600 0 $sendfs &
 661                 log_must wait
 662                 log_must zfs snapshot $sendfs@a
 663
 664                 rm_files 200 256 0 $sendfs &
 665                 rm_files 200 131072 0 $sendfs &
 666                 rm_files 20 1048576 0 $sendfs &
 667                 rm_files 2 10485760 0 $sendfs &
 668                 log_must wait
 669
 670                 mk_files 400 256 0 $sendfs &
 671                 mk_files 400 131072 0 $sendfs &
 672                 mk_files 40 1048576 0 $sendfs &
 673                 mk_files 4 10485760 0 $sendfs &
 674                 log_must wait
 675
 676                 log_must zfs snapshot $sendfs@b
 677                 log_must eval "zfs send -v $sendfs@a >/$sendpool/initial.zsend"
 678                 log_must eval "zfs send -v -i @a $sendfs@b " \
 679                     ">/$sendpool/incremental.zsend"
 680         fi
 681
 682         log_must zfs create -o compress=lz4 $streamfs
 683 }
 684
 685 #
 686 # Check to see if the specified features are set in a send stream.
 687 # The values for these features are found in include/sys/zfs_ioctl.h
 688 #
 689 # $1 The stream file
 690 # $2-$n The flags expected in the stream
 691 #
 692 function stream_has_features
 693 {
 694         typeset file=$1
 695         shift
 696
 697         [[ -f $file ]] || log_fail "Couldn't find file: $file"
 698         typeset flags=$(cat $file | zstreamdump | \
 699             awk '/features =/ {features = $3} END {print features}')
 700         typeset -A feature
 701         feature[dedup]="1"
 702         feature[dedupprops]="2"
 703         feature[sa_spill]="4"
 704         feature[embed_data]="10000"
 705         feature[lz4]="20000"
 706         feature[mooch_byteswap]="40000"
 707         feature[large_blocks]="80000"
 708         feature[resuming]="100000"
 709         feature[redacted]="200000"
 710         feature[compressed]="400000"
 711
 712         typeset flag known derived=0
 713         for flag in "$@"; do
 714                 known=${feature[$flag]}
 715                 [[ -z $known ]] && log_fail "Unknown feature: $flag"
 716
 717                 derived=$(printf "%x" $((0x${flags} & 0x${feature[$flag]})))
 718                 [[ $derived = $known ]] || return 1
 719         done
 720
 721         return 0
 722 }
 723
 724 #
 725 # Given a send stream, verify that the size of the stream matches what's
 726 # expected based on the source or target dataset. If the stream is an
 727 # incremental stream, subtract the size of the source snapshot before
 728 # comparing. This function does not currently handle incremental streams
 729 # that remove data.
 730 #
 731 # $1 The zstreamdump output file
 732 # $2 The dataset to compare against
 733 #    This can be a source of a send or recv target (fs, not snapshot)
 734 # $3 The percentage below which verification is deemed a failure
 735 # $4 The source snapshot of an incremental send
 736 #
 737
 738 function verify_stream_size
 739 {
 740         typeset stream=$1
 741         typeset ds=$2
 742         typeset percent=${3:-90}
 743         typeset inc_src=$4
 744
 745         [[ -f $stream ]] || log_fail "No such file: $stream"
 746         datasetexists $ds || log_fail "No such dataset: $ds"
 747
 748         typeset stream_size=$(cat $stream | zstreamdump | sed -n \
 749             's/ Total write size = \(.*\) (0x.*)/\1/p')
 750
 751         typeset inc_size=0
 752         if [[ -n $inc_src ]]; then
 753                 inc_size=$(get_prop lrefer $inc_src)
 754                 if stream_has_features $stream compressed; then
 755                         inc_size=$(get_prop refer $inc_src)
 756                 fi
 757         fi
 758
 759         if stream_has_features $stream compressed; then
 760                 ds_size=$(get_prop refer $ds)
 761         else
 762                 ds_size=$(get_prop lrefer $ds)
 763         fi
 764         ds_size=$((ds_size - inc_size))
 765
 766         within_percent $stream_size $ds_size $percent || log_fail \
 767             "$stream_size $ds_size differed by too much"
 768 }
 769
 770 # Cleanup function for tests involving resumable send
 771 function resume_cleanup
 772 {
 773         typeset sendfs=$1
 774         typeset streamfs=$2
 775         typeset sendpool=${sendfs%%/*}
 776
 777         datasetexists $sendfs && log_must_busy zfs destroy -r $sendfs
 778         datasetexists $streamfs && log_must_busy zfs destroy -r $streamfs
 779         cleanup_pool $POOL2
 780         rm -f /$sendpool/initial.zsend /$sendpool/incremental.zsend
 781 }
 782
 783 # Randomly set the property to one of the enumerated values.
 784 function rand_set_prop
 785 {
 786         typeset dtst=$1
 787         typeset prop=$2
 788         shift 2
 789         typeset value=$(random_get $@)
 790
 791         log_must eval "zfs set $prop='$value' $dtst"
 792 }
 793
 794 # Generate a recursive checksum of a filesystems contents.  Only file
 795 # data is included in the checksum (no meta data, or xattrs).
 796 function recursive_cksum
 797 {
 798         find $1 -type f -exec sha256sum {} \; | \
 799             sort -k 2 | awk '{ print $1 }' | sha256sum
 800 }