tests/zfs-tests/tests/functional/rsend/rsend.kshlib

   1 #
   2 # CDDL HEADER START
   3 #
   4 # The contents of this file are subject to the terms of the
   5 # Common Development and Distribution License (the "License").
   6 # You may not use this file except in compliance with the License.
   7 #
   8 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9 # or http://www.opensolaris.org/os/licensing.
  10 # See the License for the specific language governing permissions
  11 # and limitations under the License.
  12 #
  13 # When distributing Covered Code, include this CDDL HEADER in each
  14 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15 # If applicable, add the following below this CDDL HEADER, with the
  16 # fields enclosed by brackets "[]" replaced with your own identifying
  17 # information: Portions Copyright [yyyy] [name of copyright owner]
  18 #
  19 # CDDL HEADER END
  20 #
  21
  22 #
  23 # Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  24 # Use is subject to license terms.
  25 #
  26
  27 #
  28 # Copyright (c) 2013, 2016 by Delphix. All rights reserved.
  29 #
  30
  31 . $STF_SUITE/include/libtest.shlib
  32 . $STF_SUITE/include/math.shlib
  33 . $STF_SUITE/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib
  34 . $STF_SUITE/tests/functional/rsend/rsend.cfg
  35
  36 #
  37 # Set up test model which includes various datasets
  38 #
  39 #               @final
  40 #               @snapB
  41 #               @init
  42 #               |
  43 #   ______ pclone
  44 #  |      /
  45 #  |@psnap
  46 #  ||                         @final
  47 #  ||@final       @final      @snapC
  48 #  ||@snapC       @snapC      @snapB
  49 #  ||@snapA       @snapB      @snapA
  50 #  ||@init        @init       @init
  51 #  |||            |           |
  52 # $pool -------- $FS ------- fs1 ------- fs2
  53 #    \             \\_____     \          |
  54 #     vol           vol   \____ \         @fsnap
  55 #      |              |        \ \              \
  56 #      @init          @vsnap   |  ------------ fclone
  57 #      @snapA         @init \  |                    |
  58 #      @final         @snapB \ |                    @init
  59 #                     @snapC  vclone                @snapA
  60 #                     @final       |                @final
  61 #                                 @init
  62 #                                 @snapC
  63 #                                 @final
  64 #
  65 # $1 pool name
  66 #
  67 function setup_test_model
  68 {
  69         typeset pool=$1
  70
  71         log_must zfs create -p $pool/$FS/fs1/fs2
  72
  73         log_must zfs snapshot $pool@psnap
  74         log_must zfs clone $pool@psnap $pool/pclone
  75
  76         if is_global_zone ; then
  77                 log_must zfs create -V 16M $pool/vol
  78                 log_must zfs create -V 16M $pool/$FS/vol
  79                 block_device_wait
  80
  81                 log_must zfs snapshot $pool/$FS/vol@vsnap
  82                 log_must zfs clone $pool/$FS/vol@vsnap $pool/$FS/vclone
  83                 block_device_wait
  84         fi
  85
  86         log_must snapshot_tree $pool/$FS/fs1/fs2@fsnap
  87         log_must zfs clone $pool/$FS/fs1/fs2@fsnap $pool/$FS/fs1/fclone
  88         log_must zfs snapshot -r $pool@init
  89
  90         log_must snapshot_tree $pool@snapA
  91         log_must snapshot_tree $pool@snapC
  92         log_must snapshot_tree $pool/pclone@snapB
  93         log_must snapshot_tree $pool/$FS@snapB
  94         log_must snapshot_tree $pool/$FS@snapC
  95         log_must snapshot_tree $pool/$FS/fs1@snapA
  96         log_must snapshot_tree $pool/$FS/fs1@snapB
  97         log_must snapshot_tree $pool/$FS/fs1@snapC
  98         log_must snapshot_tree $pool/$FS/fs1/fclone@snapA
  99
 100         if is_global_zone ; then
 101                 log_must zfs snapshot $pool/vol@snapA
 102                 log_must zfs snapshot $pool/$FS/vol@snapB
 103                 log_must zfs snapshot $pool/$FS/vol@snapC
 104                 log_must zfs snapshot $pool/$FS/vclone@snapC
 105         fi
 106
 107         log_must zfs snapshot -r $pool@final
 108
 109         return 0
 110 }
 111
 112 #
 113 # Cleanup the BACKDIR and given pool content and all the sub datasets
 114 #
 115 # $1 pool name
 116 #
 117 function cleanup_pool
 118 {
 119         typeset pool=$1
 120         log_must rm -rf $BACKDIR/*
 121
 122         if is_global_zone ; then
 123                 log_must_busy zfs destroy -Rf $pool
 124         else
 125                 typeset list=$(zfs list -H -r -t all -o name $pool)
 126                 for ds in $list ; do
 127                         if [[ $ds != $pool ]] ; then
 128                                 if datasetexists $ds ; then
 129                                         log_must_busy zfs destroy -Rf $ds
 130                                 fi
 131                         fi
 132                 done
 133         fi
 134
 135         typeset mntpnt=$(get_prop mountpoint $pool)
 136         if ! ismounted $pool ; then
 137                 # Make sure mountpoint directory is empty
 138                 if [[ -d $mntpnt ]]; then
 139                         log_must rm -rf $mntpnt/*
 140                 fi
 141
 142                 log_must zfs mount $pool
 143         fi
 144         if [[ -d $mntpnt ]]; then
 145                 rm -rf $mntpnt/*
 146         fi
 147
 148         return 0
 149 }
 150
 151 function cleanup_pools
 152 {
 153         cleanup_pool $POOL2
 154         destroy_pool $POOL3
 155 }
 156
 157 function cmp_md5s {
 158         typeset file1=$1
 159         typeset file2=$2
 160
 161         eval md5sum $file1 | awk '{ print $1 }' > $BACKDIR/md5_file1
 162         eval md5sum $file2 | awk '{ print $1 }' > $BACKDIR/md5_file2
 163         diff $BACKDIR/md5_file1 $BACKDIR/md5_file2
 164         typeset -i ret=$?
 165
 166         rm -f $BACKDIR/md5_file1 $BACKDIR/md5_file2
 167
 168         return $ret
 169 }
 170
 171 #
 172 # Detect if the given two filesystems have same sub-datasets
 173 #
 174 # $1 source filesystem
 175 # $2 destination filesystem
 176 #
 177 function cmp_ds_subs
 178 {
 179         typeset src_fs=$1
 180         typeset dst_fs=$2
 181
 182         zfs list -r -H -t all -o name $src_fs > $BACKDIR/src1
 183         zfs list -r -H -t all -o name $dst_fs > $BACKDIR/dst1
 184
 185         eval sed -e 's:^$src_fs:PREFIX:g' < $BACKDIR/src1 > $BACKDIR/src
 186         eval sed -e 's:^$dst_fs:PREFIX:g' < $BACKDIR/dst1 > $BACKDIR/dst
 187
 188         diff $BACKDIR/src $BACKDIR/dst
 189         typeset -i ret=$?
 190
 191         rm -f $BACKDIR/src $BACKDIR/dst $BACKDIR/src1 $BACKDIR/dst1
 192
 193         return $ret
 194 }
 195
 196 #
 197 # Compare all the directories and files in two filesystems
 198 #
 199 # $1 source filesystem
 200 # $2 destination filesystem
 201 #
 202 function cmp_ds_cont
 203 {
 204         typeset src_fs=$1
 205         typeset dst_fs=$2
 206
 207         typeset srcdir dstdir
 208         srcdir=$(get_prop mountpoint $src_fs)
 209         dstdir=$(get_prop mountpoint $dst_fs)
 210
 211         diff -r $srcdir $dstdir > /dev/null 2>&1
 212         return $?
 213 }
 214
 215 #
 216 # Compare the given two dataset properties
 217 #
 218 # $1 dataset 1
 219 # $2 dataset 2
 220 #
 221 function cmp_ds_prop
 222 {
 223         typeset dtst1=$1
 224         typeset dtst2=$2
 225
 226         for item in "type" "origin" "volblocksize" "acltype" "dnodesize" \
 227             "atime" "canmount" "checksum" "compression" "copies" "devices" \
 228             "exec" "quota" "readonly" "recordsize" "reservation" "setuid" \
 229             "snapdir" "version" "volsize" "xattr" "zoned" "mountpoint";
 230         do
 231                 zfs get -H -o property,value,source $item $dtst1 >> \
 232                     $BACKDIR/dtst1
 233                 zfs get -H -o property,value,source $item $dtst2 >> \
 234                     $BACKDIR/dtst2
 235         done
 236
 237         eval sed -e 's:$dtst1:PREFIX:g' < $BACKDIR/dtst1 > $BACKDIR/dtst1
 238         eval sed -e 's:$dtst2:PREFIX:g' < $BACKDIR/dtst2 > $BACKDIR/dtst2
 239
 240         diff $BACKDIR/dtst1 $BACKDIR/dtst2
 241         typeset -i ret=$?
 242
 243         rm -f $BACKDIR/dtst1 $BACKDIR/dtst2
 244
 245         return $ret
 246
 247 }
 248
 249 #
 250 # Random create directories and files
 251 #
 252 # $1 directory
 253 #
 254 function random_tree
 255 {
 256         typeset dir=$1
 257
 258         if [[ -d $dir ]]; then
 259                 rm -rf $dir
 260         fi
 261         mkdir -p $dir
 262         typeset -i ret=$?
 263
 264         typeset -i nl nd nf
 265         ((nl = RANDOM % 6 + 1))
 266         ((nd = RANDOM % 3 ))
 267         ((nf = RANDOM % 5 ))
 268         mktree -b $dir -l $nl -d $nd -f $nf
 269         ((ret |= $?))
 270
 271         return $ret
 272 }
 273
 274 #
 275 # Put data in filesystem and take snapshot
 276 #
 277 # $1 snapshot name
 278 #
 279 function snapshot_tree
 280 {
 281         typeset snap=$1
 282         typeset ds=${snap%%@*}
 283         typeset type=$(get_prop "type" $ds)
 284
 285         typeset -i ret=0
 286         if [[ $type == "filesystem" ]]; then
 287                 typeset mntpnt=$(get_prop mountpoint $ds)
 288                 ((ret |= $?))
 289
 290                 if ((ret == 0)) ; then
 291                         eval random_tree $mntpnt/${snap##$ds}
 292                         ((ret |= $?))
 293                 fi
 294         fi
 295
 296         if ((ret == 0)) ; then
 297                 zfs snapshot $snap
 298                 ((ret |= $?))
 299         fi
 300
 301         return $ret
 302 }
 303
 304 #
 305 # Destroy the given snapshot and stuff
 306 #
 307 # $1 snapshot
 308 #
 309 function destroy_tree
 310 {
 311         typeset -i ret=0
 312         typeset snap
 313         for snap in "$@" ; do
 314                 log_must_busy zfs destroy $snap
 315
 316                 typeset ds=${snap%%@*}
 317                 typeset type=$(get_prop "type" $ds)
 318                 if [[ $type == "filesystem" ]]; then
 319                         typeset mntpnt=$(get_prop mountpoint $ds)
 320                         if [[ -n $mntpnt ]]; then
 321                                 rm -rf $mntpnt/$snap
 322                         fi
 323                 fi
 324         done
 325
 326         return 0
 327 }
 328
 329 #
 330 # Get all the sub-datasets of give dataset with specific suffix
 331 #
 332 # $1 Given dataset
 333 # $2 Suffix
 334 #
 335 function getds_with_suffix
 336 {
 337         typeset ds=$1
 338         typeset suffix=$2
 339
 340         typeset list=$(zfs list -r -H -t all -o name $ds | grep "$suffix$")
 341
 342         echo $list
 343 }
 344
 345 #
 346 # Output inherited properties whitch is edited for file system
 347 #
 348 function fs_inherit_prop
 349 {
 350         typeset fs_prop
 351         if is_global_zone ; then
 352                 fs_prop=$(zfs inherit 2>&1 | \
 353                     awk '$2=="YES" && $3=="YES" {print $1}')
 354                 if ! is_te_enabled ; then
 355                         fs_prop=$(echo $fs_prop | grep -v "mlslabel")
 356                 fi
 357         else
 358                 fs_prop=$(zfs inherit 2>&1 | \
 359                     awk '$2=="YES" && $3=="YES" {print $1}'|
 360                     egrep -v "devices|mlslabel|sharenfs|sharesmb|zoned")
 361         fi
 362
 363         echo $fs_prop
 364 }
 365
 366 #
 367 # Output inherited properties for volume
 368 #
 369 function vol_inherit_prop
 370 {
 371         echo "checksum readonly"
 372 }
 373
 374 #
 375 # Get the destination dataset to compare
 376 #
 377 function get_dst_ds
 378 {
 379         typeset srcfs=$1
 380         typeset dstfs=$2
 381
 382         #
 383         # If the srcfs is not pool
 384         #
 385         if ! zpool list $srcfs > /dev/null 2>&1 ; then
 386                 eval dstfs="$dstfs/${srcfs#*/}"
 387         fi
 388
 389         echo $dstfs
 390 }
 391
 392 #
 393 # Make test files
 394 #
 395 # $1 Number of files to create
 396 # $2 Maximum file size
 397 # $3 File ID offset
 398 # $4 File system to create the files on
 399 #
 400 function mk_files
 401 {
 402         nfiles=$1
 403         maxsize=$2
 404         file_id_offset=$3
 405         fs=$4
 406         bs=512
 407
 408         for ((i=0; i<$nfiles; i=i+1)); do
 409                 file_name="/$fs/file-$maxsize-$((i+$file_id_offset))"
 410                 file_size=$((($RANDOM * $RANDOM % ($maxsize - 1)) + 1))
 411
 412                 #
 413                 # Create an interesting mix of files which contain both
 414                 # data blocks and holes for more realistic test coverage.
 415                 # Half the files are created as sparse then partially filled,
 416                 # the other half is dense then a hole is punched in the file.
 417                 #
 418                 if [ $((RANDOM % 2)) -eq 0 ]; then
 419                         truncate -s $file_size $file_name || \
 420                             log_fail "Failed to create $file_name"
 421                         dd if=/dev/urandom of=$file_name \
 422                             bs=$bs count=$(($file_size / 2 / $bs)) \
 423                             seek=$(($RANDOM % (($file_size / 2 / $bs) + 1))) \
 424                             conv=notrunc >/dev/null 2>&1 || \
 425                             log_fail "Failed to create $file_name"
 426                 else
 427                         dd if=/dev/urandom of=$file_name \
 428                             bs=$file_size count=1 >/dev/null 2>&1 || \
 429                             log_fail "Failed to create $file_name"
 430                         dd if=/dev/zero of=$file_name \
 431                             bs=$bs count=$(($file_size / 2 / $bs)) \
 432                             seek=$(($RANDOM % (($file_size / 2 / $bs) + 1))) \
 433                             conv=notrunc >/dev/null 2>&1 || \
 434                             log_fail "Failed to create $file_name"
 435                 fi
 436         done
 437         echo Created $nfiles files of random sizes up to $maxsize bytes
 438 }
 439
 440 #
 441 # Remove test files
 442 #
 443 # $1 Number of files to remove
 444 # $2 Maximum file size
 445 # $3 File ID offset
 446 # $4 File system to remove the files from
 447 #
 448 function rm_files
 449 {
 450         nfiles=$1
 451         maxsize=$2
 452         file_id_offset=$3
 453         fs=$4
 454
 455         for ((i=0; i<$nfiles; i=i+1)); do
 456                 rm -f /$fs/file-$maxsize-$((i+$file_id_offset))
 457         done
 458         echo Removed $nfiles files of random sizes up to $maxsize bytes
 459 }
 460
 461 #
 462 # Simulate a random set of operations which could be reasonably expected
 463 # to occur on an average filesystem.
 464 #
 465 # $1 Number of files to modify
 466 # $2 Maximum file size
 467 # $3 File system to modify the file on
 468 # $4 Enabled xattrs (optional)
 469 #
 470 function churn_files
 471 {
 472         nfiles=$1
 473         maxsize=$2
 474         fs=$3
 475         xattrs=${4:-1}
 476
 477         #
 478         # Remove roughly half of the files in order to make it more
 479         # likely that a dnode will be reallocated.
 480         #
 481         for ((i=0; i<$nfiles; i=i+1)); do
 482                 file_name="/$fs/file-$i"
 483
 484                 if [[ -e $file_name ]]; then
 485                         if [ $((RANDOM % 2)) -eq 0 ]; then
 486                                 rm $file_name || \
 487                                     log_fail "Failed to remove $file_name"
 488                         fi
 489                 fi
 490         done
 491
 492         #
 493         # Remount the filesystem to simulate normal usage.  This resets
 494         # the last allocated object id allowing for new objects to be
 495         # reallocated in the locations of previously freed objects.
 496         #
 497         log_must zfs unmount $fs
 498         log_must zfs mount $fs
 499
 500         for i in {0..$nfiles}; do
 501                 file_name="/$fs/file-$i"
 502                 file_size=$((($RANDOM * $RANDOM % ($maxsize - 1)) + 1))
 503
 504                 #
 505                 # When the file exists modify it in one of five ways to
 506                 # simulate normal usage:
 507                 # - (20%) Remove and set and extended attribute on the file
 508                 # - (20%) Overwrite the existing file
 509                 # - (20%) Truncate the existing file to a random length
 510                 # - (20%) Truncate the existing file to zero length
 511                 # - (20%) Remove the file
 512                 #
 513                 # Otherwise create the missing file.  20% of the created
 514                 # files will be small and use embedded block pointers, the
 515                 # remainder with have random sizes up to the maximum size.
 516                 # Three extended attributes are attached to all of the files.
 517                 #
 518                 if [[ -e $file_name ]]; then
 519                         value=$((RANDOM % 5))
 520                         if [ $value -eq 0 -a $xattrs -ne 0 ]; then
 521                                 attrname="testattr$((RANDOM % 3))"
 522                                 attrlen="$(((RANDOM % 1000) + 1))"
 523                                 attrvalue="$(random_string VALID_NAME_CHAR \
 524                                     $attrlen)"
 525                                 attr -qr $attrname $file_name || \
 526                                     log_fail "Failed to remove $attrname"
 527                                 attr -qs $attrname \
 528                                     -V "$attrvalue" $file_name || \
 529                                     log_fail "Failed to set $attrname"
 530                         elif [ $value -eq 1 ]; then
 531                                 dd if=/dev/urandom of=$file_name \
 532                                     bs=$file_size count=1 >/dev/null 2>&1 || \
 533                                     log_fail "Failed to overwrite $file_name"
 534                         elif [ $value -eq 2 ]; then
 535                                 truncate -s $file_size $file_name || \
 536                                     log_fail "Failed to truncate $file_name"
 537                         elif [ $value -eq 3 ]; then
 538                                 truncate -s 0 $file_name || \
 539                                     log_fail "Failed to truncate $file_name"
 540                         else
 541                                 rm $file_name || \
 542                                     log_fail "Failed to remove $file_name"
 543                         fi
 544                 else
 545                         if [ $((RANDOM % 5)) -eq 0 ]; then
 546                                 file_size=$((($RANDOM % 64) + 1))
 547                         fi
 548
 549                         dd if=/dev/urandom of=$file_name \
 550                             bs=$file_size count=1 >/dev/null 2>&1 || \
 551                             log_fail "Failed to create $file_name"
 552
 553                         if [ $xattrs -ne 0 ]; then
 554                                 for j in {0..2}; do
 555                                         attrname="testattr$j"
 556                                         attrlen="$(((RANDOM % 1000) + 1))"
 557                                         attrvalue="$(random_string \
 558                                             VALID_NAME_CHAR $attrlen)"
 559                                         attr -qs $attrname \
 560                                             -V "$attrvalue" $file_name || \
 561                                             log_fail "Failed to set $attrname"
 562                                 done
 563                         fi
 564                 fi
 565         done
 566
 567         return 0
 568 }
 569
 570 #
 571 # Mess up file contents
 572 #
 573 # $1 The file path
 574 #
 575 function mess_file
 576 {
 577         file=$1
 578
 579         filesize=$(stat -c '%s' $file)
 580         offset=$(($RANDOM * $RANDOM % $filesize))
 581         if (($RANDOM % 7 <= 1)); then
 582                 #
 583                 # We corrupt 2 bytes to minimize the chance that we
 584                 # write the same value that's already there.
 585                 #
 586                 log_must eval "dd if=/dev/urandom of=$file conv=notrunc " \
 587                     "bs=1 count=2 seek=$offset >/dev/null 2>&1"
 588         else
 589                 log_must truncate -s $offset $file
 590         fi
 591 }
 592
 593 #
 594 # Diff the send/receive filesystems
 595 #
 596 # $1 The sent filesystem
 597 # $2 The received filesystem
 598 #
 599 function file_check
 600 {
 601         sendfs=$1
 602         recvfs=$2
 603
 604         if [[ -d /$recvfs/.zfs/snapshot/a && -d \
 605             /$sendfs/.zfs/snapshot/a ]]; then
 606                 diff -r /$recvfs/.zfs/snapshot/a /$sendfs/.zfs/snapshot/a
 607                 [[ $? -eq 0 ]] || log_fail "Differences found in snap a"
 608         fi
 609         if [[ -d /$recvfs/.zfs/snapshot/b && -d \
 610             /$sendfs/.zfs/snapshot/b ]]; then
 611                 diff -r /$recvfs/.zfs/snapshot/b /$sendfs/.zfs/snapshot/b
 612                 [[ $? -eq 0 ]] || log_fail "Differences found in snap b"
 613         fi
 614 }
 615
 616 #
 617 # Resume test helper
 618 #
 619 # $1 The ZFS send command
 620 # $2 The filesystem where the streams are sent
 621 # $3 The receive filesystem
 622 #
 623 function resume_test
 624 {
 625         sendcmd=$1
 626         streamfs=$2
 627         recvfs=$3
 628
 629         stream_num=1
 630         log_must eval "$sendcmd >/$streamfs/$stream_num"
 631
 632         for ((i=0; i<2; i=i+1)); do
 633                 mess_file /$streamfs/$stream_num
 634                 log_mustnot zfs recv -suv $recvfs </$streamfs/$stream_num
 635                 stream_num=$((stream_num+1))
 636
 637                 token=$(zfs get -Hp -o value receive_resume_token $recvfs)
 638                 log_must eval "zfs send -v -t $token >/$streamfs/$stream_num"
 639                 [[ -f /$streamfs/$stream_num ]] || \
 640                     log_fail "NO FILE /$streamfs/$stream_num"
 641         done
 642         log_must zfs recv -suv $recvfs </$streamfs/$stream_num
 643 }
 644
 645 #
 646 # Setup filesystems for the resumable send/receive tests
 647 #
 648 # $1 The "send" filesystem
 649 # $2 The "recv" filesystem
 650 #
 651 function test_fs_setup
 652 {
 653         typeset sendfs=$1
 654         typeset recvfs=$2
 655         typeset streamfs=$3
 656         typeset sendpool=${sendfs%%/*}
 657         typeset recvpool=${recvfs%%/*}
 658
 659         datasetexists $sendfs && log_must_busy zfs destroy -r $sendpool
 660         datasetexists $recvfs && log_must_busy zfs destroy -r $recvpool
 661         datasetexists $streamfs && log_must_busy zfs destroy -r $streamfs
 662
 663         if $(datasetexists $sendfs || zfs create -o compress=lz4 $sendfs); then
 664                 mk_files 1000 256 0 $sendfs &
 665                 mk_files 1000 131072 0 $sendfs &
 666                 mk_files 100 1048576 0 $sendfs &
 667                 mk_files 10 10485760 0 $sendfs &
 668                 mk_files 1 104857600 0 $sendfs &
 669                 log_must wait
 670                 log_must zfs snapshot $sendfs@a
 671
 672                 rm_files 200 256 0 $sendfs &
 673                 rm_files 200 131072 0 $sendfs &
 674                 rm_files 20 1048576 0 $sendfs &
 675                 rm_files 2 10485760 0 $sendfs &
 676                 log_must wait
 677
 678                 mk_files 400 256 0 $sendfs &
 679                 mk_files 400 131072 0 $sendfs &
 680                 mk_files 40 1048576 0 $sendfs &
 681                 mk_files 4 10485760 0 $sendfs &
 682                 log_must wait
 683
 684                 log_must zfs snapshot $sendfs@b
 685                 log_must eval "zfs send -v $sendfs@a >/$sendpool/initial.zsend"
 686                 log_must eval "zfs send -v -i @a $sendfs@b " \
 687                     ">/$sendpool/incremental.zsend"
 688         fi
 689
 690         log_must zfs create -o compress=lz4 $streamfs
 691 }
 692
 693 #
 694 # Check to see if the specified features are set in a send stream.
 695 # The values for these features are found in include/sys/zfs_ioctl.h
 696 #
 697 # $1 The stream file
 698 # $2-$n The flags expected in the stream
 699 #
 700 function stream_has_features
 701 {
 702         typeset file=$1
 703         shift
 704
 705         [[ -f $file ]] || log_fail "Couldn't find file: $file"
 706         typeset flags=$(cat $file | zstreamdump | \
 707             awk '/features =/ {features = $3} END {print features}')
 708         typeset -A feature
 709         feature[dedup]="1"
 710         feature[dedupprops]="2"
 711         feature[sa_spill]="4"
 712         feature[embed_data]="10000"
 713         feature[lz4]="20000"
 714         feature[mooch_byteswap]="40000"
 715         feature[large_blocks]="80000"
 716         feature[resuming]="100000"
 717         feature[redacted]="200000"
 718         feature[compressed]="400000"
 719
 720         typeset flag known derived=0
 721         for flag in "$@"; do
 722                 known=${feature[$flag]}
 723                 [[ -z $known ]] && log_fail "Unknown feature: $flag"
 724
 725                 derived=$(printf "%x" $((0x${flags} & 0x${feature[$flag]})))
 726                 [[ $derived = $known ]] || return 1
 727         done
 728
 729         return 0
 730 }
 731
 732 #
 733 # Given a send stream, verify that the size of the stream matches what's
 734 # expected based on the source or target dataset. If the stream is an
 735 # incremental stream, subtract the size of the source snapshot before
 736 # comparing. This function does not currently handle incremental streams
 737 # that remove data.
 738 #
 739 # $1 The zstreamdump output file
 740 # $2 The dataset to compare against
 741 #    This can be a source of a send or recv target (fs, not snapshot)
 742 # $3 The percentage below which verification is deemed a failure
 743 # $4 The source snapshot of an incremental send
 744 #
 745
 746 function verify_stream_size
 747 {
 748         typeset stream=$1
 749         typeset ds=$2
 750         typeset percent=${3:-90}
 751         typeset inc_src=$4
 752
 753         [[ -f $stream ]] || log_fail "No such file: $stream"
 754         datasetexists $ds || log_fail "No such dataset: $ds"
 755
 756         typeset stream_size=$(cat $stream | zstreamdump | sed -n \
 757             's/ Total write size = \(.*\) (0x.*)/\1/p')
 758
 759         typeset inc_size=0
 760         if [[ -n $inc_src ]]; then
 761                 inc_size=$(get_prop lrefer $inc_src)
 762                 if stream_has_features $stream compressed; then
 763                         inc_size=$(get_prop refer $inc_src)
 764                 fi
 765         fi
 766
 767         if stream_has_features $stream compressed; then
 768                 ds_size=$(get_prop refer $ds)
 769         else
 770                 ds_size=$(get_prop lrefer $ds)
 771         fi
 772         ds_size=$((ds_size - inc_size))
 773
 774         within_percent $stream_size $ds_size $percent || log_fail \
 775             "$stream_size $ds_size differed by too much"
 776 }
 777
 778 # Cleanup function for tests involving resumable send
 779 function resume_cleanup
 780 {
 781         typeset sendfs=$1
 782         typeset streamfs=$2
 783         typeset sendpool=${sendfs%%/*}
 784
 785         datasetexists $sendfs && log_must_busy zfs destroy -r $sendfs
 786         datasetexists $streamfs && log_must_busy zfs destroy -r $streamfs
 787         cleanup_pool $POOL2
 788         rm -f /$sendpool/initial.zsend /$sendpool/incremental.zsend
 789 }
 790
 791 # Randomly set the property to one of the enumerated values.
 792 function rand_set_prop
 793 {
 794         typeset dtst=$1
 795         typeset prop=$2
 796         shift 2
 797         typeset value=$(random_get $@)
 798
 799         log_must eval "zfs set $prop='$value' $dtst"
 800 }
 801
 802 # Generate a recursive checksum of a filesystem which includes the file
 803 # contents and any associated xattrs.
 804 function recursive_cksum
 805 {
 806         find $1 -type f -exec sh -c 'sha256sum {}; getfattr \
 807             --absolute-names --only-values -d {} | sha256sum' \; | \
 808             sort -k 2 | awk '{ print $1 }' | sha256sum
 809 }