scripts/zimport.sh

   1 #!/bin/bash
   2 #
   3 # Verify that an assortment of known good reference pools can be imported
   4 # using different versions of the ZoL code.
   5 #
   6 # By default references pools for the major ZFS implementation will be
   7 # checked against the most recent ZoL tags and the master development branch.
   8 # Alternate tags or branches may be verified with the '-s <src-tag> option.
   9 # Passing the keyword "installed" will instruct the script to test whatever
  10 # version is installed.
  11 #
  12 # Preferentially a reference pool is used for all tests.  However, if one
  13 # does not exist and the pool-tag matches one of the src-tags then a new
  14 # reference pool will be created using binaries from that source build.
  15 # This is particularly useful when you need to test your changes before
  16 # opening a pull request.  The keyword 'all' can be used as short hand
  17 # refer to all available reference pools.
  18 #
  19 # New reference pools may be added by placing a bzip2 compressed tarball
  20 # of the pool in the scripts/zfs-images directory and then passing
  21 # the -p <pool-tag> option.  To increase the test coverage reference pools
  22 # should be collected for all the major ZFS implementations.  Having these
  23 # pools easily available is also helpful to the developers.
  24 #
  25 # Care should be taken to run these tests with a kernel supported by all
  26 # the listed tags.  Otherwise build failure will cause false positives.
  27 #
  28 #
  29 # EXAMPLES:
  30 #
  31 # The following example will verify the zfs-0.6.2 tag, the master branch,
  32 # and the installed zfs version can correctly import the listed pools.
  33 # Note there is no reference pool available for master and installed but
  34 # because binaries are available one is automatically constructed.  The
  35 # working directory is also preserved between runs (-k) preventing the
  36 # need to rebuild from source for multiple runs.
  37 #
  38 #  zimport.sh -k -f /var/tmp/zimport \
  39 #      -s "zfs-0.6.2 master installed" \
  40 #      -p "zevo-1.1.1 zol-0.6.2 zol-0.6.2-173 master installed"
  41 #
  42 # --------------------- ZFS on Linux Source Versions --------------
  43 #                 zfs-0.6.2       master          0.6.2-175_g36eb554
  44 # -----------------------------------------------------------------
  45 # Clone SPL       Local         Local           Skip
  46 # Clone ZFS       Local         Local           Skip
  47 # Build SPL       Pass          Pass            Skip
  48 # Build ZFS       Pass          Pass            Skip
  49 # -----------------------------------------------------------------
  50 # zevo-1.1.1      Pass          Pass            Pass
  51 # zol-0.6.2       Pass          Pass            Pass
  52 # zol-0.6.2-173   Fail          Pass            Pass
  53 # master          Pass          Pass            Pass
  54 # installed       Pass          Pass            Pass
  55 #
  56 basedir="$(dirname $0)"
  57
  58 SCRIPT_COMMON=common.sh
  59 if [ -f "${basedir}/${SCRIPT_COMMON}" ]; then
  60 . "${basedir}/${SCRIPT_COMMON}"
  61 else
  62 echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
  63 fi
  64
  65 PROG=zimport.sh
  66
  67 SRC_TAGS="zfs-0.6.1 zfs-0.6.2 master"
  68 POOL_TAGS="all master"
  69 TEST_DIR=`mktemp -u -d -p /var/tmp zimport.XXXXXXXX`
  70 KEEP=0
  71 VERBOSE=0
  72 COLOR=1
  73 REPO="https://github.com/zfsonlinux"
  74 IMAGES_DIR="$SCRIPTDIR/zfs-images/"
  75 IMAGES_TAR="https://github.com/zfsonlinux/zfs-images/tarball/master"
  76 CPUS=`grep -c ^processor /proc/cpuinfo`
  77 ERROR=0
  78
  79 usage() {
  80 cat << EOF
  81 USAGE:
  82 zimport.sh [hvl] [-r repo] [-s src-tag] [-i pool-dir] [-p pool-tag] [-f path]
  83
  84 DESCRIPTION:
  85         ZPOOL import verification tests
  86
  87 OPTIONS:
  88         -h                Show this message
  89         -v                Verbose
  90         -c                No color
  91         -k                Keep temporary directory
  92         -r <repo>         Source repository ($REPO)
  93         -s <src-tag>...   Verify ZoL versions with the listed tags
  94         -i <pool-dir>     Pool image directory
  95         -p <pool-tag>...  Verify pools created with the listed tags
  96         -f <path>         Temporary directory to use
  97
  98 EOF
  99 }
 100
 101 while getopts 'hvckr:s:i:p:f:?' OPTION; do
 102         case $OPTION in
 103         h)
 104                 usage
 105                 exit 1
 106                 ;;
 107         v)
 108                 VERBOSE=1
 109                 ;;
 110         c)
 111                 COLOR=0
 112                 ;;
 113         k)
 114                 KEEP=1
 115                 ;;
 116         r)
 117                 REPO="$OPTARG"
 118                 ;;
 119         s)
 120                 SRC_TAGS="$OPTARG"
 121                 ;;
 122         i)
 123                 IMAGES_DIR="$OPTARG"
 124                 ;;
 125         p)
 126                 POOL_TAGS="$OPTARG"
 127                 ;;
 128         f)
 129                 TEST_DIR="$OPTARG"
 130                 ;;
 131         ?)
 132                 usage
 133                 exit
 134                 ;;
 135         esac
 136 done
 137
 138 # Initialize the test suite
 139 init
 140 check_modules || die "ZFS modules must be unloaded"
 141
 142 SRC_DIR="$TEST_DIR/src"
 143 SRC_DIR_SPL="$SRC_DIR/spl"
 144 SRC_DIR_ZFS="$SRC_DIR/zfs"
 145
 146 if [ $COLOR -eq 0 ]; then
 147         COLOR_GREEN=""
 148         COLOR_BROWN=""
 149         COLOR_RED=""
 150         COLOR_RESET=""
 151 fi
 152
 153 pass_nonewline() {
 154         echo -n -e "${COLOR_GREEN}Pass${COLOR_RESET}\t\t"
 155 }
 156
 157 skip_nonewline() {
 158         echo -n -e "${COLOR_BROWN}Skip${COLOR_RESET}\t\t"
 159 }
 160
 161 fail_nonewline() {
 162         echo -n -e "${COLOR_RED}Fail${COLOR_RESET}\t\t"
 163 }
 164
 165 #
 166 # Set several helper variables which are derived from a source tag.
 167 #
 168 # SPL_TAG - The tag zfs-x.y.z is translated to spl-x.y.z.
 169 # SPL_DIR - The spl directory name.
 170 # SPL_URL - The spl github URL to fetch the tarball.
 171 # ZFS_TAG - The passed zfs-x.y.z tag
 172 # ZFS_DIR - The zfs directory name
 173 # ZFS_URL - The zfs github URL to fetch the tarball
 174 #
 175 src_set_vars() {
 176         local TAG=$1
 177
 178         SPL_TAG=`echo $TAG | sed -e 's/zfs/spl/'`
 179         SPL_DIR=$SRC_DIR_SPL/$SPL_TAG
 180         SPL_URL=$REPO/spl/tarball/$SPL_TAG
 181
 182         ZFS_TAG=$TAG
 183         ZFS_DIR=$SRC_DIR_ZFS/$ZFS_TAG
 184         ZFS_URL=$REPO/zfs/tarball/$ZFS_TAG
 185
 186         if [ "$TAG" = "installed" ]; then
 187                 ZPOOL_CMD=`which zpool`
 188                 ZFS_CMD=`which zfs`
 189                 ZFS_SH="/usr/share/zfs/zfs.sh"
 190                 ZPOOL_CREATE="/usr/share/zfs/zpool-create.sh"
 191         else
 192                 ZPOOL_CMD="./cmd/zpool/zpool"
 193                 ZFS_CMD="./cmd/zfs/zfs"
 194                 ZFS_SH="./scripts/zfs.sh"
 195                 ZPOOL_CREATE="./scripts/zpool-create.sh"
 196         fi
 197 }
 198
 199 #
 200 # Set several helper variables which are derived from a pool name such
 201 # as zol-0.6.x, zevo-1.1.1, etc.  These refer to example pools from various
 202 # ZFS implementations which are used to verify compatibility.
 203 #
 204 # POOL_TAG          - The example pools name in scripts/zfs-images/.
 205 # POOL_BZIP         - The full path to the example bzip2 compressed pool.
 206 # POOL_DIR          - The top level test path for this pool.
 207 # POOL_DIR_PRISTINE - The directory containing a pristine version of the pool.
 208 # POOL_DIR_COPY     - The directory containing a working copy of the pool.
 209 # POOL_DIR_SRC      - Location of a source build if it exists for this pool.
 210 #
 211 pool_set_vars() {
 212         local TAG=$1
 213
 214         POOL_TAG=$TAG
 215         POOL_BZIP=$IMAGES_DIR/$POOL_TAG.tar.bz2
 216         POOL_DIR=$TEST_DIR/pools/$POOL_TAG
 217         POOL_DIR_PRISTINE=$POOL_DIR/pristine
 218         POOL_DIR_COPY=$POOL_DIR/copy
 219         POOL_DIR_SRC=`echo -n "$SRC_DIR_ZFS/"; \
 220             echo "$POOL_TAG" | sed -e 's/zol/zfs/'`
 221 }
 222
 223 #
 224 # Construct a non-trivial pool given a specific version of the source.  More
 225 # interesting pools provide better test coverage so this function should
 226 # extended as needed to create more realistic pools.
 227 #
 228 pool_create() {
 229         pool_set_vars $1
 230         src_set_vars $1
 231
 232         if [ "$POOL_TAG" != "installed" ]; then
 233                 cd $POOL_DIR_SRC
 234         fi
 235
 236         $ZFS_SH zfs="spa_config_path=$POOL_DIR_PRISTINE" || fail 1
 237
 238         # Create a file vdev RAIDZ pool.
 239         FILEDIR="$POOL_DIR_PRISTINE" $ZPOOL_CREATE \
 240             -c file-raidz -p $POOL_TAG -v -x >/dev/null || fail 2
 241
 242         # Create a pool/fs filesystem with some random contents.
 243         $ZFS_CMD create $POOL_TAG/fs || fail 3
 244         populate /$POOL_TAG/fs/ 10 100
 245
 246         # Snapshot that filesystem, clone it, remove the files/dirs,
 247         # replace them with new files/dirs.
 248         $ZFS_CMD snap $POOL_TAG/fs@snap || fail 4
 249         $ZFS_CMD clone $POOL_TAG/fs@snap $POOL_TAG/clone || fail 5
 250         rm -Rf /$POOL_TAG/clone/* || fail 6
 251         populate /$POOL_TAG/clone/ 10 100
 252
 253         # Scrub the pool, delay slightly, then export it.  It is now
 254         # somewhat interesting for testing purposes.
 255         $ZPOOL_CMD scrub $POOL_TAG || fail 7
 256         sleep 10
 257         $ZPOOL_CMD export $POOL_TAG || fail 8
 258
 259         $ZFS_SH -u || fail 9
 260 }
 261
 262 # If the zfs-images directory doesn't exist fetch a copy from Github then
 263 # cache it in the $TEST_DIR and update $IMAGES_DIR.
 264 if [ ! -d $IMAGES_DIR ]; then
 265         IMAGES_DIR="$TEST_DIR/zfs-images"
 266         mkdir -p $IMAGES_DIR
 267         curl -sL $IMAGES_TAR | \
 268             tar -xz -C $IMAGES_DIR --strip-components=1 || fail 10
 269 fi
 270
 271 # Given the available images in the zfs-images directory substitute the
 272 # list of available images for the reserved keywork 'all'.
 273 for TAG in $POOL_TAGS; do
 274
 275         if  [ "$TAG" = "all" ]; then
 276                 ALL_TAGS=`ls $IMAGES_DIR | grep "tar.bz2" | \
 277                     sed 's/.tar.bz2//' | tr '\n' ' '`
 278                 NEW_TAGS="$NEW_TAGS $ALL_TAGS"
 279         else
 280                 NEW_TAGS="$NEW_TAGS $TAG"
 281         fi
 282 done
 283 POOL_TAGS="$NEW_TAGS"
 284
 285 if [ $VERBOSE -ne 0 ]; then
 286         echo "---------------------------- Options ----------------------------"
 287         echo "VERBOSE=$VERBOSE"
 288         echo "KEEP=$KEEP"
 289         echo "REPO=$REPO"
 290         echo "SRC_TAGS="$SRC_TAGS""
 291         echo "POOL_TAGS="$POOL_TAGS""
 292         echo "PATH=$TEST_DIR"
 293         echo
 294 fi
 295
 296 if [ ! -d $TEST_DIR ]; then
 297         mkdir -p $TEST_DIR
 298 fi
 299
 300 if [ ! -d $SRC_DIR ]; then
 301         mkdir -p $SRC_DIR
 302 fi
 303
 304 # Print a header for all tags which are being tested.
 305 echo "--------------------- ZFS on Linux Source Versions --------------"
 306 printf "%-16s" " "
 307 for TAG in $SRC_TAGS; do
 308         src_set_vars $TAG
 309
 310         if [ "$TAG" = "installed" ]; then
 311                 ZFS_VERSION=`modinfo zfs | awk '/version:/ { print $2; exit }'`
 312                 if [ -n "$ZFS_VERSION" ]; then
 313                         printf "%-16s" $ZFS_VERSION
 314                 else
 315                         echo "ZFS is not installed\n"
 316                         fail
 317                 fi
 318         else
 319                 printf "%-16s" $TAG
 320         fi
 321 done
 322 echo -e "\n-----------------------------------------------------------------"
 323
 324 #
 325 # Attempt to generate the tarball from your local git repository, if that
 326 # fails then attempt to download the tarball from Github.
 327 #
 328 printf "%-16s" "Clone SPL"
 329 for TAG in $SRC_TAGS; do
 330         src_set_vars $TAG
 331
 332         if [ -d $SPL_DIR ]; then
 333                 skip_nonewline
 334         elif  [ "$SPL_TAG" = "installed" ]; then
 335                 skip_nonewline
 336         else
 337                 cd $SRC_DIR
 338
 339                 if [ ! -d $SRC_DIR_SPL ]; then
 340                         mkdir -p $SRC_DIR_SPL
 341                 fi
 342
 343                 git archive --format=tar --prefix=$SPL_TAG/ $SPL_TAG \
 344                     -o $SRC_DIR_SPL/$SPL_TAG.tar &>/dev/nul || \
 345                     rm $SRC_DIR_SPL/$SPL_TAG.tar
 346                 if [ -s $SRC_DIR_SPL/$SPL_TAG.tar ]; then
 347                         tar -xf $SRC_DIR_SPL/$SPL_TAG.tar -C $SRC_DIR_SPL
 348                         rm $SRC_DIR_SPL/$SPL_TAG.tar
 349                         echo -n -e "${COLOR_GREEN}Local${COLOR_RESET}\t\t"
 350                 else
 351                         mkdir -p $SPL_DIR || fail 1
 352                         curl -sL $SPL_URL | tar -xz -C $SPL_DIR \
 353                             --strip-components=1 || fail 2
 354                         echo -n -e "${COLOR_GREEN}Remote${COLOR_RESET}\t\t"
 355                 fi
 356         fi
 357 done
 358 printf "\n"
 359
 360 #
 361 # Attempt to generate the tarball from your local git repository, if that
 362 # fails then attempt to download the tarball from Github.
 363 #
 364 printf "%-16s" "Clone ZFS"
 365 for TAG in $SRC_TAGS; do
 366         src_set_vars $TAG
 367
 368         if [ -d $ZFS_DIR ]; then
 369                 skip_nonewline
 370         elif  [ "$ZFS_TAG" = "installed" ]; then
 371                 skip_nonewline
 372         else
 373                 cd $SRC_DIR
 374
 375                 if [ ! -d $SRC_DIR_ZFS ]; then
 376                         mkdir -p $SRC_DIR_ZFS
 377                 fi
 378
 379                 git archive --format=tar --prefix=$ZFS_TAG/ $ZFS_TAG \
 380                     -o $SRC_DIR_ZFS/$ZFS_TAG.tar &>/dev/nul || \
 381                     rm $SRC_DIR_ZFS/$ZFS_TAG.tar
 382                 if [ -s $SRC_DIR_ZFS/$ZFS_TAG.tar ]; then
 383                         tar -xf $SRC_DIR_ZFS/$ZFS_TAG.tar -C $SRC_DIR_ZFS
 384                         rm $SRC_DIR_ZFS/$ZFS_TAG.tar
 385                         echo -n -e "${COLOR_GREEN}Local${COLOR_RESET}\t\t"
 386                 else
 387                         mkdir -p $ZFS_DIR || fail 1
 388                         curl -sL $ZFS_URL | tar -xz -C $ZFS_DIR \
 389                             --strip-components=1 || fail 2
 390                         echo -n -e "${COLOR_GREEN}Remote${COLOR_RESET}\t\t"
 391                 fi
 392         fi
 393 done
 394 printf "\n"
 395
 396 # Build the listed tags
 397 printf "%-16s" "Build SPL"
 398 for TAG in $SRC_TAGS; do
 399         src_set_vars $TAG
 400
 401         if [ -f $SPL_DIR/module/spl/spl.ko ]; then
 402                 skip_nonewline
 403         elif  [ "$SPL_TAG" = "installed" ]; then
 404                 skip_nonewline
 405         else
 406                 cd $SPL_DIR
 407                 make distclean &>/dev/null
 408                 sh ./autogen.sh &>/dev/null || fail 1
 409                 ./configure &>/dev/null || fail 2
 410                 make -s -j$CPUS &>/dev/null || fail 3
 411                 pass_nonewline
 412         fi
 413 done
 414 printf "\n"
 415
 416 # Build the listed tags
 417 printf "%-16s" "Build ZFS"
 418 for TAG in $SRC_TAGS; do
 419         src_set_vars $TAG
 420
 421         if [ -f $ZFS_DIR/module/zfs/zfs.ko ]; then
 422                 skip_nonewline
 423         elif  [ "$ZFS_TAG" = "installed" ]; then
 424                 skip_nonewline
 425         else
 426                 cd $ZFS_DIR
 427                 make distclean &>/dev/null
 428                 sh ./autogen.sh &>/dev/null || fail 1
 429                 ./configure --with-spl=$SPL_DIR &>/dev/null || fail 2
 430                 make -s -j$CPUS &>/dev/null || fail 3
 431                 pass_nonewline
 432         fi
 433 done
 434 printf "\n"
 435 echo "-----------------------------------------------------------------"
 436
 437 # Either create a new pool using 'zpool create', or alternately restore an
 438 # existing pool from another ZFS implementation for compatibility testing.
 439 for TAG in $POOL_TAGS; do
 440         pool_set_vars $TAG
 441         SKIP=0
 442
 443         printf "%-16s" $POOL_TAG
 444         rm -Rf $POOL_DIR
 445         mkdir -p $POOL_DIR_PRISTINE
 446
 447         # Use the existing compressed image if available.
 448         if [ -f $POOL_BZIP ]; then
 449                 tar -xjf $POOL_BZIP -C $POOL_DIR_PRISTINE \
 450                     --strip-components=1 || fail 1
 451         # Use the installed version to create the pool.
 452         elif  [ "$TAG" = "installed" ]; then
 453                 pool_create $TAG
 454         # A source build is available to create the pool.
 455         elif [ -d $POOL_DIR_SRC ]; then
 456                 pool_create $TAG
 457         else
 458                 SKIP=1
 459         fi
 460
 461         # Verify 'zpool import' works for all listed source versions.
 462         for TAG in $SRC_TAGS; do
 463
 464                 if [ $SKIP -eq 1 ]; then
 465                         skip_nonewline
 466                         continue
 467                 fi
 468
 469                 src_set_vars $TAG
 470                 if [ "$TAG" != "installed" ]; then
 471                         cd $ZFS_DIR
 472                 fi
 473                 $ZFS_SH zfs="spa_config_path=$POOL_DIR_COPY"
 474
 475                 cp -a --sparse=always $POOL_DIR_PRISTINE \
 476                     $POOL_DIR_COPY || fail 2
 477                 POOL_NAME=`$ZPOOL_CMD import -d $POOL_DIR_COPY | \
 478                     awk '/pool:/ { print $2; exit 0 }'`
 479
 480                 $ZPOOL_CMD import -N -d $POOL_DIR_COPY $POOL_NAME &>/dev/null
 481                 if [ $? -ne 0 ]; then
 482                         fail_nonewline
 483                         ERROR=1
 484                 else
 485                         $ZPOOL_CMD export $POOL_NAME || fail 3
 486                         pass_nonewline
 487                 fi
 488
 489                 rm -Rf $POOL_DIR_COPY
 490
 491                 $ZFS_SH -u || fail 4
 492         done
 493         printf "\n"
 494 done
 495
 496 if [ ! $KEEP ]; then
 497         rm -Rf $TEST_DIR
 498 fi
 499
 500 exit $ERROR