[mirror_zfs.git] / scripts / zimport.sh

#!/bin/bash
#
# Verify that an assortment of known good reference pools can be imported
# using different versions of the ZoL code.
#
# By default references pools for the major ZFS implementation will be
# checked against the most recent ZoL tags and the master development branch.
# Alternate tags or branches may be verified with the '-s <src-tag> option.
# Passing the keyword "installed" will instruct the script to test whatever
# version is installed.
#
# Preferentially a reference pool is used for all tests.  However, if one
# does not exist and the pool-tag matches one of the src-tags then a new
# reference pool will be created using binaries from that source build.
# This is particularly useful when you need to test your changes before
# opening a pull request.  The keyword 'all' can be used as short hand
# refer to all available reference pools.
#
# New reference pools may be added by placing a bzip2 compressed tarball
# of the pool in the scripts/zfs-images directory and then passing
# the -p <pool-tag> option.  To increase the test coverage reference pools
# should be collected for all the major ZFS implementations.  Having these
# pools easily available is also helpful to the developers.
#
# Care should be taken to run these tests with a kernel supported by all
# the listed tags.  Otherwise build failure will cause false positives.
#
#
# EXAMPLES:
#
# The following example will verify the zfs-0.6.2 tag, the master branch,
# and the installed zfs version can correctly import the listed pools.
# Note there is no reference pool available for master and installed but
# because binaries are available one is automatically constructed.  The
# working directory is also preserved between runs (-k) preventing the
# need to rebuild from source for multiple runs.
#
#  zimport.sh -k -f /var/tmp/zimport \
#      -s "zfs-0.6.2 master installed" \
#      -p "zevo-1.1.1 zol-0.6.2 zol-0.6.2-173 master installed"
#
# --------------------- ZFS on Linux Source Versions --------------
#                 zfs-0.6.2       master          0.6.2-175_g36eb554
# -----------------------------------------------------------------
# Clone SPL       Local		Local		Skip
# Clone ZFS       Local		Local		Skip
# Build SPL       Pass		Pass		Skip
# Build ZFS       Pass		Pass		Skip
# -----------------------------------------------------------------
# zevo-1.1.1      Pass		Pass		Pass
# zol-0.6.2       Pass		Pass		Pass
# zol-0.6.2-173   Fail		Pass		Pass
# master          Pass		Pass		Pass
# installed       Pass		Pass		Pass
#

BASE_DIR=$(dirname "$0")
SCRIPT_COMMON=common.sh
if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then
	. "${BASE_DIR}/${SCRIPT_COMMON}"
else
	echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
fi

SRC_TAGS="zfs-0.6.5.11 master"
POOL_TAGS="all master"
TEST_DIR=$(mktemp -u -d -p /var/tmp zimport.XXXXXXXX)
KEEP="no"
VERBOSE="no"
COLOR="yes"
REPO="https://github.com/zfsonlinux"
IMAGES_DIR="$SCRIPTDIR/zfs-images/"
IMAGES_TAR="https://github.com/zfsonlinux/zfs-images/tarball/master"
ERROR=0

CONFIG_LOG="configure.log"
CONFIG_OPTIONS=${CONFIG_OPTIONS:-""}
MAKE_LOG="make.log"
MAKE_OPTIONS=${MAKE_OPTIONS:-"-s -j$(nproc)"}

COLOR_GREEN="\033[0;32m"
COLOR_RED="\033[0;31m"
COLOR_BROWN="\033[0;33m"
COLOR_RESET="\033[0m"

usage() {
cat << EOF
USAGE:
zimport.sh [hvl] [-r repo] [-s src-tag] [-i pool-dir] [-p pool-tag] [-f path]

DESCRIPTION:
	ZPOOL import verification tests

OPTIONS:
	-h                Show this message
	-v                Verbose
	-c                No color
	-k                Keep temporary directory
	-r <repo>         Source repository ($REPO)
	-s <src-tag>...   Verify ZoL versions with the listed tags
	-i <pool-dir>     Pool image directory
	-p <pool-tag>...  Verify pools created with the listed tags
	-f <path>         Temporary directory to use

EOF
}

while getopts 'hvckr:s:i:p:f:?' OPTION; do
	case $OPTION in
	h)
		usage
		exit 1
		;;
	v)
		VERBOSE="yes"
		;;
	c)
		COLOR="no"
		;;
	k)
		KEEP="yes"
		;;
	r)
		REPO="$OPTARG"
		;;
	s)
		SRC_TAGS="$OPTARG"
		;;
	i)
		IMAGES_DIR="$OPTARG"
		;;
	p)
		POOL_TAGS="$OPTARG"
		;;
	f)
		TEST_DIR="$OPTARG"
		;;
	?)
		usage
		exit
		;;
	esac
done

#
# Verify the module start is not loaded
#
if lsmod | grep zfs >/dev/null; then
	echo "ZFS modules must be unloaded"
	exit 1
fi

#
# Create a random directory tree of files and sub-directories to
# to act as a copy source for the various regression tests.
#
populate() {
	local ROOT=$1
	local MAX_DIR_SIZE=$2
	local MAX_FILE_SIZE=$3

	# shellcheck disable=SC2086
	mkdir -p $ROOT/{a,b,c,d,e,f,g}/{h,i}
	DIRS=$(find "$ROOT")

	for DIR in $DIRS; do
		COUNT=$((RANDOM % MAX_DIR_SIZE))

		# shellcheck disable=SC2034
		for i in $(seq $COUNT); do
			FILE=$(mktemp -p "$DIR")
			SIZE=$((RANDOM % MAX_FILE_SIZE))
			dd if=/dev/urandom of="$FILE" bs=1k \
			    count="$SIZE" &>/dev/null
		done
	done

	return 0
}

SRC_DIR=$(mktemp -d -p /var/tmp/ zfs.src.XXXXXXXX)
trap 'rm -Rf "$SRC_DIR"' INT TERM EXIT
populate "$SRC_DIR" 10 100

SRC_DIR="$TEST_DIR/src"
SRC_DIR_SPL="$SRC_DIR/spl"
SRC_DIR_ZFS="$SRC_DIR/zfs"

if [ "$COLOR" = "no" ]; then
	COLOR_GREEN=""
	COLOR_BROWN=""
	COLOR_RED=""
	COLOR_RESET=""
fi

pass_nonewline() {
	echo -n -e "${COLOR_GREEN}Pass${COLOR_RESET}\t\t"
}

skip_nonewline() {
	echo -n -e "${COLOR_BROWN}Skip${COLOR_RESET}\t\t"
}

fail_nonewline() {
	echo -n -e "${COLOR_RED}Fail${COLOR_RESET}\t\t"
}

fail() {
	echo -e "${COLOR_RED}Fail${COLOR_RESET} ($1)"
	exit "$1"
}

#
# Set several helper variables which are derived from a source tag.
#
# SPL_TAG - The tag zfs-x.y.z is translated to spl-x.y.z.
# SPL_DIR - The spl directory name.
# SPL_URL - The spl github URL to fetch the tarball.
# ZFS_TAG - The passed zfs-x.y.z tag
# ZFS_DIR - The zfs directory name
# ZFS_URL - The zfs github URL to fetch the tarball
#
src_set_vars() {
	local TAG=$1

	SPL_TAG="${TAG//zfs/spl}"
	SPL_DIR="$SRC_DIR_SPL/$SPL_TAG"
	SPL_URL="$REPO/spl/tarball/$SPL_TAG"

	ZFS_TAG="$TAG"
	ZFS_DIR="$SRC_DIR_ZFS/$ZFS_TAG"
	ZFS_URL="$REPO/zfs/tarball/$ZFS_TAG"

	if [ "$TAG" = "installed" ]; then
		ZPOOL_CMD=$(which zpool)
		ZFS_CMD=$(which zfs)
		ZFS_SH="/usr/share/zfs/zfs.sh"
	else
		ZPOOL_CMD="./cmd/zpool/zpool"
		ZFS_CMD="./cmd/zfs/zfs"
		ZFS_SH="./scripts/zfs.sh"
	fi
}

#
# Set several helper variables which are derived from a pool name such
# as zol-0.6.x, zevo-1.1.1, etc.  These refer to example pools from various
# ZFS implementations which are used to verify compatibility.
#
# POOL_TAG          - The example pools name in scripts/zfs-images/.
# POOL_BZIP         - The full path to the example bzip2 compressed pool.
# POOL_DIR          - The top level test path for this pool.
# POOL_DIR_PRISTINE - The directory containing a pristine version of the pool.
# POOL_DIR_COPY     - The directory containing a working copy of the pool.
# POOL_DIR_SRC      - Location of a source build if it exists for this pool.
#
pool_set_vars() {
	local TAG=$1

	POOL_TAG=$TAG
	POOL_BZIP=$IMAGES_DIR/$POOL_TAG.tar.bz2
	POOL_DIR=$TEST_DIR/pools/$POOL_TAG
	POOL_DIR_PRISTINE=$POOL_DIR/pristine
	POOL_DIR_COPY=$POOL_DIR/copy
	POOL_DIR_SRC="$SRC_DIR_ZFS/${POOL_TAG//zol/zfs}"
}

#
# Construct a non-trivial pool given a specific version of the source.  More
# interesting pools provide better test coverage so this function should
# extended as needed to create more realistic pools.
#
pool_create() {
	pool_set_vars "$1"
	src_set_vars "$1"

	if [ "$POOL_TAG" != "installed" ]; then
		cd "$POOL_DIR_SRC"
	fi

	$ZFS_SH zfs="spa_config_path=$POOL_DIR_PRISTINE" || fail 1

	# Create a file vdev RAIDZ pool.
	truncate -s 1G \
	    "$POOL_DIR_PRISTINE/vdev1" "$POOL_DIR_PRISTINE/vdev2" \
	    "$POOL_DIR_PRISTINE/vdev3" "$POOL_DIR_PRISTINE/vdev4"
	$ZPOOL_CMD create "$POOL_TAG" raidz \
	    "$POOL_DIR_PRISTINE/vdev1" "$POOL_DIR_PRISTINE/vdev2" \
            "$POOL_DIR_PRISTINE/vdev3" "$POOL_DIR_PRISTINE/vdev4"

	# Create a pool/fs filesystem with some random contents.
	$ZFS_CMD create "$POOL_TAG/fs" || fail 3
	populate "/$POOL_TAG/fs/" 10 100

	# Snapshot that filesystem, clone it, remove the files/dirs,
	# replace them with new files/dirs.
	$ZFS_CMD snap "$POOL_TAG/fs@snap" || fail 4
	$ZFS_CMD clone "$POOL_TAG/fs@snap" "$POOL_TAG/clone" || fail 5
	# shellcheck disable=SC2086
	rm -Rf /$POOL_TAG/clone/* || fail 6
	populate "/$POOL_TAG/clone/" 10 100

	# Scrub the pool, delay slightly, then export it.  It is now
	# somewhat interesting for testing purposes.
	$ZPOOL_CMD scrub "$POOL_TAG" || fail 7
	sleep 10
	$ZPOOL_CMD export "$POOL_TAG" || fail 8

	$ZFS_SH -u || fail 9
}

# If the zfs-images directory doesn't exist fetch a copy from Github then
# cache it in the $TEST_DIR and update $IMAGES_DIR.
if [ ! -d "$IMAGES_DIR" ]; then
	IMAGES_DIR="$TEST_DIR/zfs-images"
	mkdir -p "$IMAGES_DIR"
	curl -sL "$IMAGES_TAR" | \
	    tar -xz -C "$IMAGES_DIR" --strip-components=1 || fail 10
fi

# Given the available images in the zfs-images directory substitute the
# list of available images for the reserved keyword 'all'.
for TAG in $POOL_TAGS; do

	if  [ "$TAG" = "all" ]; then
		# shellcheck disable=SC2010
		ALL_TAGS=$(ls "$IMAGES_DIR" | grep "tar.bz2" | \
		    sed 's/.tar.bz2//' | tr '\n' ' ')
		NEW_TAGS="$NEW_TAGS $ALL_TAGS"
	else
		NEW_TAGS="$NEW_TAGS $TAG"
	fi
done
POOL_TAGS="$NEW_TAGS"

if [ "$VERBOSE" = "yes" ]; then
	echo "---------------------------- Options ----------------------------"
	echo "VERBOSE=$VERBOSE"
	echo "KEEP=$KEEP"
	echo "REPO=$REPO"
	echo "SRC_TAGS=$SRC_TAGS"
	echo "POOL_TAGS=$POOL_TAGS"
	echo "PATH=$TEST_DIR"
	echo
fi

if [ ! -d "$TEST_DIR" ]; then
	mkdir -p "$TEST_DIR"
fi

if [ ! -d "$SRC_DIR" ]; then
	mkdir -p "$SRC_DIR"
fi

# Print a header for all tags which are being tested.
echo "--------------------- ZFS on Linux Source Versions --------------"
printf "%-16s" " "
for TAG in $SRC_TAGS; do
	src_set_vars "$TAG"

	if [ "$TAG" = "installed" ]; then
		ZFS_VERSION=$(modinfo zfs | awk '/version:/ { print $2; exit }')
		if [ -n "$ZFS_VERSION" ]; then
			printf "%-16s" "$ZFS_VERSION"
		else
			echo -e "ZFS is not installed\n"
			fail
		fi
	else
		printf "%-16s" "$TAG"
	fi
done
echo -e "\n-----------------------------------------------------------------"

#
# Attempt to generate the tarball from your local git repository, if that
# fails then attempt to download the tarball from Github.
#
printf "%-16s" "Clone SPL"
for TAG in $SRC_TAGS; do
	src_set_vars "$TAG"

	if [ -d "$SPL_DIR" ]; then
		skip_nonewline
	elif  [ "$SPL_TAG" = "installed" ]; then
		skip_nonewline
	else
		cd "$SRC_DIR"

		if [ ! -d "$SRC_DIR_SPL" ]; then
			mkdir -p "$SRC_DIR_SPL"
		fi

		git archive --format=tar --prefix="$SPL_TAG/ $SPL_TAG" \
		    -o "$SRC_DIR_SPL/$SPL_TAG.tar" &>/dev/nul || \
		    rm "$SRC_DIR_SPL/$SPL_TAG.tar"
		if [ -s "$SRC_DIR_SPL/$SPL_TAG.tar" ]; then
			tar -xf "$SRC_DIR_SPL/$SPL_TAG.tar" -C "$SRC_DIR_SPL"
			rm "$SRC_DIR_SPL/$SPL_TAG.tar"
			echo -n -e "${COLOR_GREEN}Local${COLOR_RESET}\t\t"
		else
			mkdir -p "$SPL_DIR" || fail 1
			curl -sL "$SPL_URL" | tar -xz -C "$SPL_DIR" \
			    --strip-components=1 || fail 2
			echo -n -e "${COLOR_GREEN}Remote${COLOR_RESET}\t\t"
		fi
	fi
done
printf "\n"

#
# Attempt to generate the tarball from your local git repository, if that
# fails then attempt to download the tarball from Github.
#
printf "%-16s" "Clone ZFS"
for TAG in $SRC_TAGS; do
	src_set_vars "$TAG"

	if [ -d "$ZFS_DIR" ]; then
		skip_nonewline
	elif  [ "$ZFS_TAG" = "installed" ]; then
		skip_nonewline
	else
		cd "$SRC_DIR"

		if [ ! -d "$SRC_DIR_ZFS" ]; then
			mkdir -p "$SRC_DIR_ZFS"
		fi

		git archive --format=tar --prefix="$ZFS_TAG/ $ZFS_TAG" \
		    -o "$SRC_DIR_ZFS/$ZFS_TAG.tar" &>/dev/nul || \
		    rm "$SRC_DIR_ZFS/$ZFS_TAG.tar"
		if [ -s "$SRC_DIR_ZFS/$ZFS_TAG.tar" ]; then
			tar -xf "$SRC_DIR_ZFS/$ZFS_TAG.tar" -C "$SRC_DIR_ZFS"
			rm "$SRC_DIR_ZFS/$ZFS_TAG.tar"
			echo -n -e "${COLOR_GREEN}Local${COLOR_RESET}\t\t"
		else
			mkdir -p "$ZFS_DIR" || fail 1
			curl -sL "$ZFS_URL" | tar -xz -C "$ZFS_DIR" \
			    --strip-components=1 || fail 2
			echo -n -e "${COLOR_GREEN}Remote${COLOR_RESET}\t\t"
		fi
	fi
done
printf "\n"

# Build the listed tags
printf "%-16s" "Build SPL"
for TAG in $SRC_TAGS; do
	src_set_vars "$TAG"

	if [ -f "$SPL_DIR/module/spl/spl.ko" ]; then
		skip_nonewline
	elif  [ "$SPL_TAG" = "installed" ]; then
		skip_nonewline
	else
		cd "$SPL_DIR"
		make distclean &>/dev/null
		./autogen.sh >>"$CONFIG_LOG" 2>&1 || fail 1
		# shellcheck disable=SC2086
		./configure $CONFIG_OPTIONS >>"$CONFIG_LOG" 2>&1 || fail 2
		# shellcheck disable=SC2086
		make $MAKE_OPTIONS >>"$MAKE_LOG" 2>&1 || fail 3
		pass_nonewline
	fi
done
printf "\n"

# Build the listed tags
printf "%-16s" "Build ZFS"
for TAG in $SRC_TAGS; do
	src_set_vars "$TAG"

	if [ -f "$ZFS_DIR/module/zfs/zfs.ko" ]; then
		skip_nonewline
	elif  [ "$ZFS_TAG" = "installed" ]; then
		skip_nonewline
	else
		cd "$ZFS_DIR"
		make distclean &>/dev/null
		./autogen.sh >>"$CONFIG_LOG" 2>&1 || fail 1
		# shellcheck disable=SC2086
		./configure --with-spl="$SPL_DIR" $CONFIG_OPTIONS \
                    >>"$CONFIG_LOG" 2>&1 || fail 2
		# shellcheck disable=SC2086
		make $MAKE_OPTIONS >>"$MAKE_LOG" 2>&1 || fail 3
		pass_nonewline
	fi
done
printf "\n"
echo "-----------------------------------------------------------------"

# Either create a new pool using 'zpool create', or alternately restore an
# existing pool from another ZFS implementation for compatibility testing.
for TAG in $POOL_TAGS; do
	pool_set_vars "$TAG"
	SKIP=0

	printf "%-16s" "$POOL_TAG"
	rm -Rf "$POOL_DIR"
	mkdir -p "$POOL_DIR_PRISTINE"

	# Use the existing compressed image if available.
	if [ -f "$POOL_BZIP" ]; then
		tar -xjf "$POOL_BZIP" -C "$POOL_DIR_PRISTINE" \
		    --strip-components=1 || fail 1
	# Use the installed version to create the pool.
	elif  [ "$TAG" = "installed" ]; then
		pool_create "$TAG"
	# A source build is available to create the pool.
	elif [ -d "$POOL_DIR_SRC" ]; then
		pool_create "$TAG"
	else
		SKIP=1
	fi

	# Verify 'zpool import' works for all listed source versions.
	for TAG in $SRC_TAGS; do

		if [ $SKIP -eq 1 ]; then
			skip_nonewline
			continue
		fi

		src_set_vars "$TAG"
		if [ "$TAG" != "installed" ]; then
			cd "$ZFS_DIR"
		fi
		$ZFS_SH zfs="spa_config_path=$POOL_DIR_COPY"

		cp -a --sparse=always "$POOL_DIR_PRISTINE" \
		    "$POOL_DIR_COPY" || fail 2
		POOL_NAME=$($ZPOOL_CMD import -d "$POOL_DIR_COPY" | \
		    awk '/pool:/ { print $2; exit 0 }')

		$ZPOOL_CMD import -N -d "$POOL_DIR_COPY" \
		   "$POOL_NAME" &>/dev/null
		if [ $? -ne 0 ]; then
			fail_nonewline
			ERROR=1
		else
			$ZPOOL_CMD export "$POOL_NAME" || fail 3
			pass_nonewline
		fi

		rm -Rf "$POOL_DIR_COPY"

		$ZFS_SH -u || fail 4
	done
	printf "\n"
done

if [ "$KEEP" = "no" ]; then
	rm -Rf "$TEST_DIR"
fi

exit $ERROR
Commit	Line	Data
888f7141	1	#!/bin/bash
a16bc6bd BB	2	#
	3	# Verify that an assortment of known good reference pools can be imported
	4	# using different versions of the ZoL code.
	5	#
	6	# By default references pools for the major ZFS implementation will be
	7	# checked against the most recent ZoL tags and the master development branch.
	8	# Alternate tags or branches may be verified with the '-s <src-tag> option.
	9	# Passing the keyword "installed" will instruct the script to test whatever
	10	# version is installed.
	11	#
	12	# Preferentially a reference pool is used for all tests. However, if one
	13	# does not exist and the pool-tag matches one of the src-tags then a new
	14	# reference pool will be created using binaries from that source build.
	15	# This is particularly useful when you need to test your changes before
	16	# opening a pull request. The keyword 'all' can be used as short hand
	17	# refer to all available reference pools.
	18	#
	19	# New reference pools may be added by placing a bzip2 compressed tarball
	20	# of the pool in the scripts/zfs-images directory and then passing
	21	# the -p <pool-tag> option. To increase the test coverage reference pools
	22	# should be collected for all the major ZFS implementations. Having these
	23	# pools easily available is also helpful to the developers.
	24	#
	25	# Care should be taken to run these tests with a kernel supported by all
	26	# the listed tags. Otherwise build failure will cause false positives.
	27	#
	28	#
	29	# EXAMPLES:
	30	#
	31	# The following example will verify the zfs-0.6.2 tag, the master branch,
	32	# and the installed zfs version can correctly import the listed pools.
	33	# Note there is no reference pool available for master and installed but
	34	# because binaries are available one is automatically constructed. The
	35	# working directory is also preserved between runs (-k) preventing the
	36	# need to rebuild from source for multiple runs.
	37	#
	38	# zimport.sh -k -f /var/tmp/zimport \
	39	# -s "zfs-0.6.2 master installed" \
	40	# -p "zevo-1.1.1 zol-0.6.2 zol-0.6.2-173 master installed"
	41	#
	42	# --------------------- ZFS on Linux Source Versions --------------
	43	# zfs-0.6.2 master 0.6.2-175_g36eb554
	44	# -----------------------------------------------------------------
	45	# Clone SPL Local Local Skip
	46	# Clone ZFS Local Local Skip
	47	# Build SPL Pass Pass Skip
	48	# Build ZFS Pass Pass Skip
	49	# -----------------------------------------------------------------
	50	# zevo-1.1.1 Pass Pass Pass
	51	# zol-0.6.2 Pass Pass Pass
	52	# zol-0.6.2-173 Fail Pass Pass
	53	# master Pass Pass Pass
	54	# installed Pass Pass Pass
	55	#
a16bc6bd	56
c8f9061f	57	BASE_DIR=$(dirname "$0")
a16bc6bd	58	SCRIPT_COMMON=common.sh
c8f9061f BB	59	if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then
c8f9061f BB	60	. "${BASE_DIR}/${SCRIPT_COMMON}"
a16bc6bd	61	else
c8f9061f	62	echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
a16bc6bd BB	63	fi
a16bc6bd BB	64
c8f9061f	65	SRC_TAGS="zfs-0.6.5.11 master"
a16bc6bd	66	POOL_TAGS="all master"
c8f9061f BB	67	TEST_DIR=$(mktemp -u -d -p /var/tmp zimport.XXXXXXXX)
	68	KEEP="no"
	69	VERBOSE="no"
	70	COLOR="yes"
a16bc6bd BB	71	REPO="https://github.com/zfsonlinux"
	72	IMAGES_DIR="$SCRIPTDIR/zfs-images/"
	73	IMAGES_TAR="https://github.com/zfsonlinux/zfs-images/tarball/master"
a16bc6bd BB	74	ERROR=0
a16bc6bd BB	75
27a19a0d BB	76	CONFIG_LOG="configure.log"
	77	CONFIG_OPTIONS=${CONFIG_OPTIONS:-""}
	78	MAKE_LOG="make.log"
	79	MAKE_OPTIONS=${MAKE_OPTIONS:-"-s -j$(nproc)"}
	80
c8f9061f BB	81	COLOR_GREEN="\033[0;32m"
	82	COLOR_RED="\033[0;31m"
	83	COLOR_BROWN="\033[0;33m"
	84	COLOR_RESET="\033[0m"
	85
a16bc6bd BB	86	usage() {
	87	cat << EOF
	88	USAGE:
	89	zimport.sh [hvl] [-r repo] [-s src-tag] [-i pool-dir] [-p pool-tag] [-f path]
	90
	91	DESCRIPTION:
	92	ZPOOL import verification tests
	93
	94	OPTIONS:
	95	-h Show this message
	96	-v Verbose
	97	-c No color
	98	-k Keep temporary directory
	99	-r <repo> Source repository ($REPO)
	100	-s <src-tag>... Verify ZoL versions with the listed tags
	101	-i <pool-dir> Pool image directory
	102	-p <pool-tag>... Verify pools created with the listed tags
	103	-f <path> Temporary directory to use
	104
	105	EOF
	106	}
	107
	108	while getopts 'hvckr:s:i:p:f:?' OPTION; do
	109	case $OPTION in
	110	h)
	111	usage
	112	exit 1
	113	;;
	114	v)
c8f9061f	115	VERBOSE="yes"
a16bc6bd BB	116	;;
a16bc6bd BB	117	c)
c8f9061f	118	COLOR="no"
a16bc6bd BB	119	;;
a16bc6bd BB	120	k)
c8f9061f	121	KEEP="yes"
a16bc6bd BB	122	;;
	123	r)
	124	REPO="$OPTARG"
	125	;;
	126	s)
	127	SRC_TAGS="$OPTARG"
	128	;;
	129	i)
	130	IMAGES_DIR="$OPTARG"
	131	;;
	132	p)
	133	POOL_TAGS="$OPTARG"
	134	;;
	135	f)
	136	TEST_DIR="$OPTARG"
	137	;;
	138	?)
	139	usage
	140	exit
	141	;;
	142	esac
	143	done
	144
c8f9061f BB	145	#
	146	# Verify the module start is not loaded
	147	#
	148	if lsmod \| grep zfs >/dev/null; then
	149	echo "ZFS modules must be unloaded"
	150	exit 1
	151	fi
	152
	153	#
	154	# Create a random directory tree of files and sub-directories to
	155	# to act as a copy source for the various regression tests.
	156	#
	157	populate() {
	158	local ROOT=$1
	159	local MAX_DIR_SIZE=$2
	160	local MAX_FILE_SIZE=$3
	161
	162	# shellcheck disable=SC2086
	163	mkdir -p $ROOT/{a,b,c,d,e,f,g}/{h,i}
	164	DIRS=$(find "$ROOT")
	165
	166	for DIR in $DIRS; do
	167	COUNT=$((RANDOM % MAX_DIR_SIZE))
	168
	169	# shellcheck disable=SC2034
	170	for i in $(seq $COUNT); do
	171	FILE=$(mktemp -p "$DIR")
	172	SIZE=$((RANDOM % MAX_FILE_SIZE))
	173	dd if=/dev/urandom of="$FILE" bs=1k \
	174	count="$SIZE" &>/dev/null
	175	done
	176	done
	177
	178	return 0
	179	}
	180
	181	SRC_DIR=$(mktemp -d -p /var/tmp/ zfs.src.XXXXXXXX)
	182	trap 'rm -Rf "$SRC_DIR"' INT TERM EXIT
	183	populate "$SRC_DIR" 10 100
a16bc6bd BB	184
	185	SRC_DIR="$TEST_DIR/src"
	186	SRC_DIR_SPL="$SRC_DIR/spl"
	187	SRC_DIR_ZFS="$SRC_DIR/zfs"
	188
c8f9061f	189	if [ "$COLOR" = "no" ]; then
a16bc6bd BB	190	COLOR_GREEN=""
	191	COLOR_BROWN=""
	192	COLOR_RED=""
	193	COLOR_RESET=""
	194	fi
	195
	196	pass_nonewline() {
	197	echo -n -e "${COLOR_GREEN}Pass${COLOR_RESET}\t\t"
	198	}
	199
	200	skip_nonewline() {
	201	echo -n -e "${COLOR_BROWN}Skip${COLOR_RESET}\t\t"
	202	}
	203
	204	fail_nonewline() {
	205	echo -n -e "${COLOR_RED}Fail${COLOR_RESET}\t\t"
	206	}
	207
c8f9061f BB	208	fail() {
	209	echo -e "${COLOR_RED}Fail${COLOR_RESET} ($1)"
	210	exit "$1"
	211	}
	212
a16bc6bd BB	213	#
	214	# Set several helper variables which are derived from a source tag.
	215	#
	216	# SPL_TAG - The tag zfs-x.y.z is translated to spl-x.y.z.
	217	# SPL_DIR - The spl directory name.
	218	# SPL_URL - The spl github URL to fetch the tarball.
	219	# ZFS_TAG - The passed zfs-x.y.z tag
	220	# ZFS_DIR - The zfs directory name
	221	# ZFS_URL - The zfs github URL to fetch the tarball
	222	#
	223	src_set_vars() {
	224	local TAG=$1
	225
c8f9061f BB	226	SPL_TAG="${TAG//zfs/spl}"
	227	SPL_DIR="$SRC_DIR_SPL/$SPL_TAG"
	228	SPL_URL="$REPO/spl/tarball/$SPL_TAG"
a16bc6bd	229
c8f9061f BB	230	ZFS_TAG="$TAG"
	231	ZFS_DIR="$SRC_DIR_ZFS/$ZFS_TAG"
	232	ZFS_URL="$REPO/zfs/tarball/$ZFS_TAG"
a16bc6bd BB	233
a16bc6bd BB	234	if [ "$TAG" = "installed" ]; then
c8f9061f BB	235	ZPOOL_CMD=$(which zpool)
c8f9061f BB	236	ZFS_CMD=$(which zfs)
a16bc6bd	237	ZFS_SH="/usr/share/zfs/zfs.sh"
a16bc6bd BB	238	else
	239	ZPOOL_CMD="./cmd/zpool/zpool"
	240	ZFS_CMD="./cmd/zfs/zfs"
	241	ZFS_SH="./scripts/zfs.sh"
a16bc6bd BB	242	fi
	243	}
	244
	245	#
	246	# Set several helper variables which are derived from a pool name such
	247	# as zol-0.6.x, zevo-1.1.1, etc. These refer to example pools from various
	248	# ZFS implementations which are used to verify compatibility.
	249	#
	250	# POOL_TAG - The example pools name in scripts/zfs-images/.
	251	# POOL_BZIP - The full path to the example bzip2 compressed pool.
	252	# POOL_DIR - The top level test path for this pool.
	253	# POOL_DIR_PRISTINE - The directory containing a pristine version of the pool.
	254	# POOL_DIR_COPY - The directory containing a working copy of the pool.
	255	# POOL_DIR_SRC - Location of a source build if it exists for this pool.
	256	#
	257	pool_set_vars() {
	258	local TAG=$1
	259
	260	POOL_TAG=$TAG
	261	POOL_BZIP=$IMAGES_DIR/$POOL_TAG.tar.bz2
	262	POOL_DIR=$TEST_DIR/pools/$POOL_TAG
	263	POOL_DIR_PRISTINE=$POOL_DIR/pristine
	264	POOL_DIR_COPY=$POOL_DIR/copy
c8f9061f	265	POOL_DIR_SRC="$SRC_DIR_ZFS/${POOL_TAG//zol/zfs}"
a16bc6bd BB	266	}
	267
	268	#
	269	# Construct a non-trivial pool given a specific version of the source. More
	270	# interesting pools provide better test coverage so this function should
	271	# extended as needed to create more realistic pools.
	272	#
	273	pool_create() {
c8f9061f BB	274	pool_set_vars "$1"
c8f9061f BB	275	src_set_vars "$1"
a16bc6bd BB	276
a16bc6bd BB	277	if [ "$POOL_TAG" != "installed" ]; then
c8f9061f	278	cd "$POOL_DIR_SRC"
a16bc6bd BB	279	fi
a16bc6bd BB	280
e0b8f629	281	$ZFS_SH zfs="spa_config_path=$POOL_DIR_PRISTINE" \|\| fail 1
a16bc6bd BB	282
a16bc6bd BB	283	# Create a file vdev RAIDZ pool.
c8f9061f BB	284	truncate -s 1G \
	285	"$POOL_DIR_PRISTINE/vdev1" "$POOL_DIR_PRISTINE/vdev2" \
	286	"$POOL_DIR_PRISTINE/vdev3" "$POOL_DIR_PRISTINE/vdev4"
	287	$ZPOOL_CMD create "$POOL_TAG" raidz \
	288	"$POOL_DIR_PRISTINE/vdev1" "$POOL_DIR_PRISTINE/vdev2" \
	289	"$POOL_DIR_PRISTINE/vdev3" "$POOL_DIR_PRISTINE/vdev4"
a16bc6bd BB	290
a16bc6bd BB	291	# Create a pool/fs filesystem with some random contents.
c8f9061f BB	292	$ZFS_CMD create "$POOL_TAG/fs" \|\| fail 3
c8f9061f BB	293	populate "/$POOL_TAG/fs/" 10 100
a16bc6bd BB	294
	295	# Snapshot that filesystem, clone it, remove the files/dirs,
	296	# replace them with new files/dirs.
c8f9061f BB	297	$ZFS_CMD snap "$POOL_TAG/fs@snap" \|\| fail 4
	298	$ZFS_CMD clone "$POOL_TAG/fs@snap" "$POOL_TAG/clone" \|\| fail 5
	299	# shellcheck disable=SC2086
e0b8f629	300	rm -Rf /$POOL_TAG/clone/* \|\| fail 6
c8f9061f	301	populate "/$POOL_TAG/clone/" 10 100
a16bc6bd BB	302
	303	# Scrub the pool, delay slightly, then export it. It is now
	304	# somewhat interesting for testing purposes.
c8f9061f	305	$ZPOOL_CMD scrub "$POOL_TAG" \|\| fail 7
a16bc6bd	306	sleep 10
c8f9061f	307	$ZPOOL_CMD export "$POOL_TAG" \|\| fail 8
a16bc6bd	308
e0b8f629	309	$ZFS_SH -u \|\| fail 9
a16bc6bd BB	310	}
	311
	312	# If the zfs-images directory doesn't exist fetch a copy from Github then
	313	# cache it in the $TEST_DIR and update $IMAGES_DIR.
c8f9061f	314	if [ ! -d "$IMAGES_DIR" ]; then
a16bc6bd	315	IMAGES_DIR="$TEST_DIR/zfs-images"
c8f9061f BB	316	mkdir -p "$IMAGES_DIR"
	317	curl -sL "$IMAGES_TAR" \| \
	318	tar -xz -C "$IMAGES_DIR" --strip-components=1 \|\| fail 10
a16bc6bd BB	319	fi
	320
	321	# Given the available images in the zfs-images directory substitute the
4e33ba4c	322	# list of available images for the reserved keyword 'all'.
a16bc6bd BB	323	for TAG in $POOL_TAGS; do
	324
	325	if [ "$TAG" = "all" ]; then
c8f9061f BB	326	# shellcheck disable=SC2010
	327	ALL_TAGS=$(ls "$IMAGES_DIR" \| grep "tar.bz2" \| \
	328	sed 's/.tar.bz2//' \| tr '\n' ' ')
a16bc6bd BB	329	NEW_TAGS="$NEW_TAGS $ALL_TAGS"
	330	else
	331	NEW_TAGS="$NEW_TAGS $TAG"
	332	fi
	333	done
	334	POOL_TAGS="$NEW_TAGS"
	335
c8f9061f	336	if [ "$VERBOSE" = "yes" ]; then
a16bc6bd BB	337	echo "---------------------------- Options ----------------------------"
	338	echo "VERBOSE=$VERBOSE"
	339	echo "KEEP=$KEEP"
	340	echo "REPO=$REPO"
c8f9061f BB	341	echo "SRC_TAGS=$SRC_TAGS"
c8f9061f BB	342	echo "POOL_TAGS=$POOL_TAGS"
a16bc6bd BB	343	echo "PATH=$TEST_DIR"
	344	echo
	345	fi
	346
c8f9061f BB	347	if [ ! -d "$TEST_DIR" ]; then
c8f9061f BB	348	mkdir -p "$TEST_DIR"
a16bc6bd BB	349	fi
a16bc6bd BB	350
c8f9061f BB	351	if [ ! -d "$SRC_DIR" ]; then
c8f9061f BB	352	mkdir -p "$SRC_DIR"
e0b8f629 BB	353	fi
e0b8f629 BB	354
a16bc6bd BB	355	# Print a header for all tags which are being tested.
	356	echo "--------------------- ZFS on Linux Source Versions --------------"
	357	printf "%-16s" " "
	358	for TAG in $SRC_TAGS; do
c8f9061f	359	src_set_vars "$TAG"
a16bc6bd BB	360
a16bc6bd BB	361	if [ "$TAG" = "installed" ]; then
c8f9061f	362	ZFS_VERSION=$(modinfo zfs \| awk '/version:/ { print $2; exit }')
a16bc6bd	363	if [ -n "$ZFS_VERSION" ]; then
c8f9061f	364	printf "%-16s" "$ZFS_VERSION"
a16bc6bd	365	else
c8f9061f	366	echo -e "ZFS is not installed\n"
a16bc6bd BB	367	fail
	368	fi
	369	else
c8f9061f	370	printf "%-16s" "$TAG"
a16bc6bd BB	371	fi
	372	done
	373	echo -e "\n-----------------------------------------------------------------"
	374
	375	#
	376	# Attempt to generate the tarball from your local git repository, if that
	377	# fails then attempt to download the tarball from Github.
	378	#
	379	printf "%-16s" "Clone SPL"
	380	for TAG in $SRC_TAGS; do
c8f9061f	381	src_set_vars "$TAG"
a16bc6bd	382
c8f9061f	383	if [ -d "$SPL_DIR" ]; then
a16bc6bd BB	384	skip_nonewline
	385	elif [ "$SPL_TAG" = "installed" ]; then
	386	skip_nonewline
	387	else
c8f9061f	388	cd "$SRC_DIR"
a16bc6bd	389
c8f9061f BB	390	if [ ! -d "$SRC_DIR_SPL" ]; then
c8f9061f BB	391	mkdir -p "$SRC_DIR_SPL"
a16bc6bd BB	392	fi
a16bc6bd BB	393
c8f9061f BB	394	git archive --format=tar --prefix="$SPL_TAG/ $SPL_TAG" \
	395	-o "$SRC_DIR_SPL/$SPL_TAG.tar" &>/dev/nul \|\| \
	396	rm "$SRC_DIR_SPL/$SPL_TAG.tar"
	397	if [ -s "$SRC_DIR_SPL/$SPL_TAG.tar" ]; then
	398	tar -xf "$SRC_DIR_SPL/$SPL_TAG.tar" -C "$SRC_DIR_SPL"
	399	rm "$SRC_DIR_SPL/$SPL_TAG.tar"
a16bc6bd BB	400	echo -n -e "${COLOR_GREEN}Local${COLOR_RESET}\t\t"
a16bc6bd BB	401	else
c8f9061f BB	402	mkdir -p "$SPL_DIR" \|\| fail 1
c8f9061f BB	403	curl -sL "$SPL_URL" \| tar -xz -C "$SPL_DIR" \
e0b8f629	404	--strip-components=1 \|\| fail 2
a16bc6bd BB	405	echo -n -e "${COLOR_GREEN}Remote${COLOR_RESET}\t\t"
	406	fi
	407	fi
	408	done
	409	printf "\n"
	410
	411	#
	412	# Attempt to generate the tarball from your local git repository, if that
	413	# fails then attempt to download the tarball from Github.
	414	#
	415	printf "%-16s" "Clone ZFS"
	416	for TAG in $SRC_TAGS; do
c8f9061f	417	src_set_vars "$TAG"
a16bc6bd	418
c8f9061f	419	if [ -d "$ZFS_DIR" ]; then
a16bc6bd BB	420	skip_nonewline
	421	elif [ "$ZFS_TAG" = "installed" ]; then
	422	skip_nonewline
	423	else
c8f9061f	424	cd "$SRC_DIR"
a16bc6bd	425
c8f9061f BB	426	if [ ! -d "$SRC_DIR_ZFS" ]; then
c8f9061f BB	427	mkdir -p "$SRC_DIR_ZFS"
a16bc6bd BB	428	fi
a16bc6bd BB	429
c8f9061f BB	430	git archive --format=tar --prefix="$ZFS_TAG/ $ZFS_TAG" \
	431	-o "$SRC_DIR_ZFS/$ZFS_TAG.tar" &>/dev/nul \|\| \
	432	rm "$SRC_DIR_ZFS/$ZFS_TAG.tar"
	433	if [ -s "$SRC_DIR_ZFS/$ZFS_TAG.tar" ]; then
	434	tar -xf "$SRC_DIR_ZFS/$ZFS_TAG.tar" -C "$SRC_DIR_ZFS"
	435	rm "$SRC_DIR_ZFS/$ZFS_TAG.tar"
a16bc6bd BB	436	echo -n -e "${COLOR_GREEN}Local${COLOR_RESET}\t\t"
a16bc6bd BB	437	else
c8f9061f BB	438	mkdir -p "$ZFS_DIR" \|\| fail 1
c8f9061f BB	439	curl -sL "$ZFS_URL" \| tar -xz -C "$ZFS_DIR" \
e0b8f629	440	--strip-components=1 \|\| fail 2
a16bc6bd BB	441	echo -n -e "${COLOR_GREEN}Remote${COLOR_RESET}\t\t"
	442	fi
	443	fi
	444	done
	445	printf "\n"
	446
	447	# Build the listed tags
	448	printf "%-16s" "Build SPL"
	449	for TAG in $SRC_TAGS; do
c8f9061f	450	src_set_vars "$TAG"
a16bc6bd	451
c8f9061f	452	if [ -f "$SPL_DIR/module/spl/spl.ko" ]; then
a16bc6bd BB	453	skip_nonewline
	454	elif [ "$SPL_TAG" = "installed" ]; then
	455	skip_nonewline
	456	else
c8f9061f	457	cd "$SPL_DIR"
a16bc6bd	458	make distclean &>/dev/null
c8f9061f BB	459	./autogen.sh >>"$CONFIG_LOG" 2>&1 \|\| fail 1
	460	# shellcheck disable=SC2086
	461	./configure $CONFIG_OPTIONS >>"$CONFIG_LOG" 2>&1 \|\| fail 2
	462	# shellcheck disable=SC2086
	463	make $MAKE_OPTIONS >>"$MAKE_LOG" 2>&1 \|\| fail 3
a16bc6bd BB	464	pass_nonewline
	465	fi
	466	done
	467	printf "\n"
	468
	469	# Build the listed tags
	470	printf "%-16s" "Build ZFS"
	471	for TAG in $SRC_TAGS; do
c8f9061f	472	src_set_vars "$TAG"
a16bc6bd	473
c8f9061f	474	if [ -f "$ZFS_DIR/module/zfs/zfs.ko" ]; then
a16bc6bd BB	475	skip_nonewline
	476	elif [ "$ZFS_TAG" = "installed" ]; then
	477	skip_nonewline
	478	else
c8f9061f	479	cd "$ZFS_DIR"
a16bc6bd	480	make distclean &>/dev/null
c8f9061f BB	481	./autogen.sh >>"$CONFIG_LOG" 2>&1 \|\| fail 1
	482	# shellcheck disable=SC2086
	483	./configure --with-spl="$SPL_DIR" $CONFIG_OPTIONS \
	484	>>"$CONFIG_LOG" 2>&1 \|\| fail 2
	485	# shellcheck disable=SC2086
	486	make $MAKE_OPTIONS >>"$MAKE_LOG" 2>&1 \|\| fail 3
a16bc6bd BB	487	pass_nonewline
	488	fi
	489	done
	490	printf "\n"
	491	echo "-----------------------------------------------------------------"
	492
	493	# Either create a new pool using 'zpool create', or alternately restore an
	494	# existing pool from another ZFS implementation for compatibility testing.
	495	for TAG in $POOL_TAGS; do
c8f9061f	496	pool_set_vars "$TAG"
a16bc6bd BB	497	SKIP=0
a16bc6bd BB	498
c8f9061f BB	499	printf "%-16s" "$POOL_TAG"
	500	rm -Rf "$POOL_DIR"
	501	mkdir -p "$POOL_DIR_PRISTINE"
a16bc6bd BB	502
a16bc6bd BB	503	# Use the existing compressed image if available.
c8f9061f BB	504	if [ -f "$POOL_BZIP" ]; then
c8f9061f BB	505	tar -xjf "$POOL_BZIP" -C "$POOL_DIR_PRISTINE" \
e0b8f629	506	--strip-components=1 \|\| fail 1
a16bc6bd BB	507	# Use the installed version to create the pool.
a16bc6bd BB	508	elif [ "$TAG" = "installed" ]; then
c8f9061f	509	pool_create "$TAG"
a16bc6bd	510	# A source build is available to create the pool.
c8f9061f BB	511	elif [ -d "$POOL_DIR_SRC" ]; then
c8f9061f BB	512	pool_create "$TAG"
a16bc6bd BB	513	else
	514	SKIP=1
	515	fi
	516
	517	# Verify 'zpool import' works for all listed source versions.
	518	for TAG in $SRC_TAGS; do
	519
	520	if [ $SKIP -eq 1 ]; then
	521	skip_nonewline
	522	continue
	523	fi
	524
c8f9061f	525	src_set_vars "$TAG"
a16bc6bd	526	if [ "$TAG" != "installed" ]; then
c8f9061f	527	cd "$ZFS_DIR"
a16bc6bd BB	528	fi
	529	$ZFS_SH zfs="spa_config_path=$POOL_DIR_COPY"
	530
c8f9061f BB	531	cp -a --sparse=always "$POOL_DIR_PRISTINE" \
	532	"$POOL_DIR_COPY" \|\| fail 2
	533	POOL_NAME=$($ZPOOL_CMD import -d "$POOL_DIR_COPY" \| \
	534	awk '/pool:/ { print $2; exit 0 }')
a16bc6bd	535
c8f9061f BB	536	$ZPOOL_CMD import -N -d "$POOL_DIR_COPY" \
c8f9061f BB	537	"$POOL_NAME" &>/dev/null
a16bc6bd BB	538	if [ $? -ne 0 ]; then
	539	fail_nonewline
	540	ERROR=1
	541	else
c8f9061f	542	$ZPOOL_CMD export "$POOL_NAME" \|\| fail 3
a16bc6bd BB	543	pass_nonewline
	544	fi
	545
c8f9061f	546	rm -Rf "$POOL_DIR_COPY"
a16bc6bd	547
e0b8f629	548	$ZFS_SH -u \|\| fail 4
a16bc6bd BB	549	done
	550	printf "\n"
	551	done
	552
c8f9061f BB	553	if [ "$KEEP" = "no" ]; then
c8f9061f BB	554	rm -Rf "$TEST_DIR"
a16bc6bd BB	555	fi
	556
	557	exit $ERROR