[mirror_zfs.git] / tests / zfs-tests / tests / functional / cli_root / zdb / zdb_block_size_histogram.ksh

#!/bin/ksh -p

#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source.  A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#

#
# Copyright (c) 2017 by Delphix. All rights reserved.
# Copyright (c) 2020 by Lawrence Livermore National Security LLC.

. $STF_SUITE/include/libtest.shlib


#
# DESCRIPTION:
#	Create a pool and populate it with files of various
#	recordsizes
#
# STRATEGY:
#	1. Create pool
#	2. Populate it
#	3. Run zdb -Pbbb on pool
#	4. Verify variance on blocksizes
#
function cleanup
{
	datasetexists $TESTPOOL && destroy_pool $TESTPOOL
}

SPA_MAXBLOCKSHIFT=24

function histo_populate_test_pool
{
	if [ $# -ne 1 ]; then
		log_note "histo_populate_test_pool: insufficient parameters"
		log_fail "hptp: 1 requested $# received"
	fi
	typeset pool=$1

	set -A recordsizes
	typeset -i min_rsbits=9 #512
	typeset -i max_rsbits=SPA_MAXBLOCKSHIFT #16 MiB
	typeset -i sum_filesizes=0
	re_number='^[0-9]+$'

	let histo_pool_size=$(get_pool_prop size ${pool})
	if [[ ! ${histo_pool_size} =~ ${re_number} ]]; then
		log_fail "histo_pool_size is not numeric ${pool_size}"
	fi
	let max_pool_record_size=$(get_prop recordsize ${pool})
	if [[ ! ${max_pool_record_size} =~ ${re_number} ]]; then
		log_fail "hptp: max_pool_record_size is not numeric ${max_pool_record_size}"
	fi

	sum_filesizes=$(echo "2^21"|bc)
	((min_pool_size=12*sum_filesizes))
	if [ ${histo_pool_size} -lt ${min_pool_size} ]; then
		log_note "hptp: Your pool size ${histo_pool_size}"
		log_fail "hptp: is less than minimum ${min_pool_size}"
	fi
	this_ri=min_rsbits
	file_num=0
	total_count=0
	###################
	# generate 10% + 20% + 30% + 31% = 91% of the filespace
	# attempting to use 100% will lead to no space left on device
	# Heuristic testing showed that 91% was the practical upper
	# bound on the default 4G zpool (mirrored) that is used in
	# testing.
	#
	# In order to expedite testing, we will only fill 2G (of 4G)
	# of the test pool.  You may want to modify this for
	# standalone testing.
	# 
	# In filling only 50% of the pool, we create one object on
	# each "pass" below to achieve multiple objects per record
	# size.  Creating one file per object would lead to 
	# excessive file creation time.
	###################
	# for pass in 10 20 30 31  # 91%
	for pass in 20 20 10 # 50%
	do
		((thiscount=(((histo_pool_size*pass)/100)/sum_filesizes)))

		((total_count+=thiscount))
		for rb in $(seq ${min_rsbits} ${max_rsbits})
		do
			this_rs=$(echo "2^${rb}" | bc)
			if [ ${this_rs} -gt ${max_pool_record_size} ]; then
				continue
			fi
	
			if [ ! -d /${pool}/B_${this_rs} ]; then
				zfs create ${pool}/B_${this_rs}
				zfs set recordsize=${this_rs} \
				    ${pool}/B_${this_rs}
			fi
			####################
			# Create the files in the devices and datasets
			# of the right size.  The files are filled
			# with random data to defeat the compression
			#
			# Note that the dd output is suppressed unless
			# there are errors
			####################

			dd if=/dev/urandom \
			    of=/${pool}/B_${this_rs}/file_${filenum} \
			    bs=${this_rs} count=${thiscount} \
			    iflag=fullblock 2>&1 | \
			    grep -ve "records in" -e "records out" -e "bytes.*copied"
			((filenum+=1))
		done
	done

	####################
	# Testing showed that on some devices, unless the pool is 
	# synchronized, that the block counts will be below the 
	# anticipated sizes since not all of the blocks will be flushed
	# to the device.  This 'sync' command prevents that from 
	# happening.
	####################
	sync_pool ${pool}
}
function histo_check_test_pool
{
	if [ $# -ne 1 ]; then
		log_note "histo_check_test_pool: insufficient parameters"
		log_fail "hctp: 1 requested $# received"
	fi	
	typeset pool=$1

	set -A recordsizes
	set -A recordcounts
	typeset -i rb
	typeset -i min_rsbits=9 #512
	typeset -i max_rsbits=SPA_MAXBLOCKSHIFT+1
	typeset -i this_rs
	typeset -i this_ri
	typeset -i sum_filesizes=0

	let histo_check_pool_size=$(get_pool_prop size ${pool})
	if [[ ! ${histo_check_pool_size} =~ ${re_number} ]]; then
		log_fail "histo_check_pool_size is not numeric ${histo_check_pool_size}"
	fi
	let max_pool_record_size=$(get_prop recordsize ${pool})
	if [[ ! ${max_pool_record_size} =~ ${re_number} ]]; then
		log_fail "hctp: max_pool_record_size is not numeric ${max_pool_record_size}"
	fi

	stripped="${TEST_BASE_DIR}/${pool}_stripped.txt"

	zdb -Pbbb ${pool} | \
	    sed -e '1,/^block[ 	][ 	]*psize[ 	][ 	]*lsize.*$/d' \
	    -e '/^size[ 	]*Count/d' -e '/^$/,$d' \
	    > ${stripped}

	sum_filesizes=$(echo "2^21"|bc)

	###################
	# generate 10% + 20% + 30% + 31% = 91% of the filespace
	# attempting to use 100% will lead to no space left on device
	# attempting to use 100% will lead to no space left on device
	# Heuristic testing showed that 91% was the practical upper
	# bound on the default 4G zpool (mirrored) that is used in
	# testing.
	#
	# In order to expedite testing, we will only fill 2G (of 4G)
	# of the test pool.  You may want to modify this for
	# standalone testing.
	# 
	# In filling only 50% of the pool, we create one object on
	# each "pass" below to achieve multiple objects per record
	# size.  Creating one file per object would lead to 
	# excessive file creation time.
	###################
	# for pass in 10 20 30 31  # 91%
	for pass in 20 20 10 # 50%
	do
		((thiscount=(((histo_check_pool_size*pass)/100)/sum_filesizes)))

		for rb in $(seq ${min_rsbits} ${max_rsbits})
		do
			blksize=$(echo "2^$rb"|bc)
			if [ $blksize -le $max_pool_record_size ]; then
				((recordcounts[$blksize]+=thiscount))
			fi
		done
	done

	###################
	# compare the above computed counts for blocks against
	# lsize count.  Since some devices have a minimum hardware
	# blocksize > 512, we cannot compare against the asize count.
	# E.G., if the HWBlocksize = 4096, then the asize counts for
	# 512, 1024 and 2048 will be zero and rolled up into the 
	# 4096 blocksize count for asize.   For verification we stick
	# to just lsize counts.
	#
	# The max_variance is hard-coded here at 12% to leave us some
	# margin.  Testing has shown this normally to be in the range
	# of 2%-8%, but it may be as large as 11%.
	###################
	let max_variance=12
	let fail_value=0
	let error_count=0
	log_note "Comparisons for ${pool}"
	log_note "Bsize is the blocksize, Count is predicted value"
	log_note "Bsize\tCount\tpsize\tlsize\tasize"
	while read -r blksize pc pl pm lc ll lm ac al am
	do
		if [ $blksize -gt $max_pool_record_size ]; then
			continue
		fi
		log_note \
		    "$blksize\t${recordcounts[${blksize}]}\t$pc\t$lc\t$ac"

		###################
		# get the computer record count and compute the
		# difference percentage in integer arithmetic
		###################
		rc=${recordcounts[${blksize}]}
		((rclc=(rc-lc)<0?lc-rc:rc-lc)) # absolute value
		((dp=(rclc*100)/rc))

		###################
		# Check against the allowed variance
		###################
		if [ $dp -gt ${max_variance} ]; then
			log_note \
			"Expected variance < ${max_variance}% observed ${dp}%"
			if [ ${dp} -gt ${fail_value} ]; then
				fail_value=${dp}
				((error_count++))
			fi
		fi
	done < ${stripped}
	rm "${stripped}"

	if [ ${fail_value} -gt 0 ]; then
		if [ ${error_count} -eq 1 ]; then
			log_note "hctp: There was ${error_count} error"
		else
			log_note "hctp:There were a total of ${error_count} errors"
		fi
		log_fail \
		"hctp: Max variance of ${max_variance}% exceeded, saw ${fail_value}%"
	fi
}

log_assert "Verify zdb -Pbbb (block histogram) works as expected"
log_onexit cleanup
verify_runnable "global"
verify_disk_count "$DISKS" 2

default_mirror_setup_noexit $DISKS

histo_populate_test_pool $TESTPOOL

histo_check_test_pool $TESTPOOL

log_pass "Histogram for zdb"
Commit	Line	Data
bfcbec6f RN	1	#!/bin/ksh -p
	2
	3	#
	4	# This file and its contents are supplied under the terms of the
	5	# Common Development and Distribution License ("CDDL"), version 1.0.
	6	# You may only use this file in accordance with the terms of version
	7	# 1.0 of the CDDL.
	8	#
	9	# A full copy of the text of the CDDL should have accompanied this
	10	# source. A copy of the CDDL is also available via the Internet at
	11	# http://www.illumos.org/license/CDDL.
	12	#
	13
	14	#
	15	# Copyright (c) 2017 by Delphix. All rights reserved.
	16	# Copyright (c) 2020 by Lawrence Livermore National Security LLC.
	17
	18	. $STF_SUITE/include/libtest.shlib
	19
	20
	21	#
	22	# DESCRIPTION:
	23	# Create a pool and populate it with files of various
	24	# recordsizes
	25	#
	26	# STRATEGY:
	27	# 1. Create pool
	28	# 2. Populate it
	29	# 3. Run zdb -Pbbb on pool
	30	# 4. Verify variance on blocksizes
	31	#
	32	function cleanup
	33	{
	34	datasetexists $TESTPOOL && destroy_pool $TESTPOOL
	35	}
	36
	37	SPA_MAXBLOCKSHIFT=24
	38
	39	function histo_populate_test_pool
	40	{
	41	if [ $# -ne 1 ]; then
	42	log_note "histo_populate_test_pool: insufficient parameters"
	43	log_fail "hptp: 1 requested $# received"
	44	fi
	45	typeset pool=$1
	46
	47	set -A recordsizes
	48	typeset -i min_rsbits=9 #512
	49	typeset -i max_rsbits=SPA_MAXBLOCKSHIFT #16 MiB
	50	typeset -i sum_filesizes=0
	51	re_number='^[0-9]+$'
	52
	53	let histo_pool_size=$(get_pool_prop size ${pool})
	54	if [[ ! ${histo_pool_size} =~ ${re_number} ]]; then
	55	log_fail "histo_pool_size is not numeric ${pool_size}"
	56	fi
	57	let max_pool_record_size=$(get_prop recordsize ${pool})
	58	if [[ ! ${max_pool_record_size} =~ ${re_number} ]]; then
	59	log_fail "hptp: max_pool_record_size is not numeric ${max_pool_record_size}"
	60	fi
	61
	62	sum_filesizes=$(echo "2^21"\|bc)
	63	((min_pool_size=12*sum_filesizes))
	64	if [ ${histo_pool_size} -lt ${min_pool_size} ]; then
65	log_note "hptp: Your pool size ${histo_pool_size}"
66	log_fail "hptp: is less than minimum ${min_pool_size}"
67	fi
68	this_ri=min_rsbits
69	file_num=0
70	total_count=0
71	###################
72	# generate 10% + 20% + 30% + 31% = 91% of the filespace
73	# attempting to use 100% will lead to no space left on device
74	# Heuristic testing showed that 91% was the practical upper
75	# bound on the default 4G zpool (mirrored) that is used in
76	# testing.
77	#
78	# In order to expedite testing, we will only fill 2G (of 4G)
79	# of the test pool. You may want to modify this for
80	# standalone testing.
81	#
82	# In filling only 50% of the pool, we create one object on
83	# each "pass" below to achieve multiple objects per record
84	# size. Creating one file per object would lead to
85	# excessive file creation time.
86	###################
87	# for pass in 10 20 30 31 # 91%
88	for pass in 20 20 10 # 50%
89	do
90	((thiscount=(((histo_pool_size*pass)/100)/sum_filesizes)))
91
92	((total_count+=thiscount))
93	for rb in $(seq ${min_rsbits} ${max_rsbits})
94	do
95	this_rs=$(echo "2^${rb}" \| bc)
96	if [ ${this_rs} -gt ${max_pool_record_size} ]; then
97	continue
98	fi
99
100	if [ ! -d /${pool}/B_${this_rs} ]; then
101	zfs create ${pool}/B_${this_rs}
102	zfs set recordsize=${this_rs} \
103	${pool}/B_${this_rs}
104	fi
105	####################
106	# Create the files in the devices and datasets
107	# of the right size. The files are filled
108	# with random data to defeat the compression
109	#
110	# Note that the dd output is suppressed unless
111	# there are errors
112	####################
113
114	dd if=/dev/urandom \
115	of=/${pool}/B_${this_rs}/file_${filenum} \
116	bs=${this_rs} count=${thiscount} \
117	iflag=fullblock 2>&1 \| \
f63c9dc7	118	grep -ve "records in" -e "records out" -e "bytes.*copied"
bfcbec6f RN	119	((filenum+=1))
	120	done
	121	done
	122
	123	####################
	124	# Testing showed that on some devices, unless the pool is
	125	# synchronized, that the block counts will be below the
	126	# anticipated sizes since not all of the blocks will be flushed
	127	# to the device. This 'sync' command prevents that from
	128	# happening.
	129	####################
7454275a	130	sync_pool ${pool}
bfcbec6f RN	131	}
	132	function histo_check_test_pool
	133	{
	134	if [ $# -ne 1 ]; then
	135	log_note "histo_check_test_pool: insufficient parameters"
	136	log_fail "hctp: 1 requested $# received"
	137	fi
	138	typeset pool=$1
	139
	140	set -A recordsizes
	141	set -A recordcounts
	142	typeset -i rb
	143	typeset -i min_rsbits=9 #512
	144	typeset -i max_rsbits=SPA_MAXBLOCKSHIFT+1
	145	typeset -i this_rs
	146	typeset -i this_ri
	147	typeset -i sum_filesizes=0
bfcbec6f RN	148
	149	let histo_check_pool_size=$(get_pool_prop size ${pool})
	150	if [[ ! ${histo_check_pool_size} =~ ${re_number} ]]; then
	151	log_fail "histo_check_pool_size is not numeric ${histo_check_pool_size}"
	152	fi
	153	let max_pool_record_size=$(get_prop recordsize ${pool})
	154	if [[ ! ${max_pool_record_size} =~ ${re_number} ]]; then
	155	log_fail "hctp: max_pool_record_size is not numeric ${max_pool_record_size}"
	156	fi
	157
bfcbec6f RN	158	stripped="${TEST_BASE_DIR}/${pool}_stripped.txt"
	159
	160	zdb -Pbbb ${pool} \| \
bfcbec6f RN	161	sed -e '1,/^block[ ][ ]psize[ ][ ]lsize.*$/d' \
	162	-e '/^size[ ]*Count/d' -e '/^$/,$d' \
	163	> ${stripped}
	164
	165	sum_filesizes=$(echo "2^21"\|bc)
	166
	167	###################
	168	# generate 10% + 20% + 30% + 31% = 91% of the filespace
	169	# attempting to use 100% will lead to no space left on device
	170	# attempting to use 100% will lead to no space left on device
	171	# Heuristic testing showed that 91% was the practical upper
	172	# bound on the default 4G zpool (mirrored) that is used in
	173	# testing.
	174	#
	175	# In order to expedite testing, we will only fill 2G (of 4G)
	176	# of the test pool. You may want to modify this for
	177	# standalone testing.
	178	#
	179	# In filling only 50% of the pool, we create one object on
	180	# each "pass" below to achieve multiple objects per record
	181	# size. Creating one file per object would lead to
	182	# excessive file creation time.
	183	###################
	184	# for pass in 10 20 30 31 # 91%
	185	for pass in 20 20 10 # 50%
	186	do
	187	((thiscount=(((histo_check_pool_size*pass)/100)/sum_filesizes)))
	188
	189	for rb in $(seq ${min_rsbits} ${max_rsbits})
	190	do
	191	blksize=$(echo "2^$rb"\|bc)
	192	if [ $blksize -le $max_pool_record_size ]; then
	193	((recordcounts[$blksize]+=thiscount))
	194	fi
	195	done
	196	done
	197
	198	###################
	199	# compare the above computed counts for blocks against
	200	# lsize count. Since some devices have a minimum hardware
	201	# blocksize > 512, we cannot compare against the asize count.
	202	# E.G., if the HWBlocksize = 4096, then the asize counts for
	203	# 512, 1024 and 2048 will be zero and rolled up into the
	204	# 4096 blocksize count for asize. For verification we stick
	205	# to just lsize counts.
	206	#
2d784340 BB	207	# The max_variance is hard-coded here at 12% to leave us some
	208	# margin. Testing has shown this normally to be in the range
	209	# of 2%-8%, but it may be as large as 11%.
bfcbec6f	210	###################
2d784340	211	let max_variance=12
bfcbec6f RN	212	let fail_value=0
	213	let error_count=0
	214	log_note "Comparisons for ${pool}"
	215	log_note "Bsize is the blocksize, Count is predicted value"
	216	log_note "Bsize\tCount\tpsize\tlsize\tasize"
	217	while read -r blksize pc pl pm lc ll lm ac al am
	218	do
	219	if [ $blksize -gt $max_pool_record_size ]; then
	220	continue
	221	fi
	222	log_note \
	223	"$blksize\t${recordcounts[${blksize}]}\t$pc\t$lc\t$ac"
	224
	225	###################
	226	# get the computer record count and compute the
	227	# difference percentage in integer arithmetic
	228	###################
	229	rc=${recordcounts[${blksize}]}
	230	((rclc=(rc-lc)<0?lc-rc:rc-lc)) # absolute value
	231	((dp=(rclc*100)/rc))
	232
	233	###################
	234	# Check against the allowed variance
	235	###################
	236	if [ $dp -gt ${max_variance} ]; then
	237	log_note \
	238	"Expected variance < ${max_variance}% observed ${dp}%"
	239	if [ ${dp} -gt ${fail_value} ]; then
	240	fail_value=${dp}
	241	((error_count++))
	242	fi
	243	fi
	244	done < ${stripped}
6ef2151c AZ	245	rm "${stripped}"
6ef2151c AZ	246
bfcbec6f RN	247	if [ ${fail_value} -gt 0 ]; then
	248	if [ ${error_count} -eq 1 ]; then
	249	log_note "hctp: There was ${error_count} error"
	250	else
	251	log_note "hctp:There were a total of ${error_count} errors"
	252	fi
	253	log_fail \
	254	"hctp: Max variance of ${max_variance}% exceeded, saw ${fail_value}%"
	255	fi
	256	}
	257
	258	log_assert "Verify zdb -Pbbb (block histogram) works as expected"
	259	log_onexit cleanup
	260	verify_runnable "global"
	261	verify_disk_count "$DISKS" 2
	262
	263	default_mirror_setup_noexit $DISKS
	264
	265	histo_populate_test_pool $TESTPOOL
	266
	267	histo_check_test_pool $TESTPOOL
	268
	269	log_pass "Histogram for zdb"