[ceph.git] / ceph / qa / workunits / rgw / test_rgw_orphan_list.sh

#!/usr/bin/env bash

set -ex

# if defined, debug messages will be displayed and prepended with the string
# debug="DEBUG"

huge_size=2222 # in megabytes
big_size=6 # in megabytes

huge_obj=/tmp/huge_obj.temp.$$
big_obj=/tmp/big_obj.temp.$$
empty_obj=/tmp/empty_obj.temp.$$

fifo=/tmp/orphan-fifo.$$
awscli_dir=${HOME}/awscli_temp
export PATH=${PATH}:${awscli_dir}

rgw_host=$(hostname --fqdn)
if echo "$rgw_host" | grep -q '\.' ; then
    :
else
    host_domain=".front.sepia.ceph.com"
    echo "WARNING: rgw hostname -- $rgw_host -- does not appear to be fully qualified; PUNTING and appending $host_domain"
    rgw_host="${rgw_host}${host_domain}"
fi
rgw_port=80

echo "Fully Qualified Domain Name: $rgw_host"

success() {
    echo OK.
    exit 0
}

########################################################################
# INSTALL AND CONFIGURE TOOLING

install_awscli() {
    # NB: this does verify authenticity and integrity of downloaded
    # file; see
    # https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html
    here="$(pwd)"
    cd "$HOME"
    curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
    unzip awscliv2.zip
    mkdir -p $awscli_dir
    ./aws/install -i $awscli_dir
    cd "$here"
}

uninstall_awscli() {
    here="$(pwd)"
    cd "$HOME"
    rm -rf $awscli_dir ./aws awscliv2.zip
    cd "$here"
}

sudo yum -y install s3cmd
sudo yum -y install python3-setuptools
sudo yum -y install python3-pip
sudo pip3 install --upgrade setuptools
sudo pip3 install python-swiftclient

# get ready for transition from s3cmd to awscli
if false ;then
    install_awscli
    aws --version
    uninstall_awscli
fi

s3config=/tmp/s3config.$$

# do not include the port when it is 80; the host base is used in the
# v4 signature and it needs to follow this convention for signatures
# to match
if [ "$rgw_port" -ne 80 ] ;then
    s3_host_base="${rgw_host}:${rgw_port}"
else
    s3_host_base="$rgw_host"
fi

cat >${s3config} <<EOF
[default]
host_base = $s3_host_base
access_key = 0555b35654ad1656d804
secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==
bucket_location = us-east-1
check_ssl_certificate = True
check_ssl_hostname = True
default_mime_type = binary/octet-stream
delete_removed = False
dry_run = False
enable_multipart = True
encoding = UTF-8
encrypt = False
follow_symlinks = False
force = False
guess_mime_type = True
host_bucket = anything.with.three.dots
multipart_chunk_size_mb = 15
multipart_max_chunks = 10000
recursive = False
recv_chunk = 65536
send_chunk = 65536
signature_v2 = False
socket_timeout = 300
use_https = False
use_mime_magic = True
verbosity = WARNING
EOF


# set up swift authentication
export ST_AUTH=http://${rgw_host}:${rgw_port}/auth/v1.0
export ST_USER=test:tester
export ST_KEY=testing

create_users() {
    # Create S3 user
    local akey='0555b35654ad1656d804'
    local skey='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=='
    radosgw-admin user create --uid testid \
		  --access-key $akey --secret $skey \
		  --display-name 'M. Tester' --email tester@ceph.com

    # Create Swift user
    radosgw-admin user create --subuser=test:tester \
		  --display-name=Tester-Subuser --key-type=swift \
		  --secret=testing --access=full
}

myswift() {
    if [ -n "$debug" ] ;then
	echo "${debug}: swift --verbose --debug $@"
    fi
    swift --verbose --debug "$@"
    local code=$?
    if [ $code -ne 0 ] ;then
	echo "ERROR: code = $code ; command = s3cmd --config=${s3config} --verbose --debug "$@""
	exit $code
    fi
}

mys3cmd() {
    if [ -n "$debug" ] ;then
	echo "${debug}: s3cmd --config=${s3config} --verbose --debug $@"
    fi
    s3cmd --config=${s3config} --verbose --debug "$@"
    local code=$?
    if [ $code -ne 0 ] ;then
	echo "ERROR: code = $code ; command = s3cmd --config=${s3config} --verbose --debug "$@""
	exit $code
    fi
}

mys3uploadkill() {
    if [ $# -ne 5 ] ;then
	echo "$0: error expecting 5 arguments"
	exit 1
    fi

    set -v
    local_file="$1"
    remote_bkt="$2"
    remote_obj="$3"
    fifo="$4"
    stop_part="$5"
    
    mkfifo $fifo

    s3cmd --config=${s3config} put $local_file \
	  s3://${remote_bkt}/${remote_obj} \
	  --progress \
	  --multipart-chunk-size-mb=5 >$fifo &
    set +e # don't allow errors to stop script
    while read line ;do
        echo "$line" | grep --quiet "part $stop_part "
        if [ ${PIPESTATUS[1]} -eq 0 ] ;then
	    kill -9 $(jobs -p)
            break
        fi
    done <$fifo
    set -e

    rm -f $fifo
}

mys3upload() {
    obj=$1
    bucket=$2
    dest_obj=$3

    mys3cmd put -q $obj s3://${bucket}/$dest_obj
}

########################################################################
# PREP

create_users
dd if=/dev/urandom of=$big_obj bs=1M count=${big_size}
dd if=/dev/urandom of=$huge_obj bs=1M count=${huge_size}
touch $empty_obj

quick_tests() {
    echo TRY A SWIFT COMMAND
    myswift upload swift-plain-ctr $big_obj --object-name swift-obj-test
    myswift list
    myswift list swift-plain-ctr

    echo TRY A RADOSGW-ADMIN COMMAND
    radosgw-admin bucket list # make sure rgw is up and running
}

########################################################################
# S3 TESTS

####################################
# regular multipart test

mys3cmd mb s3://multipart-bkt
mys3upload $huge_obj multipart-bkt multipart-obj
mys3cmd ls
mys3cmd ls s3://multipart-bkt

####################################
# multipart test with incomplete uploads

bkt="incomplete-mp-bkt-1"

mys3cmd mb s3://$bkt
mys3uploadkill $huge_obj $bkt incomplete-mp-obj-1 $fifo 20
mys3uploadkill $huge_obj $bkt incomplete-mp-obj-2 $fifo 100

####################################
# resharded bucket

bkt=resharded-bkt-1

mys3cmd mb s3://$bkt

for f in $(seq 8) ; do
    dest_obj="reshard-obj-${f}"
    mys3cmd put -q $big_obj s3://${bkt}/$dest_obj
done

radosgw-admin bucket reshard --num-shards 3 --bucket=$bkt --yes-i-really-mean-it
radosgw-admin bucket reshard --num-shards 5 --bucket=$bkt --yes-i-really-mean-it

####################################
# versioned bucket

if true ;then
    echo "WARNING: versioned bucket test currently turned off"
else
    bkt=versioned-bkt-1

    mys3cmd mb s3://$bkt

    # bucket-enable-versioning $bkt

    for f in $(seq 3) ;do
	for g in $(seq 10) ;do
	    dest_obj="versioned-obj-${g}"
	    mys3cmd put -q $big_obj s3://${bkt}/$dest_obj
	done
    done

    for g in $(seq 1 2 10) ;do
	dest_obj="versioned-obj-${g}"
	mys3cmd rm s3://${bkt}/$dest_obj
    done
fi

############################################################
# copy small objects

o_bkt="orig-bkt-1"
d_bkt="copy-bkt-1"
mys3cmd mb s3://$o_bkt

for f in $(seq 4) ;do
    dest_obj="orig-obj-$f"
    mys3cmd put -q $big_obj s3://${o_bkt}/$dest_obj
done

mys3cmd mb s3://$d_bkt

mys3cmd cp s3://${o_bkt}/orig-obj-1 s3://${d_bkt}/copied-obj-1
mys3cmd cp s3://${o_bkt}/orig-obj-3 s3://${d_bkt}/copied-obj-3

for f in $(seq 5 6) ;do
    dest_obj="orig-obj-$f"
    mys3cmd put -q $big_obj s3://${d_bkt}/$dest_obj
done

############################################################
# copy small objects and delete original

o_bkt="orig-bkt-2"
d_bkt="copy-bkt-2"

mys3cmd mb s3://$o_bkt

for f in $(seq 4) ;do
    dest_obj="orig-obj-$f"
    mys3cmd put -q $big_obj s3://${o_bkt}/$dest_obj
done

mys3cmd mb s3://$d_bkt

mys3cmd cp s3://${o_bkt}/orig-obj-1 s3://${d_bkt}/copied-obj-1
mys3cmd cp s3://${o_bkt}/orig-obj-3 s3://${d_bkt}/copied-obj-3

for f in $(seq 5 6) ;do
    dest_obj="orig-obj-$f"
    mys3cmd put -q $big_obj s3://${d_bkt}/$dest_obj
done

mys3cmd rb --recursive s3://${o_bkt}

############################################################
# copy multipart objects

o_bkt="orig-mp-bkt-3"
d_bkt="copy-mp-bkt-3"

mys3cmd mb s3://$o_bkt

for f in $(seq 2) ;do
    dest_obj="orig-multipart-obj-$f"
    mys3cmd put -q $huge_obj s3://${o_bkt}/$dest_obj
done

mys3cmd mb s3://$d_bkt

mys3cmd cp s3://${o_bkt}/orig-multipart-obj-1 \
	s3://${d_bkt}/copied-multipart-obj-1

for f in $(seq 5 5) ;do
    dest_obj="orig-multipart-obj-$f"
    mys3cmd put -q $huge_obj s3://${d_bkt}/$dest_obj
done


############################################################
# copy multipart objects and delete original

o_bkt="orig-mp-bkt-4"
d_bkt="copy-mp-bkt-4"

mys3cmd mb s3://$o_bkt

for f in $(seq 2) ;do
    dest_obj="orig-multipart-obj-$f"
    mys3cmd put -q $huge_obj s3://${o_bkt}/$dest_obj
done

mys3cmd mb s3://$d_bkt

mys3cmd cp s3://${o_bkt}/orig-multipart-obj-1 \
	s3://${d_bkt}/copied-multipart-obj-1

for f in $(seq 5 5) ;do
    dest_obj="orig-multipart-obj-$f"
    mys3cmd put -q $huge_obj s3://${d_bkt}/$dest_obj
done

mys3cmd rb --recursive s3://$o_bkt

########################################################################
# SWIFT TESTS

# 600MB
segment_size=629145600

############################################################
# plain test

for f in $(seq 4) ;do
    myswift upload swift-plain-ctr $big_obj --object-name swift-obj-$f
done

############################################################
# zero-len test

myswift upload swift-zerolen-ctr $empty_obj --object-name subdir/
myswift upload swift-zerolen-ctr $big_obj --object-name subdir/abc1
myswift upload swift-zerolen-ctr $empty_obj --object-name subdir/empty1
myswift upload swift-zerolen-ctr $big_obj --object-name subdir/xyz1

############################################################
# dlo test

# upload in 300MB segments
myswift upload swift-dlo-ctr $huge_obj --object-name dlo-obj-1 \
      -S $segment_size

############################################################
# slo test

# upload in 300MB segments
myswift upload swift-slo-ctr $huge_obj --object-name slo-obj-1 \
      -S $segment_size --use-slo

############################################################
# large object copy test

# upload in 300MB segments
o_ctr=swift-orig-ctr
o_obj=slo-orig-obj-1
d_ctr=swift-copy-ctr
d_obj=slo-copy-obj-1
myswift upload $o_ctr $big_obj --object-name $o_obj

myswift copy --destination /${d_ctr}/${d_obj} \
      $o_ctr $o_obj

myswift delete $o_ctr $o_obj

############################################################
# huge dlo object copy test

o_ctr=swift-orig-dlo-ctr-1
o_obj=dlo-orig-dlo-obj-1
d_ctr=swift-copy-dlo-ctr-1
d_obj=dlo-copy-dlo-obj-1

myswift upload $o_ctr $huge_obj --object-name $o_obj \
      -S $segment_size

myswift copy --destination /${d_ctr}/${d_obj} \
      $o_ctr $o_obj

############################################################
# huge dlo object copy and orig delete

o_ctr=swift-orig-dlo-ctr-2
o_obj=dlo-orig-dlo-obj-2
d_ctr=swift-copy-dlo-ctr-2
d_obj=dlo-copy-dlo-obj-2

myswift upload $o_ctr $huge_obj --object-name $o_obj \
      -S $segment_size

myswift copy --destination /${d_ctr}/${d_obj} \
      $o_ctr $o_obj

myswift delete $o_ctr $o_obj

############################################################
# huge slo object copy test

o_ctr=swift-orig-slo-ctr-1
o_obj=slo-orig-slo-obj-1
d_ctr=swift-copy-slo-ctr-1
d_obj=slo-copy-slo-obj-1
myswift upload $o_ctr $huge_obj --object-name $o_obj \
      -S $segment_size --use-slo

myswift copy --destination /${d_ctr}/${d_obj} $o_ctr $o_obj

############################################################
# huge slo object copy test and orig delete

o_ctr=swift-orig-slo-ctr-2
o_obj=slo-orig-slo-obj-2
d_ctr=swift-copy-slo-ctr-2
d_obj=slo-copy-slo-obj-2
myswift upload $o_ctr $huge_obj --object-name $o_obj \
      -S $segment_size --use-slo

myswift copy --destination /${d_ctr}/${d_obj} $o_ctr $o_obj

myswift delete $o_ctr $o_obj

########################################################################
# FORCE GARBAGE COLLECTION

sleep 6 # since for testing age at which gc can happen is 5 secs
radosgw-admin gc process --include-all


########################################
# DO ORPHAN LIST

pool="default.rgw.buckets.data"

rgw-orphan-list $pool

# we only expect there to be one output file, but loop just in case
ol_error=""
for f in orphan-list-*.out ; do
    if [ -s "$f"  ] ;then # if file non-empty
	ol_error="${ol_error}:$f"
	echo "One ore more orphans found in $f:"
	cat "$f"
    fi
done

if [ -n "$ol_error" ] ;then
    echo "ERROR: orphans found when none expected"
    exit 1
fi

########################################################################
# CLEAN UP

rm -f $empty_obj $big_obj $huge_obj $s3config

success
Commit	Line	Data
e306af50 TL	1	#!/usr/bin/env bash
	2
	3	set -ex
	4
	5	# if defined, debug messages will be displayed and prepended with the string
	6	# debug="DEBUG"
	7
	8	huge_size=2222 # in megabytes
	9	big_size=6 # in megabytes
	10
	11	huge_obj=/tmp/huge_obj.temp.$$
	12	big_obj=/tmp/big_obj.temp.$$
	13	empty_obj=/tmp/empty_obj.temp.$$
	14
	15	fifo=/tmp/orphan-fifo.$$
	16	awscli_dir=${HOME}/awscli_temp
	17	export PATH=${PATH}:${awscli_dir}
	18
	19	rgw_host=$(hostname --fqdn)
f6b5b4d7 TL	20	if echo "$rgw_host" \| grep -q '\.' ; then
	21	:
	22	else
	23	host_domain=".front.sepia.ceph.com"
	24	echo "WARNING: rgw hostname -- $rgw_host -- does not appear to be fully qualified; PUNTING and appending $host_domain"
	25	rgw_host="${rgw_host}${host_domain}"
	26	fi
e306af50 TL	27	rgw_port=80
	28
	29	echo "Fully Qualified Domain Name: $rgw_host"
	30
	31	success() {
	32	echo OK.
	33	exit 0
	34	}
	35
	36	########################################################################
	37	# INSTALL AND CONFIGURE TOOLING
	38
	39	install_awscli() {
	40	# NB: this does verify authenticity and integrity of downloaded
	41	# file; see
	42	# https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html
	43	here="$(pwd)"
	44	cd "$HOME"
	45	curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
	46	unzip awscliv2.zip
	47	mkdir -p $awscli_dir
	48	./aws/install -i $awscli_dir
	49	cd "$here"
	50	}
	51
	52	uninstall_awscli() {
	53	here="$(pwd)"
	54	cd "$HOME"
	55	rm -rf $awscli_dir ./aws awscliv2.zip
	56	cd "$here"
	57	}
	58
f91f0fd5 TL	59	sudo yum -y install s3cmd
f91f0fd5 TL	60	sudo yum -y install python3-setuptools
e306af50 TL	61	sudo yum -y install python3-pip
	62	sudo pip3 install --upgrade setuptools
	63	sudo pip3 install python-swiftclient
	64
	65	# get ready for transition from s3cmd to awscli
	66	if false ;then
	67	install_awscli
	68	aws --version
	69	uninstall_awscli
	70	fi
	71
	72	s3config=/tmp/s3config.$$
	73
	74	# do not include the port when it is 80; the host base is used in the
	75	# v4 signature and it needs to follow this convention for signatures
	76	# to match
	77	if [ "$rgw_port" -ne 80 ] ;then
	78	s3_host_base="${rgw_host}:${rgw_port}"
	79	else
	80	s3_host_base="$rgw_host"
	81	fi
	82
	83	cat >${s3config} <<EOF
	84	[default]
	85	host_base = $s3_host_base
	86	access_key = 0555b35654ad1656d804
	87	secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==
	88	bucket_location = us-east-1
	89	check_ssl_certificate = True
	90	check_ssl_hostname = True
	91	default_mime_type = binary/octet-stream
	92	delete_removed = False
	93	dry_run = False
	94	enable_multipart = True
	95	encoding = UTF-8
	96	encrypt = False
	97	follow_symlinks = False
	98	force = False
	99	guess_mime_type = True
	100	host_bucket = anything.with.three.dots
	101	multipart_chunk_size_mb = 15
	102	multipart_max_chunks = 10000
	103	recursive = False
	104	recv_chunk = 65536
	105	send_chunk = 65536
	106	signature_v2 = False
	107	socket_timeout = 300
	108	use_https = False
	109	use_mime_magic = True
	110	verbosity = WARNING
	111	EOF
	112
	113
	114	# set up swift authentication
	115	export ST_AUTH=http://${rgw_host}:${rgw_port}/auth/v1.0
	116	export ST_USER=test:tester
	117	export ST_KEY=testing
	118
	119	create_users() {
	120	# Create S3 user
	121	local akey='0555b35654ad1656d804'
	122	local skey='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=='
	123	radosgw-admin user create --uid testid \
	124	--access-key $akey --secret $skey \
125	--display-name 'M. Tester' --email tester@ceph.com
126
127	# Create Swift user
128	radosgw-admin user create --subuser=test:tester \
129	--display-name=Tester-Subuser --key-type=swift \
130	--secret=testing --access=full
131	}
132
133	myswift() {
134	if [ -n "$debug" ] ;then
135	echo "${debug}: swift --verbose --debug $@"
136	fi
137	swift --verbose --debug "$@"
138	local code=$?
139	if [ $code -ne 0 ] ;then
140	echo "ERROR: code = $code ; command = s3cmd --config=${s3config} --verbose --debug "$@""
141	exit $code
142	fi
143	}
144
145	mys3cmd() {
146	if [ -n "$debug" ] ;then
147	echo "${debug}: s3cmd --config=${s3config} --verbose --debug $@"
148	fi
149	s3cmd --config=${s3config} --verbose --debug "$@"
150	local code=$?
151	if [ $code -ne 0 ] ;then
152	echo "ERROR: code = $code ; command = s3cmd --config=${s3config} --verbose --debug "$@""
153	exit $code
154	fi
155	}
156
157	mys3uploadkill() {
158	if [ $# -ne 5 ] ;then
159	echo "$0: error expecting 5 arguments"
160	exit 1
161	fi
162
163	set -v
164	local_file="$1"
165	remote_bkt="$2"
166	remote_obj="$3"
167	fifo="$4"
168	stop_part="$5"
169
170	mkfifo $fifo
171
172	s3cmd --config=${s3config} put $local_file \
173	s3://${remote_bkt}/${remote_obj} \
174	--progress \
175	--multipart-chunk-size-mb=5 >$fifo &
176	set +e # don't allow errors to stop script
177	while read line ;do
178	echo "$line" \| grep --quiet "part $stop_part "
179	if [ ${PIPESTATUS[1]} -eq 0 ] ;then
180	kill -9 $(jobs -p)
181	break
182	fi
183	done <$fifo
184	set -e
185
186	rm -f $fifo
187	}
188
189	mys3upload() {
190	obj=$1
191	bucket=$2
192	dest_obj=$3
193
194	mys3cmd put -q $obj s3://${bucket}/$dest_obj
195	}
196
197	########################################################################
198	# PREP
199
200	create_users
201	dd if=/dev/urandom of=$big_obj bs=1M count=${big_size}
202	dd if=/dev/urandom of=$huge_obj bs=1M count=${huge_size}
203	touch $empty_obj
204
205	quick_tests() {
206	echo TRY A SWIFT COMMAND
207	myswift upload swift-plain-ctr $big_obj --object-name swift-obj-test
208	myswift list
209	myswift list swift-plain-ctr
210
211	echo TRY A RADOSGW-ADMIN COMMAND
212	radosgw-admin bucket list # make sure rgw is up and running
213	}
214
215	########################################################################
216	# S3 TESTS
217
218	####################################
219	# regular multipart test
220
221	mys3cmd mb s3://multipart-bkt
222	mys3upload $huge_obj multipart-bkt multipart-obj
223	mys3cmd ls
224	mys3cmd ls s3://multipart-bkt
225
226	####################################
227	# multipart test with incomplete uploads
228
229	bkt="incomplete-mp-bkt-1"
230
231	mys3cmd mb s3://$bkt
232	mys3uploadkill $huge_obj $bkt incomplete-mp-obj-1 $fifo 20
233	mys3uploadkill $huge_obj $bkt incomplete-mp-obj-2 $fifo 100
234
235	####################################
236	# resharded bucket
237
238	bkt=resharded-bkt-1
239
240	mys3cmd mb s3://$bkt
241
242	for f in $(seq 8) ; do
243	dest_obj="reshard-obj-${f}"
244	mys3cmd put -q $big_obj s3://${bkt}/$dest_obj
245	done
246
247	radosgw-admin bucket reshard --num-shards 3 --bucket=$bkt --yes-i-really-mean-it
248	radosgw-admin bucket reshard --num-shards 5 --bucket=$bkt --yes-i-really-mean-it
249
250	####################################
251	# versioned bucket
252
253	if true ;then
254	echo "WARNING: versioned bucket test currently turned off"
255	else
256	bkt=versioned-bkt-1
257
258	mys3cmd mb s3://$bkt
259
260	# bucket-enable-versioning $bkt
261
262	for f in $(seq 3) ;do
263	for g in $(seq 10) ;do
264	dest_obj="versioned-obj-${g}"
265	mys3cmd put -q $big_obj s3://${bkt}/$dest_obj
266	done
267	done
268
269	for g in $(seq 1 2 10) ;do
270	dest_obj="versioned-obj-${g}"
271	mys3cmd rm s3://${bkt}/$dest_obj
272	done
273	fi
274
275	############################################################
276	# copy small objects
277
278	o_bkt="orig-bkt-1"
279	d_bkt="copy-bkt-1"
280	mys3cmd mb s3://$o_bkt
281
282	for f in $(seq 4) ;do
283	dest_obj="orig-obj-$f"
284	mys3cmd put -q $big_obj s3://${o_bkt}/$dest_obj
285	done
286
287	mys3cmd mb s3://$d_bkt
288
289	mys3cmd cp s3://${o_bkt}/orig-obj-1 s3://${d_bkt}/copied-obj-1
290	mys3cmd cp s3://${o_bkt}/orig-obj-3 s3://${d_bkt}/copied-obj-3
291
292	for f in $(seq 5 6) ;do
293	dest_obj="orig-obj-$f"
294	mys3cmd put -q $big_obj s3://${d_bkt}/$dest_obj
295	done
296
297	############################################################
298	# copy small objects and delete original
299
300	o_bkt="orig-bkt-2"
301	d_bkt="copy-bkt-2"
302
303	mys3cmd mb s3://$o_bkt
304
305	for f in $(seq 4) ;do
306	dest_obj="orig-obj-$f"
307	mys3cmd put -q $big_obj s3://${o_bkt}/$dest_obj
308	done
309
310	mys3cmd mb s3://$d_bkt
311
312	mys3cmd cp s3://${o_bkt}/orig-obj-1 s3://${d_bkt}/copied-obj-1
313	mys3cmd cp s3://${o_bkt}/orig-obj-3 s3://${d_bkt}/copied-obj-3
314
315	for f in $(seq 5 6) ;do
316	dest_obj="orig-obj-$f"
317	mys3cmd put -q $big_obj s3://${d_bkt}/$dest_obj
318	done
319
320	mys3cmd rb --recursive s3://${o_bkt}
321
322	############################################################
323	# copy multipart objects
324
325	o_bkt="orig-mp-bkt-3"
326	d_bkt="copy-mp-bkt-3"
327
328	mys3cmd mb s3://$o_bkt
329
330	for f in $(seq 2) ;do
331	dest_obj="orig-multipart-obj-$f"
332	mys3cmd put -q $huge_obj s3://${o_bkt}/$dest_obj
333	done
334
335	mys3cmd mb s3://$d_bkt
336
337	mys3cmd cp s3://${o_bkt}/orig-multipart-obj-1 \
338	s3://${d_bkt}/copied-multipart-obj-1
339
340	for f in $(seq 5 5) ;do
341	dest_obj="orig-multipart-obj-$f"
342	mys3cmd put -q $huge_obj s3://${d_bkt}/$dest_obj
343	done
344
345
346	############################################################
347	# copy multipart objects and delete original
348
349	o_bkt="orig-mp-bkt-4"
350	d_bkt="copy-mp-bkt-4"
351
352	mys3cmd mb s3://$o_bkt
353
354	for f in $(seq 2) ;do
355	dest_obj="orig-multipart-obj-$f"
356	mys3cmd put -q $huge_obj s3://${o_bkt}/$dest_obj
357	done
358
359	mys3cmd mb s3://$d_bkt
360
361	mys3cmd cp s3://${o_bkt}/orig-multipart-obj-1 \
362	s3://${d_bkt}/copied-multipart-obj-1
363
364	for f in $(seq 5 5) ;do
365	dest_obj="orig-multipart-obj-$f"
366	mys3cmd put -q $huge_obj s3://${d_bkt}/$dest_obj
367	done
368
369	mys3cmd rb --recursive s3://$o_bkt
370
371	########################################################################
372	# SWIFT TESTS
373
374	# 600MB
375	segment_size=629145600
376
377	############################################################
378	# plain test
379
380	for f in $(seq 4) ;do
381	myswift upload swift-plain-ctr $big_obj --object-name swift-obj-$f
382	done
383
384	############################################################
385	# zero-len test
386
387	myswift upload swift-zerolen-ctr $empty_obj --object-name subdir/
388	myswift upload swift-zerolen-ctr $big_obj --object-name subdir/abc1
389	myswift upload swift-zerolen-ctr $empty_obj --object-name subdir/empty1
390	myswift upload swift-zerolen-ctr $big_obj --object-name subdir/xyz1
391
392	############################################################
393	# dlo test
394
395	# upload in 300MB segments
396	myswift upload swift-dlo-ctr $huge_obj --object-name dlo-obj-1 \
397	-S $segment_size
398
399	############################################################
400	# slo test
401
402	# upload in 300MB segments
403	myswift upload swift-slo-ctr $huge_obj --object-name slo-obj-1 \
404	-S $segment_size --use-slo
405
406	############################################################
407	# large object copy test
408
409	# upload in 300MB segments
410	o_ctr=swift-orig-ctr
411	o_obj=slo-orig-obj-1
412	d_ctr=swift-copy-ctr
413	d_obj=slo-copy-obj-1
414	myswift upload $o_ctr $big_obj --object-name $o_obj
415
416	myswift copy --destination /${d_ctr}/${d_obj} \
417	$o_ctr $o_obj
418
419	myswift delete $o_ctr $o_obj
420
421	############################################################
422	# huge dlo object copy test
423
424	o_ctr=swift-orig-dlo-ctr-1
425	o_obj=dlo-orig-dlo-obj-1
426	d_ctr=swift-copy-dlo-ctr-1
427	d_obj=dlo-copy-dlo-obj-1
428
429	myswift upload $o_ctr $huge_obj --object-name $o_obj \
430	-S $segment_size
431
432	myswift copy --destination /${d_ctr}/${d_obj} \
433	$o_ctr $o_obj
434
435	############################################################
436	# huge dlo object copy and orig delete
437
438	o_ctr=swift-orig-dlo-ctr-2
439	o_obj=dlo-orig-dlo-obj-2
440	d_ctr=swift-copy-dlo-ctr-2
441	d_obj=dlo-copy-dlo-obj-2
442
443	myswift upload $o_ctr $huge_obj --object-name $o_obj \
444	-S $segment_size
445
446	myswift copy --destination /${d_ctr}/${d_obj} \
447	$o_ctr $o_obj
448
449	myswift delete $o_ctr $o_obj
450
451	############################################################
452	# huge slo object copy test
453
454	o_ctr=swift-orig-slo-ctr-1
455	o_obj=slo-orig-slo-obj-1
456	d_ctr=swift-copy-slo-ctr-1
457	d_obj=slo-copy-slo-obj-1
458	myswift upload $o_ctr $huge_obj --object-name $o_obj \
459	-S $segment_size --use-slo
460
461	myswift copy --destination /${d_ctr}/${d_obj} $o_ctr $o_obj
462
463	############################################################
464	# huge slo object copy test and orig delete
465
466	o_ctr=swift-orig-slo-ctr-2
467	o_obj=slo-orig-slo-obj-2
468	d_ctr=swift-copy-slo-ctr-2
469	d_obj=slo-copy-slo-obj-2
470	myswift upload $o_ctr $huge_obj --object-name $o_obj \
471	-S $segment_size --use-slo
472
473	myswift copy --destination /${d_ctr}/${d_obj} $o_ctr $o_obj
474
475	myswift delete $o_ctr $o_obj
476
477	########################################################################
478	# FORCE GARBAGE COLLECTION
479
480	sleep 6 # since for testing age at which gc can happen is 5 secs
481	radosgw-admin gc process --include-all
482
483
484	########################################
485	# DO ORPHAN LIST
486
487	pool="default.rgw.buckets.data"
488
489	rgw-orphan-list $pool
490
491	# we only expect there to be one output file, but loop just in case
492	ol_error=""
493	for f in orphan-list-*.out ; do
494	if [ -s "$f" ] ;then # if file non-empty
495	ol_error="${ol_error}:$f"
496	echo "One ore more orphans found in $f:"
497	cat "$f"
498	fi
499	done
500
501	if [ -n "$ol_error" ] ;then
502	echo "ERROR: orphans found when none expected"
503	exit 1
504	fi
505
506	########################################################################
507	# CLEAN UP
508
509	rm -f $empty_obj $big_obj $huge_obj $s3config
510
511	success