[ceph.git] / ceph / qa / workunits / rgw / test_rgw_orphan_list.sh

#!/usr/bin/env bash

set -ex

# if defined, debug messages will be displayed and prepended with the string
# debug="DEBUG"

huge_size=2222 # in megabytes
big_size=6 # in megabytes

huge_obj=/tmp/huge_obj.temp.$$
big_obj=/tmp/big_obj.temp.$$
empty_obj=/tmp/empty_obj.temp.$$

fifo=/tmp/orphan-fifo.$$
awscli_dir=${HOME}/awscli_temp
export PATH=${PATH}:${awscli_dir}

rgw_host=$(hostname --fqdn)
if echo "$rgw_host" | grep -q '\.' ; then
    :
else
    host_domain=".front.sepia.ceph.com"
    echo "WARNING: rgw hostname -- $rgw_host -- does not appear to be fully qualified; PUNTING and appending $host_domain"
    rgw_host="${rgw_host}${host_domain}"
fi
rgw_port=80

echo "Fully Qualified Domain Name: $rgw_host"

success() {
    echo OK.
    exit 0
}

########################################################################
# INSTALL AND CONFIGURE TOOLING

install_awscli() {
    # NB: this does verify authenticity and integrity of downloaded
    # file; see
    # https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html
    here="$(pwd)"
    cd "$HOME"
    curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
    unzip awscliv2.zip
    mkdir -p $awscli_dir
    ./aws/install -i $awscli_dir
    cd "$here"
}

uninstall_awscli() {
    here="$(pwd)"
    cd "$HOME"
    rm -rf $awscli_dir ./aws awscliv2.zip
    cd "$here"
}

sudo dnf install -y s3cmd

sudo yum install python3-setuptools
sudo yum -y install python3-pip
sudo pip3 install --upgrade setuptools
sudo pip3 install python-swiftclient

# get ready for transition from s3cmd to awscli
if false ;then
    install_awscli
    aws --version
    uninstall_awscli
fi

s3config=/tmp/s3config.$$

# do not include the port when it is 80; the host base is used in the
# v4 signature and it needs to follow this convention for signatures
# to match
if [ "$rgw_port" -ne 80 ] ;then
    s3_host_base="${rgw_host}:${rgw_port}"
else
    s3_host_base="$rgw_host"
fi

cat >${s3config} <<EOF
[default]
host_base = $s3_host_base
access_key = 0555b35654ad1656d804
secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==
bucket_location = us-east-1
check_ssl_certificate = True
check_ssl_hostname = True
default_mime_type = binary/octet-stream
delete_removed = False
dry_run = False
enable_multipart = True
encoding = UTF-8
encrypt = False
follow_symlinks = False
force = False
guess_mime_type = True
host_bucket = anything.with.three.dots
multipart_chunk_size_mb = 15
multipart_max_chunks = 10000
recursive = False
recv_chunk = 65536
send_chunk = 65536
signature_v2 = False
socket_timeout = 300
use_https = False
use_mime_magic = True
verbosity = WARNING
EOF


# set up swift authentication
export ST_AUTH=http://${rgw_host}:${rgw_port}/auth/v1.0
export ST_USER=test:tester
export ST_KEY=testing

create_users() {
    # Create S3 user
    local akey='0555b35654ad1656d804'
    local skey='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=='
    radosgw-admin user create --uid testid \
		  --access-key $akey --secret $skey \
		  --display-name 'M. Tester' --email tester@ceph.com

    # Create Swift user
    radosgw-admin user create --subuser=test:tester \
		  --display-name=Tester-Subuser --key-type=swift \
		  --secret=testing --access=full
}

myswift() {
    if [ -n "$debug" ] ;then
	echo "${debug}: swift --verbose --debug $@"
    fi
    swift --verbose --debug "$@"
    local code=$?
    if [ $code -ne 0 ] ;then
	echo "ERROR: code = $code ; command = s3cmd --config=${s3config} --verbose --debug "$@""
	exit $code
    fi
}

mys3cmd() {
    if [ -n "$debug" ] ;then
	echo "${debug}: s3cmd --config=${s3config} --verbose --debug $@"
    fi
    s3cmd --config=${s3config} --verbose --debug "$@"
    local code=$?
    if [ $code -ne 0 ] ;then
	echo "ERROR: code = $code ; command = s3cmd --config=${s3config} --verbose --debug "$@""
	exit $code
    fi
}

mys3uploadkill() {
    if [ $# -ne 5 ] ;then
	echo "$0: error expecting 5 arguments"
	exit 1
    fi

    set -v
    local_file="$1"
    remote_bkt="$2"
    remote_obj="$3"
    fifo="$4"
    stop_part="$5"
    
    mkfifo $fifo

    s3cmd --config=${s3config} put $local_file \
	  s3://${remote_bkt}/${remote_obj} \
	  --progress \
	  --multipart-chunk-size-mb=5 >$fifo &
    set +e # don't allow errors to stop script
    while read line ;do
        echo "$line" | grep --quiet "part $stop_part "
        if [ ${PIPESTATUS[1]} -eq 0 ] ;then
	    kill -9 $(jobs -p)
            break
        fi
    done <$fifo
    set -e

    rm -f $fifo
}

mys3upload() {
    obj=$1
    bucket=$2
    dest_obj=$3

    mys3cmd put -q $obj s3://${bucket}/$dest_obj
}

########################################################################
# PREP

create_users
dd if=/dev/urandom of=$big_obj bs=1M count=${big_size}
dd if=/dev/urandom of=$huge_obj bs=1M count=${huge_size}
touch $empty_obj

quick_tests() {
    echo TRY A SWIFT COMMAND
    myswift upload swift-plain-ctr $big_obj --object-name swift-obj-test
    myswift list
    myswift list swift-plain-ctr

    echo TRY A RADOSGW-ADMIN COMMAND
    radosgw-admin bucket list # make sure rgw is up and running
}

########################################################################
# S3 TESTS

####################################
# regular multipart test

mys3cmd mb s3://multipart-bkt
mys3upload $huge_obj multipart-bkt multipart-obj
mys3cmd ls
mys3cmd ls s3://multipart-bkt

####################################
# multipart test with incomplete uploads

bkt="incomplete-mp-bkt-1"

mys3cmd mb s3://$bkt
mys3uploadkill $huge_obj $bkt incomplete-mp-obj-1 $fifo 20
mys3uploadkill $huge_obj $bkt incomplete-mp-obj-2 $fifo 100

####################################
# resharded bucket

bkt=resharded-bkt-1

mys3cmd mb s3://$bkt

for f in $(seq 8) ; do
    dest_obj="reshard-obj-${f}"
    mys3cmd put -q $big_obj s3://${bkt}/$dest_obj
done

radosgw-admin bucket reshard --num-shards 3 --bucket=$bkt --yes-i-really-mean-it
radosgw-admin bucket reshard --num-shards 5 --bucket=$bkt --yes-i-really-mean-it

####################################
# versioned bucket

if true ;then
    echo "WARNING: versioned bucket test currently turned off"
else
    bkt=versioned-bkt-1

    mys3cmd mb s3://$bkt

    # bucket-enable-versioning $bkt

    for f in $(seq 3) ;do
	for g in $(seq 10) ;do
	    dest_obj="versioned-obj-${g}"
	    mys3cmd put -q $big_obj s3://${bkt}/$dest_obj
	done
    done

    for g in $(seq 1 2 10) ;do
	dest_obj="versioned-obj-${g}"
	mys3cmd rm s3://${bkt}/$dest_obj
    done
fi

############################################################
# copy small objects

o_bkt="orig-bkt-1"
d_bkt="copy-bkt-1"
mys3cmd mb s3://$o_bkt

for f in $(seq 4) ;do
    dest_obj="orig-obj-$f"
    mys3cmd put -q $big_obj s3://${o_bkt}/$dest_obj
done

mys3cmd mb s3://$d_bkt

mys3cmd cp s3://${o_bkt}/orig-obj-1 s3://${d_bkt}/copied-obj-1
mys3cmd cp s3://${o_bkt}/orig-obj-3 s3://${d_bkt}/copied-obj-3

for f in $(seq 5 6) ;do
    dest_obj="orig-obj-$f"
    mys3cmd put -q $big_obj s3://${d_bkt}/$dest_obj
done

############################################################
# copy small objects and delete original

o_bkt="orig-bkt-2"
d_bkt="copy-bkt-2"

mys3cmd mb s3://$o_bkt

for f in $(seq 4) ;do
    dest_obj="orig-obj-$f"
    mys3cmd put -q $big_obj s3://${o_bkt}/$dest_obj
done

mys3cmd mb s3://$d_bkt

mys3cmd cp s3://${o_bkt}/orig-obj-1 s3://${d_bkt}/copied-obj-1
mys3cmd cp s3://${o_bkt}/orig-obj-3 s3://${d_bkt}/copied-obj-3

for f in $(seq 5 6) ;do
    dest_obj="orig-obj-$f"
    mys3cmd put -q $big_obj s3://${d_bkt}/$dest_obj
done

mys3cmd rb --recursive s3://${o_bkt}

############################################################
# copy multipart objects

o_bkt="orig-mp-bkt-3"
d_bkt="copy-mp-bkt-3"

mys3cmd mb s3://$o_bkt

for f in $(seq 2) ;do
    dest_obj="orig-multipart-obj-$f"
    mys3cmd put -q $huge_obj s3://${o_bkt}/$dest_obj
done

mys3cmd mb s3://$d_bkt

mys3cmd cp s3://${o_bkt}/orig-multipart-obj-1 \
	s3://${d_bkt}/copied-multipart-obj-1

for f in $(seq 5 5) ;do
    dest_obj="orig-multipart-obj-$f"
    mys3cmd put -q $huge_obj s3://${d_bkt}/$dest_obj
done


############################################################
# copy multipart objects and delete original

o_bkt="orig-mp-bkt-4"
d_bkt="copy-mp-bkt-4"

mys3cmd mb s3://$o_bkt

for f in $(seq 2) ;do
    dest_obj="orig-multipart-obj-$f"
    mys3cmd put -q $huge_obj s3://${o_bkt}/$dest_obj
done

mys3cmd mb s3://$d_bkt

mys3cmd cp s3://${o_bkt}/orig-multipart-obj-1 \
	s3://${d_bkt}/copied-multipart-obj-1

for f in $(seq 5 5) ;do
    dest_obj="orig-multipart-obj-$f"
    mys3cmd put -q $huge_obj s3://${d_bkt}/$dest_obj
done

mys3cmd rb --recursive s3://$o_bkt

########################################################################
# SWIFT TESTS

# 600MB
segment_size=629145600

############################################################
# plain test

for f in $(seq 4) ;do
    myswift upload swift-plain-ctr $big_obj --object-name swift-obj-$f
done

############################################################
# zero-len test

myswift upload swift-zerolen-ctr $empty_obj --object-name subdir/
myswift upload swift-zerolen-ctr $big_obj --object-name subdir/abc1
myswift upload swift-zerolen-ctr $empty_obj --object-name subdir/empty1
myswift upload swift-zerolen-ctr $big_obj --object-name subdir/xyz1

############################################################
# dlo test

# upload in 300MB segments
myswift upload swift-dlo-ctr $huge_obj --object-name dlo-obj-1 \
      -S $segment_size

############################################################
# slo test

# upload in 300MB segments
myswift upload swift-slo-ctr $huge_obj --object-name slo-obj-1 \
      -S $segment_size --use-slo

############################################################
# large object copy test

# upload in 300MB segments
o_ctr=swift-orig-ctr
o_obj=slo-orig-obj-1
d_ctr=swift-copy-ctr
d_obj=slo-copy-obj-1
myswift upload $o_ctr $big_obj --object-name $o_obj

myswift copy --destination /${d_ctr}/${d_obj} \
      $o_ctr $o_obj

myswift delete $o_ctr $o_obj

############################################################
# huge dlo object copy test

o_ctr=swift-orig-dlo-ctr-1
o_obj=dlo-orig-dlo-obj-1
d_ctr=swift-copy-dlo-ctr-1
d_obj=dlo-copy-dlo-obj-1

myswift upload $o_ctr $huge_obj --object-name $o_obj \
      -S $segment_size

myswift copy --destination /${d_ctr}/${d_obj} \
      $o_ctr $o_obj

############################################################
# huge dlo object copy and orig delete

o_ctr=swift-orig-dlo-ctr-2
o_obj=dlo-orig-dlo-obj-2
d_ctr=swift-copy-dlo-ctr-2
d_obj=dlo-copy-dlo-obj-2

myswift upload $o_ctr $huge_obj --object-name $o_obj \
      -S $segment_size

myswift copy --destination /${d_ctr}/${d_obj} \
      $o_ctr $o_obj

myswift delete $o_ctr $o_obj

############################################################
# huge slo object copy test

o_ctr=swift-orig-slo-ctr-1
o_obj=slo-orig-slo-obj-1
d_ctr=swift-copy-slo-ctr-1
d_obj=slo-copy-slo-obj-1
myswift upload $o_ctr $huge_obj --object-name $o_obj \
      -S $segment_size --use-slo

myswift copy --destination /${d_ctr}/${d_obj} $o_ctr $o_obj

############################################################
# huge slo object copy test and orig delete

o_ctr=swift-orig-slo-ctr-2
o_obj=slo-orig-slo-obj-2
d_ctr=swift-copy-slo-ctr-2
d_obj=slo-copy-slo-obj-2
myswift upload $o_ctr $huge_obj --object-name $o_obj \
      -S $segment_size --use-slo

myswift copy --destination /${d_ctr}/${d_obj} $o_ctr $o_obj

myswift delete $o_ctr $o_obj

########################################################################
# FORCE GARBAGE COLLECTION

sleep 6 # since for testing age at which gc can happen is 5 secs
radosgw-admin gc process --include-all


########################################
# DO ORPHAN LIST

pool="default.rgw.buckets.data"

rgw-orphan-list $pool

# we only expect there to be one output file, but loop just in case
ol_error=""
for f in orphan-list-*.out ; do
    if [ -s "$f"  ] ;then # if file non-empty
	ol_error="${ol_error}:$f"
	echo "One ore more orphans found in $f:"
	cat "$f"
    fi
done

if [ -n "$ol_error" ] ;then
    echo "ERROR: orphans found when none expected"
    exit 1
fi

########################################################################
# CLEAN UP

rm -f $empty_obj $big_obj $huge_obj $s3config

success
Commit	Line	Data
e306af50 TL	1	#!/usr/bin/env bash
	2
	3	set -ex
	4
	5	# if defined, debug messages will be displayed and prepended with the string
	6	# debug="DEBUG"
	7
	8	huge_size=2222 # in megabytes
	9	big_size=6 # in megabytes
	10
	11	huge_obj=/tmp/huge_obj.temp.$$
	12	big_obj=/tmp/big_obj.temp.$$
	13	empty_obj=/tmp/empty_obj.temp.$$
	14
	15	fifo=/tmp/orphan-fifo.$$
	16	awscli_dir=${HOME}/awscli_temp
	17	export PATH=${PATH}:${awscli_dir}
	18
	19	rgw_host=$(hostname --fqdn)
f6b5b4d7 TL	20	if echo "$rgw_host" \| grep -q '\.' ; then
	21	:
	22	else
	23	host_domain=".front.sepia.ceph.com"
	24	echo "WARNING: rgw hostname -- $rgw_host -- does not appear to be fully qualified; PUNTING and appending $host_domain"
	25	rgw_host="${rgw_host}${host_domain}"
	26	fi
e306af50 TL	27	rgw_port=80
	28
	29	echo "Fully Qualified Domain Name: $rgw_host"
	30
	31	success() {
	32	echo OK.
	33	exit 0
	34	}
	35
	36	########################################################################
	37	# INSTALL AND CONFIGURE TOOLING
	38
	39	install_awscli() {
	40	# NB: this does verify authenticity and integrity of downloaded
	41	# file; see
	42	# https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html
	43	here="$(pwd)"
	44	cd "$HOME"
	45	curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
	46	unzip awscliv2.zip
	47	mkdir -p $awscli_dir
	48	./aws/install -i $awscli_dir
	49	cd "$here"
	50	}
	51
	52	uninstall_awscli() {
	53	here="$(pwd)"
	54	cd "$HOME"
	55	rm -rf $awscli_dir ./aws awscliv2.zip
	56	cd "$here"
	57	}
	58
	59	sudo dnf install -y s3cmd
	60
	61	sudo yum install python3-setuptools
	62	sudo yum -y install python3-pip
	63	sudo pip3 install --upgrade setuptools
	64	sudo pip3 install python-swiftclient
	65
	66	# get ready for transition from s3cmd to awscli
	67	if false ;then
	68	install_awscli
	69	aws --version
	70	uninstall_awscli
	71	fi
	72
	73	s3config=/tmp/s3config.$$
	74
	75	# do not include the port when it is 80; the host base is used in the
	76	# v4 signature and it needs to follow this convention for signatures
	77	# to match
	78	if [ "$rgw_port" -ne 80 ] ;then
	79	s3_host_base="${rgw_host}:${rgw_port}"
	80	else
	81	s3_host_base="$rgw_host"
	82	fi
	83
	84	cat >${s3config} <<EOF
	85	[default]
	86	host_base = $s3_host_base
	87	access_key = 0555b35654ad1656d804
	88	secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==
	89	bucket_location = us-east-1
	90	check_ssl_certificate = True
91	check_ssl_hostname = True
92	default_mime_type = binary/octet-stream
93	delete_removed = False
94	dry_run = False
95	enable_multipart = True
96	encoding = UTF-8
97	encrypt = False
98	follow_symlinks = False
99	force = False
100	guess_mime_type = True
101	host_bucket = anything.with.three.dots
102	multipart_chunk_size_mb = 15
103	multipart_max_chunks = 10000
104	recursive = False
105	recv_chunk = 65536
106	send_chunk = 65536
107	signature_v2 = False
108	socket_timeout = 300
109	use_https = False
110	use_mime_magic = True
111	verbosity = WARNING
112	EOF
113
114
115	# set up swift authentication
116	export ST_AUTH=http://${rgw_host}:${rgw_port}/auth/v1.0
117	export ST_USER=test:tester
118	export ST_KEY=testing
119
120	create_users() {
121	# Create S3 user
122	local akey='0555b35654ad1656d804'
123	local skey='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=='
124	radosgw-admin user create --uid testid \
125	--access-key $akey --secret $skey \
126	--display-name 'M. Tester' --email tester@ceph.com
127
128	# Create Swift user
129	radosgw-admin user create --subuser=test:tester \
130	--display-name=Tester-Subuser --key-type=swift \
131	--secret=testing --access=full
132	}
133
134	myswift() {
135	if [ -n "$debug" ] ;then
136	echo "${debug}: swift --verbose --debug $@"
137	fi
138	swift --verbose --debug "$@"
139	local code=$?
140	if [ $code -ne 0 ] ;then
141	echo "ERROR: code = $code ; command = s3cmd --config=${s3config} --verbose --debug "$@""
142	exit $code
143	fi
144	}
145
146	mys3cmd() {
147	if [ -n "$debug" ] ;then
148	echo "${debug}: s3cmd --config=${s3config} --verbose --debug $@"
149	fi
150	s3cmd --config=${s3config} --verbose --debug "$@"
151	local code=$?
152	if [ $code -ne 0 ] ;then
153	echo "ERROR: code = $code ; command = s3cmd --config=${s3config} --verbose --debug "$@""
154	exit $code
155	fi
156	}
157
158	mys3uploadkill() {
159	if [ $# -ne 5 ] ;then
160	echo "$0: error expecting 5 arguments"
161	exit 1
162	fi
163
164	set -v
165	local_file="$1"
166	remote_bkt="$2"
167	remote_obj="$3"
168	fifo="$4"
169	stop_part="$5"
170
171	mkfifo $fifo
172
173	s3cmd --config=${s3config} put $local_file \
174	s3://${remote_bkt}/${remote_obj} \
175	--progress \
176	--multipart-chunk-size-mb=5 >$fifo &
177	set +e # don't allow errors to stop script
178	while read line ;do
179	echo "$line" \| grep --quiet "part $stop_part "
180	if [ ${PIPESTATUS[1]} -eq 0 ] ;then
181	kill -9 $(jobs -p)
182	break
183	fi
184	done <$fifo
185	set -e
186
187	rm -f $fifo
188	}
189
190	mys3upload() {
191	obj=$1
192	bucket=$2
193	dest_obj=$3
194
195	mys3cmd put -q $obj s3://${bucket}/$dest_obj
196	}
197
198	########################################################################
199	# PREP
200
201	create_users
202	dd if=/dev/urandom of=$big_obj bs=1M count=${big_size}
203	dd if=/dev/urandom of=$huge_obj bs=1M count=${huge_size}
204	touch $empty_obj
205
206	quick_tests() {
207	echo TRY A SWIFT COMMAND
208	myswift upload swift-plain-ctr $big_obj --object-name swift-obj-test
209	myswift list
210	myswift list swift-plain-ctr
211
212	echo TRY A RADOSGW-ADMIN COMMAND
213	radosgw-admin bucket list # make sure rgw is up and running
214	}
215
216	########################################################################
217	# S3 TESTS
218
219	####################################
220	# regular multipart test
221
222	mys3cmd mb s3://multipart-bkt
223	mys3upload $huge_obj multipart-bkt multipart-obj
224	mys3cmd ls
225	mys3cmd ls s3://multipart-bkt
226
227	####################################
228	# multipart test with incomplete uploads
229
230	bkt="incomplete-mp-bkt-1"
231
232	mys3cmd mb s3://$bkt
233	mys3uploadkill $huge_obj $bkt incomplete-mp-obj-1 $fifo 20
234	mys3uploadkill $huge_obj $bkt incomplete-mp-obj-2 $fifo 100
235
236	####################################
237	# resharded bucket
238
239	bkt=resharded-bkt-1
240
241	mys3cmd mb s3://$bkt
242
243	for f in $(seq 8) ; do
244	dest_obj="reshard-obj-${f}"
245	mys3cmd put -q $big_obj s3://${bkt}/$dest_obj
246	done
247
248	radosgw-admin bucket reshard --num-shards 3 --bucket=$bkt --yes-i-really-mean-it
249	radosgw-admin bucket reshard --num-shards 5 --bucket=$bkt --yes-i-really-mean-it
250
251	####################################
252	# versioned bucket
253
254	if true ;then
255	echo "WARNING: versioned bucket test currently turned off"
256	else
257	bkt=versioned-bkt-1
258
259	mys3cmd mb s3://$bkt
260
261	# bucket-enable-versioning $bkt
262
263	for f in $(seq 3) ;do
264	for g in $(seq 10) ;do
265	dest_obj="versioned-obj-${g}"
266	mys3cmd put -q $big_obj s3://${bkt}/$dest_obj
267	done
268	done
269
270	for g in $(seq 1 2 10) ;do
271	dest_obj="versioned-obj-${g}"
272	mys3cmd rm s3://${bkt}/$dest_obj
273	done
274	fi
275
276	############################################################
277	# copy small objects
278
279	o_bkt="orig-bkt-1"
280	d_bkt="copy-bkt-1"
281	mys3cmd mb s3://$o_bkt
282
283	for f in $(seq 4) ;do
284	dest_obj="orig-obj-$f"
285	mys3cmd put -q $big_obj s3://${o_bkt}/$dest_obj
286	done
287
288	mys3cmd mb s3://$d_bkt
289
290	mys3cmd cp s3://${o_bkt}/orig-obj-1 s3://${d_bkt}/copied-obj-1
291	mys3cmd cp s3://${o_bkt}/orig-obj-3 s3://${d_bkt}/copied-obj-3
292
293	for f in $(seq 5 6) ;do
294	dest_obj="orig-obj-$f"
295	mys3cmd put -q $big_obj s3://${d_bkt}/$dest_obj
296	done
297
298	############################################################
299	# copy small objects and delete original
300
301	o_bkt="orig-bkt-2"
302	d_bkt="copy-bkt-2"
303
304	mys3cmd mb s3://$o_bkt
305
306	for f in $(seq 4) ;do
307	dest_obj="orig-obj-$f"
308	mys3cmd put -q $big_obj s3://${o_bkt}/$dest_obj
309	done
310
311	mys3cmd mb s3://$d_bkt
312
313	mys3cmd cp s3://${o_bkt}/orig-obj-1 s3://${d_bkt}/copied-obj-1
314	mys3cmd cp s3://${o_bkt}/orig-obj-3 s3://${d_bkt}/copied-obj-3
315
316	for f in $(seq 5 6) ;do
317	dest_obj="orig-obj-$f"
318	mys3cmd put -q $big_obj s3://${d_bkt}/$dest_obj
319	done
320
321	mys3cmd rb --recursive s3://${o_bkt}
322
323	############################################################
324	# copy multipart objects
325
326	o_bkt="orig-mp-bkt-3"
327	d_bkt="copy-mp-bkt-3"
328
329	mys3cmd mb s3://$o_bkt
330
331	for f in $(seq 2) ;do
332	dest_obj="orig-multipart-obj-$f"
333	mys3cmd put -q $huge_obj s3://${o_bkt}/$dest_obj
334	done
335
336	mys3cmd mb s3://$d_bkt
337
338	mys3cmd cp s3://${o_bkt}/orig-multipart-obj-1 \
339	s3://${d_bkt}/copied-multipart-obj-1
340
341	for f in $(seq 5 5) ;do
342	dest_obj="orig-multipart-obj-$f"
343	mys3cmd put -q $huge_obj s3://${d_bkt}/$dest_obj
344	done
345
346
347	############################################################
348	# copy multipart objects and delete original
349
350	o_bkt="orig-mp-bkt-4"
351	d_bkt="copy-mp-bkt-4"
352
353	mys3cmd mb s3://$o_bkt
354
355	for f in $(seq 2) ;do
356	dest_obj="orig-multipart-obj-$f"
357	mys3cmd put -q $huge_obj s3://${o_bkt}/$dest_obj
358	done
359
360	mys3cmd mb s3://$d_bkt
361
362	mys3cmd cp s3://${o_bkt}/orig-multipart-obj-1 \
363	s3://${d_bkt}/copied-multipart-obj-1
364
365	for f in $(seq 5 5) ;do
366	dest_obj="orig-multipart-obj-$f"
367	mys3cmd put -q $huge_obj s3://${d_bkt}/$dest_obj
368	done
369
370	mys3cmd rb --recursive s3://$o_bkt
371
372	########################################################################
373	# SWIFT TESTS
374
375	# 600MB
376	segment_size=629145600
377
378	############################################################
379	# plain test
380
381	for f in $(seq 4) ;do
382	myswift upload swift-plain-ctr $big_obj --object-name swift-obj-$f
383	done
384
385	############################################################
386	# zero-len test
387
388	myswift upload swift-zerolen-ctr $empty_obj --object-name subdir/
389	myswift upload swift-zerolen-ctr $big_obj --object-name subdir/abc1
390	myswift upload swift-zerolen-ctr $empty_obj --object-name subdir/empty1
391	myswift upload swift-zerolen-ctr $big_obj --object-name subdir/xyz1
392
393	############################################################
394	# dlo test
395
396	# upload in 300MB segments
397	myswift upload swift-dlo-ctr $huge_obj --object-name dlo-obj-1 \
398	-S $segment_size
399
400	############################################################
401	# slo test
402
403	# upload in 300MB segments
404	myswift upload swift-slo-ctr $huge_obj --object-name slo-obj-1 \
405	-S $segment_size --use-slo
406
407	############################################################
408	# large object copy test
409
410	# upload in 300MB segments
411	o_ctr=swift-orig-ctr
412	o_obj=slo-orig-obj-1
413	d_ctr=swift-copy-ctr
414	d_obj=slo-copy-obj-1
415	myswift upload $o_ctr $big_obj --object-name $o_obj
416
417	myswift copy --destination /${d_ctr}/${d_obj} \
418	$o_ctr $o_obj
419
420	myswift delete $o_ctr $o_obj
421
422	############################################################
423	# huge dlo object copy test
424
425	o_ctr=swift-orig-dlo-ctr-1
426	o_obj=dlo-orig-dlo-obj-1
427	d_ctr=swift-copy-dlo-ctr-1
428	d_obj=dlo-copy-dlo-obj-1
429
430	myswift upload $o_ctr $huge_obj --object-name $o_obj \
431	-S $segment_size
432
433	myswift copy --destination /${d_ctr}/${d_obj} \
434	$o_ctr $o_obj
435
436	############################################################
437	# huge dlo object copy and orig delete
438
439	o_ctr=swift-orig-dlo-ctr-2
440	o_obj=dlo-orig-dlo-obj-2
441	d_ctr=swift-copy-dlo-ctr-2
442	d_obj=dlo-copy-dlo-obj-2
443
444	myswift upload $o_ctr $huge_obj --object-name $o_obj \
445	-S $segment_size
446
447	myswift copy --destination /${d_ctr}/${d_obj} \
448	$o_ctr $o_obj
449
450	myswift delete $o_ctr $o_obj
451
452	############################################################
453	# huge slo object copy test
454
455	o_ctr=swift-orig-slo-ctr-1
456	o_obj=slo-orig-slo-obj-1
457	d_ctr=swift-copy-slo-ctr-1
458	d_obj=slo-copy-slo-obj-1
459	myswift upload $o_ctr $huge_obj --object-name $o_obj \
460	-S $segment_size --use-slo
461
462	myswift copy --destination /${d_ctr}/${d_obj} $o_ctr $o_obj
463
464	############################################################
465	# huge slo object copy test and orig delete
466
467	o_ctr=swift-orig-slo-ctr-2
468	o_obj=slo-orig-slo-obj-2
469	d_ctr=swift-copy-slo-ctr-2
470	d_obj=slo-copy-slo-obj-2
471	myswift upload $o_ctr $huge_obj --object-name $o_obj \
472	-S $segment_size --use-slo
473
474	myswift copy --destination /${d_ctr}/${d_obj} $o_ctr $o_obj
475
476	myswift delete $o_ctr $o_obj
477
478	########################################################################
479	# FORCE GARBAGE COLLECTION
480
481	sleep 6 # since for testing age at which gc can happen is 5 secs
482	radosgw-admin gc process --include-all
483
484
485	########################################
486	# DO ORPHAN LIST
487
488	pool="default.rgw.buckets.data"
489
490	rgw-orphan-list $pool
491
492	# we only expect there to be one output file, but loop just in case
493	ol_error=""
494	for f in orphan-list-*.out ; do
495	if [ -s "$f" ] ;then # if file non-empty
496	ol_error="${ol_error}:$f"
497	echo "One ore more orphans found in $f:"
498	cat "$f"
499	fi
500	done
501
502	if [ -n "$ol_error" ] ;then
503	echo "ERROR: orphans found when none expected"
504	exit 1
505	fi
506
507	########################################################################
508	# CLEAN UP
509
510	rm -f $empty_obj $big_obj $huge_obj $s3config
511
512	success