ceph/qa/workunits/rgw/test_rgw_orphan_list.sh

   1 #!/usr/bin/env bash
   2
   3 set -ex
   4
   5 # if defined, debug messages will be displayed and prepended with the string
   6 # debug="DEBUG"
   7
   8 huge_size=2222 # in megabytes
   9 big_size=6 # in megabytes
  10
  11 huge_obj=/tmp/huge_obj.temp.$$
  12 big_obj=/tmp/big_obj.temp.$$
  13 empty_obj=/tmp/empty_obj.temp.$$
  14
  15 fifo=/tmp/orphan-fifo.$$
  16 awscli_dir=${HOME}/awscli_temp
  17 export PATH=${PATH}:${awscli_dir}
  18
  19 rgw_host=$(hostname --fqdn)
  20 rgw_port=80
  21
  22 echo "Fully Qualified Domain Name: $rgw_host"
  23
  24 success() {
  25     echo OK.
  26     exit 0
  27 }
  28
  29 ########################################################################
  30 # INSTALL AND CONFIGURE TOOLING
  31
  32 install_awscli() {
  33     # NB: this does verify authenticity and integrity of downloaded
  34     # file; see
  35     # https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html
  36     here="$(pwd)"
  37     cd "$HOME"
  38     curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
  39     unzip awscliv2.zip
  40     mkdir -p $awscli_dir
  41     ./aws/install -i $awscli_dir
  42     cd "$here"
  43 }
  44
  45 uninstall_awscli() {
  46     here="$(pwd)"
  47     cd "$HOME"
  48     rm -rf $awscli_dir ./aws awscliv2.zip
  49     cd "$here"
  50 }
  51
  52 sudo dnf install -y s3cmd
  53
  54 sudo yum install python3-setuptools
  55 sudo yum -y install python3-pip
  56 sudo pip3 install --upgrade setuptools
  57 sudo pip3 install python-swiftclient
  58
  59 # get ready for transition from s3cmd to awscli
  60 if false ;then
  61     install_awscli
  62     aws --version
  63     uninstall_awscli
  64 fi
  65
  66 s3config=/tmp/s3config.$$
  67
  68 # do not include the port when it is 80; the host base is used in the
  69 # v4 signature and it needs to follow this convention for signatures
  70 # to match
  71 if [ "$rgw_port" -ne 80 ] ;then
  72     s3_host_base="${rgw_host}:${rgw_port}"
  73 else
  74     s3_host_base="$rgw_host"
  75 fi
  76
  77 cat >${s3config} <<EOF
  78 [default]
  79 host_base = $s3_host_base
  80 access_key = 0555b35654ad1656d804
  81 secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==
  82 bucket_location = us-east-1
  83 check_ssl_certificate = True
  84 check_ssl_hostname = True
  85 default_mime_type = binary/octet-stream
  86 delete_removed = False
  87 dry_run = False
  88 enable_multipart = True
  89 encoding = UTF-8
  90 encrypt = False
  91 follow_symlinks = False
  92 force = False
  93 guess_mime_type = True
  94 host_bucket = anything.with.three.dots
  95 multipart_chunk_size_mb = 15
  96 multipart_max_chunks = 10000
  97 recursive = False
  98 recv_chunk = 65536
  99 send_chunk = 65536
 100 signature_v2 = False
 101 socket_timeout = 300
 102 use_https = False
 103 use_mime_magic = True
 104 verbosity = WARNING
 105 EOF
 106
 107
 108 # set up swift authentication
 109 export ST_AUTH=http://${rgw_host}:${rgw_port}/auth/v1.0
 110 export ST_USER=test:tester
 111 export ST_KEY=testing
 112
 113 create_users() {
 114     # Create S3 user
 115     local akey='0555b35654ad1656d804'
 116     local skey='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=='
 117     radosgw-admin user create --uid testid \
 118                   --access-key $akey --secret $skey \
 119                   --display-name 'M. Tester' --email tester@ceph.com
 120
 121     # Create Swift user
 122     radosgw-admin user create --subuser=test:tester \
 123                   --display-name=Tester-Subuser --key-type=swift \
 124                   --secret=testing --access=full
 125 }
 126
 127 myswift() {
 128     if [ -n "$debug" ] ;then
 129         echo "${debug}: swift --verbose --debug $@"
 130     fi
 131     swift --verbose --debug "$@"
 132     local code=$?
 133     if [ $code -ne 0 ] ;then
 134         echo "ERROR: code = $code ; command = s3cmd --config=${s3config} --verbose --debug "$@""
 135         exit $code
 136     fi
 137 }
 138
 139 mys3cmd() {
 140     if [ -n "$debug" ] ;then
 141         echo "${debug}: s3cmd --config=${s3config} --verbose --debug $@"
 142     fi
 143     s3cmd --config=${s3config} --verbose --debug "$@"
 144     local code=$?
 145     if [ $code -ne 0 ] ;then
 146         echo "ERROR: code = $code ; command = s3cmd --config=${s3config} --verbose --debug "$@""
 147         exit $code
 148     fi
 149 }
 150
 151 mys3uploadkill() {
 152     if [ $# -ne 5 ] ;then
 153         echo "$0: error expecting 5 arguments"
 154         exit 1
 155     fi
 156
 157     set -v
 158     local_file="$1"
 159     remote_bkt="$2"
 160     remote_obj="$3"
 161     fifo="$4"
 162     stop_part="$5"
 163
 164     mkfifo $fifo
 165
 166     s3cmd --config=${s3config} put $local_file \
 167           s3://${remote_bkt}/${remote_obj} \
 168           --progress \
 169           --multipart-chunk-size-mb=5 >$fifo &
 170     set +e # don't allow errors to stop script
 171     while read line ;do
 172         echo "$line" | grep --quiet "part $stop_part "
 173         if [ ${PIPESTATUS[1]} -eq 0 ] ;then
 174             kill -9 $(jobs -p)
 175             break
 176         fi
 177     done <$fifo
 178     set -e
 179
 180     rm -f $fifo
 181 }
 182
 183 mys3upload() {
 184     obj=$1
 185     bucket=$2
 186     dest_obj=$3
 187
 188     mys3cmd put -q $obj s3://${bucket}/$dest_obj
 189 }
 190
 191 ########################################################################
 192 # PREP
 193
 194 create_users
 195 dd if=/dev/urandom of=$big_obj bs=1M count=${big_size}
 196 dd if=/dev/urandom of=$huge_obj bs=1M count=${huge_size}
 197 touch $empty_obj
 198
 199 quick_tests() {
 200     echo TRY A SWIFT COMMAND
 201     myswift upload swift-plain-ctr $big_obj --object-name swift-obj-test
 202     myswift list
 203     myswift list swift-plain-ctr
 204
 205     echo TRY A RADOSGW-ADMIN COMMAND
 206     radosgw-admin bucket list # make sure rgw is up and running
 207 }
 208
 209 ########################################################################
 210 # S3 TESTS
 211
 212 ####################################
 213 # regular multipart test
 214
 215 mys3cmd mb s3://multipart-bkt
 216 mys3upload $huge_obj multipart-bkt multipart-obj
 217 mys3cmd ls
 218 mys3cmd ls s3://multipart-bkt
 219
 220 ####################################
 221 # multipart test with incomplete uploads
 222
 223 bkt="incomplete-mp-bkt-1"
 224
 225 mys3cmd mb s3://$bkt
 226 mys3uploadkill $huge_obj $bkt incomplete-mp-obj-1 $fifo 20
 227 mys3uploadkill $huge_obj $bkt incomplete-mp-obj-2 $fifo 100
 228
 229 ####################################
 230 # resharded bucket
 231
 232 bkt=resharded-bkt-1
 233
 234 mys3cmd mb s3://$bkt
 235
 236 for f in $(seq 8) ; do
 237     dest_obj="reshard-obj-${f}"
 238     mys3cmd put -q $big_obj s3://${bkt}/$dest_obj
 239 done
 240
 241 radosgw-admin bucket reshard --num-shards 3 --bucket=$bkt --yes-i-really-mean-it
 242 radosgw-admin bucket reshard --num-shards 5 --bucket=$bkt --yes-i-really-mean-it
 243
 244 ####################################
 245 # versioned bucket
 246
 247 if true ;then
 248     echo "WARNING: versioned bucket test currently turned off"
 249 else
 250     bkt=versioned-bkt-1
 251
 252     mys3cmd mb s3://$bkt
 253
 254     # bucket-enable-versioning $bkt
 255
 256     for f in $(seq 3) ;do
 257         for g in $(seq 10) ;do
 258             dest_obj="versioned-obj-${g}"
 259             mys3cmd put -q $big_obj s3://${bkt}/$dest_obj
 260         done
 261     done
 262
 263     for g in $(seq 1 2 10) ;do
 264         dest_obj="versioned-obj-${g}"
 265         mys3cmd rm s3://${bkt}/$dest_obj
 266     done
 267 fi
 268
 269 ############################################################
 270 # copy small objects
 271
 272 o_bkt="orig-bkt-1"
 273 d_bkt="copy-bkt-1"
 274 mys3cmd mb s3://$o_bkt
 275
 276 for f in $(seq 4) ;do
 277     dest_obj="orig-obj-$f"
 278     mys3cmd put -q $big_obj s3://${o_bkt}/$dest_obj
 279 done
 280
 281 mys3cmd mb s3://$d_bkt
 282
 283 mys3cmd cp s3://${o_bkt}/orig-obj-1 s3://${d_bkt}/copied-obj-1
 284 mys3cmd cp s3://${o_bkt}/orig-obj-3 s3://${d_bkt}/copied-obj-3
 285
 286 for f in $(seq 5 6) ;do
 287     dest_obj="orig-obj-$f"
 288     mys3cmd put -q $big_obj s3://${d_bkt}/$dest_obj
 289 done
 290
 291 ############################################################
 292 # copy small objects and delete original
 293
 294 o_bkt="orig-bkt-2"
 295 d_bkt="copy-bkt-2"
 296
 297 mys3cmd mb s3://$o_bkt
 298
 299 for f in $(seq 4) ;do
 300     dest_obj="orig-obj-$f"
 301     mys3cmd put -q $big_obj s3://${o_bkt}/$dest_obj
 302 done
 303
 304 mys3cmd mb s3://$d_bkt
 305
 306 mys3cmd cp s3://${o_bkt}/orig-obj-1 s3://${d_bkt}/copied-obj-1
 307 mys3cmd cp s3://${o_bkt}/orig-obj-3 s3://${d_bkt}/copied-obj-3
 308
 309 for f in $(seq 5 6) ;do
 310     dest_obj="orig-obj-$f"
 311     mys3cmd put -q $big_obj s3://${d_bkt}/$dest_obj
 312 done
 313
 314 mys3cmd rb --recursive s3://${o_bkt}
 315
 316 ############################################################
 317 # copy multipart objects
 318
 319 o_bkt="orig-mp-bkt-3"
 320 d_bkt="copy-mp-bkt-3"
 321
 322 mys3cmd mb s3://$o_bkt
 323
 324 for f in $(seq 2) ;do
 325     dest_obj="orig-multipart-obj-$f"
 326     mys3cmd put -q $huge_obj s3://${o_bkt}/$dest_obj
 327 done
 328
 329 mys3cmd mb s3://$d_bkt
 330
 331 mys3cmd cp s3://${o_bkt}/orig-multipart-obj-1 \
 332         s3://${d_bkt}/copied-multipart-obj-1
 333
 334 for f in $(seq 5 5) ;do
 335     dest_obj="orig-multipart-obj-$f"
 336     mys3cmd put -q $huge_obj s3://${d_bkt}/$dest_obj
 337 done
 338
 339
 340 ############################################################
 341 # copy multipart objects and delete original
 342
 343 o_bkt="orig-mp-bkt-4"
 344 d_bkt="copy-mp-bkt-4"
 345
 346 mys3cmd mb s3://$o_bkt
 347
 348 for f in $(seq 2) ;do
 349     dest_obj="orig-multipart-obj-$f"
 350     mys3cmd put -q $huge_obj s3://${o_bkt}/$dest_obj
 351 done
 352
 353 mys3cmd mb s3://$d_bkt
 354
 355 mys3cmd cp s3://${o_bkt}/orig-multipart-obj-1 \
 356         s3://${d_bkt}/copied-multipart-obj-1
 357
 358 for f in $(seq 5 5) ;do
 359     dest_obj="orig-multipart-obj-$f"
 360     mys3cmd put -q $huge_obj s3://${d_bkt}/$dest_obj
 361 done
 362
 363 mys3cmd rb --recursive s3://$o_bkt
 364
 365 ########################################################################
 366 # SWIFT TESTS
 367
 368 # 600MB
 369 segment_size=629145600
 370
 371 ############################################################
 372 # plain test
 373
 374 for f in $(seq 4) ;do
 375     myswift upload swift-plain-ctr $big_obj --object-name swift-obj-$f
 376 done
 377
 378 ############################################################
 379 # zero-len test
 380
 381 myswift upload swift-zerolen-ctr $empty_obj --object-name subdir/
 382 myswift upload swift-zerolen-ctr $big_obj --object-name subdir/abc1
 383 myswift upload swift-zerolen-ctr $empty_obj --object-name subdir/empty1
 384 myswift upload swift-zerolen-ctr $big_obj --object-name subdir/xyz1
 385
 386 ############################################################
 387 # dlo test
 388
 389 # upload in 300MB segments
 390 myswift upload swift-dlo-ctr $huge_obj --object-name dlo-obj-1 \
 391       -S $segment_size
 392
 393 ############################################################
 394 # slo test
 395
 396 # upload in 300MB segments
 397 myswift upload swift-slo-ctr $huge_obj --object-name slo-obj-1 \
 398       -S $segment_size --use-slo
 399
 400 ############################################################
 401 # large object copy test
 402
 403 # upload in 300MB segments
 404 o_ctr=swift-orig-ctr
 405 o_obj=slo-orig-obj-1
 406 d_ctr=swift-copy-ctr
 407 d_obj=slo-copy-obj-1
 408 myswift upload $o_ctr $big_obj --object-name $o_obj
 409
 410 myswift copy --destination /${d_ctr}/${d_obj} \
 411       $o_ctr $o_obj
 412
 413 myswift delete $o_ctr $o_obj
 414
 415 ############################################################
 416 # huge dlo object copy test
 417
 418 o_ctr=swift-orig-dlo-ctr-1
 419 o_obj=dlo-orig-dlo-obj-1
 420 d_ctr=swift-copy-dlo-ctr-1
 421 d_obj=dlo-copy-dlo-obj-1
 422
 423 myswift upload $o_ctr $huge_obj --object-name $o_obj \
 424       -S $segment_size
 425
 426 myswift copy --destination /${d_ctr}/${d_obj} \
 427       $o_ctr $o_obj
 428
 429 ############################################################
 430 # huge dlo object copy and orig delete
 431
 432 o_ctr=swift-orig-dlo-ctr-2
 433 o_obj=dlo-orig-dlo-obj-2
 434 d_ctr=swift-copy-dlo-ctr-2
 435 d_obj=dlo-copy-dlo-obj-2
 436
 437 myswift upload $o_ctr $huge_obj --object-name $o_obj \
 438       -S $segment_size
 439
 440 myswift copy --destination /${d_ctr}/${d_obj} \
 441       $o_ctr $o_obj
 442
 443 myswift delete $o_ctr $o_obj
 444
 445 ############################################################
 446 # huge slo object copy test
 447
 448 o_ctr=swift-orig-slo-ctr-1
 449 o_obj=slo-orig-slo-obj-1
 450 d_ctr=swift-copy-slo-ctr-1
 451 d_obj=slo-copy-slo-obj-1
 452 myswift upload $o_ctr $huge_obj --object-name $o_obj \
 453       -S $segment_size --use-slo
 454
 455 myswift copy --destination /${d_ctr}/${d_obj} $o_ctr $o_obj
 456
 457 ############################################################
 458 # huge slo object copy test and orig delete
 459
 460 o_ctr=swift-orig-slo-ctr-2
 461 o_obj=slo-orig-slo-obj-2
 462 d_ctr=swift-copy-slo-ctr-2
 463 d_obj=slo-copy-slo-obj-2
 464 myswift upload $o_ctr $huge_obj --object-name $o_obj \
 465       -S $segment_size --use-slo
 466
 467 myswift copy --destination /${d_ctr}/${d_obj} $o_ctr $o_obj
 468
 469 myswift delete $o_ctr $o_obj
 470
 471 ########################################################################
 472 # FORCE GARBAGE COLLECTION
 473
 474 sleep 6 # since for testing age at which gc can happen is 5 secs
 475 radosgw-admin gc process --include-all
 476
 477
 478 ########################################
 479 # DO ORPHAN LIST
 480
 481 pool="default.rgw.buckets.data"
 482
 483 rgw-orphan-list $pool
 484
 485 # we only expect there to be one output file, but loop just in case
 486 ol_error=""
 487 for f in orphan-list-*.out ; do
 488     if [ -s "$f"  ] ;then # if file non-empty
 489         ol_error="${ol_error}:$f"
 490         echo "One ore more orphans found in $f:"
 491         cat "$f"
 492     fi
 493 done
 494
 495 if [ -n "$ol_error" ] ;then
 496     echo "ERROR: orphans found when none expected"
 497     exit 1
 498 fi
 499
 500 ########################################################################
 501 # CLEAN UP
 502
 503 rm -f $empty_obj $big_obj $huge_obj $s3config
 504
 505 success