ceph/qa/workunits/rgw/test_rgw_orphan_list.sh

   1 #!/usr/bin/env bash
   2
   3 # set -x
   4 set -e
   5
   6 # if defined, debug messages will be displayed and prepended with the string
   7 # debug="DEBUG"
   8
   9 huge_size=5100 # in megabytes
  10 big_size=7 # in megabytes
  11
  12 huge_obj=/tmp/huge_obj.temp.$$
  13 big_obj=/tmp/big_obj.temp.$$
  14 empty_obj=/tmp/empty_obj.temp.$$
  15
  16 fifo=/tmp/orphan-fifo.$$
  17 awscli_dir=${HOME}/awscli_temp
  18 export PATH=${PATH}:${awscli_dir}
  19
  20 rgw_host=$(hostname --fqdn)
  21 if echo "$rgw_host" | grep -q '\.' ; then
  22     :
  23 else
  24     host_domain=".front.sepia.ceph.com"
  25     echo "WARNING: rgw hostname -- $rgw_host -- does not appear to be fully qualified; PUNTING and appending $host_domain"
  26     rgw_host="${rgw_host}${host_domain}"
  27 fi
  28 rgw_port=80
  29
  30 echo "Fully Qualified Domain Name: $rgw_host"
  31
  32 success() {
  33     echo OK.
  34     exit 0
  35 }
  36
  37 ########################################################################
  38 # INSTALL AND CONFIGURE TOOLING
  39
  40 install_awscli() {
  41     # NB: this does verify authenticity and integrity of downloaded
  42     # file; see
  43     # https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html
  44     here="$(pwd)"
  45     cd "$HOME"
  46     curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
  47     unzip awscliv2.zip
  48     mkdir -p $awscli_dir
  49     ./aws/install -i $awscli_dir
  50     cd "$here"
  51 }
  52
  53 uninstall_awscli() {
  54     here="$(pwd)"
  55     cd "$HOME"
  56     rm -rf $awscli_dir ./aws awscliv2.zip
  57     cd "$here"
  58 }
  59
  60 sudo yum -y install s3cmd
  61 sudo yum -y install python3-setuptools
  62 sudo yum -y install python3-pip
  63 sudo pip3 install --upgrade setuptools
  64 sudo pip3 install python-swiftclient
  65
  66 # get ready for transition from s3cmd to awscli
  67 if false ;then
  68     install_awscli
  69     aws --version
  70     uninstall_awscli
  71 fi
  72
  73 s3config=/tmp/s3config.$$
  74
  75 # do not include the port when it is 80; the host base is used in the
  76 # v4 signature and it needs to follow this convention for signatures
  77 # to match
  78 if [ "$rgw_port" -ne 80 ] ;then
  79     s3_host_base="${rgw_host}:${rgw_port}"
  80 else
  81     s3_host_base="$rgw_host"
  82 fi
  83
  84 cat >${s3config} <<EOF
  85 [default]
  86 host_base = $s3_host_base
  87 access_key = 0555b35654ad1656d804
  88 secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==
  89 bucket_location = us-east-1
  90 check_ssl_certificate = True
  91 check_ssl_hostname = True
  92 default_mime_type = binary/octet-stream
  93 delete_removed = False
  94 dry_run = False
  95 enable_multipart = True
  96 encoding = UTF-8
  97 encrypt = False
  98 follow_symlinks = False
  99 force = False
 100 guess_mime_type = True
 101 host_bucket = anything.with.three.dots
 102 multipart_chunk_size_mb = 15
 103 multipart_max_chunks = 10000
 104 recursive = False
 105 recv_chunk = 65536
 106 send_chunk = 65536
 107 signature_v2 = False
 108 socket_timeout = 300
 109 use_https = False
 110 use_mime_magic = True
 111 verbosity = WARNING
 112 EOF
 113
 114
 115 # set up swift authentication
 116 export ST_AUTH=http://${rgw_host}:${rgw_port}/auth/v1.0
 117 export ST_USER=test:tester
 118 export ST_KEY=testing
 119
 120 create_users() {
 121     # Create S3 user
 122     local akey='0555b35654ad1656d804'
 123     local skey='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=='
 124     radosgw-admin user create --uid testid \
 125                   --access-key $akey --secret $skey \
 126                   --display-name 'M. Tester' --email tester@ceph.com
 127
 128     # Create Swift user
 129     radosgw-admin user create --subuser=test:tester \
 130                   --display-name=Tester-Subuser --key-type=swift \
 131                   --secret=testing --access=full
 132 }
 133
 134 myswift() {
 135     if [ -n "$debug" ] ;then
 136         echo "${debug}: swift --verbose --debug $@"
 137     fi
 138     swift --verbose --debug "$@"
 139     local code=$?
 140     if [ $code -ne 0 ] ;then
 141         echo "ERROR: code = $code ; command = s3cmd --config=${s3config} --verbose --debug "$@""
 142         exit $code
 143     fi
 144 }
 145
 146 mys3cmd() {
 147     if [ -n "$debug" ] ;then
 148         echo "${debug}: s3cmd --config=${s3config} --verbose --debug $@"
 149     fi
 150     s3cmd --config=${s3config} --verbose --debug "$@"
 151     local code=$?
 152     if [ $code -ne 0 ] ;then
 153         echo "ERROR: code = $code ; command = s3cmd --config=${s3config} --verbose --debug "$@""
 154         exit $code
 155     fi
 156 }
 157
 158 mys3uploadkill() {
 159     if [ $# -ne 5 ] ;then
 160         echo "$0: error expecting 5 arguments"
 161         exit 1
 162     fi
 163
 164     local_file="$1"
 165     remote_bkt="$2"
 166     remote_obj="$3"
 167     fifo="$4"
 168     stop_part="$5"
 169
 170     mkfifo $fifo
 171
 172     s3cmd --config=${s3config} put $local_file \
 173           s3://${remote_bkt}/${remote_obj} \
 174           --progress \
 175           --multipart-chunk-size-mb=5 >$fifo &
 176     set +e # don't allow errors to stop script
 177     while read line ;do
 178         echo "$line" | grep --quiet "part $stop_part "
 179         if [ ${PIPESTATUS[1]} -eq 0 ] ;then
 180             kill -9 $(jobs -p)
 181             break
 182         fi
 183     done <$fifo
 184     set -e
 185
 186     rm -f $fifo
 187 }
 188
 189 mys3upload() {
 190     obj=$1
 191     bucket=$2
 192     dest_obj=$3
 193
 194     mys3cmd put -q $obj s3://${bucket}/$dest_obj
 195 }
 196
 197 ########################################################################
 198 # PREP
 199
 200 create_users
 201 dd if=/dev/urandom of=$big_obj bs=1M count=${big_size}
 202 dd if=/dev/urandom of=$huge_obj bs=1M count=${huge_size}
 203 touch $empty_obj
 204
 205 quick_tests() {
 206     echo TRY A SWIFT COMMAND
 207     myswift upload swift-plain-ctr $big_obj --object-name swift-obj-test
 208     myswift list
 209     myswift list swift-plain-ctr
 210
 211     echo TRY A RADOSGW-ADMIN COMMAND
 212     radosgw-admin bucket list # make sure rgw is up and running
 213 }
 214
 215 ########################################################################
 216 # S3 TESTS
 217
 218 ####################################
 219 # regular multipart test
 220
 221 mys3cmd mb s3://multipart-bkt
 222 mys3upload $huge_obj multipart-bkt multipart-obj
 223 mys3cmd ls
 224 mys3cmd ls s3://multipart-bkt
 225
 226 ####################################
 227 # multipart test with incomplete uploads
 228
 229 bkt="incomplete-mp-bkt-1"
 230
 231 mys3cmd mb s3://$bkt
 232
 233 mys3uploadkill $huge_obj $bkt incomplete-mp-obj-c $fifo 20
 234
 235 # generate an incomplete multipart with more than 1,000 parts
 236 mys3uploadkill $huge_obj $bkt incomplete-mp-obj-b $fifo 1005
 237
 238 # generate more than 1000 incomplet multiparts
 239 for c in $(seq 1005) ;do
 240     mys3uploadkill $huge_obj $bkt incomplete-mp-obj-c-$c $fifo 3
 241 done
 242
 243 ####################################
 244 # resharded bucket
 245
 246 bkt=resharded-bkt-1
 247
 248 mys3cmd mb s3://$bkt
 249
 250 for f in $(seq 8) ; do
 251     dest_obj="reshard-obj-${f}"
 252     mys3cmd put -q $big_obj s3://${bkt}/$dest_obj
 253 done
 254
 255 radosgw-admin bucket reshard --num-shards 3 --bucket=$bkt --yes-i-really-mean-it
 256 radosgw-admin bucket reshard --num-shards 5 --bucket=$bkt --yes-i-really-mean-it
 257
 258 ####################################
 259 # versioned bucket
 260
 261 if true ;then
 262     echo "WARNING: versioned bucket test currently turned off"
 263 else
 264     bkt=versioned-bkt-1
 265
 266     mys3cmd mb s3://$bkt
 267
 268     # bucket-enable-versioning $bkt
 269
 270     for f in $(seq 3) ;do
 271         for g in $(seq 10) ;do
 272             dest_obj="versioned-obj-${g}"
 273             mys3cmd put -q $big_obj s3://${bkt}/$dest_obj
 274         done
 275     done
 276
 277     for g in $(seq 1 2 10) ;do
 278         dest_obj="versioned-obj-${g}"
 279         mys3cmd rm s3://${bkt}/$dest_obj
 280     done
 281 fi
 282
 283 ############################################################
 284 # copy small objects
 285
 286 o_bkt="orig-bkt-1"
 287 d_bkt="copy-bkt-1"
 288 mys3cmd mb s3://$o_bkt
 289
 290 for f in $(seq 4) ;do
 291     dest_obj="orig-obj-$f"
 292     mys3cmd put -q $big_obj s3://${o_bkt}/$dest_obj
 293 done
 294
 295 mys3cmd mb s3://$d_bkt
 296
 297 mys3cmd cp s3://${o_bkt}/orig-obj-1 s3://${d_bkt}/copied-obj-1
 298 mys3cmd cp s3://${o_bkt}/orig-obj-3 s3://${d_bkt}/copied-obj-3
 299
 300 for f in $(seq 5 6) ;do
 301     dest_obj="orig-obj-$f"
 302     mys3cmd put -q $big_obj s3://${d_bkt}/$dest_obj
 303 done
 304
 305 ############################################################
 306 # copy small objects and delete original
 307
 308 o_bkt="orig-bkt-2"
 309 d_bkt="copy-bkt-2"
 310
 311 mys3cmd mb s3://$o_bkt
 312
 313 for f in $(seq 4) ;do
 314     dest_obj="orig-obj-$f"
 315     mys3cmd put -q $big_obj s3://${o_bkt}/$dest_obj
 316 done
 317
 318 mys3cmd mb s3://$d_bkt
 319
 320 mys3cmd cp s3://${o_bkt}/orig-obj-1 s3://${d_bkt}/copied-obj-1
 321 mys3cmd cp s3://${o_bkt}/orig-obj-3 s3://${d_bkt}/copied-obj-3
 322
 323 for f in $(seq 5 6) ;do
 324     dest_obj="orig-obj-$f"
 325     mys3cmd put -q $big_obj s3://${d_bkt}/$dest_obj
 326 done
 327
 328 mys3cmd rb --recursive s3://${o_bkt}
 329
 330 ############################################################
 331 # copy multipart objects
 332
 333 o_bkt="orig-mp-bkt-3"
 334 d_bkt="copy-mp-bkt-3"
 335
 336 mys3cmd mb s3://$o_bkt
 337
 338 for f in $(seq 2) ;do
 339     dest_obj="orig-multipart-obj-$f"
 340     mys3cmd put -q $huge_obj s3://${o_bkt}/$dest_obj
 341 done
 342
 343 mys3cmd mb s3://$d_bkt
 344
 345 mys3cmd cp s3://${o_bkt}/orig-multipart-obj-1 \
 346         s3://${d_bkt}/copied-multipart-obj-1
 347
 348 for f in $(seq 5 5) ;do
 349     dest_obj="orig-multipart-obj-$f"
 350     mys3cmd put -q $huge_obj s3://${d_bkt}/$dest_obj
 351 done
 352
 353
 354 ############################################################
 355 # copy multipart objects and delete original
 356
 357 o_bkt="orig-mp-bkt-4"
 358 d_bkt="copy-mp-bkt-4"
 359
 360 mys3cmd mb s3://$o_bkt
 361
 362 for f in $(seq 2) ;do
 363     dest_obj="orig-multipart-obj-$f"
 364     mys3cmd put -q $huge_obj s3://${o_bkt}/$dest_obj
 365 done
 366
 367 mys3cmd mb s3://$d_bkt
 368
 369 mys3cmd cp s3://${o_bkt}/orig-multipart-obj-1 \
 370         s3://${d_bkt}/copied-multipart-obj-1
 371
 372 for f in $(seq 5 5) ;do
 373     dest_obj="orig-multipart-obj-$f"
 374     mys3cmd put -q $huge_obj s3://${d_bkt}/$dest_obj
 375 done
 376
 377 mys3cmd rb --recursive s3://$o_bkt
 378
 379 ########################################################################
 380 # SWIFT TESTS
 381
 382 # 600MB
 383 segment_size=629145600
 384
 385 ############################################################
 386 # plain test
 387
 388 for f in $(seq 4) ;do
 389     myswift upload swift-plain-ctr $big_obj --object-name swift-obj-$f
 390 done
 391
 392 ############################################################
 393 # zero-len test
 394
 395 myswift upload swift-zerolen-ctr $empty_obj --object-name subdir/
 396 myswift upload swift-zerolen-ctr $big_obj --object-name subdir/abc1
 397 myswift upload swift-zerolen-ctr $empty_obj --object-name subdir/empty1
 398 myswift upload swift-zerolen-ctr $big_obj --object-name subdir/xyz1
 399
 400 ############################################################
 401 # dlo test
 402
 403 # upload in 300MB segments
 404 myswift upload swift-dlo-ctr $huge_obj --object-name dlo-obj-1 \
 405       -S $segment_size
 406
 407 ############################################################
 408 # slo test
 409
 410 # upload in 300MB segments
 411 myswift upload swift-slo-ctr $huge_obj --object-name slo-obj-1 \
 412       -S $segment_size --use-slo
 413
 414 ############################################################
 415 # large object copy test
 416
 417 # upload in 300MB segments
 418 o_ctr=swift-orig-ctr
 419 o_obj=slo-orig-obj-1
 420 d_ctr=swift-copy-ctr
 421 d_obj=slo-copy-obj-1
 422 myswift upload $o_ctr $big_obj --object-name $o_obj
 423
 424 myswift copy --destination /${d_ctr}/${d_obj} \
 425       $o_ctr $o_obj
 426
 427 myswift delete $o_ctr $o_obj
 428
 429 ############################################################
 430 # huge dlo object copy test
 431
 432 o_ctr=swift-orig-dlo-ctr-1
 433 o_obj=dlo-orig-dlo-obj-1
 434 d_ctr=swift-copy-dlo-ctr-1
 435 d_obj=dlo-copy-dlo-obj-1
 436
 437 myswift upload $o_ctr $huge_obj --object-name $o_obj \
 438       -S $segment_size
 439
 440 myswift copy --destination /${d_ctr}/${d_obj} \
 441       $o_ctr $o_obj
 442
 443 ############################################################
 444 # huge dlo object copy and orig delete
 445
 446 o_ctr=swift-orig-dlo-ctr-2
 447 o_obj=dlo-orig-dlo-obj-2
 448 d_ctr=swift-copy-dlo-ctr-2
 449 d_obj=dlo-copy-dlo-obj-2
 450
 451 myswift upload $o_ctr $huge_obj --object-name $o_obj \
 452       -S $segment_size
 453
 454 myswift copy --destination /${d_ctr}/${d_obj} \
 455       $o_ctr $o_obj
 456
 457 myswift delete $o_ctr $o_obj
 458
 459 ############################################################
 460 # huge slo object copy test
 461
 462 o_ctr=swift-orig-slo-ctr-1
 463 o_obj=slo-orig-slo-obj-1
 464 d_ctr=swift-copy-slo-ctr-1
 465 d_obj=slo-copy-slo-obj-1
 466 myswift upload $o_ctr $huge_obj --object-name $o_obj \
 467       -S $segment_size --use-slo
 468
 469 myswift copy --destination /${d_ctr}/${d_obj} $o_ctr $o_obj
 470
 471 ############################################################
 472 # huge slo object copy test and orig delete
 473
 474 o_ctr=swift-orig-slo-ctr-2
 475 o_obj=slo-orig-slo-obj-2
 476 d_ctr=swift-copy-slo-ctr-2
 477 d_obj=slo-copy-slo-obj-2
 478 myswift upload $o_ctr $huge_obj --object-name $o_obj \
 479       -S $segment_size --use-slo
 480
 481 myswift copy --destination /${d_ctr}/${d_obj} $o_ctr $o_obj
 482
 483 myswift delete $o_ctr $o_obj
 484
 485 ########################################################################
 486 # FORCE GARBAGE COLLECTION
 487
 488 sleep 6 # since for testing age at which gc can happen is 5 secs
 489 radosgw-admin gc process --include-all
 490
 491
 492 ########################################
 493 # DO ORPHAN LIST
 494
 495 pool="default.rgw.buckets.data"
 496
 497 rgw-orphan-list $pool
 498
 499 # we only expect there to be one output file, but loop just in case
 500 ol_error=""
 501 for f in orphan-list-*.out ; do
 502     if [ -s "$f"  ] ;then # if file non-empty
 503         ol_error="${ol_error}:$f"
 504         echo "One ore more orphans found in $f:"
 505         cat "$f"
 506     fi
 507 done
 508
 509 if [ -n "$ol_error" ] ;then
 510     echo "ERROR: orphans found when none expected"
 511     exit 1
 512 fi
 513
 514 ########################################################################
 515 # CLEAN UP
 516
 517 rm -f $empty_obj $big_obj $huge_obj $s3config
 518
 519 success