ceph/qa/workunits/rgw/test_rgw_orphan_list.sh

   1 #!/usr/bin/env bash
   2
   3 set -ex
   4
   5 # if defined, debug messages will be displayed and prepended with the string
   6 # debug="DEBUG"
   7
   8 huge_size=2222 # in megabytes
   9 big_size=6 # in megabytes
  10
  11 huge_obj=/tmp/huge_obj.temp.$$
  12 big_obj=/tmp/big_obj.temp.$$
  13 empty_obj=/tmp/empty_obj.temp.$$
  14
  15 fifo=/tmp/orphan-fifo.$$
  16 awscli_dir=${HOME}/awscli_temp
  17 export PATH=${PATH}:${awscli_dir}
  18
  19 rgw_host=$(hostname --fqdn)
  20 if echo "$rgw_host" | grep -q '\.' ; then
  21     :
  22 else
  23     host_domain=".front.sepia.ceph.com"
  24     echo "WARNING: rgw hostname -- $rgw_host -- does not appear to be fully qualified; PUNTING and appending $host_domain"
  25     rgw_host="${rgw_host}${host_domain}"
  26 fi
  27 rgw_port=80
  28
  29 echo "Fully Qualified Domain Name: $rgw_host"
  30
  31 success() {
  32     echo OK.
  33     exit 0
  34 }
  35
  36 ########################################################################
  37 # INSTALL AND CONFIGURE TOOLING
  38
  39 install_awscli() {
  40     # NB: this does verify authenticity and integrity of downloaded
  41     # file; see
  42     # https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html
  43     here="$(pwd)"
  44     cd "$HOME"
  45     curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
  46     unzip awscliv2.zip
  47     mkdir -p $awscli_dir
  48     ./aws/install -i $awscli_dir
  49     cd "$here"
  50 }
  51
  52 uninstall_awscli() {
  53     here="$(pwd)"
  54     cd "$HOME"
  55     rm -rf $awscli_dir ./aws awscliv2.zip
  56     cd "$here"
  57 }
  58
  59 sudo dnf install -y s3cmd
  60
  61 sudo yum install python3-setuptools
  62 sudo yum -y install python3-pip
  63 sudo pip3 install --upgrade setuptools
  64 sudo pip3 install python-swiftclient
  65
  66 # get ready for transition from s3cmd to awscli
  67 if false ;then
  68     install_awscli
  69     aws --version
  70     uninstall_awscli
  71 fi
  72
  73 s3config=/tmp/s3config.$$
  74
  75 # do not include the port when it is 80; the host base is used in the
  76 # v4 signature and it needs to follow this convention for signatures
  77 # to match
  78 if [ "$rgw_port" -ne 80 ] ;then
  79     s3_host_base="${rgw_host}:${rgw_port}"
  80 else
  81     s3_host_base="$rgw_host"
  82 fi
  83
  84 cat >${s3config} <<EOF
  85 [default]
  86 host_base = $s3_host_base
  87 access_key = 0555b35654ad1656d804
  88 secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==
  89 bucket_location = us-east-1
  90 check_ssl_certificate = True
  91 check_ssl_hostname = True
  92 default_mime_type = binary/octet-stream
  93 delete_removed = False
  94 dry_run = False
  95 enable_multipart = True
  96 encoding = UTF-8
  97 encrypt = False
  98 follow_symlinks = False
  99 force = False
 100 guess_mime_type = True
 101 host_bucket = anything.with.three.dots
 102 multipart_chunk_size_mb = 15
 103 multipart_max_chunks = 10000
 104 recursive = False
 105 recv_chunk = 65536
 106 send_chunk = 65536
 107 signature_v2 = False
 108 socket_timeout = 300
 109 use_https = False
 110 use_mime_magic = True
 111 verbosity = WARNING
 112 EOF
 113
 114
 115 # set up swift authentication
 116 export ST_AUTH=http://${rgw_host}:${rgw_port}/auth/v1.0
 117 export ST_USER=test:tester
 118 export ST_KEY=testing
 119
 120 create_users() {
 121     # Create S3 user
 122     local akey='0555b35654ad1656d804'
 123     local skey='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=='
 124     radosgw-admin user create --uid testid \
 125                   --access-key $akey --secret $skey \
 126                   --display-name 'M. Tester' --email tester@ceph.com
 127
 128     # Create Swift user
 129     radosgw-admin user create --subuser=test:tester \
 130                   --display-name=Tester-Subuser --key-type=swift \
 131                   --secret=testing --access=full
 132 }
 133
 134 myswift() {
 135     if [ -n "$debug" ] ;then
 136         echo "${debug}: swift --verbose --debug $@"
 137     fi
 138     swift --verbose --debug "$@"
 139     local code=$?
 140     if [ $code -ne 0 ] ;then
 141         echo "ERROR: code = $code ; command = s3cmd --config=${s3config} --verbose --debug "$@""
 142         exit $code
 143     fi
 144 }
 145
 146 mys3cmd() {
 147     if [ -n "$debug" ] ;then
 148         echo "${debug}: s3cmd --config=${s3config} --verbose --debug $@"
 149     fi
 150     s3cmd --config=${s3config} --verbose --debug "$@"
 151     local code=$?
 152     if [ $code -ne 0 ] ;then
 153         echo "ERROR: code = $code ; command = s3cmd --config=${s3config} --verbose --debug "$@""
 154         exit $code
 155     fi
 156 }
 157
 158 mys3uploadkill() {
 159     if [ $# -ne 5 ] ;then
 160         echo "$0: error expecting 5 arguments"
 161         exit 1
 162     fi
 163
 164     set -v
 165     local_file="$1"
 166     remote_bkt="$2"
 167     remote_obj="$3"
 168     fifo="$4"
 169     stop_part="$5"
 170
 171     mkfifo $fifo
 172
 173     s3cmd --config=${s3config} put $local_file \
 174           s3://${remote_bkt}/${remote_obj} \
 175           --progress \
 176           --multipart-chunk-size-mb=5 >$fifo &
 177     set +e # don't allow errors to stop script
 178     while read line ;do
 179         echo "$line" | grep --quiet "part $stop_part "
 180         if [ ${PIPESTATUS[1]} -eq 0 ] ;then
 181             kill -9 $(jobs -p)
 182             break
 183         fi
 184     done <$fifo
 185     set -e
 186
 187     rm -f $fifo
 188 }
 189
 190 mys3upload() {
 191     obj=$1
 192     bucket=$2
 193     dest_obj=$3
 194
 195     mys3cmd put -q $obj s3://${bucket}/$dest_obj
 196 }
 197
 198 ########################################################################
 199 # PREP
 200
 201 create_users
 202 dd if=/dev/urandom of=$big_obj bs=1M count=${big_size}
 203 dd if=/dev/urandom of=$huge_obj bs=1M count=${huge_size}
 204 touch $empty_obj
 205
 206 quick_tests() {
 207     echo TRY A SWIFT COMMAND
 208     myswift upload swift-plain-ctr $big_obj --object-name swift-obj-test
 209     myswift list
 210     myswift list swift-plain-ctr
 211
 212     echo TRY A RADOSGW-ADMIN COMMAND
 213     radosgw-admin bucket list # make sure rgw is up and running
 214 }
 215
 216 ########################################################################
 217 # S3 TESTS
 218
 219 ####################################
 220 # regular multipart test
 221
 222 mys3cmd mb s3://multipart-bkt
 223 mys3upload $huge_obj multipart-bkt multipart-obj
 224 mys3cmd ls
 225 mys3cmd ls s3://multipart-bkt
 226
 227 ####################################
 228 # multipart test with incomplete uploads
 229
 230 bkt="incomplete-mp-bkt-1"
 231
 232 mys3cmd mb s3://$bkt
 233 mys3uploadkill $huge_obj $bkt incomplete-mp-obj-1 $fifo 20
 234 mys3uploadkill $huge_obj $bkt incomplete-mp-obj-2 $fifo 100
 235
 236 ####################################
 237 # resharded bucket
 238
 239 bkt=resharded-bkt-1
 240
 241 mys3cmd mb s3://$bkt
 242
 243 for f in $(seq 8) ; do
 244     dest_obj="reshard-obj-${f}"
 245     mys3cmd put -q $big_obj s3://${bkt}/$dest_obj
 246 done
 247
 248 radosgw-admin bucket reshard --num-shards 3 --bucket=$bkt --yes-i-really-mean-it
 249 radosgw-admin bucket reshard --num-shards 5 --bucket=$bkt --yes-i-really-mean-it
 250
 251 ####################################
 252 # versioned bucket
 253
 254 if true ;then
 255     echo "WARNING: versioned bucket test currently turned off"
 256 else
 257     bkt=versioned-bkt-1
 258
 259     mys3cmd mb s3://$bkt
 260
 261     # bucket-enable-versioning $bkt
 262
 263     for f in $(seq 3) ;do
 264         for g in $(seq 10) ;do
 265             dest_obj="versioned-obj-${g}"
 266             mys3cmd put -q $big_obj s3://${bkt}/$dest_obj
 267         done
 268     done
 269
 270     for g in $(seq 1 2 10) ;do
 271         dest_obj="versioned-obj-${g}"
 272         mys3cmd rm s3://${bkt}/$dest_obj
 273     done
 274 fi
 275
 276 ############################################################
 277 # copy small objects
 278
 279 o_bkt="orig-bkt-1"
 280 d_bkt="copy-bkt-1"
 281 mys3cmd mb s3://$o_bkt
 282
 283 for f in $(seq 4) ;do
 284     dest_obj="orig-obj-$f"
 285     mys3cmd put -q $big_obj s3://${o_bkt}/$dest_obj
 286 done
 287
 288 mys3cmd mb s3://$d_bkt
 289
 290 mys3cmd cp s3://${o_bkt}/orig-obj-1 s3://${d_bkt}/copied-obj-1
 291 mys3cmd cp s3://${o_bkt}/orig-obj-3 s3://${d_bkt}/copied-obj-3
 292
 293 for f in $(seq 5 6) ;do
 294     dest_obj="orig-obj-$f"
 295     mys3cmd put -q $big_obj s3://${d_bkt}/$dest_obj
 296 done
 297
 298 ############################################################
 299 # copy small objects and delete original
 300
 301 o_bkt="orig-bkt-2"
 302 d_bkt="copy-bkt-2"
 303
 304 mys3cmd mb s3://$o_bkt
 305
 306 for f in $(seq 4) ;do
 307     dest_obj="orig-obj-$f"
 308     mys3cmd put -q $big_obj s3://${o_bkt}/$dest_obj
 309 done
 310
 311 mys3cmd mb s3://$d_bkt
 312
 313 mys3cmd cp s3://${o_bkt}/orig-obj-1 s3://${d_bkt}/copied-obj-1
 314 mys3cmd cp s3://${o_bkt}/orig-obj-3 s3://${d_bkt}/copied-obj-3
 315
 316 for f in $(seq 5 6) ;do
 317     dest_obj="orig-obj-$f"
 318     mys3cmd put -q $big_obj s3://${d_bkt}/$dest_obj
 319 done
 320
 321 mys3cmd rb --recursive s3://${o_bkt}
 322
 323 ############################################################
 324 # copy multipart objects
 325
 326 o_bkt="orig-mp-bkt-3"
 327 d_bkt="copy-mp-bkt-3"
 328
 329 mys3cmd mb s3://$o_bkt
 330
 331 for f in $(seq 2) ;do
 332     dest_obj="orig-multipart-obj-$f"
 333     mys3cmd put -q $huge_obj s3://${o_bkt}/$dest_obj
 334 done
 335
 336 mys3cmd mb s3://$d_bkt
 337
 338 mys3cmd cp s3://${o_bkt}/orig-multipart-obj-1 \
 339         s3://${d_bkt}/copied-multipart-obj-1
 340
 341 for f in $(seq 5 5) ;do
 342     dest_obj="orig-multipart-obj-$f"
 343     mys3cmd put -q $huge_obj s3://${d_bkt}/$dest_obj
 344 done
 345
 346
 347 ############################################################
 348 # copy multipart objects and delete original
 349
 350 o_bkt="orig-mp-bkt-4"
 351 d_bkt="copy-mp-bkt-4"
 352
 353 mys3cmd mb s3://$o_bkt
 354
 355 for f in $(seq 2) ;do
 356     dest_obj="orig-multipart-obj-$f"
 357     mys3cmd put -q $huge_obj s3://${o_bkt}/$dest_obj
 358 done
 359
 360 mys3cmd mb s3://$d_bkt
 361
 362 mys3cmd cp s3://${o_bkt}/orig-multipart-obj-1 \
 363         s3://${d_bkt}/copied-multipart-obj-1
 364
 365 for f in $(seq 5 5) ;do
 366     dest_obj="orig-multipart-obj-$f"
 367     mys3cmd put -q $huge_obj s3://${d_bkt}/$dest_obj
 368 done
 369
 370 mys3cmd rb --recursive s3://$o_bkt
 371
 372 ########################################################################
 373 # SWIFT TESTS
 374
 375 # 600MB
 376 segment_size=629145600
 377
 378 ############################################################
 379 # plain test
 380
 381 for f in $(seq 4) ;do
 382     myswift upload swift-plain-ctr $big_obj --object-name swift-obj-$f
 383 done
 384
 385 ############################################################
 386 # zero-len test
 387
 388 myswift upload swift-zerolen-ctr $empty_obj --object-name subdir/
 389 myswift upload swift-zerolen-ctr $big_obj --object-name subdir/abc1
 390 myswift upload swift-zerolen-ctr $empty_obj --object-name subdir/empty1
 391 myswift upload swift-zerolen-ctr $big_obj --object-name subdir/xyz1
 392
 393 ############################################################
 394 # dlo test
 395
 396 # upload in 300MB segments
 397 myswift upload swift-dlo-ctr $huge_obj --object-name dlo-obj-1 \
 398       -S $segment_size
 399
 400 ############################################################
 401 # slo test
 402
 403 # upload in 300MB segments
 404 myswift upload swift-slo-ctr $huge_obj --object-name slo-obj-1 \
 405       -S $segment_size --use-slo
 406
 407 ############################################################
 408 # large object copy test
 409
 410 # upload in 300MB segments
 411 o_ctr=swift-orig-ctr
 412 o_obj=slo-orig-obj-1
 413 d_ctr=swift-copy-ctr
 414 d_obj=slo-copy-obj-1
 415 myswift upload $o_ctr $big_obj --object-name $o_obj
 416
 417 myswift copy --destination /${d_ctr}/${d_obj} \
 418       $o_ctr $o_obj
 419
 420 myswift delete $o_ctr $o_obj
 421
 422 ############################################################
 423 # huge dlo object copy test
 424
 425 o_ctr=swift-orig-dlo-ctr-1
 426 o_obj=dlo-orig-dlo-obj-1
 427 d_ctr=swift-copy-dlo-ctr-1
 428 d_obj=dlo-copy-dlo-obj-1
 429
 430 myswift upload $o_ctr $huge_obj --object-name $o_obj \
 431       -S $segment_size
 432
 433 myswift copy --destination /${d_ctr}/${d_obj} \
 434       $o_ctr $o_obj
 435
 436 ############################################################
 437 # huge dlo object copy and orig delete
 438
 439 o_ctr=swift-orig-dlo-ctr-2
 440 o_obj=dlo-orig-dlo-obj-2
 441 d_ctr=swift-copy-dlo-ctr-2
 442 d_obj=dlo-copy-dlo-obj-2
 443
 444 myswift upload $o_ctr $huge_obj --object-name $o_obj \
 445       -S $segment_size
 446
 447 myswift copy --destination /${d_ctr}/${d_obj} \
 448       $o_ctr $o_obj
 449
 450 myswift delete $o_ctr $o_obj
 451
 452 ############################################################
 453 # huge slo object copy test
 454
 455 o_ctr=swift-orig-slo-ctr-1
 456 o_obj=slo-orig-slo-obj-1
 457 d_ctr=swift-copy-slo-ctr-1
 458 d_obj=slo-copy-slo-obj-1
 459 myswift upload $o_ctr $huge_obj --object-name $o_obj \
 460       -S $segment_size --use-slo
 461
 462 myswift copy --destination /${d_ctr}/${d_obj} $o_ctr $o_obj
 463
 464 ############################################################
 465 # huge slo object copy test and orig delete
 466
 467 o_ctr=swift-orig-slo-ctr-2
 468 o_obj=slo-orig-slo-obj-2
 469 d_ctr=swift-copy-slo-ctr-2
 470 d_obj=slo-copy-slo-obj-2
 471 myswift upload $o_ctr $huge_obj --object-name $o_obj \
 472       -S $segment_size --use-slo
 473
 474 myswift copy --destination /${d_ctr}/${d_obj} $o_ctr $o_obj
 475
 476 myswift delete $o_ctr $o_obj
 477
 478 ########################################################################
 479 # FORCE GARBAGE COLLECTION
 480
 481 sleep 6 # since for testing age at which gc can happen is 5 secs
 482 radosgw-admin gc process --include-all
 483
 484
 485 ########################################
 486 # DO ORPHAN LIST
 487
 488 pool="default.rgw.buckets.data"
 489
 490 rgw-orphan-list $pool
 491
 492 # we only expect there to be one output file, but loop just in case
 493 ol_error=""
 494 for f in orphan-list-*.out ; do
 495     if [ -s "$f"  ] ;then # if file non-empty
 496         ol_error="${ol_error}:$f"
 497         echo "One ore more orphans found in $f:"
 498         cat "$f"
 499     fi
 500 done
 501
 502 if [ -n "$ol_error" ] ;then
 503     echo "ERROR: orphans found when none expected"
 504     exit 1
 505 fi
 506
 507 ########################################################################
 508 # CLEAN UP
 509
 510 rm -f $empty_obj $big_obj $huge_obj $s3config
 511
 512 success