ceph/qa/standalone/osd/osd-recovery-stats.sh

   1 #!/usr/bin/env bash
   2 #
   3 # Copyright (C) 2017 Red Hat <contact@redhat.com>
   4 #
   5 # Author: David Zafman <dzafman@redhat.com>
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU Library Public License as published by
   9 # the Free Software Foundation; either version 2, or (at your option)
  10 # any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Library Public License for more details.
  16 #
  17
  18 source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
  19
  20 function run() {
  21     local dir=$1
  22     shift
  23
  24     # Fix port????
  25     export CEPH_MON="127.0.0.1:7115" # git grep '\<7115\>' : there must be only one
  26     export CEPH_ARGS
  27     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
  28     CEPH_ARGS+="--mon-host=$CEPH_MON "
  29     # so we will not force auth_log_shard to be acting_primary
  30     CEPH_ARGS+="--osd_force_auth_primary_missing_objects=1000000 "
  31     # Use "high_recovery_ops" profile if mclock_scheduler is enabled.
  32     CEPH_ARGS+="--osd-mclock-profile=high_recovery_ops "
  33     export margin=10
  34     export objects=200
  35     export poolname=test
  36
  37     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
  38     for func in $funcs ; do
  39         setup $dir || return 1
  40         $func $dir || return 1
  41         teardown $dir || return 1
  42     done
  43 }
  44
  45 function below_margin() {
  46     local -i check=$1
  47     shift
  48     local -i target=$1
  49
  50     return $(( $check <= $target && $check >= $target - $margin ? 0 : 1 ))
  51 }
  52
  53 function above_margin() {
  54     local -i check=$1
  55     shift
  56     local -i target=$1
  57
  58     return $(( $check >= $target && $check <= $target + $margin ? 0 : 1 ))
  59 }
  60
  61 FIND_UPACT='grep "pg[[]${PG}.*recovering.*update_calc_stats " $log | tail -1 | sed "s/.*[)] \([[][^ p]*\).*$/\1/"'
  62 FIND_FIRST='grep "pg[[]${PG}.*recovering.*update_calc_stats $which " $log | grep -F " ${UPACT}${addp}" | grep -v est | head -1 | sed "s/.* \([0-9]*\)$/\1/"'
  63 FIND_LAST='grep "pg[[]${PG}.*recovering.*update_calc_stats $which " $log | tail -1 | sed "s/.* \([0-9]*\)$/\1/"'
  64
  65 function check() {
  66     local dir=$1
  67     local PG=$2
  68     local primary=$3
  69     local type=$4
  70     local degraded_start=$5
  71     local degraded_end=$6
  72     local misplaced_start=$7
  73     local misplaced_end=$8
  74     local primary_start=${9:-}
  75     local primary_end=${10:-}
  76
  77     local log=$dir/osd.${primary}.log
  78
  79     local addp=" "
  80     if [ "$type" = "erasure" ];
  81     then
  82       addp="p"
  83     fi
  84
  85     UPACT=$(eval $FIND_UPACT)
  86
  87     # Check 3rd line at start because of false recovery starts
  88     local which="degraded"
  89     FIRST=$(eval $FIND_FIRST)
  90     below_margin $FIRST $degraded_start || return 1
  91     LAST=$(eval $FIND_LAST)
  92     above_margin $LAST $degraded_end || return 1
  93
  94     # Check 3rd line at start because of false recovery starts
  95     which="misplaced"
  96     FIRST=$(eval $FIND_FIRST)
  97     below_margin $FIRST $misplaced_start || return 1
  98     LAST=$(eval $FIND_LAST)
  99     above_margin $LAST $misplaced_end || return 1
 100
 101     # This is the value of set into MISSING_ON_PRIMARY
 102     if [ -n "$primary_start" ];
 103     then
 104       which="shard $primary"
 105       FIRST=$(eval $FIND_FIRST)
 106       below_margin $FIRST $primary_start || return 1
 107       LAST=$(eval $FIND_LAST)
 108       above_margin $LAST $primary_end || return 1
 109     fi
 110 }
 111
 112 # [1,0,?] -> [1,2,4]
 113 # degraded 500 -> 0
 114 # active+recovering+degraded
 115
 116 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                      STATE_STAMP                VERSION REPORTED UP      UP_PRIMARY ACTING  ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
 117 # 1.0         500                  0      500         0       0     0 500      500 active+recovering+degraded 2017-11-17 19:27:36.493828  28'500   32:603 [1,2,4]          1 [1,2,4]              1        0'0 2017-11-17 19:27:05.915467             0'0 2017-11-17 19:27:05.915467
 118 function do_recovery_out1() {
 119     local dir=$1
 120     shift
 121     local type=$1
 122
 123     run_mon $dir a || return 1
 124     run_mgr $dir x || return 1
 125     run_osd $dir 0 || return 1
 126     run_osd $dir 1 || return 1
 127     run_osd $dir 2 || return 1
 128     run_osd $dir 3 || return 1
 129     run_osd $dir 4 || return 1
 130     run_osd $dir 5 || return 1
 131
 132     if [ $type = "erasure" ];
 133     then
 134         ceph osd erasure-code-profile set myprofile plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd
 135         create_pool $poolname 1 1 $type myprofile
 136     else
 137         create_pool $poolname 1 1 $type
 138     fi
 139
 140     wait_for_clean || return 1
 141
 142     for i in $(seq 1 $objects)
 143     do
 144         rados -p $poolname put obj$i /dev/null
 145     done
 146
 147     local primary=$(get_primary $poolname obj1)
 148     local PG=$(get_pg $poolname obj1)
 149     # Only 2 OSDs so only 1 not primary
 150     local otherosd=$(get_not_primary $poolname obj1)
 151
 152     ceph osd set norecover
 153     kill $(cat $dir/osd.${otherosd}.pid)
 154     ceph osd down osd.${otherosd}
 155     ceph osd out osd.${otherosd}
 156     ceph osd unset norecover
 157     ceph tell osd.$(get_primary $poolname obj1) debug kick_recovery_wq 0
 158     sleep 2
 159
 160     wait_for_clean || return 1
 161
 162     check $dir $PG $primary $type $objects 0 0 0 || return 1
 163
 164     delete_pool $poolname
 165     kill_daemons $dir || return 1
 166 }
 167
 168 function TEST_recovery_replicated_out1() {
 169     local dir=$1
 170
 171     do_recovery_out1 $dir replicated || return 1
 172 }
 173
 174 function TEST_recovery_erasure_out1() {
 175     local dir=$1
 176
 177     do_recovery_out1 $dir erasure || return 1
 178 }
 179
 180 # [0, 1] -> [2,3,4,5]
 181 # degraded 1000 -> 0
 182 # misplaced 1000 -> 0
 183 # missing on primary 500 -> 0
 184
 185 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                      STATE_STAMP                VERSION REPORTED UP        UP_PRIMARY ACTING    ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
 186 # 1.0         500                500     1000      1000       0     0 500      500 active+recovering+degraded 2017-10-27 09:38:37.453438  22'500   25:394 [2,4,3,5]          2 [2,4,3,5]              2        0'0 2017-10-27 09:37:58.046748             0'0 2017-10-27 09:37:58.046748
 187 function TEST_recovery_sizeup() {
 188     local dir=$1
 189
 190     run_mon $dir a || return 1
 191     run_mgr $dir x || return 1
 192     run_osd $dir 0 || return 1
 193     run_osd $dir 1 || return 1
 194     run_osd $dir 2 || return 1
 195     run_osd $dir 3 || return 1
 196     run_osd $dir 4 || return 1
 197     run_osd $dir 5 || return 1
 198
 199     create_pool $poolname 1 1
 200     ceph osd pool set $poolname size 2
 201
 202     wait_for_clean || return 1
 203
 204     for i in $(seq 1 $objects)
 205     do
 206         rados -p $poolname put obj$i /dev/null
 207     done
 208
 209     local primary=$(get_primary $poolname obj1)
 210     local PG=$(get_pg $poolname obj1)
 211     # Only 2 OSDs so only 1 not primary
 212     local otherosd=$(get_not_primary $poolname obj1)
 213
 214     ceph osd set norecover
 215     ceph osd out osd.$primary osd.$otherosd
 216     ceph osd pool set test size 4
 217     ceph osd unset norecover
 218     # Get new primary
 219     primary=$(get_primary $poolname obj1)
 220
 221     ceph tell osd.${primary} debug kick_recovery_wq 0
 222     sleep 2
 223
 224     wait_for_clean || return 1
 225
 226     local degraded=$(expr $objects \* 2)
 227     local misplaced=$(expr $objects \* 2)
 228     local log=$dir/osd.${primary}.log
 229     check $dir $PG $primary replicated $degraded 0 $misplaced 0 $objects 0 || return 1
 230
 231     delete_pool $poolname
 232     kill_daemons $dir || return 1
 233 }
 234
 235 # [0, 1, 2, 4] -> [3, 5]
 236 # misplaced 1000 -> 0
 237 # missing on primary 500 -> 0
 238 # active+recovering+degraded
 239
 240 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                      STATE_STAMP                VERSION REPORTED UP    UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
 241 # 1.0         500                500         0      1000       0     0 500      500 active+recovering+degraded 2017-10-27 09:34:50.012261  22'500   27:118 [3,5]          3  [3,5]              3        0'0 2017-10-27 09:34:08.617248             0'0 2017-10-27 09:34:08.617248
 242 function TEST_recovery_sizedown() {
 243     local dir=$1
 244
 245     run_mon $dir a || return 1
 246     run_mgr $dir x || return 1
 247     run_osd $dir 0 || return 1
 248     run_osd $dir 1 || return 1
 249     run_osd $dir 2 || return 1
 250     run_osd $dir 3 || return 1
 251     run_osd $dir 4 || return 1
 252     run_osd $dir 5 || return 1
 253
 254     create_pool $poolname 1 1
 255     ceph osd pool set $poolname size 4
 256
 257     wait_for_clean || return 1
 258
 259     for i in $(seq 1 $objects)
 260     do
 261         rados -p $poolname put obj$i /dev/null
 262     done
 263
 264     local primary=$(get_primary $poolname obj1)
 265     local PG=$(get_pg $poolname obj1)
 266     # Only 2 OSDs so only 1 not primary
 267     local allosds=$(get_osds $poolname obj1)
 268
 269     ceph osd set norecover
 270     for osd in $allosds
 271     do
 272         ceph osd out osd.$osd
 273     done
 274
 275     ceph osd pool set test size 2
 276     ceph osd unset norecover
 277     ceph tell osd.$(get_primary $poolname obj1) debug kick_recovery_wq 0
 278     sleep 2
 279
 280     wait_for_clean || return 1
 281
 282     # Get new primary
 283     primary=$(get_primary $poolname obj1)
 284
 285     local misplaced=$(expr $objects \* 2)
 286     local log=$dir/osd.${primary}.log
 287     check $dir $PG $primary replicated 0 0 $misplaced 0 || return 1
 288
 289     UPACT=$(grep "pg[[]${PG}.*recovering.*update_calc_stats " $log | tail -1 | sed "s/.*[)] \([[][^ p]*\).*$/\1/")
 290
 291     # This is the value of set into MISSING_ON_PRIMARY
 292     FIRST=$(grep "pg[[]${PG}.*recovering.*update_calc_stats shard $primary " $log | grep -F " $UPACT " | head -1 | sed "s/.* \([0-9]*\)$/\1/")
 293     below_margin $FIRST $objects || return 1
 294     LAST=$(grep "pg[[]${PG}.*recovering.*update_calc_stats shard $primary " $log | tail -1 | sed "s/.* \([0-9]*\)$/\1/")
 295     above_margin $LAST 0 || return 1
 296
 297     delete_pool $poolname
 298     kill_daemons $dir || return 1
 299 }
 300
 301 # [1] -> [1,2]
 302 # degraded 300 -> 200
 303 # active+recovering+undersized+degraded
 304
 305 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                                 STATE_STAMP                VERSION REPORTED UP    UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
 306 # 1.0         100                  0     300         0       0     0 100      100 active+recovering+undersized+degraded 2017-11-17 17:16:15.302943  13'500   16:643 [1,2]          1  [1,2]              1        0'0 2017-11-17 17:15:34.985563             0'0 2017-11-17 17:15:34.985563
 307 function TEST_recovery_undersized() {
 308     local dir=$1
 309
 310     local osds=3
 311     run_mon $dir a || return 1
 312     run_mgr $dir x || return 1
 313     for i in $(seq 0 $(expr $osds - 1))
 314     do
 315       run_osd $dir $i || return 1
 316     done
 317
 318     create_pool $poolname 1 1
 319     ceph osd pool set $poolname size 1 --yes-i-really-mean-it
 320
 321     wait_for_clean || return 1
 322
 323     for i in $(seq 1 $objects)
 324     do
 325         rados -p $poolname put obj$i /dev/null
 326     done
 327
 328     local primary=$(get_primary $poolname obj1)
 329     local PG=$(get_pg $poolname obj1)
 330
 331     ceph osd set norecover
 332     # Mark any osd not the primary (only 1 replica so also has no replica)
 333     for i in $(seq 0 $(expr $osds - 1))
 334     do
 335       if [ $i = $primary ];
 336       then
 337         continue
 338       fi
 339       ceph osd out osd.$i
 340       break
 341     done
 342     ceph osd pool set test size 4
 343     ceph osd unset norecover
 344     ceph tell osd.$(get_primary $poolname obj1) debug kick_recovery_wq 0
 345     # Give extra sleep time because code below doesn't have the sophistication of wait_for_clean()
 346     sleep 10
 347     flush_pg_stats || return 1
 348
 349     # Wait for recovery to finish
 350     # Can't use wait_for_clean() because state goes from active+recovering+undersized+degraded
 351     # to  active+undersized+degraded
 352     for i in $(seq 1 300)
 353     do
 354       if ceph pg dump pgs | grep ^$PG | grep -qv recovering
 355       then
 356           break
 357       fi
 358       if [ $i = "300" ];
 359       then
 360           echo "Timeout waiting for recovery to finish"
 361           return 1
 362       fi
 363       sleep 1
 364     done
 365
 366     # Get new primary
 367     primary=$(get_primary $poolname obj1)
 368     local log=$dir/osd.${primary}.log
 369
 370     local first_degraded=$(expr $objects \* 3)
 371     local last_degraded=$(expr $objects \* 2)
 372     check $dir $PG $primary replicated $first_degraded $last_degraded 0 0 || return 1
 373
 374     delete_pool $poolname
 375     kill_daemons $dir || return 1
 376 }
 377
 378 # [1,0,2] -> [1,3,NONE]/[1,3,2]
 379 # degraded 100 -> 0
 380 # misplaced 100 -> 100
 381 # active+recovering+degraded+remapped
 382
 383 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                               STATE_STAMP                VERSION REPORTED UP         UP_PRIMARY ACTING  ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
 384 # 1.0         100                  0      100        100       0     0 100      100 active+recovering+degraded+remapped 2017-11-27 21:24:20.851243  18'500   23:618 [1,3,NONE]          1 [1,3,2]              1        0'0 2017-11-27 21:23:39.395242             0'0 2017-11-27 21:23:39.395242
 385 function TEST_recovery_erasure_remapped() {
 386     local dir=$1
 387
 388     run_mon $dir a || return 1
 389     run_mgr $dir x || return 1
 390     run_osd $dir 0 || return 1
 391     run_osd $dir 1 || return 1
 392     run_osd $dir 2 || return 1
 393     run_osd $dir 3 || return 1
 394
 395     ceph osd erasure-code-profile set myprofile plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd
 396     create_pool $poolname 1 1 erasure myprofile
 397     ceph osd pool set $poolname min_size 2
 398
 399     wait_for_clean || return 1
 400
 401     for i in $(seq 1 $objects)
 402     do
 403         rados -p $poolname put obj$i /dev/null
 404     done
 405
 406     local primary=$(get_primary $poolname obj1)
 407     local PG=$(get_pg $poolname obj1)
 408     local otherosd=$(get_not_primary $poolname obj1)
 409
 410     ceph osd set norecover
 411     kill $(cat $dir/osd.${otherosd}.pid)
 412     ceph osd down osd.${otherosd}
 413     ceph osd out osd.${otherosd}
 414
 415     # Mark osd not the primary and not down/out osd as just out
 416     for i in 0 1 2 3
 417     do
 418       if [ $i = $primary ];
 419       then
 420         continue
 421       fi
 422       if [ $i = $otherosd ];
 423       then
 424         continue
 425       fi
 426       ceph osd out osd.$i
 427       break
 428     done
 429     ceph osd unset norecover
 430     ceph tell osd.$(get_primary $poolname obj1) debug kick_recovery_wq 0
 431     sleep 2
 432
 433     wait_for_clean || return 1
 434
 435     local log=$dir/osd.${primary}.log
 436     check $dir $PG $primary erasure $objects 0 $objects $objects || return 1
 437
 438     delete_pool $poolname
 439     kill_daemons $dir || return 1
 440 }
 441
 442 function TEST_recovery_multi() {
 443     local dir=$1
 444
 445     local osds=6
 446     run_mon $dir a || return 1
 447     run_mgr $dir x || return 1
 448     for i in $(seq 0 $(expr $osds - 1))
 449     do
 450       run_osd $dir $i || return 1
 451     done
 452
 453     create_pool $poolname 1 1
 454     ceph osd pool set $poolname size 3
 455     ceph osd pool set $poolname min_size 1
 456
 457     wait_for_clean || return 1
 458
 459     rados -p $poolname put obj1 /dev/null
 460
 461     local primary=$(get_primary $poolname obj1)
 462     local otherosd=$(get_not_primary $poolname obj1)
 463
 464     ceph osd set noout
 465     ceph osd set norecover
 466     kill $(cat $dir/osd.${otherosd}.pid)
 467     ceph osd down osd.${otherosd}
 468
 469     local half=$(expr $objects / 2)
 470     for i in $(seq 2 $half)
 471     do
 472         rados -p $poolname put obj$i /dev/null
 473     done
 474
 475     kill $(cat $dir/osd.${primary}.pid)
 476     ceph osd down osd.${primary}
 477     activate_osd $dir ${otherosd}
 478     sleep 3
 479
 480     for i in $(seq $(expr $half + 1) $objects)
 481     do
 482         rados -p $poolname put obj$i /dev/null
 483     done
 484
 485     local PG=$(get_pg $poolname obj1)
 486     local otherosd=$(get_not_primary $poolname obj$objects)
 487
 488     ceph osd unset noout
 489     ceph osd out osd.$primary osd.$otherosd
 490     activate_osd $dir ${primary}
 491     sleep 3
 492
 493     ceph osd pool set test size 4
 494     ceph osd unset norecover
 495     ceph tell osd.$(get_primary $poolname obj1) debug kick_recovery_wq 0
 496     sleep 2
 497
 498     wait_for_clean || return 1
 499
 500     # Get new primary
 501     primary=$(get_primary $poolname obj1)
 502
 503     local log=$dir/osd.${primary}.log
 504     check $dir $PG $primary replicated 399 0 300 0 99 0 || return 1
 505
 506     delete_pool $poolname
 507     kill_daemons $dir || return 1
 508 }
 509
 510 main osd-recovery-stats "$@"
 511
 512 # Local Variables:
 513 # compile-command: "make -j4 && ../qa/run-standalone.sh osd-recovery-stats.sh"
 514 # End: