ceph/qa/standalone/osd/osd-backfill-stats.sh

   1 #!/usr/bin/env bash
   2 #
   3 # Copyright (C) 2017 Red Hat <contact@redhat.com>
   4 #
   5 # Author: David Zafman <dzafman@redhat.com>
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU Library Public License as published by
   9 # the Free Software Foundation; either version 2, or (at your option)
  10 # any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Library Public License for more details.
  16 #
  17
  18 source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
  19
  20 function run() {
  21     local dir=$1
  22     shift
  23
  24     # Fix port????
  25     export CEPH_MON="127.0.0.1:7114" # git grep '\<7114\>' : there must be only one
  26     export CEPH_ARGS
  27     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
  28     CEPH_ARGS+="--mon-host=$CEPH_MON "
  29     CEPH_ARGS+="--osd_min_pg_log_entries=5 --osd_max_pg_log_entries=10 "
  30     export margin=10
  31     export objects=200
  32     export poolname=test
  33
  34     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
  35     for func in $funcs ; do
  36         setup $dir || return 1
  37         $func $dir || return 1
  38         teardown $dir || return 1
  39     done
  40 }
  41
  42 function below_margin() {
  43     local -i check=$1
  44     shift
  45     local -i target=$1
  46
  47     return $(( $check <= $target && $check >= $target - $margin ? 0 : 1 ))
  48 }
  49
  50 function above_margin() {
  51     local -i check=$1
  52     shift
  53     local -i target=$1
  54
  55     return $(( $check >= $target && $check <= $target + $margin ? 0 : 1 ))
  56 }
  57
  58 FIND_UPACT='grep "pg[[]${PG}.*backfilling.*_update_calc_stats " $log | tail -1 | sed "s/.*[)] \([[][^ p]*\).*$/\1/"'
  59 FIND_FIRST='grep "pg[[]${PG}.*backfilling.*_update_calc_stats $which " $log | grep -F " ${UPACT}${addp}" | grep -v est | head -1 | sed "s/.* \([0-9]*\)$/\1/"'
  60 FIND_LAST='grep "pg[[]${PG}.*backfilling.*_update_calc_stats $which " $log | tail -1 | sed "s/.* \([0-9]*\)$/\1/"'
  61
  62 function check() {
  63     local dir=$1
  64     local PG=$2
  65     local primary=$3
  66     local type=$4
  67     local degraded_start=$5
  68     local degraded_end=$6
  69     local misplaced_start=$7
  70     local misplaced_end=$8
  71     local primary_start=${9:-}
  72     local primary_end=${10:-}
  73
  74     local log=$(grep -l +backfilling $dir/osd.*.log)
  75     test -n "$log" || return 1
  76
  77     local addp=" "
  78     if [ "$type" = "erasure" ];
  79     then
  80       addp="p"
  81     fi
  82
  83     UPACT=$(eval $FIND_UPACT)
  84     [ -n "$UPACT" ] || return 1
  85
  86     # Check 3rd line at start because of false recovery starts
  87     local which="degraded"
  88     FIRST=$(eval $FIND_FIRST)
  89     [ -n "$FIRST" ] || return 1
  90     below_margin $FIRST $degraded_start || return 1
  91     LAST=$(eval $FIND_LAST)
  92     [ -n "$LAST" ] || return 1
  93     above_margin $LAST $degraded_end || return 1
  94
  95     # Check 3rd line at start because of false recovery starts
  96     which="misplaced"
  97     FIRST=$(eval $FIND_FIRST)
  98     [ -n "$FIRST" ] || return 1
  99     below_margin $FIRST $misplaced_start || return 1
 100     LAST=$(eval $FIND_LAST)
 101     [ -n "$LAST" ] || return 1
 102     above_margin $LAST $misplaced_end || return 1
 103
 104     # This is the value of set into MISSING_ON_PRIMARY
 105     if [ -n "$primary_start" ];
 106     then
 107       which="shard $primary"
 108       FIRST=$(eval $FIND_FIRST)
 109       [ -n "$FIRST" ] || return 1
 110       below_margin $FIRST $primary_start || return 1
 111       LAST=$(eval $FIND_LAST)
 112       [ -n "$LAST" ] || return 1
 113       above_margin $LAST $primary_end || return 1
 114     fi
 115 }
 116
 117 # [1] -> [1, 0, 2]
 118 # degraded 1000 -> 0
 119 # state: active+undersized+degraded+remapped+backfilling
 120
 121 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                                           STATE_STAMP                VERSION REPORTED UP      UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
 122 # 1.0         500                  0      1000         0       0     0 100      100 active+undersized+degraded+remapped+backfilling 2017-10-27 09:44:23.531466  22'500   26:617 [1,0,2]          1    [1]              1        0'0 2017-10-27 09:43:44.654882             0'0 2017-10-27 09:43:44.654882
 123 function TEST_backfill_sizeup() {
 124     local dir=$1
 125
 126     run_mon $dir a || return 1
 127     run_mgr $dir x || return 1
 128     export CEPH_ARGS
 129     run_osd $dir 0 || return 1
 130     run_osd $dir 1 || return 1
 131     run_osd $dir 2 || return 1
 132     run_osd $dir 3 || return 1
 133     run_osd $dir 4 || return 1
 134     run_osd $dir 5 || return 1
 135
 136     create_pool $poolname 1 1
 137     ceph osd pool set $poolname size 1
 138
 139     wait_for_clean || return 1
 140
 141     for i in $(seq 1 $objects)
 142     do
 143         rados -p $poolname put obj$i /dev/null
 144     done
 145
 146     ceph osd pool set $poolname size 3
 147     sleep 15
 148
 149     wait_for_clean || return 1
 150
 151     local primary=$(get_primary $poolname obj1)
 152     local PG=$(get_pg $poolname obj1)
 153
 154     local degraded=$(expr $objects \* 2)
 155     check $dir $PG $primary replicated $degraded 0 0 0 || return 1
 156
 157     delete_pool $poolname
 158     kill_daemons $dir || return 1
 159 }
 160
 161
 162
 163 # [1] -> [0, 2, 4]
 164 # degraded 1000 -> 0
 165 # misplaced 500 -> 0
 166 # state: active+undersized+degraded+remapped+backfilling
 167
 168 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                                           STATE_STAMP                VERSION REPORTED UP      UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
 169 # 1.0         500                  0      1000       500       0     0 100      100 active+undersized+degraded+remapped+backfilling 2017-10-27 09:48:53.326849  22'500   26:603 [0,2,4]          0    [1]              1        0'0 2017-10-27 09:48:13.236253             0'0 2017-10-27 09:48:13.236253
 170 function TEST_backfill_sizeup_out() {
 171     local dir=$1
 172
 173     run_mon $dir a || return 1
 174     run_mgr $dir x || return 1
 175     run_osd $dir 0 || return 1
 176     run_osd $dir 1 || return 1
 177     run_osd $dir 2 || return 1
 178     run_osd $dir 3 || return 1
 179     run_osd $dir 4 || return 1
 180     run_osd $dir 5 || return 1
 181
 182     create_pool $poolname 1 1
 183     ceph osd pool set $poolname size 1
 184
 185     wait_for_clean || return 1
 186
 187     for i in $(seq 1 $objects)
 188     do
 189         rados -p $poolname put obj$i /dev/null
 190     done
 191
 192     local PG=$(get_pg $poolname obj1)
 193     # Remember primary during the backfill
 194     local primary=$(get_primary $poolname obj1)
 195
 196     ceph osd out osd.$primary
 197     ceph osd pool set $poolname size 3
 198     sleep 15
 199
 200     wait_for_clean || return 1
 201
 202     local degraded=$(expr $objects \* 2)
 203     check $dir $PG $primary replicated $degraded 0 $objects 0 || return 1
 204
 205     delete_pool $poolname
 206     kill_daemons $dir || return 1
 207 }
 208
 209
 210 # [1 0] -> [1,2]/[1,0]
 211 # misplaced 500 -> 0
 212 # state: active+remapped+backfilling
 213
 214 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                       STATE_STAMP                VERSION REPORTED UP    UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
 215 # 1.0         500                  0        0       500       0     0 100      100 active+remapped+backfilling 2017-10-27 09:51:18.800517  22'500   25:570 [1,2]          1  [1,0]              1        0'0 2017-10-27 09:50:40.441274             0'0 2017-10-27 09:50:40.441274
 216 function TEST_backfill_out() {
 217     local dir=$1
 218
 219     run_mon $dir a || return 1
 220     run_mgr $dir x || return 1
 221     run_osd $dir 0 || return 1
 222     run_osd $dir 1 || return 1
 223     run_osd $dir 2 || return 1
 224     run_osd $dir 3 || return 1
 225     run_osd $dir 4 || return 1
 226     run_osd $dir 5 || return 1
 227
 228     create_pool $poolname 1 1
 229     ceph osd pool set $poolname size 2
 230     sleep 5
 231
 232     wait_for_clean || return 1
 233
 234     for i in $(seq 1 $objects)
 235     do
 236         rados -p $poolname put obj$i /dev/null
 237     done
 238
 239     local PG=$(get_pg $poolname obj1)
 240     # Remember primary during the backfill
 241     local primary=$(get_primary $poolname obj1)
 242
 243     ceph osd out osd.$(get_not_primary $poolname obj1)
 244     sleep 15
 245
 246     wait_for_clean || return 1
 247
 248     check $dir $PG $primary replicated 0 0 $objects 0 || return 1
 249
 250     delete_pool $poolname
 251     kill_daemons $dir || return 1
 252 }
 253
 254
 255 # [0, 1] -> [0, 2]/[0]
 256 # osd 1 down/out
 257 # degraded 500 -> 0
 258 # state: active+undersized+degraded+remapped+backfilling
 259
 260 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                                           STATE_STAMP                VERSION REPORTED UP    UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
 261 # 1.0         500                  0      500         0       0     0 100      100 active+undersized+degraded+remapped+backfilling 2017-10-27 09:53:24.051091  22'500   27:719 [0,2]          0    [0]              0        0'0 2017-10-27 09:52:43.188368             0'0 2017-10-27 09:52:43.188368
 262 function TEST_backfill_down_out() {
 263     local dir=$1
 264
 265     run_mon $dir a || return 1
 266     run_mgr $dir x || return 1
 267     run_osd $dir 0 || return 1
 268     run_osd $dir 1 || return 1
 269     run_osd $dir 2 || return 1
 270     run_osd $dir 3 || return 1
 271     run_osd $dir 4 || return 1
 272     run_osd $dir 5 || return 1
 273
 274     create_pool $poolname 1 1
 275     ceph osd pool set $poolname size 2
 276     sleep 5
 277
 278     wait_for_clean || return 1
 279
 280     for i in $(seq 1 $objects)
 281     do
 282         rados -p $poolname put obj$i /dev/null
 283     done
 284
 285     local PG=$(get_pg $poolname obj1)
 286     # Remember primary during the backfill
 287     local primary=$(get_primary $poolname obj1)
 288     local otherosd=$(get_not_primary $poolname obj1)
 289
 290     kill $(cat $dir/osd.${otherosd}.pid)
 291     ceph osd down osd.${otherosd}
 292     ceph osd out osd.${otherosd}
 293     sleep 15
 294
 295     wait_for_clean || return 1
 296
 297     check $dir $PG $primary replicated $objects 0 0 0 || return 1
 298
 299     delete_pool $poolname
 300     kill_daemons $dir || return 1
 301 }
 302
 303
 304 # [1, 0] -> [2, 3, 4]
 305 # degraded 500 -> 0
 306 # misplaced 1000 -> 0
 307 # state: active+undersized+degraded+remapped+backfilling
 308
 309 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                                           STATE_STAMP                VERSION REPORTED UP      UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
 310 # 1.0         500                  0      500       1000       0     0 100      100 active+undersized+degraded+remapped+backfilling 2017-10-27 09:55:50.375722  23'500   27:553 [2,4,3]          2  [1,0]              1        0'0 2017-10-27 09:55:10.230919             0'0 2017-10-27 09:55:10.230919
 311 function TEST_backfill_out2() {
 312     local dir=$1
 313
 314     run_mon $dir a || return 1
 315     run_mgr $dir x || return 1
 316     run_osd $dir 0 || return 1
 317     run_osd $dir 1 || return 1
 318     run_osd $dir 2 || return 1
 319     run_osd $dir 3 || return 1
 320     run_osd $dir 4 || return 1
 321     run_osd $dir 5 || return 1
 322
 323     create_pool $poolname 1 1
 324     ceph osd pool set $poolname size 2
 325     sleep 5
 326
 327     wait_for_clean || return 1
 328
 329     for i in $(seq 1 $objects)
 330     do
 331         rados -p $poolname put obj$i /dev/null
 332     done
 333
 334     local PG=$(get_pg $poolname obj1)
 335     # Remember primary during the backfill
 336     local primary=$(get_primary $poolname obj1)
 337     local otherosd=$(get_not_primary $poolname obj1)
 338
 339     ceph osd set nobackfill
 340     ceph osd pool set $poolname size 3
 341     ceph osd out osd.${otherosd}
 342     ceph osd out osd.${primary}
 343     ceph osd unset nobackfill
 344     ceph tell osd.$(get_primary $poolname obj1) debug kick_recovery_wq 0
 345     sleep 2
 346
 347     wait_for_clean || return 1
 348
 349     local misplaced=$(expr $objects \* 2)
 350
 351     check $dir $PG $primary replicated $objects 0 $misplaced 0 || return 1
 352
 353     delete_pool $poolname
 354     kill_daemons $dir || return 1
 355 }
 356
 357
 358 # [0,1] ->  [2,4,3]/[0,1]
 359 # degraded 1000 -> 0
 360 # misplaced 1000 -> 500
 361 # state ends at active+clean+remapped [2,4,3]/[2,4,3,0]
 362 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                                           STATE_STAMP                VERSION REPORTED UP      UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
 363 # 1.0         500                  0     1000       1000       0     0 100      100 active+undersized+degraded+remapped+backfilling 2017-10-30 18:21:45.995149  19'500  23:1817 [2,4,3]          2  [0,1]              0        0'0 2017-10-30 18:21:05.109904             0'0 2017-10-30 18:21:05.109904
 364 # ENDS:
 365 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                 STATE_STAMP                VERSION REPORTED UP      UP_PRIMARY ACTING    ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
 366 # 1.0         500                  0        0       500       0     0   5        5 active+clean+remapped 2017-10-30 18:22:42.293730  19'500  25:2557 [2,4,3]          2 [2,4,3,0]              2        0'0 2017-10-30 18:21:05.109904             0'0 2017-10-30 18:21:05.109904
 367 function TEST_backfill_sizeup4_allout() {
 368     local dir=$1
 369
 370     run_mon $dir a || return 1
 371     run_mgr $dir x || return 1
 372     run_osd $dir 0 || return 1
 373     run_osd $dir 1 || return 1
 374     run_osd $dir 2 || return 1
 375     run_osd $dir 3 || return 1
 376     run_osd $dir 4 || return 1
 377
 378     create_pool $poolname 1 1
 379     ceph osd pool set $poolname size 2
 380
 381     wait_for_clean || return 1
 382
 383     for i in $(seq 1 $objects)
 384     do
 385         rados -p $poolname put obj$i /dev/null
 386     done
 387
 388     local PG=$(get_pg $poolname obj1)
 389     # Remember primary during the backfill
 390     local primary=$(get_primary $poolname obj1)
 391     local otherosd=$(get_not_primary $poolname obj1)
 392
 393     ceph osd set nobackfill
 394     ceph osd out osd.$otherosd
 395     ceph osd out osd.$primary
 396     ceph osd pool set $poolname size 4
 397     ceph osd unset nobackfill
 398     ceph tell osd.$(get_primary $poolname obj1) debug kick_recovery_wq 0
 399     sleep 2
 400
 401     wait_for_clean || return 1
 402
 403     local misdeg=$(expr $objects \* 2)
 404     check $dir $PG $primary replicated $misdeg 0 $misdeg $objects || return 1
 405
 406     delete_pool $poolname
 407     kill_daemons $dir || return 1
 408 }
 409
 410
 411 # [1,2,0] ->  [3]/[1,2]
 412 # misplaced 1000 -> 500
 413 # state ends at active+clean+remapped [3]/[3,1]
 414 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                       STATE_STAMP                VERSION REPORTED UP  UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
 415 # 1.0         500                  0        0       1000       0     0 100      100 active+remapped+backfilling 2017-11-28 19:13:56.092439  21'500   31:790 [3]          3  [1,2]              1        0'0 2017-11-28 19:13:28.698661             0'0 2017-11-28 19:13:28.698661
 416 function TEST_backfill_remapped() {
 417     local dir=$1
 418
 419     run_mon $dir a || return 1
 420     run_mgr $dir x || return 1
 421     run_osd $dir 0 || return 1
 422     run_osd $dir 1 || return 1
 423     run_osd $dir 2 || return 1
 424     run_osd $dir 3 || return 1
 425
 426     create_pool $poolname 1 1
 427     ceph osd pool set $poolname size 3
 428     sleep 5
 429
 430     wait_for_clean || return 1
 431
 432     for i in $(seq 1 $objects)
 433     do
 434         rados -p $poolname put obj$i /dev/null
 435     done
 436
 437     local PG=$(get_pg $poolname obj1)
 438     # Remember primary during the backfill
 439     local primary=$(get_primary $poolname obj1)
 440     local otherosd=$(get_not_primary $poolname obj1)
 441
 442     ceph osd set nobackfill
 443     ceph osd out osd.${otherosd}
 444     for i in $(get_osds $poolname obj1)
 445     do
 446         if [ $i = $primary -o $i = $otherosd ];
 447         then
 448             continue
 449         fi
 450         ceph osd out osd.$i
 451         break
 452     done
 453     ceph osd out osd.${primary}
 454     ceph osd pool set $poolname size 2
 455     sleep 2
 456     ceph osd unset nobackfill
 457     ceph tell osd.$(get_primary $poolname obj1) debug kick_recovery_wq 0
 458     sleep 2
 459
 460     wait_for_clean || return 1
 461
 462     local misplaced=$(expr $objects \* 2)
 463
 464     check $dir $PG $primary replicated 0 0 $misplaced $objects || return 1
 465
 466     delete_pool $poolname
 467     kill_daemons $dir || return 1
 468 }
 469
 470 # [1,0,2] -> [4,3,NONE]/[1,0,2]
 471 # misplaced 1500 -> 500
 472 # state ends at active+clean+remapped [4,3,NONE]/[4,3,2]
 473
 474 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                                STATE_STAMP                VERSION REPORTED UP         UP_PRIMARY ACTING  ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
 475 # 1.0         500                  0      0      1500       0     0 100      100 active+degraded+remapped+backfilling 2017-10-31 16:53:39.467126  19'500   23:615 [4,3,NONE]          4 [1,0,2]              1        0'0 2017-10-31 16:52:59.624429             0'0 2017-10-31 16:52:59.624429
 476
 477
 478 # ENDS:
 479
 480 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                 STATE_STAMP                VERSION REPORTED UP         UP_PRIMARY ACTING  ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
 481 # 1.0         500                  0        0       500       0     0   5        5 active+clean+remapped 2017-10-31 16:48:34.414040  19'500  25:2049 [4,3,NONE]          4 [4,3,2]              4        0'0 2017-10-31 16:46:58.203440             0'0 2017-10-31 16:46:58.203440
 482 function TEST_backfill_ec_all_out() {
 483     local dir=$1
 484
 485     run_mon $dir a || return 1
 486     run_mgr $dir x || return 1
 487     run_osd $dir 0 || return 1
 488     run_osd $dir 1 || return 1
 489     run_osd $dir 2 || return 1
 490     run_osd $dir 3 || return 1
 491     run_osd $dir 4 || return 1
 492
 493     ceph osd erasure-code-profile set myprofile plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd
 494     create_pool $poolname 1 1 erasure myprofile
 495
 496     wait_for_clean || return 1
 497
 498     for i in $(seq 1 $objects)
 499     do
 500         rados -p $poolname put obj$i /dev/null
 501     done
 502
 503     local PG=$(get_pg $poolname obj1)
 504     # Remember primary during the backfill
 505     local primary=$(get_primary $poolname obj1)
 506
 507     ceph osd set nobackfill
 508     for o in $(get_osds $poolname obj1)
 509     do
 510         ceph osd out osd.$o
 511     done
 512     ceph osd unset nobackfill
 513     ceph tell osd.$(get_primary $poolname obj1) debug kick_recovery_wq 0
 514     sleep 2
 515
 516     wait_for_clean || return 1
 517
 518     local misplaced=$(expr $objects \* 3)
 519     check $dir $PG $primary erasure 0 0 $misplaced $objects || return 1
 520
 521     delete_pool $poolname
 522     kill_daemons $dir || return 1
 523 }
 524
 525
 526 # [1,0,2] -> [4, 0, 2]
 527 # misplaced 500 -> 0
 528 # active+remapped+backfilling
 529 #
 530 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                       STATE_STAMP                VERSION REPORTED UP      UP_PRIMARY ACTING  ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
 531 # 1.0         500                  0        0       500       0     0 100      100 active+remapped+backfilling 2017-11-08 18:05:39.036420  24'500   27:742 [4,0,2]          4 [1,0,2]              1        0'0 2017-11-08 18:04:58.697315             0'0 2017-11-08 18:04:58.697315
 532 function TEST_backfill_ec_prim_out() {
 533     local dir=$1
 534
 535     run_mon $dir a || return 1
 536     run_mgr $dir x || return 1
 537     run_osd $dir 0 || return 1
 538     run_osd $dir 1 || return 1
 539     run_osd $dir 2 || return 1
 540     run_osd $dir 3 || return 1
 541     run_osd $dir 4 || return 1
 542
 543     ceph osd erasure-code-profile set myprofile plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd
 544     create_pool $poolname 1 1 erasure myprofile
 545
 546     wait_for_clean || return 1
 547
 548     for i in $(seq 1 $objects)
 549     do
 550         rados -p $poolname put obj$i /dev/null
 551     done
 552
 553     local PG=$(get_pg $poolname obj1)
 554     # Remember primary during the backfill
 555     local primary=$(get_primary $poolname obj1)
 556
 557     ceph osd set nobackfill
 558     ceph osd out osd.$primary
 559     ceph osd unset nobackfill
 560     ceph tell osd.$(get_primary $poolname obj1) debug kick_recovery_wq 0
 561     sleep 2
 562
 563     wait_for_clean || return 1
 564
 565     local misplaced=$(expr $objects \* 3)
 566     check $dir $PG $primary erasure 0 0 $objects 0 || return 1
 567
 568     delete_pool $poolname
 569     kill_daemons $dir || return 1
 570 }
 571
 572 # [1,0] -> [1,2]
 573 # degraded 500 -> 0
 574 # misplaced 1000 -> 0
 575 #
 576 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                                           STATE_STAMP                VERSION REPORTED UP      UP_PRIMARY ACTING     ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
 577 # 1.0         500                  0      500      1000       0     0 100      100 active+undersized+degraded+remapped+backfilling 2017-11-06 14:02:29.439105  24'500  29:1020 [4,3,5]          4 [1,NONE,2]              1        0'0 2017-11-06 14:01:46.509963             0'0 2017-11-06 14:01:46.509963
 578 function TEST_backfill_ec_down_all_out() {
 579     local dir=$1
 580
 581     run_mon $dir a || return 1
 582     run_mgr $dir x || return 1
 583     run_osd $dir 0 || return 1
 584     run_osd $dir 1 || return 1
 585     run_osd $dir 2 || return 1
 586     run_osd $dir 3 || return 1
 587     run_osd $dir 4 || return 1
 588     run_osd $dir 5 || return 1
 589
 590     ceph osd erasure-code-profile set myprofile plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd
 591     create_pool $poolname 1 1 erasure myprofile
 592     ceph osd pool set $poolname min_size 2
 593
 594     wait_for_clean || return 1
 595
 596     for i in $(seq 1 $objects)
 597     do
 598         rados -p $poolname put obj$i /dev/null
 599     done
 600
 601     local PG=$(get_pg $poolname obj1)
 602     # Remember primary during the backfill
 603     local primary=$(get_primary $poolname obj1)
 604     local otherosd=$(get_not_primary $poolname obj1)
 605     local allosds=$(get_osds $poolname obj1)
 606
 607     ceph osd set nobackfill
 608     kill $(cat $dir/osd.${otherosd}.pid)
 609     ceph osd down osd.${otherosd}
 610     for o in $allosds
 611     do
 612         ceph osd out osd.$o
 613     done
 614     ceph osd unset nobackfill
 615     ceph tell osd.$(get_primary $poolname obj1) debug kick_recovery_wq 0
 616     sleep 2
 617     flush_pg_stats
 618
 619     # Wait for recovery to finish
 620     # Can't use wait_for_clean() because state goes from active+undersized+degraded+remapped+backfilling
 621     # to  active+undersized+remapped
 622     while(true)
 623     do
 624       if test "$(ceph --format json pg dump pgs |
 625          jq '[.[] | .state | select(. == "incomplete")] | length')" -ne "0"
 626       then
 627         sleep 2
 628         continue
 629       fi
 630       break
 631     done
 632     ceph pg dump pgs
 633     for i in $(seq 1 60)
 634     do
 635       if ceph pg dump pgs | grep ^$PG | grep -qv backfilling
 636       then
 637           break
 638       fi
 639       if [ $i = "60" ];
 640       then
 641           echo "Timeout waiting for recovery to finish"
 642           return 1
 643       fi
 644       sleep 1
 645     done
 646
 647     ceph pg dump pgs
 648
 649     local misplaced=$(expr $objects \* 2)
 650     check $dir $PG $primary erasure $objects 0 $misplaced 0 || return 1
 651
 652     delete_pool $poolname
 653     kill_daemons $dir || return 1
 654 }
 655
 656
 657 # [1,0,2] -> [1,3,2]
 658 # degraded 500 -> 0
 659 # active+backfilling+degraded
 660 #
 661 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                                           STATE_STAMP                VERSION REPORTED UP      UP_PRIMARY ACTING     ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
 662 # 1.0         500                  0      500         0       0     0 100      100 active+undersized+degraded+remapped+backfilling 2017-11-06 13:57:25.412322  22'500   28:794 [1,3,2]          1 [1,NONE,2]              1        0'0 2017-11-06 13:54:58.033906             0'0 2017-11-06 13:54:58.033906
 663 function TEST_backfill_ec_down_out() {
 664     local dir=$1
 665
 666     run_mon $dir a || return 1
 667     run_mgr $dir x || return 1
 668     run_osd $dir 0 || return 1
 669     run_osd $dir 1 || return 1
 670     run_osd $dir 2 || return 1
 671     run_osd $dir 3 || return 1
 672     run_osd $dir 4 || return 1
 673     run_osd $dir 5 || return 1
 674
 675     ceph osd erasure-code-profile set myprofile plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd
 676     create_pool $poolname 1 1 erasure myprofile
 677     ceph osd pool set $poolname min_size 2
 678
 679     wait_for_clean || return 1
 680
 681     for i in $(seq 1 $objects)
 682     do
 683         rados -p $poolname put obj$i /dev/null
 684     done
 685
 686     local PG=$(get_pg $poolname obj1)
 687     # Remember primary during the backfill
 688     local primary=$(get_primary $poolname obj1)
 689     local otherosd=$(get_not_primary $poolname obj1)
 690
 691     ceph osd set nobackfill
 692     kill $(cat $dir/osd.${otherosd}.pid)
 693     ceph osd down osd.${otherosd}
 694     ceph osd out osd.${otherosd}
 695     ceph osd unset nobackfill
 696     ceph tell osd.$(get_primary $poolname obj1) debug kick_recovery_wq 0
 697     sleep 2
 698
 699     wait_for_clean || return 1
 700
 701     local misplaced=$(expr $objects \* 2)
 702     check $dir $PG $primary erasure $objects 0 0 0 || return 1
 703
 704     delete_pool $poolname
 705     kill_daemons $dir || return 1
 706 }
 707
 708
 709 main osd-backfill-stats "$@"
 710
 711 # Local Variables:
 712 # compile-command: "make -j4 && ../qa/run-standalone.sh osd-backfill-stats.sh"
 713 # End: