[ceph.git] / ceph / qa / standalone / osd / osd-backfill-stats.sh

#!/usr/bin/env bash
#
# Copyright (C) 2017 Red Hat <contact@redhat.com>
#
# Author: David Zafman <dzafman@redhat.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Library Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Library Public License for more details.
#

source $CEPH_ROOT/qa/standalone/ceph-helpers.sh

function run() {
    local dir=$1
    shift

    # Fix port????
    export CEPH_MON="127.0.0.1:7114" # git grep '\<7114\>' : there must be only one
    export CEPH_ARGS
    CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
    CEPH_ARGS+="--mon-host=$CEPH_MON "
    CEPH_ARGS+="--osd_min_pg_log_entries=5 --osd_max_pg_log_entries=10 "
    export margin=10
    export objects=200
    export poolname=test

    local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
    for func in $funcs ; do
        setup $dir || return 1
        $func $dir || return 1
        teardown $dir || return 1
    done
}

function below_margin() {
    local -i check=$1
    shift
    local -i target=$1

    return $(( $check <= $target && $check >= $target - $margin ? 0 : 1 ))
}

function above_margin() {
    local -i check=$1
    shift
    local -i target=$1

    return $(( $check >= $target && $check <= $target + $margin ? 0 : 1 ))
}

function check() {
    local PG=$1
    local log=$2
    local degraded_start=$3
    local degraded_end=$4
    local misplaced_start=$5
    local misplaced_end=$6
    local type=$7

    local addp=" "
    if [ "$type" = "erasure" ];
    then
      addp="p"
    fi

    UPACT=$(grep "pg[[]${PG}.*backfilling.*_update_calc_stats " $log | tail -1 | sed "s/.*[)] \([[][^ p]*\).*$/\1/")

    # Check 3rd line at start because of false backfill starts
    FIRST=$(grep "pg[[]${PG}.*backfilling.*_update_calc_stats degraded " $log | grep -F " ${UPACT}${addp}" | head -1 | sed "s/.* \([0-9]*\)$/\1/")
    below_margin $FIRST $degraded_start || return 1
    LAST=$(grep "pg[[]${PG}.*backfilling.*_update_calc_stats degraded " $log | tail -1 | sed "s/.* \([0-9]*\)$/\1/")
    above_margin $LAST $degraded_end || return 1

    # Check 3rd line at start because of false backfill starts
    FIRST=$(grep "pg[[]${PG}.*backfilling.*_update_calc_stats misplaced " $log | grep -F " ${UPACT}${addp}" | head -1 | sed "s/.* \([0-9]*\)$/\1/")
    below_margin $FIRST $misplaced_start || return 1
    LAST=$(grep "pg[[]${PG}.*backfilling.*_update_calc_stats misplaced " $log | tail -1 | sed "s/.* \([0-9]*\)$/\1/")
    above_margin $LAST $misplaced_end || return 1
}

# [1] -> [1, 0, 2]
# degraded 1000 -> 0
# state: active+undersized+degraded+remapped+backfilling

# PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                                           STATE_STAMP                VERSION REPORTED UP      UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
# 1.0         500                  0      1000         0       0     0 100      100 active+undersized+degraded+remapped+backfilling 2017-10-27 09:44:23.531466  22'500   26:617 [1,0,2]          1    [1]              1        0'0 2017-10-27 09:43:44.654882             0'0 2017-10-27 09:43:44.654882
function TEST_backfill_sizeup() {
    local dir=$1

    run_mon $dir a || return 1
    run_mgr $dir x || return 1
    export CEPH_ARGS
    run_osd $dir 0 || return 1
    run_osd $dir 1 || return 1
    run_osd $dir 2 || return 1
    run_osd $dir 3 || return 1
    run_osd $dir 4 || return 1
    run_osd $dir 5 || return 1

    create_pool $poolname 1 1
    ceph osd pool set $poolname size 1

    wait_for_clean || return 1

    for i in $(seq 1 $objects)
    do
	rados -p $poolname put obj$i /dev/null
    done

    ceph osd pool set $poolname size 3
    sleep 15

    wait_for_clean || return 1

    local primary=$(get_primary $poolname obj1)
    local PG=$(get_pg $poolname obj1)

    local log=$(grep -l +backfilling $dir/osd.*.log)
    test -n "$log" || return 1

    local degraded=$(expr $objects \* 2)
    check $PG $log $degraded 0 0 0 || return 1

    delete_pool $poolname
    kill_daemons $dir || return 1
}


# [1] -> [0, 2, 4]
# degraded 1000 -> 0
# misplaced 500 -> 0
# state: active+undersized+degraded+remapped+backfilling

# PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                                           STATE_STAMP                VERSION REPORTED UP      UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
# 1.0         500                  0      1000       500       0     0 100      100 active+undersized+degraded+remapped+backfilling 2017-10-27 09:48:53.326849  22'500   26:603 [0,2,4]          0    [1]              1        0'0 2017-10-27 09:48:13.236253             0'0 2017-10-27 09:48:13.236253
function TEST_backfill_sizeup_out() {
    local dir=$1

    run_mon $dir a || return 1
    run_mgr $dir x || return 1
    run_osd $dir 0 || return 1
    run_osd $dir 1 || return 1
    run_osd $dir 2 || return 1
    run_osd $dir 3 || return 1
    run_osd $dir 4 || return 1
    run_osd $dir 5 || return 1

    create_pool $poolname 1 1
    ceph osd pool set $poolname size 1

    wait_for_clean || return 1

    for i in $(seq 1 $objects)
    do
	rados -p $poolname put obj$i /dev/null
    done

    local PG=$(get_pg $poolname obj1)
    # Remember primary during the backfill
    local primary=$(get_primary $poolname obj1)

    ceph osd out osd.$primary
    ceph osd pool set $poolname size 3
    sleep 15

    wait_for_clean || return 1

    local log=$(grep -l +backfilling $dir/osd.*.log)
    test -n "$log" || return 1
    local degraded=$(expr $objects \* 2)
    check $PG $log $degraded 0 $objects 0 || return 1

    delete_pool $poolname
    kill_daemons $dir || return 1
}


# [1 0] -> [1,2]/[1,0]
# misplaced 500 -> 0
# state: active+remapped+backfilling

# PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                       STATE_STAMP                VERSION REPORTED UP    UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
# 1.0         500                  0        0       500       0     0 100      100 active+remapped+backfilling 2017-10-27 09:51:18.800517  22'500   25:570 [1,2]          1  [1,0]              1        0'0 2017-10-27 09:50:40.441274             0'0 2017-10-27 09:50:40.441274
function TEST_backfill_out() {
    local dir=$1

    run_mon $dir a || return 1
    run_mgr $dir x || return 1
    run_osd $dir 0 || return 1
    run_osd $dir 1 || return 1
    run_osd $dir 2 || return 1
    run_osd $dir 3 || return 1
    run_osd $dir 4 || return 1
    run_osd $dir 5 || return 1

    create_pool $poolname 1 1
    ceph osd pool set $poolname size 2
    sleep 5

    wait_for_clean || return 1

    for i in $(seq 1 $objects)
    do
	rados -p $poolname put obj$i /dev/null
    done

    local PG=$(get_pg $poolname obj1)
    # Remember primary during the backfill
    local primary=$(get_primary $poolname obj1)

    ceph osd out osd.$(get_not_primary $poolname obj1)
    sleep 15

    wait_for_clean || return 1

    local log=$(grep -l +backfilling $dir/osd.*.log)
    test -n "$log" || return 1

    check $PG $log 0 0 $objects 0 || return 1

    delete_pool $poolname
    kill_daemons $dir || return 1
}


# [0, 1] -> [0, 2]/[0]
# osd 1 down/out
# degraded 500 -> 0
# state: active+undersized+degraded+remapped+backfilling

# PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                                           STATE_STAMP                VERSION REPORTED UP    UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
# 1.0         500                  0      500         0       0     0 100      100 active+undersized+degraded+remapped+backfilling 2017-10-27 09:53:24.051091  22'500   27:719 [0,2]          0    [0]              0        0'0 2017-10-27 09:52:43.188368             0'0 2017-10-27 09:52:43.188368
function TEST_backfill_down_out() {
    local dir=$1

    run_mon $dir a || return 1
    run_mgr $dir x || return 1
    run_osd $dir 0 || return 1
    run_osd $dir 1 || return 1
    run_osd $dir 2 || return 1
    run_osd $dir 3 || return 1
    run_osd $dir 4 || return 1
    run_osd $dir 5 || return 1

    create_pool $poolname 1 1
    ceph osd pool set $poolname size 2
    sleep 5

    wait_for_clean || return 1

    for i in $(seq 1 $objects)
    do
	rados -p $poolname put obj$i /dev/null
    done

    local PG=$(get_pg $poolname obj1)
    # Remember primary during the backfill
    local primary=$(get_primary $poolname obj1)
    local otherosd=$(get_not_primary $poolname obj1)

    kill $(cat $dir/osd.${otherosd}.pid)
    ceph osd down osd.${otherosd}
    ceph osd out osd.${otherosd}
    sleep 15

    wait_for_clean || return 1

    local log=$(grep -l +backfilling $dir/osd.*.log)
    test -n "$log" || return 1

    check $PG $log $objects 0 0 0 || return 1

    delete_pool $poolname
    kill_daemons $dir || return 1
}


# [1, 0] -> [2, 3, 4]
# degraded 500 -> 0
# misplaced 1000 -> 0
# state: active+undersized+degraded+remapped+backfilling

# PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                                           STATE_STAMP                VERSION REPORTED UP      UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
# 1.0         500                  0      500       1000       0     0 100      100 active+undersized+degraded+remapped+backfilling 2017-10-27 09:55:50.375722  23'500   27:553 [2,4,3]          2  [1,0]              1        0'0 2017-10-27 09:55:10.230919             0'0 2017-10-27 09:55:10.230919
function TEST_backfill_out2() {
    local dir=$1

    run_mon $dir a || return 1
    run_mgr $dir x || return 1
    run_osd $dir 0 || return 1
    run_osd $dir 1 || return 1
    run_osd $dir 2 || return 1
    run_osd $dir 3 || return 1
    run_osd $dir 4 || return 1
    run_osd $dir 5 || return 1

    create_pool $poolname 1 1
    ceph osd pool set $poolname size 2
    sleep 5

    wait_for_clean || return 1

    for i in $(seq 1 $objects)
    do
	rados -p $poolname put obj$i /dev/null
    done

    local PG=$(get_pg $poolname obj1)
    # Remember primary during the backfill
    local primary=$(get_primary $poolname obj1)
    local otherosd=$(get_not_primary $poolname obj1)

    ceph osd set nobackfill
    ceph osd pool set $poolname size 3
    ceph osd out osd.${otherosd}
    ceph osd out osd.${primary}
    ceph osd unset nobackfill
    ceph tell osd.$(get_primary $poolname obj1) debug kick_recovery_wq 0
    sleep 2

    wait_for_clean || return 1

    local log=$(grep -l +backfilling $dir/osd.*.log)
    test -n "$log" || return 1
    local misplaced=$(expr $objects \* 2)

    check $PG $log $objects 0 $misplaced 0 || return 1

    delete_pool $poolname
    kill_daemons $dir || return 1
}


# [0,1] ->  [2,4,3]/[0,1]
# degraded 1000 -> 0
# misplaced 1000 -> 500
# state ends at active+clean+remapped [2,4,3]/[2,4,3,0]
# PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                                           STATE_STAMP                VERSION REPORTED UP      UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
# 1.0         500                  0     1000       1000       0     0 100      100 active+undersized+degraded+remapped+backfilling 2017-10-30 18:21:45.995149  19'500  23:1817 [2,4,3]          2  [0,1]              0        0'0 2017-10-30 18:21:05.109904             0'0 2017-10-30 18:21:05.109904
# ENDS:
# PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                 STATE_STAMP                VERSION REPORTED UP      UP_PRIMARY ACTING    ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
# 1.0         500                  0        0       500       0     0   5        5 active+clean+remapped 2017-10-30 18:22:42.293730  19'500  25:2557 [2,4,3]          2 [2,4,3,0]              2        0'0 2017-10-30 18:21:05.109904             0'0 2017-10-30 18:21:05.109904
function TEST_backfill_sizeup4_allout() {
    local dir=$1

    run_mon $dir a || return 1
    run_mgr $dir x || return 1
    run_osd $dir 0 || return 1
    run_osd $dir 1 || return 1
    run_osd $dir 2 || return 1
    run_osd $dir 3 || return 1
    run_osd $dir 4 || return 1

    create_pool $poolname 1 1
    ceph osd pool set $poolname size 2

    wait_for_clean || return 1

    for i in $(seq 1 $objects)
    do
	rados -p $poolname put obj$i /dev/null
    done

    local PG=$(get_pg $poolname obj1)
    # Remember primary during the backfill
    local primary=$(get_primary $poolname obj1)
    local otherosd=$(get_not_primary $poolname obj1)

    ceph osd set nobackfill
    ceph osd out osd.$otherosd
    ceph osd out osd.$primary
    ceph osd pool set $poolname size 4
    ceph osd unset nobackfill
    ceph tell osd.$(get_primary $poolname obj1) debug kick_recovery_wq 0
    sleep 2

    wait_for_clean || return 1

    local log=$(grep -l +backfilling $dir/osd.*.log)
    test -n "$log" || return 1
    local misdeg=$(expr $objects \* 2)
    check $PG $log $misdeg 0 $misdeg $objects || return 1

    delete_pool $poolname
    kill_daemons $dir || return 1
}


# [1,2,0] ->  [3]/[1,2]
# misplaced 1000 -> 500
# state ends at active+clean+remapped [3]/[3,1]
# PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                       STATE_STAMP                VERSION REPORTED UP  UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
# 1.0         500                  0        0       1000       0     0 100      100 active+remapped+backfilling 2017-11-28 19:13:56.092439  21'500   31:790 [3]          3  [1,2]              1        0'0 2017-11-28 19:13:28.698661             0'0 2017-11-28 19:13:28.698661
function TEST_backfill_remapped() {
    local dir=$1

    run_mon $dir a || return 1
    run_mgr $dir x || return 1
    run_osd $dir 0 || return 1
    run_osd $dir 1 || return 1
    run_osd $dir 2 || return 1
    run_osd $dir 3 || return 1

    create_pool $poolname 1 1
    ceph osd pool set $poolname size 3
    sleep 5

    wait_for_clean || return 1

    for i in $(seq 1 $objects)
    do
	rados -p $poolname put obj$i /dev/null
    done

    local PG=$(get_pg $poolname obj1)
    # Remember primary during the backfill
    local primary=$(get_primary $poolname obj1)
    local otherosd=$(get_not_primary $poolname obj1)

    ceph osd set nobackfill
    ceph osd out osd.${otherosd}
    for i in $(get_osds $poolname obj1)
    do
        if [ $i = $primary -o $i = $otherosd ];
        then
            continue
        fi
        ceph osd out osd.$i
        break
    done
    ceph osd out osd.${primary}
    ceph osd pool set $poolname size 2
    sleep 2
    ceph osd unset nobackfill
    ceph tell osd.$(get_primary $poolname obj1) debug kick_recovery_wq 0
    sleep 2

    wait_for_clean || return 1

    local log=$(grep -l +backfilling $dir/osd.*.log)
    test -n "$log" || return 1
    local misplaced=$(expr $objects \* 2)

    check $PG $log 0 0 $misplaced $objects || return 1

    delete_pool $poolname
    kill_daemons $dir || return 1
}

# [1,0,2] -> [4,3,NONE]/[1,0,2]
# misplaced 1500 -> 500
# state ends at active+clean+remapped [4,3,NONE]/[4,3,2]

# PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                                STATE_STAMP                VERSION REPORTED UP         UP_PRIMARY ACTING  ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
# 1.0         500                  0      0      1500       0     0 100      100 active+degraded+remapped+backfilling 2017-10-31 16:53:39.467126  19'500   23:615 [4,3,NONE]          4 [1,0,2]              1        0'0 2017-10-31 16:52:59.624429             0'0 2017-10-31 16:52:59.624429


# ENDS:

# PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                 STATE_STAMP                VERSION REPORTED UP         UP_PRIMARY ACTING  ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
# 1.0         500                  0        0       500       0     0   5        5 active+clean+remapped 2017-10-31 16:48:34.414040  19'500  25:2049 [4,3,NONE]          4 [4,3,2]              4        0'0 2017-10-31 16:46:58.203440             0'0 2017-10-31 16:46:58.203440
function TEST_backfill_ec_all_out() {
    local dir=$1

    run_mon $dir a || return 1
    run_mgr $dir x || return 1
    run_osd $dir 0 || return 1
    run_osd $dir 1 || return 1
    run_osd $dir 2 || return 1
    run_osd $dir 3 || return 1
    run_osd $dir 4 || return 1

    ceph osd erasure-code-profile set myprofile plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd
    create_pool $poolname 1 1 erasure myprofile

    wait_for_clean || return 1

    for i in $(seq 1 $objects)
    do
	rados -p $poolname put obj$i /dev/null
    done

    local PG=$(get_pg $poolname obj1)
    # Remember primary during the backfill
    local primary=$(get_primary $poolname obj1)

    ceph osd set nobackfill
    for o in $(get_osds $poolname obj1)
    do
        ceph osd out osd.$o
    done
    ceph osd unset nobackfill
    ceph tell osd.$(get_primary $poolname obj1) debug kick_recovery_wq 0
    sleep 2

    wait_for_clean || return 1

    local log=$(grep -l +backfilling $dir/osd.*.log)
    test -n "$log" || return 1

    local misplaced=$(expr $objects \* 3)
    check $PG $log 0 0 $misplaced $objects erasure || return 1

    delete_pool $poolname
    kill_daemons $dir || return 1
}


# [1,0,2] -> [4, 0, 2]
# misplaced 500 -> 0
# active+remapped+backfilling
#
# PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                       STATE_STAMP                VERSION REPORTED UP      UP_PRIMARY ACTING  ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
# 1.0         500                  0        0       500       0     0 100      100 active+remapped+backfilling 2017-11-08 18:05:39.036420  24'500   27:742 [4,0,2]          4 [1,0,2]              1        0'0 2017-11-08 18:04:58.697315             0'0 2017-11-08 18:04:58.697315
function TEST_backfill_ec_prim_out() {
    local dir=$1

    run_mon $dir a || return 1
    run_mgr $dir x || return 1
    run_osd $dir 0 || return 1
    run_osd $dir 1 || return 1
    run_osd $dir 2 || return 1
    run_osd $dir 3 || return 1
    run_osd $dir 4 || return 1

    ceph osd erasure-code-profile set myprofile plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd
    create_pool $poolname 1 1 erasure myprofile

    wait_for_clean || return 1

    for i in $(seq 1 $objects)
    do
	rados -p $poolname put obj$i /dev/null
    done

    local PG=$(get_pg $poolname obj1)
    # Remember primary during the backfill
    local primary=$(get_primary $poolname obj1)

    ceph osd set nobackfill
    ceph osd out osd.$primary
    ceph osd unset nobackfill
    ceph tell osd.$(get_primary $poolname obj1) debug kick_recovery_wq 0
    sleep 2

    wait_for_clean || return 1

    local log=$(grep -l +backfilling $dir/osd.*.log)
    test -n "$log" || return 1

    local misplaced=$(expr $objects \* 3)
    check $PG $log 0 0 $objects 0 erasure || return 1

    delete_pool $poolname
    kill_daemons $dir || return 1
}

# [1,0] -> [1,2]
# degraded 500 -> 0
# misplaced 1000 -> 0
#
# PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                                           STATE_STAMP                VERSION REPORTED UP      UP_PRIMARY ACTING     ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
# 1.0         500                  0      500      1000       0     0 100      100 active+undersized+degraded+remapped+backfilling 2017-11-06 14:02:29.439105  24'500  29:1020 [4,3,5]          4 [1,NONE,2]              1        0'0 2017-11-06 14:01:46.509963             0'0 2017-11-06 14:01:46.509963
function TEST_backfill_ec_down_all_out() {
    local dir=$1

    run_mon $dir a || return 1
    run_mgr $dir x || return 1
    run_osd $dir 0 || return 1
    run_osd $dir 1 || return 1
    run_osd $dir 2 || return 1
    run_osd $dir 3 || return 1
    run_osd $dir 4 || return 1
    run_osd $dir 5 || return 1

    ceph osd erasure-code-profile set myprofile plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd
    create_pool $poolname 1 1 erasure myprofile
    ceph osd pool set $poolname min_size 2

    wait_for_clean || return 1

    for i in $(seq 1 $objects)
    do
	rados -p $poolname put obj$i /dev/null
    done

    local PG=$(get_pg $poolname obj1)
    # Remember primary during the backfill
    local primary=$(get_primary $poolname obj1)
    local otherosd=$(get_not_primary $poolname obj1)
    local allosds=$(get_osds $poolname obj1)

    ceph osd set nobackfill
    kill $(cat $dir/osd.${otherosd}.pid)
    ceph osd down osd.${otherosd}
    for o in $allosds
    do
        ceph osd out osd.$o
    done
    ceph osd unset nobackfill
    ceph tell osd.$(get_primary $poolname obj1) debug kick_recovery_wq 0
    sleep 2
    flush_pg_stats

    # Wait for recovery to finish
    # Can't use wait_for_clean() because state goes from active+undersized+degraded+remapped+backfilling
    # to  active+undersized+remapped
    while(true)
    do
      if test "$(ceph --format json pg dump pgs |
         jq '[.[] | .state | select(. == "incomplete")] | length')" -ne "0"
      then
        sleep 2
        continue
      fi
      break
    done
    ceph pg dump pgs
    for i in $(seq 1 60)
    do
      if ceph pg dump pgs | grep ^$PG | grep -qv backfilling
      then
          break
      fi
      if [ $i = "60" ];
      then
          echo "Timeout waiting for recovery to finish"
          return 1
      fi
      sleep 1
    done

    ceph pg dump pgs

    local log=$(grep -l +backfilling $dir/osd.*.log)
    test -n "$log" || return 1

    local misplaced=$(expr $objects \* 2)
    check $PG $log $objects 0 $misplaced 0 erasure || return 1

    delete_pool $poolname
    kill_daemons $dir || return 1
}


# [1,0,2] -> [1,3,2]
# degraded 500 -> 0
# active+backfilling+degraded
#
# PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE                                           STATE_STAMP                VERSION REPORTED UP      UP_PRIMARY ACTING     ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP                LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
# 1.0         500                  0      500         0       0     0 100      100 active+undersized+degraded+remapped+backfilling 2017-11-06 13:57:25.412322  22'500   28:794 [1,3,2]          1 [1,NONE,2]              1        0'0 2017-11-06 13:54:58.033906             0'0 2017-11-06 13:54:58.033906
function TEST_backfill_ec_down_out() {
    local dir=$1

    run_mon $dir a || return 1
    run_mgr $dir x || return 1
    run_osd $dir 0 || return 1
    run_osd $dir 1 || return 1
    run_osd $dir 2 || return 1
    run_osd $dir 3 || return 1
    run_osd $dir 4 || return 1
    run_osd $dir 5 || return 1

    ceph osd erasure-code-profile set myprofile plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd
    create_pool $poolname 1 1 erasure myprofile
    ceph osd pool set $poolname min_size 2

    wait_for_clean || return 1

    for i in $(seq 1 $objects)
    do
	rados -p $poolname put obj$i /dev/null
    done

    local PG=$(get_pg $poolname obj1)
    # Remember primary during the backfill
    local primary=$(get_primary $poolname obj1)
    local otherosd=$(get_not_primary $poolname obj1)

    ceph osd set nobackfill
    kill $(cat $dir/osd.${otherosd}.pid)
    ceph osd down osd.${otherosd}
    ceph osd out osd.${otherosd}
    ceph osd unset nobackfill
    ceph tell osd.$(get_primary $poolname obj1) debug kick_recovery_wq 0
    sleep 2

    wait_for_clean || return 1

    local log=$(grep -l +backfilling $dir/osd.*.log)
    test -n "$log" || return 1

    local misplaced=$(expr $objects \* 2)
    check $PG $log $objects 0 0 0 erasure || return 1

    delete_pool $poolname
    kill_daemons $dir || return 1
}


main osd-backfill-stats "$@"

# Local Variables:
# compile-command: "make -j4 && ../qa/run-standalone.sh osd-backfill-stats.sh"
# End: