[ceph.git] / ceph / qa / standalone / mgr / balancer.sh

#!/usr/bin/env bash
#
# Copyright (C) 2019 Red Hat <contact@redhat.com>
#
# Author: David Zafman <dzafman@redhat.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Library Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Library Public License for more details.
#
source $CEPH_ROOT/qa/standalone/ceph-helpers.sh

function run() {
    local dir=$1
    shift

    export CEPH_MON="127.0.0.1:7102" # git grep '\<7102\>' : there must be only one
    export CEPH_ARGS
    CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
    CEPH_ARGS+="--mon-host=$CEPH_MON "

    local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
    for func in $funcs ; do
        $func $dir || return 1
    done
}

TEST_POOL1=test1
TEST_POOL2=test2

function TEST_balancer() {
    local dir=$1

    setup $dir || return 1
    run_mon $dir a || return 1
    run_mgr $dir x || return 1
    run_osd $dir 0 || return 1
    run_osd $dir 1 || return 1
    run_osd $dir 2 || return 1
    create_pool $TEST_POOL1 8
    create_pool $TEST_POOL2 8

    wait_for_clean || return 1

    ceph pg dump pgs
    ceph balancer status || return 1
    eval MODE=$(ceph balancer status | jq '.mode')
    test $MODE = "upmap" || return 1
    ACTIVE=$(ceph balancer status | jq '.active')
    test $ACTIVE = "true" || return 1

    ceph balancer ls || return 1
    PLANS=$(ceph balancer ls)
    test "$PLANS" = "[]" || return 1
    ceph balancer eval || return 1
    EVAL="$(ceph balancer eval)"
    test "$EVAL" = "current cluster score 0.000000 (lower is better)"
    ceph balancer eval-verbose || return 1

    ceph balancer pool add $TEST_POOL1 || return 1
    ceph balancer pool add $TEST_POOL2 || return 1
    ceph balancer pool ls || return 1
    eval POOL=$(ceph balancer pool ls | jq 'sort | .[0]')
    test "$POOL" = "$TEST_POOL1" || return 1
    eval POOL=$(ceph balancer pool ls | jq 'sort | .[1]')
    test "$POOL" = "$TEST_POOL2" || return 1
    ceph balancer pool rm $TEST_POOL1 || return 1
    ceph balancer pool rm $TEST_POOL2 || return 1
    ceph balancer pool ls || return 1
    ceph balancer pool add $TEST_POOL1 || return 1

    ceph balancer mode crush-compat || return 1
    ceph balancer status || return 1
    eval MODE=$(ceph balancer status | jq '.mode')
    test $MODE = "crush-compat" || return 1
    ceph balancer off || return 1
    ! ceph balancer optimize plan_crush $TEST_POOL1 || return 1
    ceph balancer status || return 1
    eval RESULT=$(ceph balancer status | jq '.optimize_result')
    test "$RESULT" = "Distribution is already perfect" || return 1

    ceph balancer on || return 1
    ACTIVE=$(ceph balancer status | jq '.active')
    test $ACTIVE = "true" || return 1
    sleep 2
    ceph balancer status || return 1
    ceph balancer off || return 1
    ACTIVE=$(ceph balancer status | jq '.active')
    test $ACTIVE = "false" || return 1
    sleep 2

    ceph balancer reset || return 1

    ceph balancer mode upmap || return 1
    ceph balancer status || return 1
    eval MODE=$(ceph balancer status | jq '.mode')
    test $MODE = "upmap" || return 1
    ! ceph balancer optimize plan_upmap $TEST_POOL || return 1
    ceph balancer status || return 1
    eval RESULT=$(ceph balancer status | jq '.optimize_result')
    test "$RESULT" = "Unable to find further optimization, or pool(s) pg_num is decreasing, or distribution is already perfect" || return 1

    ceph balancer on || return 1
    ACTIVE=$(ceph balancer status | jq '.active')
    test $ACTIVE = "true" || return 1
    sleep 2
    ceph balancer status || return 1
    ceph balancer off || return 1
    ACTIVE=$(ceph balancer status | jq '.active')
    test $ACTIVE = "false" || return 1

    teardown $dir || return 1
}

function TEST_balancer2() {
    local dir=$1
    TEST_PGS1=118
    TEST_PGS2=132
    TOTAL_PGS=$(expr $TEST_PGS1 + $TEST_PGS2)
    OSDS=5
    DEFAULT_REPLICAS=3
    # Integer average of PGS per OSD (70.8), so each OSD >= this
    FINAL_PER_OSD1=$(expr \( $TEST_PGS1 \* $DEFAULT_REPLICAS \) / $OSDS)
    # Integer average of PGS per OSD (150)
    FINAL_PER_OSD2=$(expr \( \( $TEST_PGS1 + $TEST_PGS2 \) \* $DEFAULT_REPLICAS \) / $OSDS)

    CEPH_ARGS+="--osd_pool_default_pg_autoscale_mode=off "
    CEPH_ARGS+="--debug_osd=20 "
    setup $dir || return 1
    run_mon $dir a || return 1
    run_mgr $dir x || return 1
    for i in $(seq 0 $(expr $OSDS - 1))
    do
      run_osd $dir $i || return 1
    done

    ceph osd set-require-min-compat-client luminous
    ceph config set mgr mgr/balancer/upmap_max_deviation 1
    ceph balancer mode upmap || return 1
    ceph balancer on || return 1
    ceph config set mgr mgr/balancer/sleep_interval 5

    create_pool $TEST_POOL1 $TEST_PGS1

    wait_for_clean || return 1

    # Wait up to 2 minutes
    OK=no
    for i in $(seq 1 25)
    do
      sleep 5
      if grep -q "Optimization plan is almost perfect" $dir/mgr.x.log
      then
        OK=yes
        break
      fi
    done
    test $OK = "yes" || return 1
    # Plan is found, but PGs still need to move
    sleep 10
    wait_for_clean || return 1
    ceph osd df

    PGS=$(ceph osd df --format=json-pretty | jq '.nodes[0].pgs')
    test $PGS -ge $FINAL_PER_OSD1 || return 1
    PGS=$(ceph osd df --format=json-pretty | jq '.nodes[1].pgs')
    test $PGS -ge $FINAL_PER_OSD1 || return 1
    PGS=$(ceph osd df --format=json-pretty | jq '.nodes[2].pgs')
    test $PGS -ge $FINAL_PER_OSD1 || return 1
    PGS=$(ceph osd df --format=json-pretty | jq '.nodes[3].pgs')
    test $PGS -ge $FINAL_PER_OSD1 || return 1
    PGS=$(ceph osd df --format=json-pretty | jq '.nodes[4].pgs')
    test $PGS -ge $FINAL_PER_OSD1 || return 1

    create_pool $TEST_POOL2 $TEST_PGS2

    # Wait up to 2 minutes
    OK=no
    for i in $(seq 1 25)
    do
      sleep 5
      COUNT=$(grep "Optimization plan is almost perfect" $dir/mgr.x.log | wc -l)
      if test $COUNT = "2"
      then
        OK=yes
        break
      fi
    done
    test $OK = "yes" || return 1
    # Plan is found, but PGs still need to move
    sleep 10
    wait_for_clean || return 1
    ceph osd df

    # We should be with plus or minus 2 of FINAL_PER_OSD2
    # This is because here each pool is balanced independently
    MIN=$(expr $FINAL_PER_OSD2 - 2)
    MAX=$(expr $FINAL_PER_OSD2 + 2)
    PGS=$(ceph osd df --format=json-pretty | jq '.nodes[0].pgs')
    test $PGS -ge $MIN -a $PGS -le $MAX || return 1
    PGS=$(ceph osd df --format=json-pretty | jq '.nodes[1].pgs')
    test $PGS -ge $MIN -a $PGS -le $MAX || return 1
    PGS=$(ceph osd df --format=json-pretty | jq '.nodes[2].pgs')
    test $PGS -ge $MIN -a $PGS -le $MAX || return 1
    PGS=$(ceph osd df --format=json-pretty | jq '.nodes[3].pgs')
    test $PGS -ge $MIN -a $PGS -le $MAX || return 1
    PGS=$(ceph osd df --format=json-pretty | jq '.nodes[4].pgs')
    test $PGS -ge $MIN -a $PGS -le $MAX || return 1

    teardown $dir || return 1
}

main balancer "$@"

# Local Variables:
# compile-command: "make -j4 && ../qa/run-standalone.sh balancer.sh"
# End:
Commit	Line	Data
eafe8130 TL	1	#!/usr/bin/env bash
	2	#
	3	# Copyright (C) 2019 Red Hat <contact@redhat.com>
	4	#
	5	# Author: David Zafman <dzafman@redhat.com>
	6	#
	7	# This program is free software; you can redistribute it and/or modify
	8	# it under the terms of the GNU Library Public License as published by
	9	# the Free Software Foundation; either version 2, or (at your option)
	10	# any later version.
	11	#
	12	# This program is distributed in the hope that it will be useful,
	13	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	15	# GNU Library Public License for more details.
	16	#
	17	source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
	18
	19	function run() {
	20	local dir=$1
	21	shift
	22
	23	export CEPH_MON="127.0.0.1:7102" # git grep '\<7102\>' : there must be only one
	24	export CEPH_ARGS
	25	CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
	26	CEPH_ARGS+="--mon-host=$CEPH_MON "
	27
	28	local funcs=${@:-$(set \| sed -n -e 's/^\(TEST_[0-9a-z_]\) ./\1/p')}
	29	for func in $funcs ; do
	30	$func $dir \|\| return 1
	31	done
	32	}
	33
	34	TEST_POOL1=test1
	35	TEST_POOL2=test2
	36
	37	function TEST_balancer() {
	38	local dir=$1
	39
	40	setup $dir \|\| return 1
	41	run_mon $dir a \|\| return 1
	42	run_mgr $dir x \|\| return 1
	43	run_osd $dir 0 \|\| return 1
	44	run_osd $dir 1 \|\| return 1
	45	run_osd $dir 2 \|\| return 1
	46	create_pool $TEST_POOL1 8
	47	create_pool $TEST_POOL2 8
	48
	49	wait_for_clean \|\| return 1
	50
	51	ceph pg dump pgs
eafe8130 TL	52	ceph balancer status \|\| return 1
eafe8130 TL	53	eval MODE=$(ceph balancer status \| jq '.mode')
f67539c2	54	test $MODE = "upmap" \|\| return 1
eafe8130	55	ACTIVE=$(ceph balancer status \| jq '.active')
f67539c2	56	test $ACTIVE = "true" \|\| return 1
eafe8130 TL	57
	58	ceph balancer ls \|\| return 1
	59	PLANS=$(ceph balancer ls)
	60	test "$PLANS" = "[]" \|\| return 1
	61	ceph balancer eval \|\| return 1
	62	EVAL="$(ceph balancer eval)"
	63	test "$EVAL" = "current cluster score 0.000000 (lower is better)"
	64	ceph balancer eval-verbose \|\| return 1
	65
	66	ceph balancer pool add $TEST_POOL1 \|\| return 1
	67	ceph balancer pool add $TEST_POOL2 \|\| return 1
	68	ceph balancer pool ls \|\| return 1
92f5a8d4	69	eval POOL=$(ceph balancer pool ls \| jq 'sort \| .[0]')
eafe8130	70	test "$POOL" = "$TEST_POOL1" \|\| return 1
92f5a8d4	71	eval POOL=$(ceph balancer pool ls \| jq 'sort \| .[1]')
eafe8130 TL	72	test "$POOL" = "$TEST_POOL2" \|\| return 1
	73	ceph balancer pool rm $TEST_POOL1 \|\| return 1
	74	ceph balancer pool rm $TEST_POOL2 \|\| return 1
	75	ceph balancer pool ls \|\| return 1
	76	ceph balancer pool add $TEST_POOL1 \|\| return 1
	77
	78	ceph balancer mode crush-compat \|\| return 1
	79	ceph balancer status \|\| return 1
	80	eval MODE=$(ceph balancer status \| jq '.mode')
	81	test $MODE = "crush-compat" \|\| return 1
f67539c2	82	ceph balancer off \|\| return 1
eafe8130 TL	83	! ceph balancer optimize plan_crush $TEST_POOL1 \|\| return 1
	84	ceph balancer status \|\| return 1
	85	eval RESULT=$(ceph balancer status \| jq '.optimize_result')
	86	test "$RESULT" = "Distribution is already perfect" \|\| return 1
	87
	88	ceph balancer on \|\| return 1
	89	ACTIVE=$(ceph balancer status \| jq '.active')
	90	test $ACTIVE = "true" \|\| return 1
	91	sleep 2
	92	ceph balancer status \|\| return 1
	93	ceph balancer off \|\| return 1
	94	ACTIVE=$(ceph balancer status \| jq '.active')
	95	test $ACTIVE = "false" \|\| return 1
	96	sleep 2
	97
	98	ceph balancer reset \|\| return 1
	99
	100	ceph balancer mode upmap \|\| return 1
	101	ceph balancer status \|\| return 1
	102	eval MODE=$(ceph balancer status \| jq '.mode')
	103	test $MODE = "upmap" \|\| return 1
	104	! ceph balancer optimize plan_upmap $TEST_POOL \|\| return 1
	105	ceph balancer status \|\| return 1
	106	eval RESULT=$(ceph balancer status \| jq '.optimize_result')
92f5a8d4	107	test "$RESULT" = "Unable to find further optimization, or pool(s) pg_num is decreasing, or distribution is already perfect" \|\| return 1
eafe8130 TL	108
	109	ceph balancer on \|\| return 1
	110	ACTIVE=$(ceph balancer status \| jq '.active')
	111	test $ACTIVE = "true" \|\| return 1
	112	sleep 2
	113	ceph balancer status \|\| return 1
	114	ceph balancer off \|\| return 1
	115	ACTIVE=$(ceph balancer status \| jq '.active')
	116	test $ACTIVE = "false" \|\| return 1
	117
	118	teardown $dir \|\| return 1
	119	}
	120
92f5a8d4 TL	121	function TEST_balancer2() {
	122	local dir=$1
	123	TEST_PGS1=118
	124	TEST_PGS2=132
	125	TOTAL_PGS=$(expr $TEST_PGS1 + $TEST_PGS2)
	126	OSDS=5
	127	DEFAULT_REPLICAS=3
	128	# Integer average of PGS per OSD (70.8), so each OSD >= this
	129	FINAL_PER_OSD1=$(expr \( $TEST_PGS1 \* $DEFAULT_REPLICAS \) / $OSDS)
	130	# Integer average of PGS per OSD (150)
	131	FINAL_PER_OSD2=$(expr \( \( $TEST_PGS1 + $TEST_PGS2 \) \* $DEFAULT_REPLICAS \) / $OSDS)
	132
	133	CEPH_ARGS+="--osd_pool_default_pg_autoscale_mode=off "
	134	CEPH_ARGS+="--debug_osd=20 "
	135	setup $dir \|\| return 1
	136	run_mon $dir a \|\| return 1
	137	run_mgr $dir x \|\| return 1
	138	for i in $(seq 0 $(expr $OSDS - 1))
	139	do
	140	run_osd $dir $i \|\| return 1
	141	done
	142
	143	ceph osd set-require-min-compat-client luminous
	144	ceph config set mgr mgr/balancer/upmap_max_deviation 1
	145	ceph balancer mode upmap \|\| return 1
	146	ceph balancer on \|\| return 1
	147	ceph config set mgr mgr/balancer/sleep_interval 5
	148
	149	create_pool $TEST_POOL1 $TEST_PGS1
	150
	151	wait_for_clean \|\| return 1
	152
	153	# Wait up to 2 minutes
	154	OK=no
	155	for i in $(seq 1 25)
	156	do
	157	sleep 5
	158	if grep -q "Optimization plan is almost perfect" $dir/mgr.x.log
	159	then
	160	OK=yes
	161	break
	162	fi
	163	done
	164	test $OK = "yes" \|\| return 1
	165	# Plan is found, but PGs still need to move
9f95a23c TL	166	sleep 10
9f95a23c TL	167	wait_for_clean \|\| return 1
92f5a8d4 TL	168	ceph osd df
	169
	170	PGS=$(ceph osd df --format=json-pretty \| jq '.nodes[0].pgs')
	171	test $PGS -ge $FINAL_PER_OSD1 \|\| return 1
	172	PGS=$(ceph osd df --format=json-pretty \| jq '.nodes[1].pgs')
	173	test $PGS -ge $FINAL_PER_OSD1 \|\| return 1
	174	PGS=$(ceph osd df --format=json-pretty \| jq '.nodes[2].pgs')
	175	test $PGS -ge $FINAL_PER_OSD1 \|\| return 1
	176	PGS=$(ceph osd df --format=json-pretty \| jq '.nodes[3].pgs')
	177	test $PGS -ge $FINAL_PER_OSD1 \|\| return 1
	178	PGS=$(ceph osd df --format=json-pretty \| jq '.nodes[4].pgs')
	179	test $PGS -ge $FINAL_PER_OSD1 \|\| return 1
	180
	181	create_pool $TEST_POOL2 $TEST_PGS2
	182
	183	# Wait up to 2 minutes
	184	OK=no
	185	for i in $(seq 1 25)
	186	do
	187	sleep 5
	188	COUNT=$(grep "Optimization plan is almost perfect" $dir/mgr.x.log \| wc -l)
	189	if test $COUNT = "2"
	190	then
	191	OK=yes
	192	break
	193	fi
	194	done
	195	test $OK = "yes" \|\| return 1
	196	# Plan is found, but PGs still need to move
9f95a23c TL	197	sleep 10
9f95a23c TL	198	wait_for_clean \|\| return 1
92f5a8d4 TL	199	ceph osd df
92f5a8d4 TL	200
9f95a23c	201	# We should be with plus or minus 2 of FINAL_PER_OSD2
92f5a8d4	202	# This is because here each pool is balanced independently
9f95a23c TL	203	MIN=$(expr $FINAL_PER_OSD2 - 2)
9f95a23c TL	204	MAX=$(expr $FINAL_PER_OSD2 + 2)
92f5a8d4 TL	205	PGS=$(ceph osd df --format=json-pretty \| jq '.nodes[0].pgs')
	206	test $PGS -ge $MIN -a $PGS -le $MAX \|\| return 1
	207	PGS=$(ceph osd df --format=json-pretty \| jq '.nodes[1].pgs')
	208	test $PGS -ge $MIN -a $PGS -le $MAX \|\| return 1
	209	PGS=$(ceph osd df --format=json-pretty \| jq '.nodes[2].pgs')
	210	test $PGS -ge $MIN -a $PGS -le $MAX \|\| return 1
	211	PGS=$(ceph osd df --format=json-pretty \| jq '.nodes[3].pgs')
	212	test $PGS -ge $MIN -a $PGS -le $MAX \|\| return 1
	213	PGS=$(ceph osd df --format=json-pretty \| jq '.nodes[4].pgs')
	214	test $PGS -ge $MIN -a $PGS -le $MAX \|\| return 1
	215
	216	teardown $dir \|\| return 1
	217	}
	218
eafe8130 TL	219	main balancer "$@"
	220
	221	# Local Variables:
	222	# compile-command: "make -j4 && ../qa/run-standalone.sh balancer.sh"
	223	# End: