ceph/qa/workunits/cephadm/test_cephadm.sh

   1 #!/bin/bash -ex
   2
   3 SCRIPT_NAME=$(basename ${BASH_SOURCE[0]})
   4 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
   5
   6 # cleanup during exit
   7 [ -z "$CLEANUP" ] && CLEANUP=true
   8
   9 FSID='00000000-0000-0000-0000-0000deadbeef'
  10
  11 # images that are used
  12 IMAGE_MASTER=${IMAGE_MASTER:-'quay.ceph.io/ceph-ci/ceph:master'}
  13 IMAGE_PACIFIC=${IMAGE_PACIFIC:-'quay.ceph.io/ceph-ci/ceph:pacific'}
  14 #IMAGE_OCTOPUS=${IMAGE_OCTOPUS:-'quay.ceph.io/ceph-ci/ceph:octopus'}
  15 IMAGE_DEFAULT=${IMAGE_PACIFIC}
  16
  17 OSD_IMAGE_NAME="${SCRIPT_NAME%.*}_osd.img"
  18 OSD_IMAGE_SIZE='6G'
  19 OSD_TO_CREATE=2
  20 OSD_VG_NAME=${SCRIPT_NAME%.*}
  21 OSD_LV_NAME=${SCRIPT_NAME%.*}
  22
  23 CEPHADM_SRC_DIR=${SCRIPT_DIR}/../../../src/cephadm
  24 CEPHADM_SAMPLES_DIR=${CEPHADM_SRC_DIR}/samples
  25
  26 [ -z "$SUDO" ] && SUDO=sudo
  27
  28 if [ -z "$CEPHADM" ]; then
  29     CEPHADM=${CEPHADM_SRC_DIR}/cephadm
  30 fi
  31
  32 # at this point, we need $CEPHADM set
  33 if ! [ -x "$CEPHADM" ]; then
  34     echo "cephadm not found. Please set \$CEPHADM"
  35     exit 1
  36 fi
  37
  38 # add image to args
  39 CEPHADM_ARGS="$CEPHADM_ARGS --image $IMAGE_DEFAULT"
  40
  41 # combine into a single var
  42 CEPHADM_BIN="$CEPHADM"
  43 CEPHADM="$SUDO $CEPHADM_BIN $CEPHADM_ARGS"
  44
  45 # clean up previous run(s)?
  46 $CEPHADM rm-cluster --fsid $FSID --force
  47 $SUDO vgchange -an $OSD_VG_NAME || true
  48 loopdev=$($SUDO losetup -a | grep $(basename $OSD_IMAGE_NAME) | awk -F : '{print $1}')
  49 if ! [ "$loopdev" = "" ]; then
  50     $SUDO losetup -d $loopdev
  51 fi
  52
  53 # TMPDIR for test data
  54 [ -d "$TMPDIR" ] || TMPDIR=$(mktemp -d tmp.$SCRIPT_NAME.XXXXXX)
  55 [ -d "$TMPDIR_TEST_MULTIPLE_MOUNTS" ] || TMPDIR_TEST_MULTIPLE_MOUNTS=$(mktemp -d tmp.$SCRIPT_NAME.XXXXXX)
  56
  57 function cleanup()
  58 {
  59     if [ $CLEANUP = false ]; then
  60         # preserve the TMPDIR state
  61         echo "========================"
  62         echo "!!! CLEANUP=$CLEANUP !!!"
  63         echo
  64         echo "TMPDIR=$TMPDIR"
  65         echo "========================"
  66         return
  67     fi
  68
  69     dump_all_logs $FSID
  70     rm -rf $TMPDIR
  71 }
  72 trap cleanup EXIT
  73
  74 function expect_false()
  75 {
  76         set -x
  77         if eval "$@"; then return 1; else return 0; fi
  78 }
  79
  80 function is_available()
  81 {
  82     local name="$1"
  83     local condition="$2"
  84     local tries="$3"
  85
  86     local num=0
  87     while ! eval "$condition"; do
  88         num=$(($num + 1))
  89         if [ "$num" -ge $tries ]; then
  90             echo "$name is not available"
  91             false
  92         fi
  93         sleep 5
  94     done
  95
  96     echo "$name is available"
  97     true
  98 }
  99
 100 function dump_log()
 101 {
 102     local fsid="$1"
 103     local name="$2"
 104     local num_lines="$3"
 105
 106     if [ -z $num_lines ]; then
 107         num_lines=100
 108     fi
 109
 110     echo '-------------------------'
 111     echo 'dump daemon log:' $name
 112     echo '-------------------------'
 113
 114     $CEPHADM logs --fsid $fsid --name $name -- --no-pager -n $num_lines
 115 }
 116
 117 function dump_all_logs()
 118 {
 119     local fsid="$1"
 120     local names=$($CEPHADM ls | jq -r '.[] | select(.fsid == "'$fsid'").name')
 121
 122     echo 'dumping logs for daemons: ' $names
 123     for name in $names; do
 124         dump_log $fsid $name
 125     done
 126 }
 127
 128 function nfs_stop()
 129 {
 130     # stop the running nfs server
 131     local units="nfs-server nfs-kernel-server"
 132     for unit in $units; do
 133         if systemctl status $unit < /dev/null; then
 134             $SUDO systemctl stop $unit
 135         fi
 136     done
 137
 138     # ensure the NFS port is no longer in use
 139     expect_false "$SUDO ss -tlnp '( sport = :nfs )' | grep LISTEN"
 140 }
 141
 142 ## prepare + check host
 143 $SUDO $CEPHADM check-host
 144
 145 ## run a gather-facts (output to stdout)
 146 $SUDO $CEPHADM gather-facts
 147
 148 ## version + --image
 149 $SUDO CEPHADM_IMAGE=$IMAGE_PACIFIC $CEPHADM_BIN version
 150 $SUDO CEPHADM_IMAGE=$IMAGE_PACIFIC $CEPHADM_BIN version \
 151     | grep 'ceph version 16'
 152 #$SUDO CEPHADM_IMAGE=$IMAGE_OCTOPUS $CEPHADM_BIN version
 153 #$SUDO CEPHADM_IMAGE=$IMAGE_OCTOPUS $CEPHADM_BIN version \
 154 #    | grep 'ceph version 15'
 155 $SUDO $CEPHADM_BIN --image $IMAGE_MASTER version | grep 'ceph version'
 156
 157 # try force docker; this won't work if docker isn't installed
 158 systemctl status docker > /dev/null && ( $CEPHADM --docker version | grep 'ceph version' ) || echo "docker not installed"
 159
 160 ## test shell before bootstrap, when crash dir isn't (yet) present on this host
 161 $CEPHADM shell --fsid $FSID -- ceph -v | grep 'ceph version'
 162 $CEPHADM shell --fsid $FSID -e FOO=BAR -- printenv | grep FOO=BAR
 163
 164 ## bootstrap
 165 ORIG_CONFIG=`mktemp -p $TMPDIR`
 166 CONFIG=`mktemp -p $TMPDIR`
 167 MONCONFIG=`mktemp -p $TMPDIR`
 168 KEYRING=`mktemp -p $TMPDIR`
 169 IP=127.0.0.1
 170 cat <<EOF > $ORIG_CONFIG
 171 [global]
 172         log to file = true
 173         osd crush chooseleaf type = 0
 174 EOF
 175 $CEPHADM bootstrap \
 176       --mon-id a \
 177       --mgr-id x \
 178       --mon-ip $IP \
 179       --fsid $FSID \
 180       --config $ORIG_CONFIG \
 181       --output-config $CONFIG \
 182       --output-keyring $KEYRING \
 183       --output-pub-ssh-key $TMPDIR/ceph.pub \
 184       --allow-overwrite \
 185       --skip-mon-network \
 186       --skip-monitoring-stack \
 187       --with-exporter
 188 test -e $CONFIG
 189 test -e $KEYRING
 190 rm -f $ORIG_CONFIG
 191
 192 $SUDO test -e /var/log/ceph/$FSID/ceph-mon.a.log
 193 $SUDO test -e /var/log/ceph/$FSID/ceph-mgr.x.log
 194
 195 for u in ceph.target \
 196              ceph-$FSID.target \
 197              ceph-$FSID@mon.a \
 198              ceph-$FSID@mgr.x; do
 199     systemctl is-enabled $u
 200     systemctl is-active $u
 201 done
 202 systemctl | grep system-ceph | grep -q .slice  # naming is escaped and annoying
 203
 204 # check ceph -s works (via shell w/ passed config/keyring)
 205 $CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \
 206       ceph -s | grep $FSID
 207
 208 for t in mon mgr node-exporter prometheus grafana; do
 209     $CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \
 210              ceph orch apply $t --unmanaged
 211 done
 212
 213 ## ls
 214 $CEPHADM ls | jq '.[]' | jq 'select(.name == "mon.a").fsid' \
 215     | grep $FSID
 216 $CEPHADM ls | jq '.[]' | jq 'select(.name == "mgr.x").fsid' \
 217     | grep $FSID
 218
 219 # make sure the version is returned correctly
 220 $CEPHADM ls | jq '.[]' | jq 'select(.name == "mon.a").version' | grep -q \\.
 221
 222 ## deploy
 223 # add mon.b
 224 cp $CONFIG $MONCONFIG
 225 echo "public addrv = [v2:$IP:3301,v1:$IP:6790]" >> $MONCONFIG
 226 $CEPHADM deploy --name mon.b \
 227       --fsid $FSID \
 228       --keyring /var/lib/ceph/$FSID/mon.a/keyring \
 229       --config $MONCONFIG
 230 for u in ceph-$FSID@mon.b; do
 231     systemctl is-enabled $u
 232     systemctl is-active $u
 233 done
 234 cond="$CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \
 235             ceph mon stat | grep '2 mons'"
 236 is_available "mon.b" "$cond" 30
 237
 238 # add mgr.y
 239 $CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \
 240       ceph auth get-or-create mgr.y \
 241       mon 'allow profile mgr' \
 242       osd 'allow *' \
 243       mds 'allow *' > $TMPDIR/keyring.mgr.y
 244 $CEPHADM deploy --name mgr.y \
 245       --fsid $FSID \
 246       --keyring $TMPDIR/keyring.mgr.y \
 247       --config $CONFIG
 248 for u in ceph-$FSID@mgr.y; do
 249     systemctl is-enabled $u
 250     systemctl is-active $u
 251 done
 252
 253 for f in `seq 1 30`; do
 254     if $CEPHADM shell --fsid $FSID \
 255              --config $CONFIG --keyring $KEYRING -- \
 256           ceph -s -f json-pretty \
 257         | jq '.mgrmap.num_standbys' | grep -q 1 ; then break; fi
 258     sleep 1
 259 done
 260 $CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \
 261       ceph -s -f json-pretty \
 262     | jq '.mgrmap.num_standbys' | grep -q 1
 263
 264 # add osd.{1,2,..}
 265 dd if=/dev/zero of=$TMPDIR/$OSD_IMAGE_NAME bs=1 count=0 seek=$OSD_IMAGE_SIZE
 266 loop_dev=$($SUDO losetup -f)
 267 $SUDO vgremove -f $OSD_VG_NAME || true
 268 $SUDO losetup $loop_dev $TMPDIR/$OSD_IMAGE_NAME
 269 $SUDO pvcreate $loop_dev && $SUDO vgcreate $OSD_VG_NAME $loop_dev
 270
 271 # osd boostrap keyring
 272 $CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \
 273       ceph auth get client.bootstrap-osd > $TMPDIR/keyring.bootstrap.osd
 274
 275 # create lvs first so ceph-volume doesn't overlap with lv creation
 276 for id in `seq 0 $((--OSD_TO_CREATE))`; do
 277     $SUDO lvcreate -l $((100/$OSD_TO_CREATE))%VG -n $OSD_LV_NAME.$id $OSD_VG_NAME
 278 done
 279
 280 for id in `seq 0 $((--OSD_TO_CREATE))`; do
 281     device_name=/dev/$OSD_VG_NAME/$OSD_LV_NAME.$id
 282     CEPH_VOLUME="$CEPHADM ceph-volume \
 283                        --fsid $FSID \
 284                        --config $CONFIG \
 285                        --keyring $TMPDIR/keyring.bootstrap.osd --"
 286
 287     # prepare the osd
 288     $CEPH_VOLUME lvm prepare --bluestore --data $device_name --no-systemd
 289     $CEPH_VOLUME lvm batch --no-auto $device_name --yes --no-systemd
 290
 291     # osd id and osd fsid
 292     $CEPH_VOLUME lvm list --format json $device_name > $TMPDIR/osd.map
 293     osd_id=$($SUDO cat $TMPDIR/osd.map | jq -cr '.. | ."ceph.osd_id"? | select(.)')
 294     osd_fsid=$($SUDO cat $TMPDIR/osd.map | jq -cr '.. | ."ceph.osd_fsid"? | select(.)')
 295
 296     # deploy the osd
 297     $CEPHADM deploy --name osd.$osd_id \
 298           --fsid $FSID \
 299           --keyring $TMPDIR/keyring.bootstrap.osd \
 300           --config $CONFIG \
 301           --osd-fsid $osd_fsid
 302 done
 303
 304 # add node-exporter
 305 ${CEPHADM//--image $IMAGE_DEFAULT/} deploy \
 306     --name node-exporter.a --fsid $FSID
 307 cond="curl 'http://localhost:9100' | grep -q 'Node Exporter'"
 308 is_available "node-exporter" "$cond" 10
 309
 310 # add prometheus
 311 cat ${CEPHADM_SAMPLES_DIR}/prometheus.json | \
 312         ${CEPHADM//--image $IMAGE_DEFAULT/} deploy \
 313             --name prometheus.a --fsid $FSID --config-json -
 314 cond="curl 'localhost:9095/api/v1/query?query=up'"
 315 is_available "prometheus" "$cond" 10
 316
 317 # add grafana
 318 cat ${CEPHADM_SAMPLES_DIR}/grafana.json | \
 319         ${CEPHADM//--image $IMAGE_DEFAULT/} deploy \
 320             --name grafana.a --fsid $FSID --config-json -
 321 cond="curl --insecure 'https://localhost:3000' | grep -q 'grafana'"
 322 is_available "grafana" "$cond" 50
 323
 324 # add nfs-ganesha
 325 nfs_stop
 326 nfs_rados_pool=$(cat ${CEPHADM_SAMPLES_DIR}/nfs.json | jq -r '.["pool"]')
 327 $CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \
 328         ceph osd pool create $nfs_rados_pool 64
 329 $CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \
 330         rados --pool nfs-ganesha --namespace nfs-ns create conf-nfs.a
 331 $CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \
 332          ceph orch pause
 333 $CEPHADM deploy --name nfs.a \
 334       --fsid $FSID \
 335       --keyring $KEYRING \
 336       --config $CONFIG \
 337       --config-json ${CEPHADM_SAMPLES_DIR}/nfs.json
 338 cond="$SUDO ss -tlnp '( sport = :nfs )' | grep 'ganesha.nfsd'"
 339 is_available "nfs" "$cond" 10
 340 $CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \
 341          ceph orch resume
 342
 343 # add alertmanager via custom container
 344 alertmanager_image=$(cat ${CEPHADM_SAMPLES_DIR}/custom_container.json | jq -r '.image')
 345 tcp_ports=$(cat ${CEPHADM_SAMPLES_DIR}/custom_container.json | jq -r '.ports | map_values(.|tostring) | join(" ")')
 346 cat ${CEPHADM_SAMPLES_DIR}/custom_container.json | \
 347       ${CEPHADM//--image $IMAGE_DEFAULT/} \
 348       --image $alertmanager_image \
 349       deploy \
 350       --tcp-ports "$tcp_ports" \
 351       --name container.alertmanager.a \
 352       --fsid $FSID \
 353       --config-json -
 354 cond="$CEPHADM enter --fsid $FSID --name container.alertmanager.a -- test -f \
 355       /etc/alertmanager/alertmanager.yml"
 356 is_available "alertmanager.yml" "$cond" 10
 357 cond="curl 'http://localhost:9093' | grep -q 'Alertmanager'"
 358 is_available "alertmanager" "$cond" 10
 359
 360 # Fetch the token we need to access the exporter API
 361 token=$($CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING ceph cephadm get-exporter-config | jq -r '.token')
 362 [[ ! -z "$token" ]]
 363
 364 # check all exporter threads active
 365 cond="curl -k -s -H \"Authorization: Bearer $token\" \
 366       https://localhost:9443/v1/metadata/health | \
 367       jq -r '.tasks | select(.disks == \"active\" and .daemons == \"active\" and .host == \"active\")'"
 368 is_available "exporter_threads_active" "$cond" 3
 369
 370 # check we deployed for all hosts
 371 $CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING ceph orch ls --service-type cephadm-exporter --format json
 372 host_pattern=$($CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING ceph orch ls --service-type cephadm-exporter --format json | jq -r '.[0].placement.host_pattern')
 373 [[ "$host_pattern" = "*" ]]
 374
 375 ## run
 376 # WRITE ME
 377
 378 ## unit
 379 $CEPHADM unit --fsid $FSID --name mon.a -- is-enabled
 380 $CEPHADM unit --fsid $FSID --name mon.a -- is-active
 381 expect_false $CEPHADM unit --fsid $FSID --name mon.xyz -- is-active
 382 $CEPHADM unit --fsid $FSID --name mon.a -- disable
 383 expect_false $CEPHADM unit --fsid $FSID --name mon.a -- is-enabled
 384 $CEPHADM unit --fsid $FSID --name mon.a -- enable
 385 $CEPHADM unit --fsid $FSID --name mon.a -- is-enabled
 386
 387 ## shell
 388 $CEPHADM shell --fsid $FSID -- true
 389 $CEPHADM shell --fsid $FSID -- test -d /var/log/ceph
 390 expect_false $CEPHADM --timeout 10 shell --fsid $FSID -- sleep 60
 391 $CEPHADM --timeout 60 shell --fsid $FSID -- sleep 10
 392 $CEPHADM shell --fsid $FSID --mount $TMPDIR $TMPDIR_TEST_MULTIPLE_MOUNTS -- stat /mnt/$(basename $TMPDIR)
 393
 394 ## enter
 395 expect_false $CEPHADM enter
 396 $CEPHADM enter --fsid $FSID --name mon.a -- test -d /var/lib/ceph/mon/ceph-a
 397 $CEPHADM enter --fsid $FSID --name mgr.x -- test -d /var/lib/ceph/mgr/ceph-x
 398 $CEPHADM enter --fsid $FSID --name mon.a -- pidof ceph-mon
 399 expect_false $CEPHADM enter --fsid $FSID --name mgr.x -- pidof ceph-mon
 400 $CEPHADM enter --fsid $FSID --name mgr.x -- pidof ceph-mgr
 401 # this triggers a bug in older versions of podman, including 18.04's 1.6.2
 402 #expect_false $CEPHADM --timeout 5 enter --fsid $FSID --name mon.a -- sleep 30
 403 $CEPHADM --timeout 60 enter --fsid $FSID --name mon.a -- sleep 10
 404
 405 ## ceph-volume
 406 $CEPHADM ceph-volume --fsid $FSID -- inventory --format=json \
 407       | jq '.[]'
 408
 409 ## preserve test state
 410 [ $CLEANUP = false ] && exit 0
 411
 412 ## rm-daemon
 413 # mon and osd require --force
 414 expect_false $CEPHADM rm-daemon --fsid $FSID --name mon.a
 415 # mgr does not
 416 $CEPHADM rm-daemon --fsid $FSID --name mgr.x
 417
 418 ## rm-cluster
 419 expect_false $CEPHADM rm-cluster --fsid $FSID
 420 $CEPHADM rm-cluster --fsid $FSID --force
 421
 422 echo PASS