projects / ceph.git / blame
| Commit | Line | Data |
|---|---|---|
| 9f95a23c TL |
1 | #!/bin/bash -ex |
| 2 | ||
| 3 | SCRIPT_NAME=$(basename ${BASH_SOURCE[0]}) | |
| 4 | SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" | |
| 5 | ||
| 6 | FSID='00000000-0000-0000-0000-0000deadbeef' | |
| 7 | ||
| 8 | # images that are used | |
| 9 | IMAGE_MASTER=${IMAGE_MASTER:-'quay.io/ceph-ci/ceph:octopus'} # octopus for octopus branch | |
| 10 | IMAGE_NAUTILUS=${IMAGE_NAUTILUS:-'docker.io/ceph/daemon-base:latest-nautilus'} | |
| 11 | IMAGE_MIMIC=${IMAGE_MIMIC:-'docker.io/ceph/daemon-base:latest-mimic'} | |
| 12 | ||
| 13 | TMPDIR=$(mktemp -d) | |
| 14 | ||
| 15 | function cleanup() | |
| 16 | { | |
| 17 | dump_all_logs | |
| 18 | rm -rf $TMPDIR | |
| 19 | } | |
| 20 | trap cleanup EXIT | |
| 21 | ||
| 22 | OSD_IMAGE_NAME="${SCRIPT_NAME%.*}_osd.img" | |
| 23 | OSD_IMAGE_SIZE='6G' | |
| 24 | OSD_TO_CREATE=2 | |
| 25 | OSD_VG_NAME=${SCRIPT_NAME%.*} | |
| 26 | OSD_LV_NAME=${SCRIPT_NAME%.*} | |
| 27 | ||
| 28 | CEPHADM_SRC_DIR=${SCRIPT_DIR}/../../../src/cephadm | |
| 29 | CEPHADM_SAMPLES_DIR=${CEPHADM_SRC_DIR}/samples | |
| 30 | ||
| 31 | [ -z "$SUDO" ] && SUDO=sudo | |
| 32 | ||
| 33 | if [ -z "$CEPHADM" ]; then | |
| 34 | CEPHADM=${CEPHADM_SRC_DIR}/cephadm | |
| 35 | fi | |
| 36 | ||
| 37 | # at this point, we need $CEPHADM set | |
| 38 | if ! [ -x "$CEPHADM" ]; then | |
| 39 | echo "cephadm not found. Please set \$CEPHADM" | |
| 40 | exit 1 | |
| 41 | fi | |
| 42 | ||
| 43 | # respawn ourselves with a shebang | |
| 44 | if [ -z "$PYTHON_KLUDGE" ]; then | |
| 45 | # see which pythons we should test with | |
| 46 | PYTHONS="" | |
| 47 | which python3 && PYTHONS="$PYTHONS python3" | |
| 48 | which python2 && PYTHONS="$PYTHONS python2" | |
| 49 | echo "PYTHONS $PYTHONS" | |
| 50 | if [ -z "$PYTHONS" ]; then | |
| 51 | echo "No PYTHONS found!" | |
| 52 | exit 1 | |
| 53 | fi | |
| 54 | ||
| 55 | TMPBINDIR=$(mktemp -d) | |
| 56 | trap "rm -rf $TMPBINDIR" EXIT | |
| 57 | ORIG_CEPHADM="$CEPHADM" | |
| 58 | CEPHADM="$TMPBINDIR/cephadm" | |
| 59 | for p in $PYTHONS; do | |
| 60 | echo "=== re-running with $p ===" | |
| 61 | ln -s `which $p` $TMPBINDIR/python | |
| 62 | echo "#!$TMPBINDIR/python" > $CEPHADM | |
| 63 | cat $ORIG_CEPHADM >> $CEPHADM | |
| 64 | chmod 700 $CEPHADM | |
| 65 | $TMPBINDIR/python --version | |
| 66 | PYTHON_KLUDGE=1 CEPHADM=$CEPHADM $0 | |
| 67 | rm $TMPBINDIR/python | |
| 68 | done | |
| 69 | rm -rf $TMPBINDIR | |
| 70 | echo "PASS with all of: $PYTHONS" | |
| 71 | exit 0 | |
| 72 | fi | |
| 73 | ||
| 74 | # add image to args | |
| 75 | CEPHADM_ARGS="$CEPHADM_ARGS --image $IMAGE_MASTER" | |
| 76 | ||
| 77 | # combine into a single var | |
| 78 | CEPHADM_BIN="$CEPHADM" | |
| 79 | CEPHADM="$SUDO $CEPHADM_BIN $CEPHADM_ARGS" | |
| 80 | ||
| 81 | # clean up previous run(s)? | |
| 82 | $CEPHADM rm-cluster --fsid $FSID --force | |
| 83 | $SUDO vgchange -an $OSD_VG_NAME || true | |
| 84 | loopdev=$($SUDO losetup -a | grep $(basename $OSD_IMAGE_NAME) | awk -F : '{print $1}') | |
| 85 | if ! [ "$loopdev" = "" ]; then | |
| 86 | $SUDO losetup -d $loopdev | |
| 87 | fi | |
| 88 | ||
| 89 | function expect_false() | |
| 90 | { | |
| 91 | set -x | |
| 92 | if eval "$@"; then return 1; else return 0; fi | |
| 93 | } | |
| 94 | ||
| 95 | function is_available() | |
| 96 | { | |
| 97 | local name="$1" | |
| 98 | local condition="$2" | |
| 99 | local tries="$3" | |
| 100 | ||
| 101 | local num=0 | |
| 102 | while ! eval "$condition"; do | |
| 103 | num=$(($num + 1)) | |
| 104 | if [ "$num" -ge $tries ]; then | |
| 105 | echo "$name is not available" | |
| 106 | false | |
| 107 | fi | |
| 108 | sleep 5 | |
| 109 | done | |
| 110 | ||
| 111 | echo "$name is available" | |
| 112 | true | |
| 113 | } | |
| 114 | ||
| 115 | function dump_log() | |
| 116 | { | |
| 117 | local name="$1" | |
| 118 | local num_lines="$2" | |
| 119 | ||
| 120 | if [ -z $num_lines ]; then | |
| 121 | num_lines=100 | |
| 122 | fi | |
| 123 | ||
| 124 | echo '-------------------------' | |
| 125 | echo 'dump daemon log:' $name | |
| 126 | echo '-------------------------' | |
| 127 | ||
| 128 | $CEPHADM logs --name $name -- --no-pager -n $num_lines | |
| 129 | } | |
| 130 | ||
| 131 | function dump_all_logs() | |
| 132 | { | |
| 133 | names=$($CEPHADM ls | jq -r '.[].name') | |
| 134 | ||
| 135 | echo 'dumping logs for daemons: ' $names | |
| 136 | for name in $names; do | |
| 137 | dump_log $name | |
| 138 | done | |
| 139 | } | |
| 140 | ||
| 141 | function nfs_stop() | |
| 142 | { | |
| 143 | # stop the running nfs server | |
| 144 | local units="nfs-server nfs-kernel-server" | |
| 145 | for unit in $units; do | |
| 146 | if systemctl status $unit; then | |
| 147 | $SUDO systemctl stop $unit | |
| 148 | fi | |
| 149 | done | |
| 150 | ||
| 151 | # ensure the NFS port is no longer in use | |
| 152 | expect_false "$SUDO ss -tlnp '( sport = :nfs )' | grep LISTEN" | |
| 153 | } | |
| 154 | ||
| 155 | ## prepare + check host | |
| 156 | $SUDO $CEPHADM check-host | |
| 157 | ||
| 158 | ## version + --image | |
| 159 | $SUDO CEPHADM_IMAGE=$IMAGE_NAUTILUS $CEPHADM_BIN version | |
| 160 | $SUDO CEPHADM_IMAGE=$IMAGE_NAUTILUS $CEPHADM_BIN version \ | |
| 161 | | grep 'ceph version 14' | |
| 162 | $SUDO $CEPHADM_BIN --image $IMAGE_MIMIC version | |
| 163 | $SUDO $CEPHADM_BIN --image $IMAGE_MIMIC version \ | |
| 164 | | grep 'ceph version 13' | |
| 165 | $SUDO $CEPHADM_BIN --image $IMAGE_MASTER version | grep 'ceph version' | |
| 166 | ||
| 167 | # try force docker; this won't work if docker isn't installed | |
| 168 | systemctl status docker && ( $CEPHADM --docker version | grep 'ceph version' ) | |
| 169 | ||
| 170 | ## test shell before bootstrap, when crash dir isn't (yet) present on this host | |
| 171 | $CEPHADM shell --fsid $FSID -- ceph -v | grep 'ceph version' | |
| 172 | $CEPHADM shell --fsid $FSID -e FOO=BAR -- printenv | grep FOO=BAR | |
| 173 | ||
| 174 | ## bootstrap | |
| 175 | ORIG_CONFIG=`mktemp -p $TMPDIR` | |
| 176 | CONFIG=`mktemp -p $TMPDIR` | |
| 177 | MONCONFIG=`mktemp -p $TMPDIR` | |
| 178 | KEYRING=`mktemp -p $TMPDIR` | |
| 179 | IP=127.0.0.1 | |
| 180 | cat <<EOF > $ORIG_CONFIG | |
| 181 | [global] | |
| 182 | log to file = true | |
| 183 | osd crush chooseleaf type = 0 | |
| 184 | EOF | |
| 185 | $CEPHADM bootstrap \ | |
| 186 | --mon-id a \ | |
| 187 | --mgr-id x \ | |
| 188 | --mon-ip $IP \ | |
| 189 | --fsid $FSID \ | |
| 190 | --config $ORIG_CONFIG \ | |
| 191 | --output-config $CONFIG \ | |
| 192 | --output-keyring $KEYRING \ | |
| 193 | --output-pub-ssh-key $TMPDIR/ceph.pub \ | |
| 194 | --allow-overwrite \ | |
| 195 | --skip-mon-network \ | |
| 196 | --skip-monitoring-stack | |
| 197 | test -e $CONFIG | |
| 198 | test -e $KEYRING | |
| 199 | rm -f $ORIG_CONFIG | |
| 200 | ||
| 201 | $SUDO test -e /var/log/ceph/$FSID/ceph-mon.a.log | |
| 202 | $SUDO test -e /var/log/ceph/$FSID/ceph-mgr.x.log | |
| 203 | ||
| 204 | for u in ceph.target \ | |
| 205 | ceph-$FSID.target \ | |
| 206 | ceph-$FSID@mon.a \ | |
| 207 | ceph-$FSID@mgr.x; do | |
| 208 | systemctl is-enabled $u | |
| 209 | systemctl is-active $u | |
| 210 | done | |
| 211 | systemctl | grep system-ceph | grep -q .slice # naming is escaped and annoying | |
| 212 | ||
| 213 | # check ceph -s works (via shell w/ passed config/keyring) | |
| 214 | $CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ | |
| 215 | ceph -s | grep $FSID | |
| 216 | ||
| 217 | for t in mon mgr node-exporter prometheus grafana; do | |
| 218 | $CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ | |
| 219 | ceph orch apply $t --unmanaged | |
| 220 | done | |
| 221 | ||
| 222 | ## ls | |
| 223 | $CEPHADM ls | jq '.[]' | jq 'select(.name == "mon.a").fsid' \ | |
| 224 | | grep $FSID | |
| 225 | $CEPHADM ls | jq '.[]' | jq 'select(.name == "mgr.x").fsid' \ | |
| 226 | | grep $FSID | |
| 227 | ||
| 228 | # make sure the version is returned correctly | |
| 229 | $CEPHADM ls | jq '.[]' | jq 'select(.name == "mon.a").version' | grep -q \\. | |
| 230 | ||
| 231 | ## deploy | |
| 232 | # add mon.b | |
| 233 | cp $CONFIG $MONCONFIG | |
| 234 | echo "public addr = $IP:3301" >> $MONCONFIG | |
| 235 | $CEPHADM deploy --name mon.b \ | |
| 236 | --fsid $FSID \ | |
| 237 | --keyring /var/lib/ceph/$FSID/mon.a/keyring \ | |
| 238 | --config $CONFIG | |
| 239 | for u in ceph-$FSID@mon.b; do | |
| 240 | systemctl is-enabled $u | |
| 241 | systemctl is-active $u | |
| 242 | done | |
| 243 | ||
| 244 | # add mgr.y | |
| 245 | $CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ | |
| 246 | ceph auth get-or-create mgr.y \ | |
| 247 | mon 'allow profile mgr' \ | |
| 248 | osd 'allow *' \ | |
| 249 | mds 'allow *' > $TMPDIR/keyring.mgr.y | |
| 250 | $CEPHADM deploy --name mgr.y \ | |
| 251 | --fsid $FSID \ | |
| 252 | --keyring $TMPDIR/keyring.mgr.y \ | |
| 253 | --config $CONFIG | |
| 254 | for u in ceph-$FSID@mgr.y; do | |
| 255 | systemctl is-enabled $u | |
| 256 | systemctl is-active $u | |
| 257 | done | |
| 258 | ||
| 259 | for f in `seq 1 30`; do | |
| 260 | if $CEPHADM shell --fsid $FSID \ | |
| 261 | --config $CONFIG --keyring $KEYRING -- \ | |
| 262 | ceph -s -f json-pretty \ | |
| 263 | | jq '.mgrmap.num_standbys' | grep -q 1 ; then break; fi | |
| 264 | sleep 1 | |
| 265 | done | |
| 266 | $CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ | |
| 267 | ceph -s -f json-pretty \ | |
| 268 | | jq '.mgrmap.num_standbys' | grep -q 1 | |
| 269 | ||
| 270 | # add osd.{1,2,..} | |
| 271 | dd if=/dev/zero of=$TMPDIR/$OSD_IMAGE_NAME bs=1 count=0 seek=$OSD_IMAGE_SIZE | |
| 272 | loop_dev=$($SUDO losetup -f) | |
| 273 | $SUDO vgremove -f $OSD_VG_NAME || true | |
| 274 | $SUDO losetup $loop_dev $TMPDIR/$OSD_IMAGE_NAME | |
| 275 | $SUDO pvcreate $loop_dev && $SUDO vgcreate $OSD_VG_NAME $loop_dev | |
| 276 | for id in `seq 0 $((--OSD_TO_CREATE))`; do | |
| 277 | $SUDO lvcreate -l $((100/$OSD_TO_CREATE))%VG -n $OSD_LV_NAME.$id $OSD_VG_NAME | |
| 278 | $CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ | |
| 279 | ceph orch daemon add osd \ | |
| 280 | $(hostname):/dev/$OSD_VG_NAME/$OSD_LV_NAME.$id | |
| 281 | done | |
| 282 | ||
| 283 | # add node-exporter | |
| 284 | ${CEPHADM//--image $IMAGE_MASTER/} deploy \ | |
| 285 | --name node-exporter.a --fsid $FSID | |
| 286 | cond="curl 'http://localhost:9100' | grep -q 'Node Exporter'" | |
| 287 | is_available "node-exporter" "$cond" 5 | |
| 288 | ||
| 289 | # add prometheus | |
| 290 | cat ${CEPHADM_SAMPLES_DIR}/prometheus.json | \ | |
| 291 | ${CEPHADM//--image $IMAGE_MASTER/} deploy \ | |
| 292 | --name prometheus.a --fsid $FSID --config-json - | |
| 293 | cond="curl 'localhost:9095/api/v1/query?query=up'" | |
| 294 | is_available "prometheus" "$cond" 5 | |
| 295 | ||
| 296 | # add grafana | |
| 297 | cat ${CEPHADM_SAMPLES_DIR}/grafana.json | \ | |
| 298 | ${CEPHADM//--image $IMAGE_MASTER/} deploy \ | |
| 299 | --name grafana.a --fsid $FSID --config-json - | |
| 300 | cond="curl --insecure 'https://localhost:3000' | grep -q 'grafana'" | |
| 301 | is_available "grafana" "$cond" 30 | |
| 302 | ||
| 303 | # add nfs-ganesha | |
| 304 | nfs_stop | |
| 305 | nfs_rados_pool=$(cat ${CEPHADM_SAMPLES_DIR}/nfs.json | jq -r '.["pool"]') | |
| 306 | $CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ | |
| 307 | ceph osd pool create $nfs_rados_pool 64 | |
| 308 | $CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ | |
| 309 | rados --pool nfs-ganesha --namespace nfs-ns create conf-nfs.a | |
| 310 | $CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ | |
| 311 | ceph orch pause | |
| 312 | $CEPHADM deploy --name nfs.a \ | |
| 313 | --fsid $FSID \ | |
| 314 | --keyring $KEYRING \ | |
| 315 | --config $CONFIG \ | |
| 316 | --config-json ${CEPHADM_SAMPLES_DIR}/nfs.json | |
| 317 | cond="$SUDO ss -tlnp '( sport = :nfs )' | grep 'ganesha.nfsd'" | |
| 318 | is_available "nfs" "$cond" 10 | |
| 319 | $CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ | |
| 320 | ceph orch resume | |
| 321 | ||
| 322 | ## run | |
| 323 | # WRITE ME | |
| 324 | ||
| 325 | ## unit | |
| 326 | $CEPHADM unit --fsid $FSID --name mon.a -- is-enabled | |
| 327 | $CEPHADM unit --fsid $FSID --name mon.a -- is-active | |
| 328 | expect_false $CEPHADM unit --fsid $FSID --name mon.xyz -- is-active | |
| 329 | $CEPHADM unit --fsid $FSID --name mon.a -- disable | |
| 330 | expect_false $CEPHADM unit --fsid $FSID --name mon.a -- is-enabled | |
| 331 | $CEPHADM unit --fsid $FSID --name mon.a -- enable | |
| 332 | $CEPHADM unit --fsid $FSID --name mon.a -- is-enabled | |
| 333 | ||
| 334 | ## shell | |
| 335 | $CEPHADM shell --fsid $FSID -- true | |
| 336 | $CEPHADM shell --fsid $FSID -- test -d /var/log/ceph | |
| 337 | expect_false $CEPHADM --timeout 1 shell --fsid $FSID -- sleep 10 | |
| 338 | $CEPHADM --timeout 10 shell --fsid $FSID -- sleep 1 | |
| 339 | ||
| 340 | ## enter | |
| 341 | expect_false $CEPHADM enter | |
| 342 | $CEPHADM enter --fsid $FSID --name mon.a -- test -d /var/lib/ceph/mon/ceph-a | |
| 343 | $CEPHADM enter --fsid $FSID --name mgr.x -- test -d /var/lib/ceph/mgr/ceph-x | |
| 344 | $CEPHADM enter --fsid $FSID --name mon.a -- pidof ceph-mon | |
| 345 | expect_false $CEPHADM enter --fsid $FSID --name mgr.x -- pidof ceph-mon | |
| 346 | $CEPHADM enter --fsid $FSID --name mgr.x -- pidof ceph-mgr | |
| 347 | # this triggers a bug in older versions of podman, including 18.04's 1.6.2 | |
| 348 | #expect_false $CEPHADM --timeout 1 enter --fsid $FSID --name mon.a -- sleep 10 | |
| 349 | $CEPHADM --timeout 10 enter --fsid $FSID --name mon.a -- sleep 1 | |
| 350 | ||
| 351 | ## ceph-volume | |
| 352 | $CEPHADM ceph-volume --fsid $FSID -- inventory --format=json \ | |
| 353 | | jq '.[]' | |
| 354 | ||
| 355 | ## rm-daemon | |
| 356 | # mon and osd require --force | |
| 357 | expect_false $CEPHADM rm-daemon --fsid $FSID --name mon.a | |
| 358 | # mgr does not | |
| 359 | $CEPHADM rm-daemon --fsid $FSID --name mgr.x | |
| 360 | ||
| 361 | ## rm-cluster | |
| 362 | expect_false $CEPHADM rm-cluster --fsid $FSID | |
| 363 | $CEPHADM rm-cluster --fsid $FSID --force | |
| 364 | ||
| 365 | rm -rf $TMPDIR | |
| 366 | echo PASS |