3 SCRIPT_NAME
=$
(basename ${BASH_SOURCE[0]})
4 SCRIPT_DIR
="$( cd "$
( dirname "${BASH_SOURCE[0]}" )" && pwd )"
7 [ -z "$CLEANUP" ] && CLEANUP
=true
9 FSID
='00000000-0000-0000-0000-0000deadbeef'
11 # images that are used
12 IMAGE_MASTER
=${IMAGE_MASTER:-'quay.ceph.io/ceph-ci/ceph:master'}
13 IMAGE_PACIFIC
=${IMAGE_PACIFIC:-'quay.ceph.io/ceph-ci/ceph:pacific'}
14 #IMAGE_OCTOPUS=${IMAGE_OCTOPUS:-'quay.ceph.io/ceph-ci/ceph:octopus'}
15 IMAGE_DEFAULT
=${IMAGE_MASTER}
17 OSD_IMAGE_NAME
="${SCRIPT_NAME%.*}_osd.img"
20 OSD_VG_NAME
=${SCRIPT_NAME%.*}
21 OSD_LV_NAME
=${SCRIPT_NAME%.*}
23 CEPHADM_SRC_DIR
=${SCRIPT_DIR}/..
/..
/..
/src
/cephadm
24 CEPHADM_SAMPLES_DIR
=${CEPHADM_SRC_DIR}/samples
26 [ -z "$SUDO" ] && SUDO
=sudo
28 if [ -z "$CEPHADM" ]; then
29 CEPHADM
=${CEPHADM_SRC_DIR}/cephadm
32 # at this point, we need $CEPHADM set
33 if ! [ -x "$CEPHADM" ]; then
34 echo "cephadm not found. Please set \$CEPHADM"
39 CEPHADM_ARGS
="$CEPHADM_ARGS --image $IMAGE_DEFAULT"
41 # combine into a single var
42 CEPHADM_BIN
="$CEPHADM"
43 CEPHADM
="$SUDO $CEPHADM_BIN $CEPHADM_ARGS"
45 # clean up previous run(s)?
46 $CEPHADM rm-cluster
--fsid $FSID --force
47 $SUDO vgchange
-an $OSD_VG_NAME || true
48 loopdev
=$
($SUDO losetup
-a |
grep $
(basename $OSD_IMAGE_NAME) |
awk -F : '{print $1}')
49 if ! [ "$loopdev" = "" ]; then
50 $SUDO losetup
-d $loopdev
53 # TMPDIR for test data
54 [ -d "$TMPDIR" ] || TMPDIR
=$
(mktemp
-d tmp.
$SCRIPT_NAME.XXXXXX
)
55 [ -d "$TMPDIR_TEST_MULTIPLE_MOUNTS" ] || TMPDIR_TEST_MULTIPLE_MOUNTS
=$
(mktemp
-d tmp.
$SCRIPT_NAME.XXXXXX
)
59 if [ $CLEANUP = false
]; then
60 # preserve the TMPDIR state
61 echo "========================"
62 echo "!!! CLEANUP=$CLEANUP !!!"
65 echo "========================"
74 function expect_false
()
77 if eval "$@"; then return 1; else return 0; fi
80 function is_available
()
87 while ! eval "$condition"; do
89 if [ "$num" -ge $tries ]; then
90 echo "$name is not available"
96 echo "$name is available"
106 if [ -z $num_lines ]; then
110 echo '-------------------------'
111 echo 'dump daemon log:' $name
112 echo '-------------------------'
114 $CEPHADM logs
--fsid $fsid --name $name -- --no-pager -n $num_lines
117 function dump_all_logs
()
120 local names
=$
($CEPHADM ls | jq
-r '.[] | select(.fsid == "'$fsid'").name')
122 echo 'dumping logs for daemons: ' $names
123 for name
in $names; do
130 # stop the running nfs server
131 local units
="nfs-server nfs-kernel-server"
132 for unit
in $units; do
133 if systemctl status
$unit < /dev
/null
; then
134 $SUDO systemctl stop
$unit
138 # ensure the NFS port is no longer in use
139 expect_false
"$SUDO ss -tlnp '( sport = :nfs )' | grep LISTEN"
142 ## prepare + check host
143 $SUDO $CEPHADM check-host
145 ## run a gather-facts (output to stdout)
146 $SUDO $CEPHADM gather-facts
149 $SUDO CEPHADM_IMAGE
=$IMAGE_PACIFIC $CEPHADM_BIN version
150 $SUDO CEPHADM_IMAGE
=$IMAGE_PACIFIC $CEPHADM_BIN version \
151 |
grep 'ceph version 16'
152 #$SUDO CEPHADM_IMAGE=$IMAGE_OCTOPUS $CEPHADM_BIN version
153 #$SUDO CEPHADM_IMAGE=$IMAGE_OCTOPUS $CEPHADM_BIN version \
154 # | grep 'ceph version 15'
155 $SUDO $CEPHADM_BIN --image $IMAGE_MASTER version |
grep 'ceph version'
157 # try force docker; this won't work if docker isn't installed
158 systemctl status docker
> /dev
/null
&& ( $CEPHADM --docker version |
grep 'ceph version' ) ||
echo "docker not installed"
160 ## test shell before bootstrap, when crash dir isn't (yet) present on this host
161 $CEPHADM shell
--fsid $FSID -- ceph
-v |
grep 'ceph version'
162 $CEPHADM shell
--fsid $FSID -e FOO
=BAR
-- printenv |
grep FOO
=BAR
165 echo foo |
$CEPHADM shell
-- cat |
grep -q foo
168 ORIG_CONFIG
=`mktemp -p $TMPDIR`
169 CONFIG
=`mktemp -p $TMPDIR`
170 MONCONFIG
=`mktemp -p $TMPDIR`
171 KEYRING
=`mktemp -p $TMPDIR`
173 cat <<EOF > $ORIG_CONFIG
176 osd crush chooseleaf type = 0
183 --config $ORIG_CONFIG \
184 --output-config $CONFIG \
185 --output-keyring $KEYRING \
186 --output-pub-ssh-key $TMPDIR/ceph.pub \
189 --skip-monitoring-stack
194 $SUDO test -e /var
/log
/ceph
/$FSID/ceph-mon.a.log
195 $SUDO test -e /var
/log
/ceph
/$FSID/ceph-mgr.x.log
197 for u
in ceph.target \
201 systemctl is-enabled
$u
202 systemctl is-active
$u
204 systemctl |
grep system-ceph |
grep -q .slice
# naming is escaped and annoying
206 # check ceph -s works (via shell w/ passed config/keyring)
207 $CEPHADM shell
--fsid $FSID --config $CONFIG --keyring $KEYRING -- \
210 for t
in mon mgr node-exporter prometheus grafana
; do
211 $CEPHADM shell
--fsid $FSID --config $CONFIG --keyring $KEYRING -- \
212 ceph orch apply
$t --unmanaged
216 $CEPHADM ls | jq
'.[]' | jq
'select(.name == "mon.a").fsid' \
218 $CEPHADM ls | jq
'.[]' | jq
'select(.name == "mgr.x").fsid' \
221 # make sure the version is returned correctly
222 $CEPHADM ls | jq
'.[]' | jq
'select(.name == "mon.a").version' |
grep -q \\.
226 cp $CONFIG $MONCONFIG
227 echo "public addrv = [v2:$IP:3301,v1:$IP:6790]" >> $MONCONFIG
228 $CEPHADM deploy
--name mon.b \
230 --keyring /var
/lib
/ceph
/$FSID/mon.a
/keyring \
232 for u
in ceph-
$FSID@mon.b
; do
233 systemctl is-enabled
$u
234 systemctl is-active
$u
236 cond
="$CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \
237 ceph mon stat | grep '2 mons'"
238 is_available
"mon.b" "$cond" 30
241 $CEPHADM shell
--fsid $FSID --config $CONFIG --keyring $KEYRING -- \
242 ceph auth get-or-create mgr.y \
243 mon
'allow profile mgr' \
245 mds
'allow *' > $TMPDIR/keyring.mgr.y
246 $CEPHADM deploy
--name mgr.y \
248 --keyring $TMPDIR/keyring.mgr.y \
250 for u
in ceph-
$FSID@mgr.y
; do
251 systemctl is-enabled
$u
252 systemctl is-active
$u
255 for f
in `seq 1 30`; do
256 if $CEPHADM shell
--fsid $FSID \
257 --config $CONFIG --keyring $KEYRING -- \
258 ceph
-s -f json-pretty \
259 | jq
'.mgrmap.num_standbys' |
grep -q 1 ; then break; fi
262 $CEPHADM shell
--fsid $FSID --config $CONFIG --keyring $KEYRING -- \
263 ceph
-s -f json-pretty \
264 | jq
'.mgrmap.num_standbys' |
grep -q 1
267 dd if=/dev
/zero of
=$TMPDIR/$OSD_IMAGE_NAME bs
=1 count
=0 seek
=$OSD_IMAGE_SIZE
268 loop_dev
=$
($SUDO losetup
-f)
269 $SUDO vgremove
-f $OSD_VG_NAME || true
270 $SUDO losetup
$loop_dev $TMPDIR/$OSD_IMAGE_NAME
271 $SUDO pvcreate
$loop_dev && $SUDO vgcreate
$OSD_VG_NAME $loop_dev
273 # osd boostrap keyring
274 $CEPHADM shell
--fsid $FSID --config $CONFIG --keyring $KEYRING -- \
275 ceph auth get client.bootstrap-osd
> $TMPDIR/keyring.bootstrap.osd
277 # create lvs first so ceph-volume doesn't overlap with lv creation
278 for id
in `seq 0 $((--OSD_TO_CREATE))`; do
279 $SUDO lvcreate
-l $
((100/$OSD_TO_CREATE))%VG
-n $OSD_LV_NAME.
$id $OSD_VG_NAME
282 for id
in `seq 0 $((--OSD_TO_CREATE))`; do
283 device_name
=/dev
/$OSD_VG_NAME/$OSD_LV_NAME.
$id
284 CEPH_VOLUME
="$CEPHADM ceph-volume \
287 --keyring $TMPDIR/keyring.bootstrap.osd --"
290 $CEPH_VOLUME lvm prepare
--bluestore --data $device_name --no-systemd
291 $CEPH_VOLUME lvm
batch --no-auto $device_name --yes --no-systemd
293 # osd id and osd fsid
294 $CEPH_VOLUME lvm list
--format json
$device_name > $TMPDIR/osd.map
295 osd_id
=$
($SUDO cat $TMPDIR/osd.map | jq
-cr '.. | ."ceph.osd_id"? | select(.)')
296 osd_fsid
=$
($SUDO cat $TMPDIR/osd.map | jq
-cr '.. | ."ceph.osd_fsid"? | select(.)')
299 $CEPHADM deploy
--name osd.
$osd_id \
301 --keyring $TMPDIR/keyring.bootstrap.osd \
307 ${CEPHADM//--image $IMAGE_DEFAULT/} deploy \
308 --name node-exporter.a
--fsid $FSID
309 cond
="curl 'http://localhost:9100' | grep -q 'Node Exporter'"
310 is_available
"node-exporter" "$cond" 10
313 cat ${CEPHADM_SAMPLES_DIR}/prometheus.json | \
314 ${CEPHADM//--image $IMAGE_DEFAULT/} deploy \
315 --name prometheus.a
--fsid $FSID --config-json -
316 cond
="curl 'localhost:9095/api/v1/query?query=up'"
317 is_available
"prometheus" "$cond" 10
320 cat ${CEPHADM_SAMPLES_DIR}/grafana.json | \
321 ${CEPHADM//--image $IMAGE_DEFAULT/} deploy \
322 --name grafana.a
--fsid $FSID --config-json -
323 cond
="curl --insecure 'https://localhost:3000' | grep -q 'grafana'"
324 is_available
"grafana" "$cond" 50
328 nfs_rados_pool
=$
(cat ${CEPHADM_SAMPLES_DIR}/nfs.json | jq
-r '.["pool"]')
329 $CEPHADM shell
--fsid $FSID --config $CONFIG --keyring $KEYRING -- \
330 ceph osd pool create
$nfs_rados_pool 64
331 $CEPHADM shell
--fsid $FSID --config $CONFIG --keyring $KEYRING -- \
332 rados
--pool nfs-ganesha
--namespace nfs-ns create conf-nfs.a
333 $CEPHADM shell
--fsid $FSID --config $CONFIG --keyring $KEYRING -- \
335 $CEPHADM deploy
--name nfs.a \
339 --config-json ${CEPHADM_SAMPLES_DIR}/nfs.json
340 cond
="$SUDO ss -tlnp '( sport = :nfs )' | grep 'ganesha.nfsd'"
341 is_available
"nfs" "$cond" 10
342 $CEPHADM shell
--fsid $FSID --config $CONFIG --keyring $KEYRING -- \
345 # add alertmanager via custom container
346 alertmanager_image
=$
(cat ${CEPHADM_SAMPLES_DIR}/custom_container.json | jq
-r '.image')
347 tcp_ports
=$
(cat ${CEPHADM_SAMPLES_DIR}/custom_container.json | jq
-r '.ports | map_values(.|tostring) | join(" ")')
348 cat ${CEPHADM_SAMPLES_DIR}/custom_container.json | \
349 ${CEPHADM//--image $IMAGE_DEFAULT/} \
350 --image $alertmanager_image \
352 --tcp-ports "$tcp_ports" \
353 --name container.alertmanager.a \
356 cond
="$CEPHADM enter --fsid $FSID --name container.alertmanager.a -- test -f \
357 /etc/alertmanager/alertmanager.yml"
358 is_available
"alertmanager.yml" "$cond" 10
359 cond
="curl 'http://localhost:9093' | grep -q 'Alertmanager'"
360 is_available
"alertmanager" "$cond" 10
366 $CEPHADM unit
--fsid $FSID --name mon.a
-- is-enabled
367 $CEPHADM unit
--fsid $FSID --name mon.a
-- is-active
368 expect_false
$CEPHADM unit
--fsid $FSID --name mon.xyz
-- is-active
369 $CEPHADM unit
--fsid $FSID --name mon.a
-- disable
370 expect_false
$CEPHADM unit
--fsid $FSID --name mon.a
-- is-enabled
371 $CEPHADM unit
--fsid $FSID --name mon.a
-- enable
372 $CEPHADM unit
--fsid $FSID --name mon.a
-- is-enabled
375 $CEPHADM shell
--fsid $FSID -- true
376 $CEPHADM shell
--fsid $FSID -- test -d /var
/log
/ceph
377 expect_false
$CEPHADM --timeout 10 shell
--fsid $FSID -- sleep 60
378 $CEPHADM --timeout 60 shell
--fsid $FSID -- sleep 10
379 $CEPHADM shell
--fsid $FSID --mount $TMPDIR $TMPDIR_TEST_MULTIPLE_MOUNTS -- stat
/mnt
/$
(basename $TMPDIR)
382 expect_false
$CEPHADM enter
383 $CEPHADM enter
--fsid $FSID --name mon.a
-- test -d /var
/lib
/ceph
/mon
/ceph-a
384 $CEPHADM enter
--fsid $FSID --name mgr.x
-- test -d /var
/lib
/ceph
/mgr
/ceph-x
385 $CEPHADM enter
--fsid $FSID --name mon.a
-- pidof ceph-mon
386 expect_false
$CEPHADM enter
--fsid $FSID --name mgr.x
-- pidof ceph-mon
387 $CEPHADM enter
--fsid $FSID --name mgr.x
-- pidof ceph-mgr
388 # this triggers a bug in older versions of podman, including 18.04's 1.6.2
389 #expect_false $CEPHADM --timeout 5 enter --fsid $FSID --name mon.a -- sleep 30
390 $CEPHADM --timeout 60 enter
--fsid $FSID --name mon.a
-- sleep 10
393 $CEPHADM ceph-volume
--fsid $FSID -- inventory
--format=json \
396 ## preserve test state
397 [ $CLEANUP = false
] && exit 0
400 # mon and osd require --force
401 expect_false
$CEPHADM rm-daemon
--fsid $FSID --name mon.a
403 $CEPHADM rm-daemon
--fsid $FSID --name mgr.x
405 expect_false
$CEPHADM zap-osds
--fsid $FSID
406 $CEPHADM zap-osds
--fsid $FSID --force
409 expect_false
$CEPHADM rm-cluster
--fsid $FSID --zap-osds
410 $CEPHADM rm-cluster
--fsid $FSID --force --zap-osds