[ceph.git] / ceph / qa / suites / orch / cephadm / upgrade / 3-upgrade / staggered.yaml

tasks:
- cephadm.shell:
    env: [sha1]
    mon.a:
      # setup rgw
      - radosgw-admin realm create --rgw-realm=r --default
      - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default
      - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default
      - radosgw-admin period update --rgw-realm=r --commit
      - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000
      # setup iscsi
      - ceph osd pool create foo
      - rbd pool init foo
      - ceph orch apply iscsi foo u p
      - sleep 180
      - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force
      - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force
      - ceph config set global log_to_journald false --force
      # get some good info on the state of things pre-upgrade. Useful for debugging
      - ceph orch ps
      - ceph versions
      - ceph -s
      - ceph orch ls
      # doing staggered upgrade requires mgr daemons being on a version that contains the staggered upgrade code
      # until there is a stable version that contains it, we can test by manually upgrading a mgr daemon
      - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1
      - ceph orch ps --refresh
      - sleep 180
      # gather more possible debugging info
      - ceph orch ps
      - ceph versions
      - ceph -s
      - ceph health detail
      # check that there are two different versions found for mgr daemon (which implies we upgraded one)
      - ceph versions | jq -e '.mgr | length == 2'
      - ceph mgr fail
      - sleep 180
      # now try upgrading the other mgr
      - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1
      - ceph orch ps --refresh
      - sleep 180
      # gather more possible debugging info
      - ceph orch ps
      - ceph versions
      - ceph health detail
      - ceph -s
      - ceph mgr fail
      - sleep 180
      # gather more debugging info
      - ceph orch ps
      - ceph versions
      - ceph -s
      - ceph health detail
      # now that both mgrs should have been redeployed with the new version, we should be back on only 1 version for the mgrs
      - ceph versions | jq -e '.mgr | length == 1'
      - ceph mgr fail
      - sleep 180
      # debugging info
      - ceph orch ps
      - ceph orch ls
      - ceph versions
      # to make sure mgr daemons upgrade is fully completed, including being deployed by a mgr on a new version
      # also serves as an early failure if manually upgrading the mgrs failed as --daemon-types won't be recognized
      - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr
      - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
      # verify only one version found for mgrs and that their version hash matches what we are upgrading to
      - ceph versions | jq -e '.mgr | length == 1'
      - ceph versions | jq -e '.mgr | keys' | grep $sha1
      # verify overall we still see two versions, basically to make sure --daemon-types wasn't ignored and all daemons upgraded
      - ceph versions | jq -e '.overall | length == 2'
      # check that exactly two daemons have been upgraded to the new image (our 2 mgr daemons)
      - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2'
      - ceph orch upgrade status
      - ceph health detail
      # upgrade only the mons on one of the two hosts
      - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}')
      - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
      - ceph orch ps
      # verify two different version seen for mons
      - ceph versions | jq -e '.mon | length == 2'
      - ceph orch upgrade status
      - ceph health detail
      # upgrade mons on the other hosts
      - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}')
      - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
      - ceph orch ps
      # verify all mons now on same version and version hash matches what we are upgrading to
      - ceph versions | jq -e '.mon | length == 1'
      - ceph versions | jq -e '.mon | keys' | grep $sha1
      # verify exactly 5 daemons are now upgraded (2 mgrs, 3 mons)
      - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5'
      - ceph orch upgrade status
      - ceph health detail
      # upgrade exactly 2 osd daemons
      - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2
      - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
      - ceph orch ps
      # verify two different versions now seen for osds
      - ceph versions | jq -e '.osd | length == 2'
      # verify exactly 7 daemons have been upgraded (2 mgrs, 3 mons, 2 osds)
      - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7'
      - ceph orch upgrade status
      - ceph health detail
      # upgrade one more osd
      - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1
      - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
      - ceph orch ps
      - ceph versions | jq -e '.osd | length == 2'
      # verify now 8 daemons have been upgraded
      - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8'
      # upgrade the rest of the osds
      - ceph orch upgrade status
      - ceph health detail
      - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd
      - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
      - ceph orch ps
      # verify all osds are now on same version and version hash matches what we are upgrading to
      - ceph versions | jq -e '.osd | length == 1'
      - ceph versions | jq -e '.osd | keys' | grep $sha1
      - ceph orch upgrade status
      - ceph health detail
      # upgrade the rgw daemons using --services
      - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo
      - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
      - ceph orch ps
      # verify all rgw daemons on same version and version hash matches what we are upgrading to
      - ceph versions | jq -e '.rgw | length == 1'
      - ceph versions | jq -e '.rgw | keys' | grep $sha1
      - ceph orch upgrade status
      - ceph health detail
      # run upgrade one more time with no filter parameters to make sure anything left gets upgraded
      - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
Commit	Line	Data
33c7a0ef TL	1	tasks:
	2	- cephadm.shell:
	3	env: [sha1]
	4	mon.a:
1e59de90	5	# setup rgw
33c7a0ef TL	6	- radosgw-admin realm create --rgw-realm=r --default
	7	- radosgw-admin zonegroup create --rgw-zonegroup=default --master --default
	8	- radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default
	9	- radosgw-admin period update --rgw-realm=r --commit
1e59de90 TL	10	- ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000
	11	# setup iscsi
	12	- ceph osd pool create foo
	13	- rbd pool init foo
	14	- ceph orch apply iscsi foo u p
33c7a0ef TL	15	- sleep 180
	16	- ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force
	17	- ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force
	18	- ceph config set global log_to_journald false --force
	19	# get some good info on the state of things pre-upgrade. Useful for debugging
	20	- ceph orch ps
	21	- ceph versions
	22	- ceph -s
	23	- ceph orch ls
	24	# doing staggered upgrade requires mgr daemons being on a version that contains the staggered upgrade code
	25	# until there is a stable version that contains it, we can test by manually upgrading a mgr daemon
1e59de90	26	- ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json \| jq .standbys \| jq .[] \| jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1
33c7a0ef TL	27	- ceph orch ps --refresh
	28	- sleep 180
	29	# gather more possible debugging info
	30	- ceph orch ps
	31	- ceph versions
	32	- ceph -s
1e59de90	33	- ceph health detail
33c7a0ef TL	34	# check that there are two different versions found for mgr daemon (which implies we upgraded one)
	35	- ceph versions \| jq -e '.mgr \| length == 2'
	36	- ceph mgr fail
	37	- sleep 180
	38	# now try upgrading the other mgr
33c7a0ef TL	39	- ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json \| jq .standbys \| jq .[] \| jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1
	40	- ceph orch ps --refresh
	41	- sleep 180
	42	# gather more possible debugging info
	43	- ceph orch ps
	44	- ceph versions
1e59de90	45	- ceph health detail
33c7a0ef TL	46	- ceph -s
	47	- ceph mgr fail
	48	- sleep 180
	49	# gather more debugging info
	50	- ceph orch ps
	51	- ceph versions
	52	- ceph -s
1e59de90	53	- ceph health detail
33c7a0ef TL	54	# now that both mgrs should have been redeployed with the new version, we should be back on only 1 version for the mgrs
	55	- ceph versions \| jq -e '.mgr \| length == 1'
	56	- ceph mgr fail
	57	- sleep 180
	58	# debugging info
	59	- ceph orch ps
1e59de90	60	- ceph orch ls
33c7a0ef	61	- ceph versions
1e59de90	62	# to make sure mgr daemons upgrade is fully completed, including being deployed by a mgr on a new version
33c7a0ef TL	63	# also serves as an early failure if manually upgrading the mgrs failed as --daemon-types won't be recognized
	64	- ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr
	65	- while ceph orch upgrade status \| jq '.in_progress' \| grep true && ! ceph orch upgrade status \| jq '.message' \| grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
	66	# verify only one version found for mgrs and that their version hash matches what we are upgrading to
	67	- ceph versions \| jq -e '.mgr \| length == 1'
	68	- ceph versions \| jq -e '.mgr \| keys' \| grep $sha1
1e59de90	69	# verify overall we still see two versions, basically to make sure --daemon-types wasn't ignored and all daemons upgraded
33c7a0ef TL	70	- ceph versions \| jq -e '.overall \| length == 2'
	71	# check that exactly two daemons have been upgraded to the new image (our 2 mgr daemons)
	72	- ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 \| jq -e '.up_to_date \| length == 2'
1e59de90 TL	73	- ceph orch upgrade status
1e59de90 TL	74	- ceph health detail
33c7a0ef TL	75	# upgrade only the mons on one of the two hosts
	76	- ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps \| grep mgr.x \| awk '{print $2}')
	77	- while ceph orch upgrade status \| jq '.in_progress' \| grep true && ! ceph orch upgrade status \| jq '.message' \| grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
	78	- ceph orch ps
1e59de90	79	# verify two different version seen for mons
33c7a0ef	80	- ceph versions \| jq -e '.mon \| length == 2'
1e59de90 TL	81	- ceph orch upgrade status
1e59de90 TL	82	- ceph health detail
33c7a0ef TL	83	# upgrade mons on the other hosts
	84	- ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps \| grep mgr.y \| awk '{print $2}')
	85	- while ceph orch upgrade status \| jq '.in_progress' \| grep true && ! ceph orch upgrade status \| jq '.message' \| grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
	86	- ceph orch ps
	87	# verify all mons now on same version and version hash matches what we are upgrading to
	88	- ceph versions \| jq -e '.mon \| length == 1'
	89	- ceph versions \| jq -e '.mon \| keys' \| grep $sha1
	90	# verify exactly 5 daemons are now upgraded (2 mgrs, 3 mons)
	91	- ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 \| jq -e '.up_to_date \| length == 5'
1e59de90 TL	92	- ceph orch upgrade status
1e59de90 TL	93	- ceph health detail
33c7a0ef TL	94	# upgrade exactly 2 osd daemons
	95	- ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2
	96	- while ceph orch upgrade status \| jq '.in_progress' \| grep true && ! ceph orch upgrade status \| jq '.message' \| grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
	97	- ceph orch ps
	98	# verify two different versions now seen for osds
	99	- ceph versions \| jq -e '.osd \| length == 2'
	100	# verify exactly 7 daemons have been upgraded (2 mgrs, 3 mons, 2 osds)
	101	- ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 \| jq -e '.up_to_date \| length == 7'
1e59de90 TL	102	- ceph orch upgrade status
1e59de90 TL	103	- ceph health detail
33c7a0ef TL	104	# upgrade one more osd
	105	- ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1
	106	- while ceph orch upgrade status \| jq '.in_progress' \| grep true && ! ceph orch upgrade status \| jq '.message' \| grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
	107	- ceph orch ps
	108	- ceph versions \| jq -e '.osd \| length == 2'
	109	# verify now 8 daemons have been upgraded
	110	- ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 \| jq -e '.up_to_date \| length == 8'
	111	# upgrade the rest of the osds
1e59de90 TL	112	- ceph orch upgrade status
1e59de90 TL	113	- ceph health detail
33c7a0ef TL	114	- ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd
	115	- while ceph orch upgrade status \| jq '.in_progress' \| grep true && ! ceph orch upgrade status \| jq '.message' \| grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
	116	- ceph orch ps
	117	# verify all osds are now on same version and version hash matches what we are upgrading to
	118	- ceph versions \| jq -e '.osd \| length == 1'
	119	- ceph versions \| jq -e '.osd \| keys' \| grep $sha1
1e59de90 TL	120	- ceph orch upgrade status
1e59de90 TL	121	- ceph health detail
33c7a0ef	122	# upgrade the rgw daemons using --services
1e59de90	123	- ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo
33c7a0ef TL	124	- while ceph orch upgrade status \| jq '.in_progress' \| grep true && ! ceph orch upgrade status \| jq '.message' \| grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
	125	- ceph orch ps
	126	# verify all rgw daemons on same version and version hash matches what we are upgrading to
	127	- ceph versions \| jq -e '.rgw \| length == 1'
	128	- ceph versions \| jq -e '.rgw \| keys' \| grep $sha1
1e59de90 TL	129	- ceph orch upgrade status
1e59de90 TL	130	- ceph health detail
33c7a0ef TL	131	# run upgrade one more time with no filter parameters to make sure anything left gets upgraded
33c7a0ef TL	132	- ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1