2 # -*- mode:sh; tab-width:4; sh-basic-offset:4; indent-tabs-mode:nil -*-
3 # vim: softtabstop=4 shiftwidth=4 expandtab
10 if [[ "$s" =~ \
]]; then
11 printf -- "'%s' " "$s"
24 debug quoted_print
"$@" '&'
29 debug quoted_print
"$@"
34 if [ -n "$VSTART_DEST" ]; then
36 SRC_PATH
=`(cd $SRC_PATH; pwd)`
42 CEPH_CONF_PATH
=$VSTART_DEST
43 CEPH_DEV_DIR
=$VSTART_DEST/dev
44 CEPH_OUT_DIR
=$VSTART_DEST/out
45 CEPH_ASOK_DIR
=$VSTART_DEST/out
48 get_cmake_variable
() {
50 grep "${variable}:" CMakeCache.txt | cut
-d "=" -f 2
53 # for running out of the CMake build directory
54 if [ -e CMakeCache.txt
]; then
55 # Out of tree build, learn source location from CMakeCache.txt
56 CEPH_ROOT
=$
(get_cmake_variable ceph_SOURCE_DIR
)
58 [ -z "$MGR_PYTHON_PATH" ] && MGR_PYTHON_PATH
=$CEPH_ROOT/src
/pybind
/mgr
61 # use CEPH_BUILD_ROOT to vstart from a 'make install'
62 if [ -n "$CEPH_BUILD_ROOT" ]; then
63 [ -z "$CEPH_BIN" ] && CEPH_BIN
=$CEPH_BUILD_ROOT/bin
64 [ -z "$CEPH_LIB" ] && CEPH_LIB
=$CEPH_BUILD_ROOT/lib
65 [ -z "$EC_PATH" ] && EC_PATH
=$CEPH_LIB/erasure-code
66 [ -z "$OBJCLASS_PATH" ] && OBJCLASS_PATH
=$CEPH_LIB/rados-classes
67 # make install should install python extensions into PYTHONPATH
68 elif [ -n "$CEPH_ROOT" ]; then
69 [ -z "$CEPHFS_SHELL" ] && CEPHFS_SHELL
=$CEPH_ROOT/src
/tools
/cephfs
/cephfs-shell
70 [ -z "$PYBIND" ] && PYBIND
=$CEPH_ROOT/src
/pybind
71 [ -z "$CEPH_BIN" ] && CEPH_BIN
=$CEPH_BUILD_DIR/bin
72 [ -z "$CEPH_ADM" ] && CEPH_ADM
=$CEPH_BIN/ceph
73 [ -z "$INIT_CEPH" ] && INIT_CEPH
=$CEPH_BIN/init-ceph
74 [ -z "$CEPH_LIB" ] && CEPH_LIB
=$CEPH_BUILD_DIR/lib
75 [ -z "$OBJCLASS_PATH" ] && OBJCLASS_PATH
=$CEPH_LIB
76 [ -z "$EC_PATH" ] && EC_PATH
=$CEPH_LIB
77 [ -z "$CEPH_PYTHON_COMMON" ] && CEPH_PYTHON_COMMON
=$CEPH_ROOT/src
/python-common
80 if [ -z "${CEPH_VSTART_WRAPPER}" ]; then
84 [ -z "$PYBIND" ] && PYBIND
=.
/pybind
86 [ -n "$CEPH_PYTHON_COMMON" ] && CEPH_PYTHON_COMMON
="$CEPH_PYTHON_COMMON:"
87 CYTHON_PYTHONPATH
="$CEPH_LIB/cython_modules/lib.3"
88 export PYTHONPATH
=$PYBIND:$CYTHON_PYTHONPATH:$CEPH_PYTHON_COMMON$PYTHONPATH
90 export LD_LIBRARY_PATH
=$CEPH_LIB:$LD_LIBRARY_PATH
91 export DYLD_LIBRARY_PATH
=$CEPH_LIB:$DYLD_LIBRARY_PATH
92 # Suppress logging for regular use that indicated that we are using a
93 # development version. vstart.sh is only used during testing and
97 [ -z "$CEPH_NUM_MON" ] && CEPH_NUM_MON
="$MON"
98 [ -z "$CEPH_NUM_OSD" ] && CEPH_NUM_OSD
="$OSD"
99 [ -z "$CEPH_NUM_MDS" ] && CEPH_NUM_MDS
="$MDS"
100 [ -z "$CEPH_NUM_MGR" ] && CEPH_NUM_MGR
="$MGR"
101 [ -z "$CEPH_NUM_FS" ] && CEPH_NUM_FS
="$FS"
102 [ -z "$CEPH_NUM_RGW" ] && CEPH_NUM_RGW
="$RGW"
103 [ -z "$GANESHA_DAEMON_NUM" ] && GANESHA_DAEMON_NUM
="$GANESHA"
105 # if none of the CEPH_NUM_* number is specified, kill the existing
107 if [ -z "$CEPH_NUM_MON" -a \
108 -z "$CEPH_NUM_OSD" -a \
109 -z "$CEPH_NUM_MDS" -a \
110 -z "$CEPH_NUM_MGR" -a \
111 -z "$GANESHA_DAEMON_NUM" ]; then
117 [ -z "$CEPH_NUM_MON" ] && CEPH_NUM_MON
=3
118 [ -z "$CEPH_NUM_OSD" ] && CEPH_NUM_OSD
=3
119 [ -z "$CEPH_NUM_MDS" ] && CEPH_NUM_MDS
=3
120 [ -z "$CEPH_NUM_MGR" ] && CEPH_NUM_MGR
=1
121 [ -z "$CEPH_NUM_FS" ] && CEPH_NUM_FS
=1
122 [ -z "$CEPH_MAX_MDS" ] && CEPH_MAX_MDS
=1
123 [ -z "$CEPH_NUM_RGW" ] && CEPH_NUM_RGW
=0
124 [ -z "$GANESHA_DAEMON_NUM" ] && GANESHA_DAEMON_NUM
=0
126 [ -z "$CEPH_DIR" ] && CEPH_DIR
="$PWD"
127 [ -z "$CEPH_DEV_DIR" ] && CEPH_DEV_DIR
="$CEPH_DIR/dev"
128 [ -z "$CEPH_OUT_DIR" ] && CEPH_OUT_DIR
="$CEPH_DIR/out"
129 [ -z "$CEPH_RGW_PORT" ] && CEPH_RGW_PORT
=8000
130 [ -z "$CEPH_CONF_PATH" ] && CEPH_CONF_PATH
=$CEPH_DIR
132 if [ $CEPH_NUM_OSD -gt 3 ]; then
133 OSD_POOL_DEFAULT_SIZE
=3
135 OSD_POOL_DEFAULT_SIZE
=$CEPH_NUM_OSD
152 cephx
=1 #turn cephx on by default
155 if [ `uname` = FreeBSD
]; then
156 objectstore
="filestore"
158 objectstore
="bluestore"
163 lockdep
=${LOCKDEP:-1}
164 spdk_enabled
=0 #disable SPDK by default
167 with_mgr_dashboard
=true
168 if [[ "$(get_cmake_variable WITH_MGR_DASHBOARD_FRONTEND)" != "ON" ]] ||
169 [[ "$(get_cmake_variable WITH_RBD)" != "ON" ]]; then
170 debug
echo "ceph-mgr dashboard not built - disabling."
171 with_mgr_dashboard
=false
178 VSTART_SEC
="client.vstart.sh"
184 conf_fn
="$CEPH_CONF_PATH/ceph.conf"
185 keyring_fn
="$CEPH_CONF_PATH/keyring"
186 osdmap_fn
="/tmp/ceph_osdmap.$$"
187 monmap_fn
="/tmp/ceph_monmap.$$"
192 usage
="usage: $0 [option]... \nex: MON=3 OSD=1 MDS=1 MGR=1 RGW=1 GANESHA=1 $0 -n -d\n"
193 usage
=$usage"options:\n"
194 usage
=$usage"\t-d, --debug\n"
195 usage
=$usage"\t-s, --standby_mds: Generate standby-replay MDS for each active\n"
196 usage
=$usage"\t-l, --localhost: use localhost instead of hostname\n"
197 usage
=$usage"\t-i <ip>: bind to specific ip\n"
198 usage
=$usage"\t-n, --new\n"
199 usage
=$usage"\t--valgrind[_{osd,mds,mon,rgw}] 'toolname args...'\n"
200 usage
=$usage"\t--nodaemon: use ceph-run as wrapper for mon/osd/mds\n"
201 usage
=$usage"\t--redirect-output: only useful with nodaemon, directs output to log file\n"
202 usage
=$usage"\t--smallmds: limit mds cache memory limit\n"
203 usage
=$usage"\t-m ip:port\t\tspecify monitor address\n"
204 usage
=$usage"\t-k keep old configuration files (default)\n"
205 usage
=$usage"\t-x enable cephx (on by default)\n"
206 usage
=$usage"\t-X disable cephx\n"
207 usage
=$usage"\t-g --gssapi enable Kerberos/GSSApi authentication\n"
208 usage
=$usage"\t-G disable Kerberos/GSSApi authentication\n"
209 usage
=$usage"\t--hitset <pool> <hit_set_type>: enable hitset tracking\n"
210 usage
=$usage"\t-e : create an erasure pool\n";
211 usage
=$usage"\t-o config\t\t add extra config parameters to all sections\n"
212 usage
=$usage"\t--rgw_port specify ceph rgw http listen port\n"
213 usage
=$usage"\t--rgw_frontend specify the rgw frontend configuration\n"
214 usage
=$usage"\t--rgw_compression specify the rgw compression plugin\n"
215 usage
=$usage"\t-b, --bluestore use bluestore as the osd objectstore backend (default)\n"
216 usage
=$usage"\t-f, --filestore use filestore as the osd objectstore backend\n"
217 usage
=$usage"\t-K, --kstore use kstore as the osd objectstore backend\n"
218 usage
=$usage"\t--memstore use memstore as the osd objectstore backend\n"
219 usage
=$usage"\t--cache <pool>: enable cache tiering on pool\n"
220 usage
=$usage"\t--short: short object names only; necessary for ext4 dev\n"
221 usage
=$usage"\t--nolockdep disable lockdep\n"
222 usage
=$usage"\t--multimds <count> allow multimds with maximum active count\n"
223 usage
=$usage"\t--without-dashboard: do not run using mgr dashboard\n"
224 usage
=$usage"\t--bluestore-spdk <vendor>:<device>: enable SPDK and specify the PCI-ID of the NVME device\n"
225 usage
=$usage"\t--msgr1: use msgr1 only\n"
226 usage
=$usage"\t--msgr2: use msgr2 only\n"
227 usage
=$usage"\t--msgr21: use msgr2 and msgr1\n"
228 usage
=$usage"\t--crimson: use crimson-osd instead of ceph-osd\n"
229 usage
=$usage"\t--osd-args: specify any extra osd specific options\n"
230 usage
=$usage"\t--bluestore-devs: comma-separated list of blockdevs to use for bluestore\n"
231 usage
=$usage"\t--inc-osd: append some more osds into existing vcluster\n"
232 usage
=$usage"\t--cephadm: enable cephadm orchestrator with ~/.ssh/id_rsa[.pub]\n"
233 usage
=$usage"\t--no-parallel: dont start all OSDs in parallel\n"
240 while [ $# -ge 1 ]; do
252 [ -z "$2" ] && usage_exit
266 if [ "$inc_osd_num" == "" ]; then
298 [ -z "$2" ] && usage_exit
307 [ -z "$2" ] && usage_exit
312 [ -z "$2" ] && usage_exit
317 [ -z "$2" ] && usage_exit
322 [ -z "$2" ] && usage_exit
327 [ -z "$2" ] && usage_exit
361 [ -z "$2" ] && usage_exit
366 cephx
=1 # this is on be default, flag exists for historical consistency
380 if [ ! -r $conf_fn ]; then
381 echo "cannot use old configuration: $conf_fn not readable." >&2
387 objectstore
="memstore"
390 objectstore
="bluestore"
393 objectstore
="filestore"
399 hitset
="$hitset $2 $3"
404 extra_conf
="$extra_conf $2
409 if [ -z "$cache" ]; then
424 with_mgr_dashboard
=false
427 [ -z "$2" ] && usage_exit
433 IFS
=',' read -r -a bluestore_dev
<<< "$2"
434 for dev
in "${bluestore_dev[@]}"; do
435 if [ ! -b $dev -o ! -w $dev ]; then
436 echo "All --bluestore-devs must refer to writable block devices"
448 if [ $kill_all -eq 1 ]; then
449 $SUDO $INIT_CEPH stop
452 if [ "$new" -eq 0 ]; then
453 if [ -z "$CEPH_ASOK_DIR" ]; then
454 CEPH_ASOK_DIR
=`dirname $($CEPH_BIN/ceph-conf -c $conf_fn --show-config-value admin_socket)`
456 mkdir
-p $CEPH_ASOK_DIR
457 MON
=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_mon 2>/dev/null` && \
459 OSD
=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_osd 2>/dev/null` && \
461 MDS
=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_mds 2>/dev/null` && \
463 MGR
=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_mgr 2>/dev/null` && \
465 RGW
=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_rgw 2>/dev/null` && \
467 GANESHA
=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_ganesha 2>/dev/null` && \
468 GANESHA_DAEMON_NUM
="$GANESHA"
471 if [ -e "$conf_fn" ]; then
472 asok_dir
=`dirname $($CEPH_BIN/ceph-conf -c $conf_fn --show-config-value admin_socket)`
474 if [ $asok_dir != /var
/run
/ceph
]; then
475 [ -d $asok_dir ] && rm -f $asok_dir/* && rmdir $asok_dir
478 if [ -z "$CEPH_ASOK_DIR" ]; then
479 CEPH_ASOK_DIR
=`mktemp -u -d "${TMPDIR:-/tmp}/ceph-asok.XXXXXX"`
490 eval "valg=\$valgrind_$type"
491 [ -z "$valg" ] && valg
="$valgrind"
493 if [ -n "$valg" ]; then
494 prunb valgrind
--tool="$valg" $valgrind_args "$@" -f
497 if [ "$nodaemon" -eq 0 ]; then
499 elif [ "$redirect" -eq 0 ]; then
500 prunb
${CEPH_ROOT}/src
/ceph-run
"$@" -f
502 ( prunb
${CEPH_ROOT}/src
/ceph-run
"$@" -f ) >$CEPH_OUT_DIR/$type.
$num.stdout
2>&1
508 if [ "$new" -eq 1 -o "$overwrite_conf" -eq 1 ]; then
515 lspci
-mm -n -D -d $pci_id | cut
-d ' ' -f 1 |
sed -n $which_pci'p'
518 get_pci_selector_num
() {
519 lspci
-mm -n -D -d $pci_id | cut
-d' ' -f 1 |
wc -l
524 if [ $CEPH_NUM_RGW -eq 0 ]; then
528 # setup each rgw on a sequential port, starting at $CEPH_RGW_PORT.
529 # individual rgw's ids will be their ports.
530 current_port
=$CEPH_RGW_PORT
531 for n
in $
(seq 1 $CEPH_NUM_RGW); do
533 [client.rgw.${current_port}]
534 rgw frontends = $rgw_frontend port=${current_port}
535 admin socket = ${CEPH_OUT_DIR}/radosgw.${current_port}.asok
537 current_port
=$
((current_port
+ 1))
544 log file = $CEPH_OUT_DIR/\$name.log
545 admin socket = $CEPH_ASOK_DIR/\$name.asok
547 pid file = $CEPH_OUT_DIR/\$name.pid
548 heartbeat file = $CEPH_OUT_DIR/\$name.heartbeat
551 local mgr_modules
="restful iostat"
552 if $with_mgr_dashboard; then
553 mgr_modules
="dashboard $mgr_modules"
557 if [ $msgr -eq 21 ]; then
563 if [ $msgr -eq 2 ]; then
566 ms bind msgr1 = false
569 if [ $msgr -eq 1 ]; then
571 ms bind msgr2 = false
577 ; generated by vstart.sh on `date`
579 num mon = $CEPH_NUM_MON
580 num osd = $CEPH_NUM_OSD
581 num mds = $CEPH_NUM_MDS
582 num mgr = $CEPH_NUM_MGR
583 num rgw = $CEPH_NUM_RGW
584 num ganesha = $GANESHA_DAEMON_NUM
588 osd failsafe full ratio = .99
589 mon osd full ratio = .99
590 mon osd nearfull ratio = .99
591 mon osd backfillfull ratio = .99
592 mon_max_pg_per_osd = ${MON_MAX_PG_PER_OSD:-1000}
593 erasure code dir = $EC_PATH
594 plugin dir = $CEPH_LIB
595 filestore fd cache size = 32
596 run dir = $CEPH_OUT_DIR
597 crash dir = $CEPH_OUT_DIR
598 enable experimental unrecoverable data corrupting features = *
599 osd_crush_chooseleaf_type = 0
600 debug asok assert abort = true
604 if [ "$lockdep" -eq 1 ] ; then
609 if [ "$cephx" -eq 1 ] ; then
611 auth cluster required = cephx
612 auth service required = cephx
613 auth client required = cephx
615 elif [ "$gssapi_authx" -eq 1 ] ; then
617 auth cluster required = gss
618 auth service required = gss
619 auth client required = gss
620 gss ktab client file = $CEPH_DEV_DIR/gss_\$name.keytab
624 auth cluster required = none
625 auth service required = none
626 auth client required = none
629 if [ "$short" -eq 1 ]; then
630 COSDSHORT
=" osd max object name len = 460
631 osd max object namespace len = 64"
633 if [ "$objectstore" == "bluestore" ]; then
634 if [ "$spdk_enabled" -eq 1 ]; then
635 if [ "$(get_pci_selector_num)" -eq 0 ]; then
636 echo "Not find the specified NVME device, please check." >&2
639 if [ $
(get_pci_selector_num
) -lt $CEPH_NUM_OSD ]; then
640 echo "OSD number ($CEPH_NUM_OSD) is greater than NVME SSD number ($(get_pci_selector_num)), please check." >&2
643 BLUESTORE_OPTS
=" bluestore_block_db_path = \"\"
644 bluestore_block_db_size = 0
645 bluestore_block_db_create = false
646 bluestore_block_wal_path = \"\"
647 bluestore_block_wal_size = 0
648 bluestore_block_wal_create = false
649 bluestore_spdk_mem = 2048"
651 BLUESTORE_OPTS
=" bluestore block db path = $CEPH_DEV_DIR/osd\$id/block.db.file
652 bluestore block db size = 1073741824
653 bluestore block db create = true
654 bluestore block wal path = $CEPH_DEV_DIR/osd\$id/block.wal.file
655 bluestore block wal size = 1048576000
656 bluestore block wal create = true"
661 keyring = $keyring_fn
662 log file = $CEPH_OUT_DIR/\$name.\$pid.log
663 admin socket = $CEPH_ASOK_DIR/\$name.\$pid.asok
666 rgw crypt s3 kms backend = testing
667 rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo=
668 rgw crypt require ssl = false
669 ; uncomment the following to set LC days as the value in seconds;
670 ; needed for passing lc time based s3-tests (can be verbose)
671 ; rgw lc debug interval = 10
672 ; The following settings are for SSE-KMS with Vault
673 ;rgw crypt s3 kms backend = vault
674 ;rgw crypt vault auth = token
675 ;rgw crypt vault token file = $CEPH_CONF_PATH/vault.token
676 ;rgw crypt vault addr = http://127.0.0.1:8200
677 ;rgw crypt vault secret engine = kv
678 ;rgw crypt vault prefix = /v1/kv/data
679 ;rgw crypt vault secret engine = transit
680 ;rgw crypt vault prefix = /v1/transit/export/encryption-key/
695 mds data = $CEPH_DEV_DIR/mds.\$id
696 mds root ino uid = `id -u`
697 mds root ino gid = `id -g`
700 mgr data = $CEPH_DEV_DIR/mgr.\$id
701 mgr module path = $MGR_PYTHON_PATH
702 cephadm path = $CEPH_ROOT/src/cephadm/cephadm
707 osd_check_max_object_name_len_on_startup = false
708 osd data = $CEPH_DEV_DIR/osd\$id
709 osd journal = $CEPH_DEV_DIR/osd\$id/journal
710 osd journal size = 100
712 osd class dir = $OBJCLASS_PATH
713 osd class load list = *
714 osd class default list = *
715 osd fast shutdown = false
717 filestore wbthrottle xfs ios start flusher = 10
718 filestore wbthrottle xfs ios hard limit = 20
719 filestore wbthrottle xfs inodes hard limit = 30
720 filestore wbthrottle btrfs ios start flusher = 10
721 filestore wbthrottle btrfs ios hard limit = 20
722 filestore wbthrottle btrfs inodes hard limit = 30
723 bluestore fsck on mount = true
724 bluestore block create = true
728 kstore fsck on mount = true
729 osd objectstore = $objectstore
733 mgr initial modules = $mgr_modules
737 mon cluster log file = $CEPH_OUT_DIR/cluster.mon.\$id.log
738 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd
745 for f
in a b c d e f g h i j k l m n o p q r s t u v w x y z
747 [ $count -eq $CEPH_NUM_MON ] && break;
748 count
=$
(($count + 1))
749 if [ -z "$MONS" ]; then
756 if [ "$new" -eq 1 ]; then
757 if [ `echo $IP | grep '^127\\.'` ]; then
759 echo "NOTE: hostname resolves to loopback; remote hosts will not be able to"
760 echo " connect. either adjust /etc/hosts, or edit this script to use your"
761 echo " machine's real IP."
765 prun
$SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name=mon.
"$keyring_fn" --cap mon
'allow *'
766 prun
$SUDO "$CEPH_BIN/ceph-authtool" --gen-key --name=client.admin \
767 --cap mon
'allow *' \
768 --cap osd
'allow *' \
769 --cap mds
'allow *' \
770 --cap mgr
'allow *' \
773 # build a fresh fs monmap, mon fs
779 if [ $msgr -eq 1 ]; then
780 A
="v1:$IP:$(($CEPH_PORT+$count+1))"
782 if [ $msgr -eq 2 ]; then
783 A
="v2:$IP:$(($CEPH_PORT+$count+1))"
785 if [ $msgr -eq 21 ]; then
786 A
="[v2:$IP:$(($CEPH_PORT+$count)),v1:$IP:$(($CEPH_PORT+$count+1))]"
788 params
+=("--addv" "$f" "$A")
789 mon_host
="$mon_host $A"
793 mon data = $CEPH_DEV_DIR/mon.$f
795 count
=$
(($count + 2))
801 prun
"$CEPH_BIN/monmaptool" --create --clobber "${params[@]}" --print "$monmap_fn"
805 prun
rm -rf -- "$CEPH_DEV_DIR/mon.$f"
806 prun mkdir
-p "$CEPH_DEV_DIR/mon.$f"
807 prun
"$CEPH_BIN/ceph-mon" --mkfs -c "$conf_fn" -i "$f" --monmap="$monmap_fn" --keyring="$keyring_fn"
810 prun
rm -- "$monmap_fn"
816 run
'mon' $f $CEPH_BIN/ceph-mon
-i $f $ARGS $CMON_ARGS
821 if [ $inc_osd_num -gt 0 ]; then
822 old_maxosd
=$
($CEPH_BIN/ceph osd getmaxosd |
sed -e 's/max_osd = //' -e 's/ in epoch.*//')
824 end
=$
(($start-1+$inc_osd_num))
825 overwrite_conf
=1 # fake wconf
828 end
=$
(($CEPH_NUM_OSD-1))
831 for osd
in `seq $start $end`
833 local extra_seastar_args
834 if [ "$ceph_osd" == "crimson-osd" ]; then
835 # designate a single CPU node $osd for osd.$osd
836 extra_seastar_args
="--smp 1 --cpuset $osd"
837 if [ "$debug" -ne 0 ]; then
838 extra_seastar_args
+=" --debug"
841 if [ "$new" -eq 1 -o $inc_osd_num -gt 0 ]; then
846 if [ "$spdk_enabled" -eq 1 ]; then
848 bluestore_block_path = spdk:$(get_pci_selector $((osd+1)))
852 rm -rf $CEPH_DEV_DIR/osd
$osd || true
853 if command -v btrfs
> /dev
/null
; then
854 for f
in $CEPH_DEV_DIR/osd
$osd/*; do btrfs sub delete
$f &> /dev
/null || true
; done
856 if [ -n "$filestore_path" ]; then
857 ln -s $filestore_path $CEPH_DEV_DIR/osd
$osd
858 elif [ -n "$kstore_path" ]; then
859 ln -s $kstore_path $CEPH_DEV_DIR/osd
$osd
861 mkdir
-p $CEPH_DEV_DIR/osd
$osd
862 if [ -n "${bluestore_dev[$osd]}" ]; then
863 dd if=/dev
/zero of
=${bluestore_dev[$osd]} bs
=1M count
=1
864 ln -s ${bluestore_dev[$osd]} $CEPH_DEV_DIR/osd
$osd/block
866 bluestore fsck on mount = false
872 echo "add osd$osd $uuid"
873 OSD_SECRET
=$
($CEPH_BIN/ceph-authtool
--gen-print-key)
874 echo "{\"cephx_secret\": \"$OSD_SECRET\"}" > $CEPH_DEV_DIR/osd
$osd/new.json
875 ceph_adm osd new
$uuid -i $CEPH_DEV_DIR/osd
$osd/new.json
876 rm $CEPH_DEV_DIR/osd
$osd/new.json
877 $SUDO $CEPH_BIN/$ceph_osd $extra_osd_args -i $osd $ARGS --mkfs --key $OSD_SECRET --osd-uuid $uuid $extra_seastar_args
879 local key_fn
=$CEPH_DEV_DIR/osd
$osd/keyring
887 run
'osd' $osd $SUDO $CEPH_BIN/$ceph_osd \
888 $extra_seastar_args $extra_osd_args \
889 -i $osd $ARGS $COSD_ARGS &
898 for p
in $osds_wait; do
901 debug
echo OSDs started
903 if [ $inc_osd_num -gt 0 ]; then
905 new_maxosd
=$
($CEPH_BIN/ceph osd getmaxosd |
sed -e 's/max_osd = //' -e 's/ in epoch.*//')
906 sed -i "s/num osd = .*/num osd = $new_maxosd/g" $conf_fn
912 local ssl
=${DASHBOARD_SSL:-1}
913 # avoid monitors on nearby ports (which test/*.sh use extensively)
914 MGR_PORT
=$
(($CEPH_PORT + 1000))
916 for name
in x y z a b c d e f g h i j k l m n o p
918 [ $mgr -eq $CEPH_NUM_MGR ] && break
920 if [ "$new" -eq 1 ]; then
921 mkdir
-p $CEPH_DEV_DIR/mgr.
$name
922 key_fn
=$CEPH_DEV_DIR/mgr.
$name/keyring
923 $SUDO $CEPH_BIN/ceph-authtool
--create-keyring --gen-key --name=mgr.
$name $key_fn
924 ceph_adm
-i $key_fn auth add mgr.
$name mon
'allow profile mgr' mds
'allow *' osd
'allow *'
931 if $with_mgr_dashboard ; then
932 local port_option
="ssl_server_port"
933 local http_proto
="https"
934 if [ "$ssl" == "0" ]; then
935 port_option
="server_port"
937 ceph_adm config
set mgr mgr
/dashboard
/ssl false
--force
939 ceph_adm config
set mgr mgr
/dashboard
/$name/$port_option $MGR_PORT --force
940 if [ $mgr -eq 1 ]; then
941 DASH_URLS
="$http_proto://$IP:$MGR_PORT"
943 DASH_URLS
+=", $http_proto://$IP:$MGR_PORT"
946 MGR_PORT
=$
(($MGR_PORT + 1000))
947 ceph_adm config
set mgr mgr
/prometheus
/$name/server_port
$PROMETHEUS_PORT --force
948 PROMETHEUS_PORT
=$
(($PROMETHEUS_PORT + 1000))
950 ceph_adm config
set mgr mgr
/restful
/$name/server_port
$MGR_PORT --force
951 if [ $mgr -eq 1 ]; then
952 RESTFUL_URLS
="https://$IP:$MGR_PORT"
954 RESTFUL_URLS
+=", https://$IP:$MGR_PORT"
956 MGR_PORT
=$
(($MGR_PORT + 1000))
959 debug
echo "Starting mgr.${name}"
960 run
'mgr' $name $CEPH_BIN/ceph-mgr
-i $name $ARGS
963 if [ "$new" -eq 1 ]; then
964 # setting login credentials for dashboard
965 if $with_mgr_dashboard; then
966 while ! ceph_adm
-h |
grep -c -q ^dashboard
; do
967 debug
echo 'waiting for mgr dashboard module to start'
970 ceph_adm dashboard ac-user-create
--force-password admin admin administrator
971 if [ "$ssl" != "0" ]; then
972 if ! ceph_adm dashboard create-self-signed-cert
; then
973 debug
echo dashboard module not working correctly
!
978 while ! ceph_adm
-h |
grep -c -q ^restful
; do
979 debug
echo 'waiting for mgr restful module to start'
982 if ceph_adm restful create-self-signed-cert
; then
984 ceph_adm restful create-key admin
-o $SF
985 RESTFUL_SECRET
=`cat $SF`
988 debug
echo MGR Restful is not working
, perhaps the package is not installed?
992 if [ "$cephadm" -eq 1 ]; then
993 debug
echo Enabling cephadm orchestrator
994 ceph_adm config-key
set mgr
/cephadm
/ssh_identity_key
-i ~
/.ssh
/id_rsa
995 ceph_adm config-key
set mgr
/cephadm
/ssh_identity_pub
-i ~
/.ssh
/id_rsa.pub
996 ceph_adm mgr module
enable cephadm
997 ceph_adm orch
set backend cephadm
998 ceph_adm orch
host add
$HOSTNAME
999 ceph_adm orch apply crash
'*'
1000 ceph_adm config
set mgr mgr
/cephadm
/allow_ptrace true
1006 for name
in a b c d e f g h i j k l m n o p
1008 [ $mds -eq $CEPH_NUM_MDS ] && break
1011 if [ "$new" -eq 1 ]; then
1012 prun mkdir
-p "$CEPH_DEV_DIR/mds.$name"
1013 key_fn
=$CEPH_DEV_DIR/mds.
$name/keyring
1018 if [ "$standby" -eq 1 ]; then
1019 mkdir
-p $CEPH_DEV_DIR/mds.
${name}s
1021 mds standby for rank = $mds
1023 mds standby replay = true
1024 mds standby for name = ${name}
1027 prun
$SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name="mds.$name" "$key_fn"
1028 ceph_adm
-i "$key_fn" auth add
"mds.$name" mon
'allow profile mds' osd
'allow rw tag cephfs *=*' mds
'allow' mgr
'allow profile mds'
1029 if [ "$standby" -eq 1 ]; then
1030 prun
$SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name="mds.${name}s" \
1031 "$CEPH_DEV_DIR/mds.${name}s/keyring"
1032 ceph_adm
-i "$CEPH_DEV_DIR/mds.${name}s/keyring" auth add
"mds.${name}s" \
1033 mon
'allow profile mds' osd
'allow *' mds
'allow' mgr
'allow profile mds'
1037 run
'mds' $name $CEPH_BIN/ceph-mds
-i $name $ARGS $CMDS_ARGS
1038 if [ "$standby" -eq 1 ]; then
1039 run
'mds' $name $CEPH_BIN/ceph-mds
-i ${name}s
$ARGS $CMDS_ARGS
1042 #valgrind --tool=massif $CEPH_BIN/ceph-mds $ARGS --mds_log_max_segments 2 --mds_thrash_fragments 0 --mds_thrash_exports 0 > m #--debug_ms 20
1043 #$CEPH_BIN/ceph-mds -d $ARGS --mds_thrash_fragments 0 --mds_thrash_exports 0 #--debug_ms 20
1044 #ceph_adm mds set max_mds 2
1047 if [ $new -eq 1 ]; then
1048 if [ "$CEPH_NUM_FS" -gt "0" ] ; then
1049 sleep 5 # time for MDS to come up as standby to avoid health warnings on fs creation
1050 if [ "$CEPH_NUM_FS" -gt "1" ] ; then
1051 ceph_adm fs flag
set enable_multiple true
--yes-i-really-mean-it
1054 # wait for volume module to load
1055 while ! ceph_adm fs volume
ls ; do sleep 1 ; done
1057 for name
in a b c d e f g h i j k l m n o p
1059 ceph_adm fs volume create
${name}
1060 ceph_adm fs authorize
${name} "client.fs_${name}" / rwp
>> "$keyring_fn"
1062 [ $fs -eq $CEPH_NUM_FS ] && break
1069 # Ganesha Daemons requires nfs-ganesha nfs-ganesha-ceph nfs-ganesha-rados-grace
1070 # (version 2.7.6-2 and above) packages installed. On Fedora>=30 these packages
1071 # can be installed directly with 'dnf'. For CentOS>=8 the packages need to be
1072 # downloaded first from https://download.nfs-ganesha.org/2.7/2.7.6/CentOS/ and
1073 # then install it. Similarly for Ubuntu 16.04 follow the instructions on
1074 # https://launchpad.net/~nfs-ganesha/+archive/ubuntu/nfs-ganesha-2.7
1077 GANESHA_PORT
=$
(($CEPH_PORT + 4000))
1080 for name
in a b c d e f g h i j k l m n o p
1082 [ $ganesha -eq $GANESHA_DAEMON_NUM ] && break
1084 port
=$
(($GANESHA_PORT + ganesha
))
1085 ganesha
=$
(($ganesha + 1))
1086 ganesha_dir
="$CEPH_DEV_DIR/ganesha.$name"
1088 echo "Starting ganesha.$name on port: $port"
1090 prun
rm -rf $ganesha_dir
1091 prun mkdir
-p $ganesha_dir
1093 echo "NFS_CORE_PARAM {
1095 Enable_RQUOTA = false;
1107 RecoveryBackend = 'rados_cluster';
1108 Minor_Versions = 1, 2;
1118 Attr_Expiration_Time = 0;
1126 Ceph_Conf = $conf_fn;
1130 Ceph_Conf = $conf_fn;
1131 pool = 'nfs-ganesha';
1132 namespace = 'ganesha';
1135 }" > "$ganesha_dir/ganesha.conf"
1143 ganesha data = $ganesha_dir
1144 pid file = $ganesha_dir/ganesha.pid
1147 if !($CEPH_BIN/rados lspools |
grep "nfs-ganesha"); then
1148 prun ceph_adm osd pool create nfs-ganesha
1149 prun ceph_adm osd pool application
enable nfs-ganesha nfs
1152 prun ganesha-rados-grace
-p nfs-ganesha
-n ganesha add
$name
1153 prun ganesha-rados-grace
-p nfs-ganesha
-n ganesha
1155 prun
/usr
/bin
/ganesha.nfsd
-L "$ganesha_dir/ganesha.log" -f "$ganesha_dir/ganesha.conf" -p "$ganesha_dir/ganesha.pid" -N NIV_DEBUG
1157 # Wait few seconds for grace period to be removed
1159 prun ganesha-rados-grace
-p nfs-ganesha
-n ganesha
1161 if $with_mgr_dashboard; then
1162 $CEPH_BIN/rados
-p nfs-ganesha put
"conf-$name" "$ganesha_dir/ganesha.conf"
1166 if $with_mgr_dashboard; then
1167 ceph_adm dashboard set-ganesha-clusters-rados-pool-namespace nfs-ganesha
1171 if [ "$debug" -eq 0 ]; then
1176 debug
echo "** going verbose **"
1185 if [ -n "$MON_ADDR" ]; then
1186 CMON_ARGS
=" -m "$MON_ADDR
1187 COSD_ARGS
=" -m "$MON_ADDR
1188 CMDS_ARGS
=" -m "$MON_ADDR
1191 if [ -z "$CEPH_PORT" ]; then
1194 CEPH_PORT
="$(echo $(( RANDOM % 1000 + 40000 )))"
1195 ss
-a -n |
egrep "\<LISTEN\>.+:${CEPH_PORT}\s+" 1>/dev
/null
2>&1 ||
break
1199 [ -z "$INIT_CEPH" ] && INIT_CEPH
=$CEPH_BIN/init-ceph
1202 [ -d $CEPH_DEV_DIR/osd
0/.
] && [ -e $CEPH_DEV_DIR/sudo
] && SUDO
="sudo"
1204 if [ $inc_osd_num -eq 0 ]; then
1205 prun
$SUDO rm -f core
*
1208 [ -d $CEPH_ASOK_DIR ] || mkdir
-p $CEPH_ASOK_DIR
1209 [ -d $CEPH_OUT_DIR ] || mkdir
-p $CEPH_OUT_DIR
1210 [ -d $CEPH_DEV_DIR ] || mkdir
-p $CEPH_DEV_DIR
1211 if [ $inc_osd_num -eq 0 ]; then
1212 $SUDO rm -rf $CEPH_OUT_DIR/*
1214 [ -d gmon
] && $SUDO rm -rf gmon
/*
1216 [ "$cephx" -eq 1 ] && [ "$new" -eq 1 ] && [ -e $keyring_fn ] && rm $keyring_fn
1219 # figure machine's ip
1220 HOSTNAME
=`hostname -s`
1221 if [ -n "$ip" ]; then
1224 echo hostname
$HOSTNAME
1225 if [ -x "$(which ip 2>/dev/null)" ]; then
1230 # filter out IPv4 and localhost addresses
1231 IP
="$($IP_CMD | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p' | head -n1)"
1232 # if nothing left, try using localhost address, it might work
1233 if [ -z "$IP" ]; then IP
="127.0.0.1"; fi
1236 echo "port $CEPH_PORT"
1239 [ -z $CEPH_ADM ] && CEPH_ADM
=$CEPH_BIN/ceph
1242 if [ "$cephx" -eq 1 ]; then
1243 prun
$SUDO "$CEPH_ADM" -c "$conf_fn" -k "$keyring_fn" "$@"
1245 prun
$SUDO "$CEPH_ADM" -c "$conf_fn" "$@"
1249 if [ $inc_osd_num -gt 0 ]; then
1254 if [ "$new" -eq 1 ]; then
1258 if [ $CEPH_NUM_MON -gt 0 ]; then
1261 debug
echo Populating config ...
1262 cat <<EOF | $CEPH_BIN/ceph -c $conf_fn config assimilate-conf -i -
1264 osd_pool_default_size = $OSD_POOL_DEFAULT_SIZE
1265 osd_pool_default_min_size = 1
1268 mon_osd_reporter_subtree_level = osd
1269 mon_data_avail_warn = 2
1270 mon_data_avail_crit = 1
1271 mon_allow_pool_delete = true
1274 osd_scrub_load_threshold = 2000
1275 osd_debug_op_order = true
1276 osd_debug_misdirected_ops = true
1277 osd_copyfrom_max_chunk = 524288
1280 mds_debug_frag = true
1281 mds_debug_auth_pins = true
1282 mds_debug_subtrees = true
1285 mgr/telemetry/nag = false
1286 mgr/telemetry/enable = false
1290 if [ "$debug" -ne 0 ]; then
1291 debug
echo Setting debug configs ...
1292 cat <<EOF | $CEPH_BIN/ceph -c $conf_fn config assimilate-conf -i -
1306 debug_filestore = 20
1307 debug_bluestore = 20
1319 mds_debug_scatterstat = true
1320 mds_verify_scatter = true
1325 if [ $CEPH_NUM_MGR -gt 0 ]; then
1330 if [ $CEPH_NUM_OSD -gt 0 ]; then
1335 if [ "$smallmds" -eq 1 ]; then
1338 mds log max segments = 2
1339 # Default 'mds cache memory limit' is 1GiB, and here we set it to 100MiB.
1340 mds cache memory limit = 100M
1344 if [ $CEPH_NUM_MDS -gt 0 ]; then
1346 # key with access to all FS
1347 ceph_adm fs authorize \
* "client.fs" / rwp
>> "$keyring_fn"
1350 # Don't set max_mds until all the daemons are started, otherwise
1351 # the intended standbys might end up in active roles.
1352 if [ "$CEPH_MAX_MDS" -gt 1 ]; then
1353 sleep 5 # wait for daemons to make it into FSMap before increasing max_mds
1356 for name
in a b c d e f g h i j k l m n o p
1358 [ $fs -eq $CEPH_NUM_FS ] && break
1360 if [ "$CEPH_MAX_MDS" -gt 1 ]; then
1361 ceph_adm fs
set "cephfs_${name}" max_mds
"$CEPH_MAX_MDS"
1367 if [ "$ec" -eq 1 ]; then
1369 osd erasure-code-profile set ec-profile m=2 k=2
1370 osd pool create ec erasure ec-profile
1375 if [ $GANESHA_DAEMON_NUM -gt 0 ]; then
1380 while [ -n "$*" ]; do
1383 debug
echo "creating cache for pool $p ..."
1385 osd pool create ${p}-cache
1386 osd tier add $p ${p}-cache
1387 osd tier cache-mode ${p}-cache writeback
1388 osd tier set-overlay $p ${p}-cache
1395 while [ -n "$*" ]; do
1400 debug
echo "setting hit_set on pool $pool type $type ..."
1402 osd pool set $pool hit_set_type $type
1403 osd pool set $pool hit_set_count 8
1404 osd pool set $pool hit_set_period 30
1410 do_rgw_create_users
()
1413 local akey
='0555b35654ad1656d804'
1414 local skey
='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=='
1415 debug
echo "setting up user testid"
1416 $CEPH_BIN/radosgw-admin user create
--uid testid
--access-key $akey --secret $skey --display-name 'M. Tester' --email tester@ceph.com
-c $conf_fn > /dev
/null
1418 # Create S3-test users
1419 # See: https://github.com/ceph/s3-tests
1420 debug
echo "setting up s3-test users"
1421 $CEPH_BIN/radosgw-admin user create \
1422 --uid 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef \
1423 --access-key ABCDEFGHIJKLMNOPQRST \
1424 --secret abcdefghijklmnopqrstuvwxyzabcdefghijklmn \
1425 --display-name youruseridhere \
1426 --email s3@example.com
-c $conf_fn > /dev
/null
1427 $CEPH_BIN/radosgw-admin user create \
1428 --uid 56789abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234 \
1429 --access-key NOPQRSTUVWXYZABCDEFG \
1430 --secret nopqrstuvwxyzabcdefghijklmnabcdefghijklm \
1431 --display-name john.doe \
1432 --email john.doe@example.com
-c $conf_fn > /dev
/null
1433 $CEPH_BIN/radosgw-admin user create \
1435 --uid 9876543210abcdef0123456789abcdef0123456789abcdef0123456789abcdef \
1436 --access-key HIJKLMNOPQRSTUVWXYZA \
1437 --secret opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab \
1438 --display-name tenanteduser \
1439 --email tenanteduser@example.com
-c $conf_fn > /dev
/null
1442 debug
echo "setting up user tester"
1443 $CEPH_BIN/radosgw-admin user create
-c $conf_fn --subuser=test:tester
--display-name=Tester-Subuser
--key-type=swift
--secret=testing
--access=full
> /dev
/null
1446 echo "S3 User Info:"
1447 echo " access key: $akey"
1448 echo " secret key: $skey"
1450 echo "Swift User Info:"
1451 echo " account : test"
1452 echo " user : tester"
1453 echo " password : testing"
1459 if [ "$new" -eq 1 ]; then
1461 if [ -n "$rgw_compression" ]; then
1462 debug
echo "setting compression type=$rgw_compression"
1463 $CEPH_BIN/radosgw-admin zone placement modify
-c $conf_fn --rgw-zone=default
--placement-id=default-placement
--compression=$rgw_compression > /dev
/null
1468 if [ "$debug" -ne 0 ]; then
1469 RGWDEBUG
="--debug-rgw=20 --debug-ms=1"
1472 local CEPH_RGW_PORT_NUM
="${CEPH_RGW_PORT}"
1473 local CEPH_RGW_HTTPS
="${CEPH_RGW_PORT: -1}"
1474 if [[ "${CEPH_RGW_HTTPS}" = "s" ]]; then
1475 CEPH_RGW_PORT_NUM
="${CEPH_RGW_PORT::-1}"
1480 [ $CEPH_RGW_PORT_NUM -lt 1024 ] && RGWSUDO
=sudo
1482 current_port
=$CEPH_RGW_PORT
1483 for n
in $
(seq 1 $CEPH_NUM_RGW); do
1484 rgw_name
="client.rgw.${current_port}"
1486 ceph_adm auth get-or-create
$rgw_name \
1492 debug
echo start rgw on http
${CEPH_RGW_HTTPS}://localhost
:${current_port}
1493 run
'rgw' $current_port $RGWSUDO $CEPH_BIN/radosgw
-c $conf_fn \
1494 --log-file=${CEPH_OUT_DIR}/radosgw.
${current_port}.log \
1495 --admin-socket=${CEPH_OUT_DIR}/radosgw.
${current_port}.asok \
1496 --pid-file=${CEPH_OUT_DIR}/radosgw.
${current_port}.pid \
1499 "--rgw_frontends=${rgw_frontend} port=${current_port}${CEPH_RGW_HTTPS}"
1502 [ $i -eq $CEPH_NUM_RGW ] && break
1504 current_port
=$
((current_port
+1))
1507 if [ "$CEPH_NUM_RGW" -gt 0 ]; then
1511 debug
echo "vstart cluster complete. Use stop.sh to stop. See out/* (e.g. 'tail -f out/????') for debug output."
1514 if [ "$new" -eq 1 ]; then
1515 if $with_mgr_dashboard; then
1516 echo "dashboard urls: $DASH_URLS"
1517 echo " w/ user/pass: admin / admin"
1519 echo "restful urls: $RESTFUL_URLS"
1520 echo " w/ user/pass: admin / $RESTFUL_SECRET"
1524 # add header to the environment file
1527 echo "# source this file into your shell to set up the environment."
1528 echo "# For example:"
1529 echo "# $ . $CEPH_DIR/vstart_environment.sh"
1531 } > $CEPH_DIR/vstart_environment.sh
1533 echo "export PYTHONPATH=$PYBIND:$CYTHON_PYTHONPATH:$CEPH_PYTHON_COMMON\$PYTHONPATH"
1534 echo "export LD_LIBRARY_PATH=$CEPH_LIB:\$LD_LIBRARY_PATH"
1536 if [ "$CEPH_DIR" != "$PWD" ]; then
1537 echo "export CEPH_CONF=$conf_fn"
1538 echo "export CEPH_KEYRING=$keyring_fn"
1541 if [ -n "$CEPHFS_SHELL" ]; then
1542 echo "alias cephfs-shell=$CEPHFS_SHELL"
1544 } |
tee -a $CEPH_DIR/vstart_environment.sh
1548 # always keep this section at the very bottom of this file
1549 STRAY_CONF_PATH
="/etc/ceph/ceph.conf"
1550 if [ -f "$STRAY_CONF_PATH" -a -n "$conf_fn" -a ! "$conf_fn" -ef "$STRAY_CONF_PATH" ]; then
1554 echo " Please remove stray $STRAY_CONF_PATH if not needed."
1555 echo " Your conf files $conf_fn and $STRAY_CONF_PATH may not be in sync"
1556 echo " and may lead to undesired results."
1559 echo " Remember to restart cluster after removing $STRAY_CONF_PATH"