]> git.proxmox.com Git - ceph.git/blame - ceph/src/vstart.sh
projects / ceph.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
bump version to 12.2.1-pve3
[ceph.git] / ceph / src / vstart.sh
CommitLineData
31f18b77 1#!/bin/bash
7c673cae
FG		 2
3# abort on failure
4set -e
5
6if [ -n "$VSTART_DEST" ]; then
7 SRC_PATH=`dirname $0`
8 SRC_PATH=`(cd $SRC_PATH; pwd)`
9
10 CEPH_DIR=$SRC_PATH
11 CEPH_BIN=${PWD}/bin
12 CEPH_LIB=${PWD}/lib
13
14 CEPH_CONF_PATH=$VSTART_DEST
15 CEPH_DEV_DIR=$VSTART_DEST/dev
16 CEPH_OUT_DIR=$VSTART_DEST/out
17fi
18
19# for running out of the CMake build directory
20if [ -e CMakeCache.txt ]; then
21 # Out of tree build, learn source location from CMakeCache.txt
22 CEPH_ROOT=`grep ceph_SOURCE_DIR CMakeCache.txt | cut -d "=" -f 2`
23 CEPH_BUILD_DIR=`pwd`
24 [ -z "$MGR_PYTHON_PATH" ] && MGR_PYTHON_PATH=$CEPH_ROOT/src/pybind/mgr
25fi
26
27# use CEPH_BUILD_ROOT to vstart from a 'make install'
28if [ -n "$CEPH_BUILD_ROOT" ]; then
29 [ -z "$CEPH_BIN" ] && CEPH_BIN=$CEPH_BUILD_ROOT/bin
30 [ -z "$CEPH_LIB" ] && CEPH_LIB=$CEPH_BUILD_ROOT/lib
31 [ -z "$EC_PATH" ] && EC_PATH=$CEPH_LIB/erasure-code
32 [ -z "$OBJCLASS_PATH" ] && OBJCLASS_PATH=$CEPH_LIB/rados-classes
33elif [ -n "$CEPH_ROOT" ]; then
34 [ -z "$PYBIND" ] && PYBIND=$CEPH_ROOT/src/pybind
35 [ -z "$CEPH_BIN" ] && CEPH_BIN=$CEPH_BUILD_DIR/bin
36 [ -z "$CEPH_ADM" ] && CEPH_ADM=$CEPH_BIN/ceph
37 [ -z "$INIT_CEPH" ] && INIT_CEPH=$CEPH_BUILD_DIR/bin/init-ceph
38 [ -z "$CEPH_LIB" ] && CEPH_LIB=$CEPH_BUILD_DIR/lib
39 [ -z "$OBJCLASS_PATH" ] && OBJCLASS_PATH=$CEPH_LIB
40 [ -z "$EC_PATH" ] && EC_PATH=$CEPH_LIB
41fi
42
43if [ -z "${CEPH_VSTART_WRAPPER}" ]; then
44 PATH=$(pwd):$PATH
45fi
46
47[ -z "$PYBIND" ] && PYBIND=./pybind
48
49export PYTHONPATH=$PYBIND:$CEPH_LIB/cython_modules/lib.2:$PYTHONPATH
50export LD_LIBRARY_PATH=$CEPH_LIB:$LD_LIBRARY_PATH
51export DYLD_LIBRARY_PATH=$CEPH_LIB:$DYLD_LIBRARY_PATH
31f18b77
FG		 52# Suppress logging for regular use that indicated that we are using a
53# development version. vstart.sh is only used during testing and
54# development
55export CEPH_DEV=1
7c673cae
FG		 56
57[ -z "$CEPH_NUM_MON" ] && CEPH_NUM_MON="$MON"
58[ -z "$CEPH_NUM_OSD" ] && CEPH_NUM_OSD="$OSD"
59[ -z "$CEPH_NUM_MDS" ] && CEPH_NUM_MDS="$MDS"
60[ -z "$CEPH_NUM_MGR" ] && CEPH_NUM_MGR="$MGR"
61[ -z "$CEPH_NUM_FS" ] && CEPH_NUM_FS="$FS"
62[ -z "$CEPH_NUM_RGW" ] && CEPH_NUM_RGW="$RGW"
63
64# if none of the CEPH_NUM_* number is specified, kill the existing
65# cluster.
66if [ -z "$CEPH_NUM_MON" -a \
67 -z "$CEPH_NUM_OSD" -a \
68 -z "$CEPH_NUM_MDS" -a \
69 -z "$CEPH_NUM_MGR" ]; then
70 kill_all=1
71else
72 kill_all=0
73fi
74
75[ -z "$CEPH_NUM_MON" ] && CEPH_NUM_MON=3
76[ -z "$CEPH_NUM_OSD" ] && CEPH_NUM_OSD=3
77[ -z "$CEPH_NUM_MDS" ] && CEPH_NUM_MDS=3
78[ -z "$CEPH_NUM_MGR" ] && CEPH_NUM_MGR=1
79[ -z "$CEPH_NUM_FS" ] && CEPH_NUM_FS=1
80[ -z "$CEPH_MAX_MDS" ] && CEPH_MAX_MDS=1
81[ -z "$CEPH_NUM_RGW" ] && CEPH_NUM_RGW=0
82
83[ -z "$CEPH_DIR" ] && CEPH_DIR="$PWD"
84[ -z "$CEPH_DEV_DIR" ] && CEPH_DEV_DIR="$CEPH_DIR/dev"
85[ -z "$CEPH_OUT_DIR" ] && CEPH_OUT_DIR="$CEPH_DIR/out"
86[ -z "$CEPH_RGW_PORT" ] && CEPH_RGW_PORT=8000
87[ -z "$CEPH_CONF_PATH" ] && CEPH_CONF_PATH=$CEPH_DIR
88
89if [ $CEPH_NUM_OSD -gt 3 ]; then
90 OSD_POOL_DEFAULT_SIZE=3
91else
92 OSD_POOL_DEFAULT_SIZE=$CEPH_NUM_OSD
93fi
94
95extra_conf=""
96new=0
97standby=0
98debug=0
99ip=""
100nodaemon=0
101smallmds=0
102short=0
103ec=0
104hitset=""
105overwrite_conf=1
106cephx=1 #turn cephx on by default
107cache=""
108memstore=0
109bluestore=0
110rgw_frontend="civetweb"
224ce89b 111rgw_compression=""
7c673cae
FG		 112lockdep=${LOCKDEP:-1}
113
31f18b77
FG		 114filestore_path=
115
7c673cae
FG		 116VSTART_SEC="client.vstart.sh"
117
118MON_ADDR=""
31f18b77
FG		 119DASH_URLS=""
120RESTFUL_URLS=""
7c673cae
FG		 121
122conf_fn="$CEPH_CONF_PATH/ceph.conf"
123keyring_fn="$CEPH_CONF_PATH/keyring"
124osdmap_fn="/tmp/ceph_osdmap.$$"
125monmap_fn="/tmp/ceph_monmap.$$"
126
127usage="usage: $0 [option]... \nex: $0 -n -d --mon_num 3 --osd_num 3 --mds_num 1 --rgw_num 1\n"
128usage=$usage"options:\n"
129usage=$usage"\t-d, --debug\n"
130usage=$usage"\t-s, --standby_mds: Generate standby-replay MDS for each active\n"
131usage=$usage"\t-l, --localhost: use localhost instead of hostname\n"
132usage=$usage"\t-i <ip>: bind to specific ip\n"
133usage=$usage"\t-n, --new\n"
134usage=$usage"\t-N, --not-new: reuse existing cluster config (default)\n"
135usage=$usage"\t--valgrind[_{osd,mds,mon,rgw}] 'toolname args...'\n"
136usage=$usage"\t--nodaemon: use ceph-run as wrapper for mon/osd/mds\n"
137usage=$usage"\t--smallmds: limit mds cache size\n"
138usage=$usage"\t-m ip:port\t\tspecify monitor address\n"
139usage=$usage"\t-k keep old configuration files\n"
140usage=$usage"\t-x enable cephx (on by default)\n"
141usage=$usage"\t-X disable cephx\n"
142usage=$usage"\t--hitset <pool> <hit_set_type>: enable hitset tracking\n"
143usage=$usage"\t-e : create an erasure pool\n";
144usage=$usage"\t-o config\t\t add extra config parameters to all sections\n"
145usage=$usage"\t--mon_num specify ceph monitor count\n"
146usage=$usage"\t--osd_num specify ceph osd count\n"
147usage=$usage"\t--mds_num specify ceph mds count\n"
148usage=$usage"\t--rgw_num specify ceph rgw count\n"
149usage=$usage"\t--mgr_num specify ceph mgr count\n"
150usage=$usage"\t--rgw_port specify ceph rgw http listen port\n"
151usage=$usage"\t--rgw_frontend specify the rgw frontend configuration\n"
224ce89b 152usage=$usage"\t--rgw_compression specify the rgw compression plugin\n"
7c673cae
FG		 153usage=$usage"\t-b, --bluestore use bluestore as the osd objectstore backend\n"
154usage=$usage"\t--memstore use memstore as the osd objectstore backend\n"
155usage=$usage"\t--cache <pool>: enable cache tiering on pool\n"
156usage=$usage"\t--short: short object names only; necessary for ext4 dev\n"
157usage=$usage"\t--nolockdep disable lockdep\n"
158usage=$usage"\t--multimds <count> allow multimds with maximum active count\n"
159
160usage_exit() {
161 printf "$usage"
162 exit
163}
164
165while [ $# -ge 1 ]; do
166case $1 in
167 -d | --debug )
168 debug=1
169 ;;
170 -s | --standby_mds)
171 standby=1
172 ;;
173 -l | --localhost )
174 ip="127.0.0.1"
175 ;;
176 -i )
177 [ -z "$2" ] && usage_exit
178 ip="$2"
179 shift
180 ;;
181 -e )
182 ec=1
183 ;;
184 --new | -n )
185 new=1
186 ;;
187 --not-new | -N )
188 new=0
189 ;;
190 --short )
191 short=1
192 ;;
193 --valgrind )
194 [ -z "$2" ] && usage_exit
195 valgrind=$2
196 shift
197 ;;
198 --valgrind_args )
199 valgrind_args="$2"
200 shift
201 ;;
202 --valgrind_mds )
203 [ -z "$2" ] && usage_exit
204 valgrind_mds=$2
205 shift
206 ;;
207 --valgrind_osd )
208 [ -z "$2" ] && usage_exit
209 valgrind_osd=$2
210 shift
211 ;;
212 --valgrind_mon )
213 [ -z "$2" ] && usage_exit
214 valgrind_mon=$2
215 shift
216 ;;
217 --valgrind_mgr )
218 [ -z "$2" ] && usage_exit
219 valgrind_mgr=$2
220 shift
221 ;;
222 --valgrind_rgw )
223 [ -z "$2" ] && usage_exit
224 valgrind_rgw=$2
225 shift
226 ;;
227 --nodaemon )
228 nodaemon=1
229 ;;
230 --smallmds )
231 smallmds=1
232 ;;
233 --mon_num )
234 echo "mon_num:$2"
235 CEPH_NUM_MON="$2"
236 shift
237 ;;
238 --osd_num )
239 CEPH_NUM_OSD=$2
240 shift
241 ;;
242 --mds_num )
243 CEPH_NUM_MDS=$2
244 shift
245 ;;
246 --rgw_num )
247 CEPH_NUM_RGW=$2
248 shift
249 ;;
250 --mgr_num )
251 CEPH_NUM_MGR=$2
252 shift
253 ;;
254 --rgw_port )
255 CEPH_RGW_PORT=$2
256 shift
257 ;;
258 --rgw_frontend )
259 rgw_frontend=$2
260 shift
261 ;;
224ce89b
WB		 262 --rgw_compression )
263 rgw_compression=$2
264 shift
265 ;;
31f18b77
FG		 266 --filestore_path )
267 filestore_path=$2
268 shift
269 ;;
7c673cae
FG		 270 -m )
271 [ -z "$2" ] && usage_exit
272 MON_ADDR=$2
273 shift
274 ;;
275 -x )
276 cephx=1 # this is on be default, flag exists for historical consistency
277 ;;
278 -X )
279 cephx=0
280 ;;
281 -k )
282 if [ ! -r $conf_fn ]; then
283 echo "cannot use old configuration: $conf_fn not readable." >&2
284 exit
285 fi
286 overwrite_conf=0
287 ;;
288 --memstore )
289 memstore=1
290 ;;
291 -b | --bluestore )
292 bluestore=1
293 ;;
294 --hitset )
295 hitset="$hitset $2 $3"
296 shift
297 shift
298 ;;
299 -o )
300 extra_conf="$extra_conf $2
301"
302 shift
303 ;;
304 --cache )
305 if [ -z "$cache" ]; then
306 cache="$2"
307 else
308 cache="$cache $2"
309 fi
310 shift
311 ;;
312 --nolockdep )
313 lockdep=0
314 ;;
315 --multimds)
316 CEPH_MAX_MDS="$2"
317 shift
318 ;;
319 * )
320 usage_exit
321esac
322shift
323done
324
325if [ $kill_all -eq 1 ]; then
326 $SUDO $INIT_CEPH stop
327fi
328
329if [ "$overwrite_conf" -eq 0 ]; then
c07f9fc5
FG		 330 CEPH_ASOK_DIR=`dirname $($CEPH_BIN/ceph-conf --show-config-value admin_socket)`
331 mkdir -p $CEPH_ASOK_DIR
7c673cae
FG		 332 MON=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC num_mon 2>/dev/null` && \
333 CEPH_NUM_MON="$MON"
334 OSD=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC num_osd 2>/dev/null` && \
335 CEPH_NUM_OSD="$OSD"
336 MDS=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC num_mds 2>/dev/null` && \
337 CEPH_NUM_MDS="$MDS"
338 MGR=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC num_mgr 2>/dev/null` && \
339 CEPH_NUM_MGR="$MGR"
340 RGW=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC num_rgw 2>/dev/null` && \
341 CEPH_NUM_RGW="$RGW"
342else
343 if [ "$new" -ne 0 ]; then
344 # only delete if -n
c07f9fc5
FG		 345 asok_dir=`dirname $($CEPH_BIN/ceph-conf --show-config-value admin_socket)`
346 if [ $asok_dir != /var/run/ceph ]; then
347 [ -d $asok_dir ] && rm -f $asok_dir/* && rmdir $asok_dir
348 fi
349 if [ -z "$CEPH_ASOK_DIR" ]; then
350 CEPH_ASOK_DIR=`mktemp -u -d "${TMPDIR:-/tmp}/ceph-asok.XXXXXX"`
351 fi
7c673cae
FG		 352 [ -e "$conf_fn" ] && rm -- "$conf_fn"
353 else
c07f9fc5 354 CEPH_ASOK_DIR=`dirname $($CEPH_BIN/ceph-conf --show-config-value admin_socket)`
7c673cae
FG		 355 # -k is implied... (doesn't make sense otherwise)
356 overwrite_conf=0
357 fi
358fi
359
360ARGS="-c $conf_fn"
361
362prunb() {
363 echo "$* &"
364 "$@" &
365}
366
367prun() {
368 echo "$*"
369 "$@"
370}
371
372run() {
373 type=$1
374 shift
375 eval "valg=\$valgrind_$type"
376 [ -z "$valg" ] && valg="$valgrind"
377
378 if [ -n "$valg" ]; then
379 prunb valgrind --tool="$valg" $valgrind_args "$@" -f
380 sleep 1
381 else
382 if [ "$nodaemon" -eq 0 ]; then
383 prun "$@"
384 else
385 prunb ./ceph-run "$@" -f
386 fi
387 fi
388}
389
390wconf() {
391 if [ "$overwrite_conf" -eq 1 ]; then
392 cat >> "$conf_fn"
393 fi
394}
395
396prepare_conf() {
397 local DAEMONOPTS="
398 log file = $CEPH_OUT_DIR/\$name.log
c07f9fc5 399 admin socket = $CEPH_ASOK_DIR/\$name.asok
7c673cae
FG		 400 chdir = \"\"
401 pid file = $CEPH_OUT_DIR/\$name.pid
402 heartbeat file = $CEPH_OUT_DIR/\$name.heartbeat
403"
404
405 wconf <<EOF
406; generated by vstart.sh on `date`
407[$VSTART_SEC]
408 num mon = $CEPH_NUM_MON
409 num osd = $CEPH_NUM_OSD
410 num mds = $CEPH_NUM_MDS
411 num mgr = $CEPH_NUM_MGR
412 num rgw = $CEPH_NUM_RGW
413
414[global]
415 fsid = $(uuidgen)
416 osd pg bits = 3
417 osd pgp bits = 5 ; (invalid, but ceph should cope!)
418 osd pool default size = $OSD_POOL_DEFAULT_SIZE
419 osd crush chooseleaf type = 0
420 osd pool default min size = 1
421 osd failsafe full ratio = .99
c07f9fc5
FG		 422 mon osd nearfull ratio = .99
423 mon osd backfillfull ratio = .99
7c673cae
FG		 424 mon osd reporter subtree level = osd
425 mon osd full ratio = .99
c07f9fc5 426 mon data avail warn = 2
7c673cae
FG		 427 mon data avail crit = 1
428 erasure code dir = $EC_PATH
429 plugin dir = $CEPH_LIB
224ce89b 430 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd
7c673cae 431 rgw frontends = $rgw_frontend port=$CEPH_RGW_PORT
31f18b77 432 ; needed for s3tests
31f18b77
FG		 433 rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo=
434 rgw crypt require ssl = false
435 rgw lc debug interval = 10
7c673cae
FG		 436 filestore fd cache size = 32
437 run dir = $CEPH_OUT_DIR
438 enable experimental unrecoverable data corrupting features = *
439EOF
440 if [ "$lockdep" -eq 1 ] ; then
441 wconf <<EOF
442 lockdep = true
443EOF
444 fi
445 if [ "$cephx" -eq 1 ] ; then
446 wconf <<EOF
447 auth cluster required = cephx
448 auth service required = cephx
449 auth client required = cephx
450EOF
451 else
452 wconf <<EOF
453 auth cluster required = none
454 auth service required = none
455 auth client required = none
456EOF
457 fi
458 if [ "$short" -eq 1 ]; then
459 COSDSHORT=" osd max object name len = 460
460 osd max object namespace len = 64"
461 fi
462 wconf <<EOF
463[client]
464 keyring = $keyring_fn
465 log file = $CEPH_OUT_DIR/\$name.\$pid.log
c07f9fc5 466 admin socket = $CEPH_ASOK_DIR/\$name.\$pid.asok
7c673cae 467
224ce89b
WB		 468[client.rgw]
469
7c673cae
FG		 470[mds]
471$DAEMONOPTS
472$CMDSDEBUG
473 mds debug frag = true
474 mds debug auth pins = true
475 mds debug subtrees = true
476 mds data = $CEPH_DEV_DIR/mds.\$id
477 mds root ino uid = `id -u`
478 mds root ino gid = `id -g`
479$extra_conf
480[mgr]
7c673cae
FG		 481 mgr data = $CEPH_DEV_DIR/mgr.\$id
482 mgr module path = $MGR_PYTHON_PATH
483 mon reweight min pgs per osd = 4
31f18b77 484 mon pg warn min per osd = 3
7c673cae
FG		 485$DAEMONOPTS
486$CMGRDEBUG
487$extra_conf
488[osd]
489$DAEMONOPTS
490 osd_check_max_object_name_len_on_startup = false
491 osd data = $CEPH_DEV_DIR/osd\$id
492 osd journal = $CEPH_DEV_DIR/osd\$id/journal
493 osd journal size = 100
494 osd class tmp = out
495 osd class dir = $OBJCLASS_PATH
496 osd class load list = *
497 osd class default list = *
498 osd scrub load threshold = 2000.0
499 osd debug op order = true
500 osd debug misdirected ops = true
501 filestore wbthrottle xfs ios start flusher = 10
502 filestore wbthrottle xfs ios hard limit = 20
503 filestore wbthrottle xfs inodes hard limit = 30
504 filestore wbthrottle btrfs ios start flusher = 10
505 filestore wbthrottle btrfs ios hard limit = 20
506 filestore wbthrottle btrfs inodes hard limit = 30
507 osd copyfrom max chunk = 524288
508 bluestore fsck on mount = true
509 bluestore block create = true
510 bluestore block db size = 67108864
511 bluestore block db create = true
512 bluestore block wal size = 1048576000
513 bluestore block wal create = true
514$COSDDEBUG
515$COSDMEMSTORE
516$COSDSHORT
517$extra_conf
518[mon]
224ce89b 519 mgr initial modules = restful status dashboard
7c673cae
FG		 520 mon pg warn min per osd = 3
521 mon osd allow primary affinity = true
7c673cae
FG		 522 mon reweight min pgs per osd = 4
523 mon osd prime pg temp = true
524 crushtool = $CEPH_BIN/crushtool
525 mon allow pool delete = true
526$DAEMONOPTS
527$CMONDEBUG
528$extra_conf
529 mon cluster log file = $CEPH_OUT_DIR/cluster.mon.\$id.log
530[global]
531$extra_conf
532EOF
533}
534
535start_mon() {
536 local MONS=""
537 local count=0
538 for f in a b c d e f g h i j k l m n o p q r s t u v w x y z
539 do
540 [ $count -eq $CEPH_NUM_MON ] && break;
541 count=$(($count + 1))
542 if [ -z "$MONS" ];
543 then
544 MONS="$f"
545 else
546 MONS="$MONS $f"
547 fi
548 done
549
550 if [ "$new" -eq 1 ]; then
551 if [ `echo $IP | grep '^127\\.'` ]
552 then
553 echo
554 echo "NOTE: hostname resolves to loopback; remote hosts will not be able to"
555 echo " connect. either adjust /etc/hosts, or edit this script to use your"
556 echo " machine's real IP."
557 echo
558 fi
559
560 prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name=mon. "$keyring_fn" --cap mon 'allow *'
561 prun $SUDO "$CEPH_BIN/ceph-authtool" --gen-key --name=client.admin --set-uid=0 \
562 --cap mon 'allow *' \
563 --cap osd 'allow *' \
564 --cap mds 'allow *' \
565 --cap mgr 'allow *' \
566 "$keyring_fn"
567
224ce89b
WB		 568 prun $SUDO "$CEPH_BIN/ceph-authtool" --gen-key --name=client.rgw \
569 --cap mon 'allow rw' \
570 --cap osd 'allow rwx' \
571 --cap mgr 'allow rw' \
572 "$keyring_fn"
573
7c673cae
FG		 574 # build a fresh fs monmap, mon fs
575 local str=""
576 local count=0
577 for f in $MONS
578 do
579 str="$str --add $f $IP:$(($CEPH_PORT+$count))"
580 wconf <<EOF
581[mon.$f]
582 host = $HOSTNAME
583 mon data = $CEPH_DEV_DIR/mon.$f
584 mon addr = $IP:$(($CEPH_PORT+$count))
585EOF
586 count=$(($count + 1))
587 done
588 prun "$CEPH_BIN/monmaptool" --create --clobber $str --print "$monmap_fn"
589
590 for f in $MONS
591 do
592 prun rm -rf -- "$CEPH_DEV_DIR/mon.$f"
593 prun mkdir -p "$CEPH_DEV_DIR/mon.$f"
594 prun "$CEPH_BIN/ceph-mon" --mkfs -c "$conf_fn" -i "$f" --monmap="$monmap_fn" --keyring="$keyring_fn"
595 done
596
597 prun rm -- "$monmap_fn"
598 fi
599
600 # start monitors
601 for f in $MONS
602 do
603 run 'mon' $CEPH_BIN/ceph-mon -i $f $ARGS $CMON_ARGS
604 done
605}
606
607start_osd() {
608 for osd in `seq 0 $((CEPH_NUM_OSD-1))`
609 do
610 if [ "$new" -eq 1 ]; then
611 wconf <<EOF
612[osd.$osd]
613 host = $HOSTNAME
614EOF
615
616 rm -rf $CEPH_DEV_DIR/osd$osd || true
617 if command -v btrfs > /dev/null; then
618 for f in $CEPH_DEV_DIR/osd$osd/*; do btrfs sub delete $f &> /dev/null || true; done
619 fi
31f18b77
FG		 620 if [ -n "$filestore_path" ]; then
621 ln -s $filestore_path $CEPH_DEV_DIR/osd$osd
622 else
623 mkdir -p $CEPH_DEV_DIR/osd$osd
624 fi
7c673cae
FG		 625
626 local uuid=`uuidgen`
627 echo "add osd$osd $uuid"
628 ceph_adm osd create $uuid
629 ceph_adm osd crush add osd.$osd 1.0 host=$HOSTNAME root=default
630 $SUDO $CEPH_BIN/ceph-osd -i $osd $ARGS --mkfs --mkkey --osd-uuid $uuid
631
632 local key_fn=$CEPH_DEV_DIR/osd$osd/keyring
633 echo adding osd$osd key to auth repository
634 ceph_adm -i "$key_fn" auth add osd.$osd osd "allow *" mon "allow profile osd" mgr "allow profile osd"
635 fi
31f18b77 636 echo start osd.$osd
7c673cae
FG		 637 run 'osd' $SUDO $CEPH_BIN/ceph-osd -i $osd $ARGS $COSD_ARGS
638 done
639}
640
641start_mgr() {
642 local mgr=0
31f18b77
FG		 643 # avoid monitors on nearby ports (which test/*.sh use extensively)
644 MGR_PORT=$(($CEPH_PORT + 1000))
7c673cae
FG		 645 for name in x y z a b c d e f g h i j k l m n o p
646 do
647 [ $mgr -eq $CEPH_NUM_MGR ] && break
648 mgr=$(($mgr + 1))
649 if [ "$new" -eq 1 ]; then
650 mkdir -p $CEPH_DEV_DIR/mgr.$name
651 key_fn=$CEPH_DEV_DIR/mgr.$name/keyring
652 $SUDO $CEPH_BIN/ceph-authtool --create-keyring --gen-key --name=mgr.$name $key_fn
653 ceph_adm -i $key_fn auth add mgr.$name mon 'allow profile mgr' mds 'allow *' osd 'allow *'
654 fi
655
656 wconf <<EOF
657[mgr.$name]
658 host = $HOSTNAME
659EOF
660
c07f9fc5 661 ceph_adm config-key set mgr/dashboard/$name/server_port $MGR_PORT
31f18b77
FG		 662 DASH_URLS+="http://$IP:$MGR_PORT/"
663 MGR_PORT=$(($MGR_PORT + 1000))
664
c07f9fc5 665 ceph_adm config-key set mgr/restful/$name/server_port $MGR_PORT
31f18b77
FG		 666
667 RESTFUL_URLS+="https://$IP:$MGR_PORT"
668 MGR_PORT=$(($MGR_PORT + 1000))
669
7c673cae
FG		 670 echo "Starting mgr.${name}"
671 run 'mgr' $CEPH_BIN/ceph-mgr -i $name $ARGS
672 done
31f18b77 673
c07f9fc5
FG		 674 # use tell mgr here because the first mgr might not have activated yet
675 # to register the python module commands.
224ce89b
WB		 676 if ceph_adm tell mgr restful create-self-signed-cert; then
677 SF=`mktemp`
c07f9fc5 678 ceph_adm restful create-key admin -o $SF
224ce89b
WB		 679 RESTFUL_SECRET=`cat $SF`
680 rm $SF
681 else
682 echo MGR Restful is not working, perhaps the package is not installed?
683 fi
7c673cae
FG		 684}
685
686start_mds() {
687 if [ $new -eq 1 ]; then
688 if [ "$CEPH_NUM_FS" -gt "0" ] ; then
689 if [ "$CEPH_NUM_FS" -gt "1" ] ; then
690 ceph_adm fs flag set enable_multiple true --yes-i-really-mean-it
691 fi
692
693 local fs=0
694 for name in a b c d e f g h i j k l m n o p
695 do
696 ceph_adm osd pool create "cephfs_data_${name}" 8
697 ceph_adm osd pool create "cephfs_metadata_${name}" 8
698 ceph_adm fs new "cephfs_${name}" "cephfs_metadata_${name}" "cephfs_data_${name}"
699 fs=$(($fs + 1))
700 [ $fs -eq $CEPH_NUM_FS ] && break
701 done
702 fi
703 fi
704
705 local mds=0
706 for name in a b c d e f g h i j k l m n o p
707 do
708 [ $mds -eq $CEPH_NUM_MDS ] && break
709 mds=$(($mds + 1))
710
711 if [ "$new" -eq 1 ]; then
712 prun mkdir -p "$CEPH_DEV_DIR/mds.$name"
713 key_fn=$CEPH_DEV_DIR/mds.$name/keyring
714 wconf <<EOF
715[mds.$name]
716 host = $HOSTNAME
717EOF
718 if [ "$standby" -eq 1 ]; then
719 mkdir -p $CEPH_DEV_DIR/mds.${name}s
720 wconf <<EOF
721 mds standby for rank = $mds
722[mds.${name}s]
723 mds standby replay = true
724 mds standby for name = ${name}
725EOF
726 fi
727 prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name="mds.$name" "$key_fn"
728 ceph_adm -i "$key_fn" auth add "mds.$name" mon 'allow profile mds' osd 'allow *' mds 'allow' mgr 'allow profile mds'
729 if [ "$standby" -eq 1 ]; then
730 prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name="mds.${name}s" \
731 "$CEPH_DEV_DIR/mds.${name}s/keyring"
732 ceph_adm -i "$CEPH_DEV_DIR/mds.${name}s/keyring" auth add "mds.${name}s" \
733 mon 'allow profile mds' osd 'allow *' mds 'allow' mgr 'allow profile mds'
734 fi
735 fi
736
737 run 'mds' $CEPH_BIN/ceph-mds -i $name $ARGS $CMDS_ARGS
738 if [ "$standby" -eq 1 ]; then
739 run 'mds' $CEPH_BIN/ceph-mds -i ${name}s $ARGS $CMDS_ARGS
740 fi
741
742 #valgrind --tool=massif $CEPH_BIN/ceph-mds $ARGS --mds_log_max_segments 2 --mds_thrash_fragments 0 --mds_thrash_exports 0 > m #--debug_ms 20
743 #$CEPH_BIN/ceph-mds -d $ARGS --mds_thrash_fragments 0 --mds_thrash_exports 0 #--debug_ms 20
744 #ceph_adm mds set max_mds 2
745 done
746}
747
748if [ "$debug" -eq 0 ]; then
749 CMONDEBUG='
750 debug mon = 10
751 debug ms = 1'
752 COSDDEBUG='
753 debug ms = 1'
754 CMDSDEBUG='
755 debug ms = 1'
31f18b77
FG		 756 CMGRDEBUG='
757 debug ms = 1'
7c673cae
FG		 758else
759 echo "** going verbose **"
760 CMONDEBUG='
761 debug mon = 20
762 debug paxos = 20
763 debug auth = 20
764 debug mgrc = 20
765 debug ms = 1'
766 COSDDEBUG='
767 debug ms = 1
768 debug osd = 25
769 debug objecter = 20
770 debug monc = 20
771 debug mgrc = 20
772 debug journal = 20
773 debug filestore = 20
774 debug bluestore = 30
775 debug bluefs = 20
776 debug rocksdb = 10
777 debug bdev = 20
778 debug rgw = 20
779 debug objclass = 20'
780 CMDSDEBUG='
781 debug ms = 1
782 debug mds = 20
783 debug auth = 20
784 debug monc = 20
785 debug mgrc = 20
786 mds debug scatterstat = true
787 mds verify scatter = true
788 mds log max segments = 2'
789 CMGRDEBUG='
790 debug ms = 1
791 debug monc = 20
31f18b77 792 debug mon = 20
7c673cae
FG		 793 debug mgr = 20'
794fi
795
796if [ -n "$MON_ADDR" ]; then
797 CMON_ARGS=" -m "$MON_ADDR
798 COSD_ARGS=" -m "$MON_ADDR
799 CMDS_ARGS=" -m "$MON_ADDR
800fi
801
802if [ "$memstore" -eq 1 ]; then
803 COSDMEMSTORE='
804 osd objectstore = memstore'
805fi
806if [ "$bluestore" -eq 1 ]; then
807 COSDMEMSTORE='
808 osd objectstore = bluestore'
809fi
810
811if [ -z "$CEPH_PORT" ]; then
812 CEPH_PORT=6789
813 [ -e ".ceph_port" ] && CEPH_PORT=`cat .ceph_port`
814fi
815
816[ -z "$INIT_CEPH" ] && INIT_CEPH=$CEPH_BIN/init-ceph
817
818# sudo if btrfs
819test -d $CEPH_DEV_DIR/osd0/. && test -e $CEPH_DEV_DIR/sudo && SUDO="sudo"
820
821prun $SUDO rm -f core*
822
c07f9fc5 823test -d $CEPH_ASOK_DIR || mkdir $CEPH_ASOK_DIR
7c673cae
FG		 824test -d $CEPH_OUT_DIR || mkdir $CEPH_OUT_DIR
825test -d $CEPH_DEV_DIR || mkdir $CEPH_DEV_DIR
826$SUDO rm -rf $CEPH_OUT_DIR/*
827test -d gmon && $SUDO rm -rf gmon/*
828
829[ "$cephx" -eq 1 ] && [ "$new" -eq 1 ] && test -e $keyring_fn && rm $keyring_fn
830
831
832# figure machine's ip
833HOSTNAME=`hostname -s`
834if [ -n "$ip" ]; then
835 IP="$ip"
836else
837 echo hostname $HOSTNAME
838 if [ -x "$(which ip 2>/dev/null)" ]; then
839 IP_CMD="ip addr"
840 else
841 IP_CMD="ifconfig"
842 fi
843 # filter out IPv6 and localhost addresses
844 IP="$($IP_CMD | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p' | head -n1)"
845 # if nothing left, try using localhost address, it might work
846 if [ -z "$IP" ]; then IP="127.0.0.1"; fi
847fi
848echo "ip $IP"
849echo "port $CEPH_PORT"
850
851
852[ -z $CEPH_ADM ] && CEPH_ADM=$CEPH_BIN/ceph
853
854ceph_adm() {
855 if [ "$cephx" -eq 1 ]; then
856 prun $SUDO "$CEPH_ADM" -c "$conf_fn" -k "$keyring_fn" "$@"
857 else
858 prun $SUDO "$CEPH_ADM" -c "$conf_fn" "$@"
859 fi
860}
861
862if [ "$new" -eq 1 ]; then
863 prepare_conf
864fi
865
866if [ $CEPH_NUM_MON -gt 0 ]; then
867 start_mon
868fi
869
c07f9fc5
FG		 870if [ $CEPH_NUM_MGR -gt 0 ]; then
871 start_mgr
872fi
873
7c673cae
FG		 874# osd
875if [ $CEPH_NUM_OSD -gt 0 ]; then
876 start_osd
877fi
878
879# mds
880if [ "$smallmds" -eq 1 ]; then
881 wconf <<EOF
882[mds]
883 mds log max segments = 2
884 mds cache size = 10000
885EOF
886fi
887
888if [ $CEPH_NUM_MDS -gt 0 ]; then
889 start_mds
890fi
891
892# Don't set max_mds until all the daemons are started, otherwise
893# the intended standbys might end up in active roles.
894if [ "$CEPH_MAX_MDS" -gt 1 ]; then
895 sleep 5 # wait for daemons to make it into FSMap before increasing max_mds
896fi
897fs=0
898for name in a b c d e f g h i j k l m n o p
899do
900 [ $fs -eq $CEPH_NUM_FS ] && break
901 fs=$(($fs + 1))
902 if [ "$CEPH_MAX_MDS" -gt 1 ]; then
903 ceph_adm fs set "cephfs_${name}" allow_multimds true --yes-i-really-mean-it
904 ceph_adm fs set "cephfs_${name}" max_mds "$CEPH_MAX_MDS"
905 fi
906done
907
908# mgr
909
7c673cae
FG		 910if [ "$ec" -eq 1 ]; then
911 ceph_adm <<EOF
912osd erasure-code-profile set ec-profile m=2 k=2
913osd pool create ec 8 8 erasure ec-profile
914EOF
915fi
916
917do_cache() {
918 while [ -n "$*" ]; do
919 p="$1"
920 shift
921 echo "creating cache for pool $p ..."
922 ceph_adm <<EOF
923osd pool create ${p}-cache 8
924osd tier add $p ${p}-cache
925osd tier cache-mode ${p}-cache writeback
926osd tier set-overlay $p ${p}-cache
927EOF
928 done
929}
930do_cache $cache
931
932do_hitsets() {
933 while [ -n "$*" ]; do
934 pool="$1"
935 type="$2"
936 shift
937 shift
938 echo "setting hit_set on pool $pool type $type ..."
939 ceph_adm <<EOF
940osd pool set $pool hit_set_type $type
941osd pool set $pool hit_set_count 8
942osd pool set $pool hit_set_period 30
943EOF
944 done
945}
946do_hitsets $hitset
947
224ce89b 948do_rgw_create_users()
7c673cae
FG		 949{
950 # Create S3 user
951 local akey='0555b35654ad1656d804'
952 local skey='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=='
953 echo "setting up user testid"
954 $CEPH_BIN/radosgw-admin user create --uid testid --access-key $akey --secret $skey --display-name 'M. Tester' --email tester@ceph.com -c $conf_fn > /dev/null
955
956 # Create S3-test users
957 # See: https://github.com/ceph/s3-tests
958 echo "setting up s3-test users"
959 $CEPH_BIN/radosgw-admin user create \
960 --uid 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef \
961 --access-key ABCDEFGHIJKLMNOPQRST \
962 --secret abcdefghijklmnopqrstuvwxyzabcdefghijklmn \
963 --display-name youruseridhere \
964 --email s3@example.com -c $conf_fn > /dev/null
965 $CEPH_BIN/radosgw-admin user create \
966 --uid 56789abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234 \
967 --access-key NOPQRSTUVWXYZABCDEFG \
968 --secret nopqrstuvwxyzabcdefghijklmnabcdefghijklm \
969 --display-name john.doe \
970 --email john.doe@example.com -c $conf_fn > /dev/null
31f18b77
FG		 971 $CEPH_BIN/radosgw-admin user create \
972 --tenant testx \
973 --uid 9876543210abcdef0123456789abcdef0123456789abcdef0123456789abcdef \
974 --access-key HIJKLMNOPQRSTUVWXYZA \
975 --secret opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab \
976 --display-name tenanteduser \
977 --email tenanteduser@example.com -c $conf_fn > /dev/null
7c673cae
FG		 978
979 # Create Swift user
980 echo "setting up user tester"
981 $CEPH_BIN/radosgw-admin user create -c $conf_fn --subuser=test:tester --display-name=Tester-Subuser --key-type=swift --secret=testing --access=full > /dev/null
982
983 echo ""
984 echo "S3 User Info:"
985 echo " access key: $akey"
986 echo " secret key: $skey"
987 echo ""
988 echo "Swift User Info:"
989 echo " account : test"
990 echo " user : tester"
991 echo " password : testing"
992 echo ""
224ce89b 993}
7c673cae 994
224ce89b
WB		 995do_rgw()
996{
997 if [ "$new" -eq 1 ]; then
998 do_rgw_create_users
999 if [ -n "$rgw_compression" ]; then
1000 echo "setting compression type=$rgw_compression"
1001 $CEPH_BIN/radosgw-admin zone placement modify -c $conf_fn --rgw-zone=default --placement-id=default-placement --compression=$rgw_compression > /dev/null
1002 fi
1003 fi
7c673cae 1004 # Start server
7c673cae
FG		 1005 RGWDEBUG=""
1006 if [ "$debug" -ne 0 ]; then
1007 RGWDEBUG="--debug-rgw=20"
1008 fi
1009
1010 RGWSUDO=
1011 [ $CEPH_RGW_PORT -lt 1024 ] && RGWSUDO=sudo
1012 n=$(($CEPH_NUM_RGW - 1))
224ce89b
WB		 1013 i=0
1014 for rgw in j k l m n o p q r s t u v; do
1015 echo start rgw on http://localhost:$((CEPH_RGW_PORT + i))
1016 run 'rgw' $RGWSUDO $CEPH_BIN/radosgw -c $conf_fn --log-file=${CEPH_OUT_DIR}/rgw.$rgw.log ${RGWDEBUG} --debug-ms=1 -n client.rgw "--rgw_frontends=${rgw_frontend} port=$((CEPH_RGW_PORT + i))"
1017 i=$(($i + 1))
1018 [ $i -eq $CEPH_NUM_RGW ] && break
7c673cae
FG		 1019 done
1020}
1021if [ "$CEPH_NUM_RGW" -gt 0 ]; then
1022 do_rgw
1023fi
1024
1025echo "started. stop.sh to stop. see out/* (e.g. 'tail -f out/????') for debug output."
1026
31f18b77
FG		 1027echo ""
1028echo "dashboard urls: $DASH_URLS"
1029echo " restful urls: $RESTFUL_URLS"
1030echo " w/ user/pass: admin / $RESTFUL_SECRET"
7c673cae
FG		 1031echo ""
1032echo "export PYTHONPATH=./pybind:$PYTHONPATH"
1033echo "export LD_LIBRARY_PATH=$CEPH_LIB"
1034
1035if [ "$CEPH_DIR" != "$PWD" ]; then
1036 echo "export CEPH_CONF=$conf_fn"
1037 echo "export CEPH_KEYRING=$keyring_fn"
1038fi
31f18b77
FG		 1039
1040echo "CEPH_DEV=1"
1041
Ceph