]> git.proxmox.com Git - ceph.git/blob - ceph/src/vstart.sh
projects / ceph.git / blob
? search:


58deb58d0705643a8f46be53eb488e7c1862e2ba
[ceph.git] / ceph / src / vstart.sh
1 #!/usr/bin/env bash
2 # -*- mode:sh; tab-width:4; sh-basic-offset:4; indent-tabs-mode:nil -*-
3 # vim: softtabstop=4 shiftwidth=4 expandtab
4
5 # abort on failure
6 set -e
7
8 quoted_print() {
9 for s in "$@"; do
10 if [[ "$s" =~ \ ]]; then
11 printf -- "'%s' " "$s"
12 else
13 printf -- "$s "
14 fi
15 done
16 printf '\n'
17 }
18
19 debug() {
20 "$@" >&2
21 }
22
23 prunb() {
24 debug quoted_print "$@" '&'
25 "$@" &
26 }
27
28 prun() {
29 debug quoted_print "$@"
30 "$@"
31 }
32
33
34 if [ -n "$VSTART_DEST" ]; then
35 SRC_PATH=`dirname $0`
36 SRC_PATH=`(cd $SRC_PATH; pwd)`
37
38 CEPH_DIR=$SRC_PATH
39 CEPH_BIN=${PWD}/bin
40 CEPH_LIB=${PWD}/lib
41
42 CEPH_CONF_PATH=$VSTART_DEST
43 CEPH_DEV_DIR=$VSTART_DEST/dev
44 CEPH_OUT_DIR=$VSTART_DEST/out
45 CEPH_ASOK_DIR=$VSTART_DEST/out
46 fi
47
48 get_cmake_variable() {
49 local variable=$1
50 grep "${variable}:" CMakeCache.txt | cut -d "=" -f 2
51 }
52
53 # for running out of the CMake build directory
54 if [ -e CMakeCache.txt ]; then
55 # Out of tree build, learn source location from CMakeCache.txt
56 CEPH_ROOT=$(get_cmake_variable ceph_SOURCE_DIR)
57 CEPH_BUILD_DIR=`pwd`
58 [ -z "$MGR_PYTHON_PATH" ] && MGR_PYTHON_PATH=$CEPH_ROOT/src/pybind/mgr
59 fi
60
61 # use CEPH_BUILD_ROOT to vstart from a 'make install'
62 if [ -n "$CEPH_BUILD_ROOT" ]; then
63 [ -z "$CEPH_BIN" ] && CEPH_BIN=$CEPH_BUILD_ROOT/bin
64 [ -z "$CEPH_LIB" ] && CEPH_LIB=$CEPH_BUILD_ROOT/lib
65 [ -z "$EC_PATH" ] && EC_PATH=$CEPH_LIB/erasure-code
66 [ -z "$OBJCLASS_PATH" ] && OBJCLASS_PATH=$CEPH_LIB/rados-classes
67 # make install should install python extensions into PYTHONPATH
68 elif [ -n "$CEPH_ROOT" ]; then
69 [ -z "$CEPHFS_SHELL" ] && CEPHFS_SHELL=$CEPH_ROOT/src/tools/cephfs/cephfs-shell
70 [ -z "$PYBIND" ] && PYBIND=$CEPH_ROOT/src/pybind
71 [ -z "$CEPH_BIN" ] && CEPH_BIN=$CEPH_BUILD_DIR/bin
72 [ -z "$CEPH_ADM" ] && CEPH_ADM=$CEPH_BIN/ceph
73 [ -z "$INIT_CEPH" ] && INIT_CEPH=$CEPH_BIN/init-ceph
74 [ -z "$CEPH_LIB" ] && CEPH_LIB=$CEPH_BUILD_DIR/lib
75 [ -z "$OBJCLASS_PATH" ] && OBJCLASS_PATH=$CEPH_LIB
76 [ -z "$EC_PATH" ] && EC_PATH=$CEPH_LIB
77 [ -z "$CEPH_PYTHON_COMMON" ] && CEPH_PYTHON_COMMON=$CEPH_ROOT/src/python-common
78 fi
79
80 if [ -z "${CEPH_VSTART_WRAPPER}" ]; then
81 PATH=$(pwd):$PATH
82 fi
83
84 [ -z "$PYBIND" ] && PYBIND=./pybind
85
86 [ -n "$CEPH_PYTHON_COMMON" ] && CEPH_PYTHON_COMMON="$CEPH_PYTHON_COMMON:"
87 CYTHON_PYTHONPATH="$CEPH_LIB/cython_modules/lib.3"
88 export PYTHONPATH=$PYBIND:$CYTHON_PYTHONPATH:$CEPH_PYTHON_COMMON$PYTHONPATH
89
90 export LD_LIBRARY_PATH=$CEPH_LIB:$LD_LIBRARY_PATH
91 export DYLD_LIBRARY_PATH=$CEPH_LIB:$DYLD_LIBRARY_PATH
92 # Suppress logging for regular use that indicated that we are using a
93 # development version. vstart.sh is only used during testing and
94 # development
95 export CEPH_DEV=1
96
97 [ -z "$CEPH_NUM_MON" ] && CEPH_NUM_MON="$MON"
98 [ -z "$CEPH_NUM_OSD" ] && CEPH_NUM_OSD="$OSD"
99 [ -z "$CEPH_NUM_MDS" ] && CEPH_NUM_MDS="$MDS"
100 [ -z "$CEPH_NUM_MGR" ] && CEPH_NUM_MGR="$MGR"
101 [ -z "$CEPH_NUM_FS" ] && CEPH_NUM_FS="$FS"
102 [ -z "$CEPH_NUM_RGW" ] && CEPH_NUM_RGW="$RGW"
103 [ -z "$GANESHA_DAEMON_NUM" ] && GANESHA_DAEMON_NUM="$NFS"
104
105 # if none of the CEPH_NUM_* number is specified, kill the existing
106 # cluster.
107 if [ -z "$CEPH_NUM_MON" -a \
108 -z "$CEPH_NUM_OSD" -a \
109 -z "$CEPH_NUM_MDS" -a \
110 -z "$CEPH_NUM_MGR" -a \
111 -z "$GANESHA_DAEMON_NUM" ]; then
112 kill_all=1
113 else
114 kill_all=0
115 fi
116
117 [ -z "$CEPH_NUM_MON" ] && CEPH_NUM_MON=3
118 [ -z "$CEPH_NUM_OSD" ] && CEPH_NUM_OSD=3
119 [ -z "$CEPH_NUM_MDS" ] && CEPH_NUM_MDS=3
120 [ -z "$CEPH_NUM_MGR" ] && CEPH_NUM_MGR=1
121 [ -z "$CEPH_NUM_FS" ] && CEPH_NUM_FS=1
122 [ -z "$CEPH_MAX_MDS" ] && CEPH_MAX_MDS=1
123 [ -z "$CEPH_NUM_RGW" ] && CEPH_NUM_RGW=0
124 [ -z "$GANESHA_DAEMON_NUM" ] && GANESHA_DAEMON_NUM=0
125
126 [ -z "$CEPH_DIR" ] && CEPH_DIR="$PWD"
127 [ -z "$CEPH_DEV_DIR" ] && CEPH_DEV_DIR="$CEPH_DIR/dev"
128 [ -z "$CEPH_OUT_DIR" ] && CEPH_OUT_DIR="$CEPH_DIR/out"
129 [ -z "$CEPH_RGW_PORT" ] && CEPH_RGW_PORT=8000
130 [ -z "$CEPH_CONF_PATH" ] && CEPH_CONF_PATH=$CEPH_DIR
131
132 if [ $CEPH_NUM_OSD -gt 3 ]; then
133 OSD_POOL_DEFAULT_SIZE=3
134 else
135 OSD_POOL_DEFAULT_SIZE=$CEPH_NUM_OSD
136 fi
137
138 extra_conf=""
139 new=0
140 standby=0
141 debug=0
142 ip=""
143 nodaemon=0
144 redirect=0
145 smallmds=0
146 short=0
147 ec=0
148 cephadm=0
149 parallel=true
150 hitset=""
151 overwrite_conf=0
152 cephx=1 #turn cephx on by default
153 gssapi_authx=0
154 cache=""
155 if [ `uname` = FreeBSD ]; then
156 objectstore="filestore"
157 else
158 objectstore="bluestore"
159 fi
160 ceph_osd=ceph-osd
161 rgw_frontend="beast"
162 rgw_compression=""
163 lockdep=${LOCKDEP:-1}
164 spdk_enabled=0 #disable SPDK by default
165 pci_id=""
166
167 with_mgr_dashboard=true
168 if [[ "$(get_cmake_variable WITH_MGR_DASHBOARD_FRONTEND)" != "ON" ]] ||
169 [[ "$(get_cmake_variable WITH_RBD)" != "ON" ]]; then
170 debug echo "ceph-mgr dashboard not built - disabling."
171 with_mgr_dashboard=false
172 fi
173
174 filestore_path=
175 kstore_path=
176 bluestore_dev=
177
178 VSTART_SEC="client.vstart.sh"
179
180 MON_ADDR=""
181 DASH_URLS=""
182 RESTFUL_URLS=""
183
184 conf_fn="$CEPH_CONF_PATH/ceph.conf"
185 keyring_fn="$CEPH_CONF_PATH/keyring"
186 osdmap_fn="/tmp/ceph_osdmap.$$"
187 monmap_fn="/tmp/ceph_monmap.$$"
188 inc_osd_num=0
189
190 msgr="21"
191
192 usage="usage: $0 [option]... \nex: MON=3 OSD=1 MDS=1 MGR=1 RGW=1 NFS=1 $0 -n -d\n"
193 usage=$usage"options:\n"
194 usage=$usage"\t-d, --debug\n"
195 usage=$usage"\t-s, --standby_mds: Generate standby-replay MDS for each active\n"
196 usage=$usage"\t-l, --localhost: use localhost instead of hostname\n"
197 usage=$usage"\t-i <ip>: bind to specific ip\n"
198 usage=$usage"\t-n, --new\n"
199 usage=$usage"\t--valgrind[_{osd,mds,mon,rgw}] 'toolname args...'\n"
200 usage=$usage"\t--nodaemon: use ceph-run as wrapper for mon/osd/mds\n"
201 usage=$usage"\t--redirect-output: only useful with nodaemon, directs output to log file\n"
202 usage=$usage"\t--smallmds: limit mds cache memory limit\n"
203 usage=$usage"\t-m ip:port\t\tspecify monitor address\n"
204 usage=$usage"\t-k keep old configuration files (default)\n"
205 usage=$usage"\t-x enable cephx (on by default)\n"
206 usage=$usage"\t-X disable cephx\n"
207 usage=$usage"\t-g --gssapi enable Kerberos/GSSApi authentication\n"
208 usage=$usage"\t-G disable Kerberos/GSSApi authentication\n"
209 usage=$usage"\t--hitset <pool> <hit_set_type>: enable hitset tracking\n"
210 usage=$usage"\t-e : create an erasure pool\n";
211 usage=$usage"\t-o config\t\t add extra config parameters to all sections\n"
212 usage=$usage"\t--rgw_port specify ceph rgw http listen port\n"
213 usage=$usage"\t--rgw_frontend specify the rgw frontend configuration\n"
214 usage=$usage"\t--rgw_compression specify the rgw compression plugin\n"
215 usage=$usage"\t-b, --bluestore use bluestore as the osd objectstore backend (default)\n"
216 usage=$usage"\t-f, --filestore use filestore as the osd objectstore backend\n"
217 usage=$usage"\t-K, --kstore use kstore as the osd objectstore backend\n"
218 usage=$usage"\t--memstore use memstore as the osd objectstore backend\n"
219 usage=$usage"\t--cache <pool>: enable cache tiering on pool\n"
220 usage=$usage"\t--short: short object names only; necessary for ext4 dev\n"
221 usage=$usage"\t--nolockdep disable lockdep\n"
222 usage=$usage"\t--multimds <count> allow multimds with maximum active count\n"
223 usage=$usage"\t--without-dashboard: do not run using mgr dashboard\n"
224 usage=$usage"\t--bluestore-spdk <vendor>:<device>: enable SPDK and specify the PCI-ID of the NVME device\n"
225 usage=$usage"\t--msgr1: use msgr1 only\n"
226 usage=$usage"\t--msgr2: use msgr2 only\n"
227 usage=$usage"\t--msgr21: use msgr2 and msgr1\n"
228 usage=$usage"\t--crimson: use crimson-osd instead of ceph-osd\n"
229 usage=$usage"\t--osd-args: specify any extra osd specific options\n"
230 usage=$usage"\t--bluestore-devs: comma-separated list of blockdevs to use for bluestore\n"
231 usage=$usage"\t--inc-osd: append some more osds into existing vcluster\n"
232 usage=$usage"\t--cephadm: enable cephadm orchestrator with ~/.ssh/id_rsa[.pub]\n"
233 usage=$usage"\t--no-parallel: dont start all OSDs in parallel\n"
234
235 usage_exit() {
236 printf "$usage"
237 exit
238 }
239
240 while [ $# -ge 1 ]; do
241 case $1 in
242 -d | --debug )
243 debug=1
244 ;;
245 -s | --standby_mds)
246 standby=1
247 ;;
248 -l | --localhost )
249 ip="127.0.0.1"
250 ;;
251 -i )
252 [ -z "$2" ] && usage_exit
253 ip="$2"
254 shift
255 ;;
256 -e )
257 ec=1
258 ;;
259 --new | -n )
260 new=1
261 ;;
262 --inc-osd )
263 new=0
264 kill_all=0
265 inc_osd_num=$2
266 if [ "$inc_osd_num" == "" ]; then
267 inc_osd_num=1
268 else
269 shift
270 fi
271 ;;
272 --short )
273 short=1
274 ;;
275 --crimson )
276 ceph_osd=crimson-osd
277 ;;
278 --osd-args )
279 extra_osd_args="$2"
280 shift
281 ;;
282 --msgr1 )
283 msgr="1"
284 ;;
285 --msgr2 )
286 msgr="2"
287 ;;
288 --msgr21 )
289 msgr="21"
290 ;;
291 --cephadm )
292 cephadm=1
293 ;;
294 --no-parallel )
295 parallel=false
296 ;;
297 --valgrind )
298 [ -z "$2" ] && usage_exit
299 valgrind=$2
300 shift
301 ;;
302 --valgrind_args )
303 valgrind_args="$2"
304 shift
305 ;;
306 --valgrind_mds )
307 [ -z "$2" ] && usage_exit
308 valgrind_mds=$2
309 shift
310 ;;
311 --valgrind_osd )
312 [ -z "$2" ] && usage_exit
313 valgrind_osd=$2
314 shift
315 ;;
316 --valgrind_mon )
317 [ -z "$2" ] && usage_exit
318 valgrind_mon=$2
319 shift
320 ;;
321 --valgrind_mgr )
322 [ -z "$2" ] && usage_exit
323 valgrind_mgr=$2
324 shift
325 ;;
326 --valgrind_rgw )
327 [ -z "$2" ] && usage_exit
328 valgrind_rgw=$2
329 shift
330 ;;
331 --nodaemon )
332 nodaemon=1
333 ;;
334 --redirect-output)
335 redirect=1
336 ;;
337 --smallmds )
338 smallmds=1
339 ;;
340 --rgw_port )
341 CEPH_RGW_PORT=$2
342 shift
343 ;;
344 --rgw_frontend )
345 rgw_frontend=$2
346 shift
347 ;;
348 --rgw_compression )
349 rgw_compression=$2
350 shift
351 ;;
352 --kstore_path )
353 kstore_path=$2
354 shift
355 ;;
356 --filestore_path )
357 filestore_path=$2
358 shift
359 ;;
360 -m )
361 [ -z "$2" ] && usage_exit
362 MON_ADDR=$2
363 shift
364 ;;
365 -x )
366 cephx=1 # this is on be default, flag exists for historical consistency
367 ;;
368 -X )
369 cephx=0
370 ;;
371
372 -g | --gssapi)
373 gssapi_authx=1
374 ;;
375 -G)
376 gssapi_authx=0
377 ;;
378
379 -k )
380 if [ ! -r $conf_fn ]; then
381 echo "cannot use old configuration: $conf_fn not readable." >&2
382 exit
383 fi
384 new=0
385 ;;
386 --memstore )
387 objectstore="memstore"
388 ;;
389 -b | --bluestore )
390 objectstore="bluestore"
391 ;;
392 -f | --filestore )
393 objectstore="filestore"
394 ;;
395 -K | --kstore )
396 objectstore="kstore"
397 ;;
398 --hitset )
399 hitset="$hitset $2 $3"
400 shift
401 shift
402 ;;
403 -o )
404 extra_conf="$extra_conf $2
405 "
406 shift
407 ;;
408 --cache )
409 if [ -z "$cache" ]; then
410 cache="$2"
411 else
412 cache="$cache $2"
413 fi
414 shift
415 ;;
416 --nolockdep )
417 lockdep=0
418 ;;
419 --multimds)
420 CEPH_MAX_MDS="$2"
421 shift
422 ;;
423 --without-dashboard)
424 with_mgr_dashboard=false
425 ;;
426 --bluestore-spdk )
427 [ -z "$2" ] && usage_exit
428 pci_id="$2"
429 spdk_enabled=1
430 shift
431 ;;
432 --bluestore-devs )
433 IFS=',' read -r -a bluestore_dev <<< "$2"
434 for dev in "${bluestore_dev[@]}"; do
435 if [ ! -b $dev -o ! -w $dev ]; then
436 echo "All --bluestore-devs must refer to writable block devices"
437 exit 1
438 fi
439 done
440 shift
441 ;;
442 * )
443 usage_exit
444 esac
445 shift
446 done
447
448 if [ $kill_all -eq 1 ]; then
449 $SUDO $INIT_CEPH stop
450 fi
451
452 if [ "$new" -eq 0 ]; then
453 if [ -z "$CEPH_ASOK_DIR" ]; then
454 CEPH_ASOK_DIR=`dirname $($CEPH_BIN/ceph-conf -c $conf_fn --show-config-value admin_socket)`
455 fi
456 mkdir -p $CEPH_ASOK_DIR
457 MON=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_mon 2>/dev/null` && \
458 CEPH_NUM_MON="$MON"
459 OSD=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_osd 2>/dev/null` && \
460 CEPH_NUM_OSD="$OSD"
461 MDS=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_mds 2>/dev/null` && \
462 CEPH_NUM_MDS="$MDS"
463 MGR=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_mgr 2>/dev/null` && \
464 CEPH_NUM_MGR="$MGR"
465 RGW=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_rgw 2>/dev/null` && \
466 CEPH_NUM_RGW="$RGW"
467 NFS=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_ganesha 2>/dev/null` && \
468 GANESHA_DAEMON_NUM="$NFS"
469 else
470 # only delete if -n
471 if [ -e "$conf_fn" ]; then
472 asok_dir=`dirname $($CEPH_BIN/ceph-conf -c $conf_fn --show-config-value admin_socket)`
473 rm -- "$conf_fn"
474 if [ $asok_dir != /var/run/ceph ]; then
475 [ -d $asok_dir ] && rm -f $asok_dir/* && rmdir $asok_dir
476 fi
477 fi
478 if [ -z "$CEPH_ASOK_DIR" ]; then
479 CEPH_ASOK_DIR=`mktemp -u -d "${TMPDIR:-/tmp}/ceph-asok.XXXXXX"`
480 fi
481 fi
482
483 ARGS="-c $conf_fn"
484
485 run() {
486 type=$1
487 shift
488 num=$1
489 shift
490 eval "valg=\$valgrind_$type"
491 [ -z "$valg" ] && valg="$valgrind"
492
493 if [ -n "$valg" ]; then
494 prunb valgrind --tool="$valg" $valgrind_args "$@" -f
495 sleep 1
496 else
497 if [ "$nodaemon" -eq 0 ]; then
498 prun "$@"
499 elif [ "$redirect" -eq 0 ]; then
500 prunb ${CEPH_ROOT}/src/ceph-run "$@" -f
501 else
502 ( prunb ${CEPH_ROOT}/src/ceph-run "$@" -f ) >$CEPH_OUT_DIR/$type.$num.stdout 2>&1
503 fi
504 fi
505 }
506
507 wconf() {
508 if [ "$new" -eq 1 -o "$overwrite_conf" -eq 1 ]; then
509 cat >> "$conf_fn"
510 fi
511 }
512
513 get_pci_selector() {
514 which_pci=$1
515 lspci -mm -n -D -d $pci_id | cut -d ' ' -f 1 | sed -n $which_pci'p'
516 }
517
518 get_pci_selector_num() {
519 lspci -mm -n -D -d $pci_id | cut -d' ' -f 1 | wc -l
520 }
521
522 do_rgw_conf() {
523
524 if [ $CEPH_NUM_RGW -eq 0 ]; then
525 return 0
526 fi
527
528 # setup each rgw on a sequential port, starting at $CEPH_RGW_PORT.
529 # individual rgw's ids will be their ports.
530 current_port=$CEPH_RGW_PORT
531 for n in $(seq 1 $CEPH_NUM_RGW); do
532 wconf << EOF
533 [client.rgw.${current_port}]
534 rgw frontends = $rgw_frontend port=${current_port}
535 admin socket = ${CEPH_OUT_DIR}/radosgw.${current_port}.asok
536 EOF
537 current_port=$((current_port + 1))
538 done
539
540 }
541
542 prepare_conf() {
543 local DAEMONOPTS="
544 log file = $CEPH_OUT_DIR/\$name.log
545 admin socket = $CEPH_ASOK_DIR/\$name.asok
546 chdir = \"\"
547 pid file = $CEPH_OUT_DIR/\$name.pid
548 heartbeat file = $CEPH_OUT_DIR/\$name.heartbeat
549 "
550
551 local mgr_modules="restful iostat"
552 if $with_mgr_dashboard; then
553 mgr_modules="dashboard $mgr_modules"
554 fi
555
556 local msgr_conf=''
557 if [ $msgr -eq 21 ]; then
558 msgr_conf="
559 ms bind msgr2 = true
560 ms bind msgr1 = true
561 ";
562 fi
563 if [ $msgr -eq 2 ]; then
564 msgr_conf="
565 ms bind msgr2 = true
566 ms bind msgr1 = false
567 ";
568 fi
569 if [ $msgr -eq 1 ]; then
570 msgr_conf="
571 ms bind msgr2 = false
572 ms bind msgr1 = true
573 ";
574 fi
575
576 wconf <<EOF
577 ; generated by vstart.sh on `date`
578 [$VSTART_SEC]
579 num mon = $CEPH_NUM_MON
580 num osd = $CEPH_NUM_OSD
581 num mds = $CEPH_NUM_MDS
582 num mgr = $CEPH_NUM_MGR
583 num rgw = $CEPH_NUM_RGW
584 num ganesha = $GANESHA_DAEMON_NUM
585
586 [global]
587 fsid = $(uuidgen)
588 osd failsafe full ratio = .99
589 mon osd full ratio = .99
590 mon osd nearfull ratio = .99
591 mon osd backfillfull ratio = .99
592 mon_max_pg_per_osd = ${MON_MAX_PG_PER_OSD:-1000}
593 erasure code dir = $EC_PATH
594 plugin dir = $CEPH_LIB
595 filestore fd cache size = 32
596 run dir = $CEPH_OUT_DIR
597 crash dir = $CEPH_OUT_DIR
598 enable experimental unrecoverable data corrupting features = *
599 osd_crush_chooseleaf_type = 0
600 debug asok assert abort = true
601 $msgr_conf
602 $extra_conf
603 EOF
604 if [ "$lockdep" -eq 1 ] ; then
605 wconf <<EOF
606 lockdep = true
607 EOF
608 fi
609 if [ "$cephx" -eq 1 ] ; then
610 wconf <<EOF
611 auth cluster required = cephx
612 auth service required = cephx
613 auth client required = cephx
614 EOF
615 elif [ "$gssapi_authx" -eq 1 ] ; then
616 wconf <<EOF
617 auth cluster required = gss
618 auth service required = gss
619 auth client required = gss
620 gss ktab client file = $CEPH_DEV_DIR/gss_\$name.keytab
621 EOF
622 else
623 wconf <<EOF
624 auth cluster required = none
625 auth service required = none
626 auth client required = none
627 EOF
628 fi
629 if [ "$short" -eq 1 ]; then
630 COSDSHORT=" osd max object name len = 460
631 osd max object namespace len = 64"
632 fi
633 if [ "$objectstore" == "bluestore" ]; then
634 if [ "$spdk_enabled" -eq 1 ]; then
635 if [ "$(get_pci_selector_num)" -eq 0 ]; then
636 echo "Not find the specified NVME device, please check." >&2
637 exit
638 fi
639 if [ $(get_pci_selector_num) -lt $CEPH_NUM_OSD ]; then
640 echo "OSD number ($CEPH_NUM_OSD) is greater than NVME SSD number ($(get_pci_selector_num)), please check." >&2
641 exit
642 fi
643 BLUESTORE_OPTS=" bluestore_block_db_path = \"\"
644 bluestore_block_db_size = 0
645 bluestore_block_db_create = false
646 bluestore_block_wal_path = \"\"
647 bluestore_block_wal_size = 0
648 bluestore_block_wal_create = false
649 bluestore_spdk_mem = 2048"
650 else
651 BLUESTORE_OPTS=" bluestore block db path = $CEPH_DEV_DIR/osd\$id/block.db.file
652 bluestore block db size = 1073741824
653 bluestore block db create = true
654 bluestore block wal path = $CEPH_DEV_DIR/osd\$id/block.wal.file
655 bluestore block wal size = 1048576000
656 bluestore block wal create = true"
657 fi
658 fi
659 wconf <<EOF
660 [client]
661 keyring = $keyring_fn
662 log file = $CEPH_OUT_DIR/\$name.\$pid.log
663 admin socket = $CEPH_ASOK_DIR/\$name.\$pid.asok
664
665 ; needed for s3tests
666 rgw crypt s3 kms backend = testing
667 rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo=
668 rgw crypt require ssl = false
669 ; uncomment the following to set LC days as the value in seconds;
670 ; needed for passing lc time based s3-tests (can be verbose)
671 ; rgw lc debug interval = 10
672
673 $extra_conf
674 EOF
675 wconf <<EOF
676 [cephfs-shell]
677 debug shell = true
678
679 $extra_conf
680 EOF
681
682 do_rgw_conf
683 wconf << EOF
684 [mds]
685 $DAEMONOPTS
686 mds data = $CEPH_DEV_DIR/mds.\$id
687 mds root ino uid = `id -u`
688 mds root ino gid = `id -g`
689 $extra_conf
690 [mgr]
691 mgr data = $CEPH_DEV_DIR/mgr.\$id
692 mgr module path = $MGR_PYTHON_PATH
693 cephadm path = $CEPH_ROOT/src/cephadm/cephadm
694 $DAEMONOPTS
695 $extra_conf
696 [osd]
697 $DAEMONOPTS
698 osd_check_max_object_name_len_on_startup = false
699 osd data = $CEPH_DEV_DIR/osd\$id
700 osd journal = $CEPH_DEV_DIR/osd\$id/journal
701 osd journal size = 100
702 osd class tmp = out
703 osd class dir = $OBJCLASS_PATH
704 osd class load list = *
705 osd class default list = *
706 osd fast shutdown = false
707
708 filestore wbthrottle xfs ios start flusher = 10
709 filestore wbthrottle xfs ios hard limit = 20
710 filestore wbthrottle xfs inodes hard limit = 30
711 filestore wbthrottle btrfs ios start flusher = 10
712 filestore wbthrottle btrfs ios hard limit = 20
713 filestore wbthrottle btrfs inodes hard limit = 30
714 bluestore fsck on mount = true
715 bluestore block create = true
716 $BLUESTORE_OPTS
717
718 ; kstore
719 kstore fsck on mount = true
720 osd objectstore = $objectstore
721 $COSDSHORT
722 $extra_conf
723 [mon]
724 mgr initial modules = $mgr_modules
725 $DAEMONOPTS
726 $CMONDEBUG
727 $extra_conf
728 mon cluster log file = $CEPH_OUT_DIR/cluster.mon.\$id.log
729 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd
730 EOF
731 }
732
733 start_mon() {
734 local MONS=""
735 local count=0
736 for f in a b c d e f g h i j k l m n o p q r s t u v w x y z
737 do
738 [ $count -eq $CEPH_NUM_MON ] && break;
739 count=$(($count + 1))
740 if [ -z "$MONS" ]; then
741 MONS="$f"
742 else
743 MONS="$MONS $f"
744 fi
745 done
746
747 if [ "$new" -eq 1 ]; then
748 if [ `echo $IP | grep '^127\\.'` ]; then
749 echo
750 echo "NOTE: hostname resolves to loopback; remote hosts will not be able to"
751 echo " connect. either adjust /etc/hosts, or edit this script to use your"
752 echo " machine's real IP."
753 echo
754 fi
755
756 prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name=mon. "$keyring_fn" --cap mon 'allow *'
757 prun $SUDO "$CEPH_BIN/ceph-authtool" --gen-key --name=client.admin \
758 --cap mon 'allow *' \
759 --cap osd 'allow *' \
760 --cap mds 'allow *' \
761 --cap mgr 'allow *' \
762 "$keyring_fn"
763
764 # build a fresh fs monmap, mon fs
765 local params=()
766 local count=0
767 local mon_host=""
768 for f in $MONS
769 do
770 if [ $msgr -eq 1 ]; then
771 A="v1:$IP:$(($CEPH_PORT+$count+1))"
772 fi
773 if [ $msgr -eq 2 ]; then
774 A="v2:$IP:$(($CEPH_PORT+$count+1))"
775 fi
776 if [ $msgr -eq 21 ]; then
777 A="[v2:$IP:$(($CEPH_PORT+$count)),v1:$IP:$(($CEPH_PORT+$count+1))]"
778 fi
779 params+=("--addv" "$f" "$A")
780 mon_host="$mon_host $A"
781 wconf <<EOF
782 [mon.$f]
783 host = $HOSTNAME
784 mon data = $CEPH_DEV_DIR/mon.$f
785 EOF
786 count=$(($count + 2))
787 done
788 wconf <<EOF
789 [global]
790 mon host = $mon_host
791 EOF
792 prun "$CEPH_BIN/monmaptool" --create --clobber "${params[@]}" --print "$monmap_fn"
793
794 for f in $MONS
795 do
796 prun rm -rf -- "$CEPH_DEV_DIR/mon.$f"
797 prun mkdir -p "$CEPH_DEV_DIR/mon.$f"
798 prun "$CEPH_BIN/ceph-mon" --mkfs -c "$conf_fn" -i "$f" --monmap="$monmap_fn" --keyring="$keyring_fn"
799 done
800
801 prun rm -- "$monmap_fn"
802 fi
803
804 # start monitors
805 for f in $MONS
806 do
807 run 'mon' $f $CEPH_BIN/ceph-mon -i $f $ARGS $CMON_ARGS
808 done
809 }
810
811 start_osd() {
812 if [ $inc_osd_num -gt 0 ]; then
813 old_maxosd=$($CEPH_BIN/ceph osd getmaxosd | sed -e 's/max_osd = //' -e 's/ in epoch.*//')
814 start=$old_maxosd
815 end=$(($start-1+$inc_osd_num))
816 overwrite_conf=1 # fake wconf
817 else
818 start=0
819 end=$(($CEPH_NUM_OSD-1))
820 fi
821 local osds_wait
822 for osd in `seq $start $end`
823 do
824 local extra_seastar_args
825 if [ "$ceph_osd" == "crimson-osd" ]; then
826 # designate a single CPU node $osd for osd.$osd
827 extra_seastar_args="--smp 1 --cpuset $osd"
828 if [ "$debug" -ne 0 ]; then
829 extra_seastar_args+=" --debug"
830 fi
831 fi
832 if [ "$new" -eq 1 -o $inc_osd_num -gt 0 ]; then
833 wconf <<EOF
834 [osd.$osd]
835 host = $HOSTNAME
836 EOF
837 if [ "$spdk_enabled" -eq 1 ]; then
838 wconf <<EOF
839 bluestore_block_path = spdk:$(get_pci_selector $((osd+1)))
840 EOF
841 fi
842
843 rm -rf $CEPH_DEV_DIR/osd$osd || true
844 if command -v btrfs > /dev/null; then
845 for f in $CEPH_DEV_DIR/osd$osd/*; do btrfs sub delete $f &> /dev/null || true; done
846 fi
847 if [ -n "$filestore_path" ]; then
848 ln -s $filestore_path $CEPH_DEV_DIR/osd$osd
849 elif [ -n "$kstore_path" ]; then
850 ln -s $kstore_path $CEPH_DEV_DIR/osd$osd
851 else
852 mkdir -p $CEPH_DEV_DIR/osd$osd
853 if [ -n "${bluestore_dev[$osd]}" ]; then
854 dd if=/dev/zero of=${bluestore_dev[$osd]} bs=1M count=1
855 ln -s ${bluestore_dev[$osd]} $CEPH_DEV_DIR/osd$osd/block
856 wconf <<EOF
857 bluestore fsck on mount = false
858 EOF
859 fi
860 fi
861
862 local uuid=`uuidgen`
863 echo "add osd$osd $uuid"
864 OSD_SECRET=$($CEPH_BIN/ceph-authtool --gen-print-key)
865 echo "{\"cephx_secret\": \"$OSD_SECRET\"}" > $CEPH_DEV_DIR/osd$osd/new.json
866 ceph_adm osd new $uuid -i $CEPH_DEV_DIR/osd$osd/new.json
867 rm $CEPH_DEV_DIR/osd$osd/new.json
868 $SUDO $CEPH_BIN/$ceph_osd $extra_osd_args -i $osd $ARGS --mkfs --key $OSD_SECRET --osd-uuid $uuid $extra_seastar_args
869
870 local key_fn=$CEPH_DEV_DIR/osd$osd/keyring
871 cat > $key_fn<<EOF
872 [osd.$osd]
873 key = $OSD_SECRET
874 EOF
875 fi
876 echo start osd.$osd
877 local osd_pid
878 run 'osd' $osd $SUDO $CEPH_BIN/$ceph_osd \
879 $extra_seastar_args $extra_osd_args \
880 -i $osd $ARGS $COSD_ARGS &
881 osd_pid=$!
882 if $parallel; then
883 osds_wait=$osd_pid
884 else
885 wait $osd_pid
886 fi
887 done
888 if $parallel; then
889 for p in $osds_wait; do
890 wait $p
891 done
892 debug echo OSDs started
893 fi
894 if [ $inc_osd_num -gt 0 ]; then
895 # update num osd
896 new_maxosd=$($CEPH_BIN/ceph osd getmaxosd | sed -e 's/max_osd = //' -e 's/ in epoch.*//')
897 sed -i "s/num osd = .*/num osd = $new_maxosd/g" $conf_fn
898 fi
899 }
900
901 start_mgr() {
902 local mgr=0
903 local ssl=${DASHBOARD_SSL:-1}
904 # avoid monitors on nearby ports (which test/*.sh use extensively)
905 MGR_PORT=$(($CEPH_PORT + 1000))
906 PROMETHEUS_PORT=9283
907 for name in x y z a b c d e f g h i j k l m n o p
908 do
909 [ $mgr -eq $CEPH_NUM_MGR ] && break
910 mgr=$(($mgr + 1))
911 if [ "$new" -eq 1 ]; then
912 mkdir -p $CEPH_DEV_DIR/mgr.$name
913 key_fn=$CEPH_DEV_DIR/mgr.$name/keyring
914 $SUDO $CEPH_BIN/ceph-authtool --create-keyring --gen-key --name=mgr.$name $key_fn
915 ceph_adm -i $key_fn auth add mgr.$name mon 'allow profile mgr' mds 'allow *' osd 'allow *'
916
917 wconf <<EOF
918 [mgr.$name]
919 host = $HOSTNAME
920 EOF
921
922 if $with_mgr_dashboard ; then
923 local port_option="ssl_server_port"
924 local http_proto="https"
925 if [ "$ssl" == "0" ]; then
926 port_option="server_port"
927 http_proto="http"
928 ceph_adm config set mgr mgr/dashboard/ssl false --force
929 fi
930 ceph_adm config set mgr mgr/dashboard/$name/$port_option $MGR_PORT --force
931 if [ $mgr -eq 1 ]; then
932 DASH_URLS="$http_proto://$IP:$MGR_PORT"
933 else
934 DASH_URLS+=", $http_proto://$IP:$MGR_PORT"
935 fi
936 fi
937 MGR_PORT=$(($MGR_PORT + 1000))
938 ceph_adm config set mgr mgr/prometheus/$name/server_port $PROMETHEUS_PORT --force
939 PROMETHEUS_PORT=$(($PROMETHEUS_PORT + 1000))
940
941 ceph_adm config set mgr mgr/restful/$name/server_port $MGR_PORT --force
942 if [ $mgr -eq 1 ]; then
943 RESTFUL_URLS="https://$IP:$MGR_PORT"
944 else
945 RESTFUL_URLS+=", https://$IP:$MGR_PORT"
946 fi
947 MGR_PORT=$(($MGR_PORT + 1000))
948 fi
949
950 debug echo "Starting mgr.${name}"
951 run 'mgr' $name $CEPH_BIN/ceph-mgr -i $name $ARGS
952 done
953
954 if [ "$new" -eq 1 ]; then
955 # setting login credentials for dashboard
956 if $with_mgr_dashboard; then
957 while ! ceph_adm -h | grep -c -q ^dashboard ; do
958 debug echo 'waiting for mgr dashboard module to start'
959 sleep 1
960 done
961 ceph_adm dashboard ac-user-create --force-password admin admin administrator
962 if [ "$ssl" != "0" ]; then
963 if ! ceph_adm dashboard create-self-signed-cert; then
964 debug echo dashboard module not working correctly!
965 fi
966 fi
967 fi
968
969 while ! ceph_adm -h | grep -c -q ^restful ; do
970 debug echo 'waiting for mgr restful module to start'
971 sleep 1
972 done
973 if ceph_adm restful create-self-signed-cert; then
974 SF=`mktemp`
975 ceph_adm restful create-key admin -o $SF
976 RESTFUL_SECRET=`cat $SF`
977 rm $SF
978 else
979 debug echo MGR Restful is not working, perhaps the package is not installed?
980 fi
981 fi
982
983 if [ "$cephadm" -eq 1 ]; then
984 debug echo Enabling cephadm orchestrator
985 if [ "$new" -eq 1 ]; then
986 digest=$(curl -s \
987 https://registry.hub.docker.com/v2/repositories/ceph/daemon-base/tags/latest-master-devel \
988 | jq -r '.images[].digest')
989 ceph_adm config set global container_image "docker.io/ceph/daemon-base@$digest"
990 fi
991 ceph_adm config-key set mgr/cephadm/ssh_identity_key -i ~/.ssh/id_rsa
992 ceph_adm config-key set mgr/cephadm/ssh_identity_pub -i ~/.ssh/id_rsa.pub
993 ceph_adm mgr module enable cephadm
994 ceph_adm orch set backend cephadm
995 ceph_adm orch host add "$(hostname)"
996 ceph_adm orch apply crash '*'
997 ceph_adm config set mgr mgr/cephadm/allow_ptrace true
998 fi
999 }
1000
1001 start_mds() {
1002 local mds=0
1003 for name in a b c d e f g h i j k l m n o p
1004 do
1005 [ $mds -eq $CEPH_NUM_MDS ] && break
1006 mds=$(($mds + 1))
1007
1008 if [ "$new" -eq 1 ]; then
1009 prun mkdir -p "$CEPH_DEV_DIR/mds.$name"
1010 key_fn=$CEPH_DEV_DIR/mds.$name/keyring
1011 wconf <<EOF
1012 [mds.$name]
1013 host = $HOSTNAME
1014 EOF
1015 if [ "$standby" -eq 1 ]; then
1016 mkdir -p $CEPH_DEV_DIR/mds.${name}s
1017 wconf <<EOF
1018 mds standby for rank = $mds
1019 [mds.${name}s]
1020 mds standby replay = true
1021 mds standby for name = ${name}
1022 EOF
1023 fi
1024 prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name="mds.$name" "$key_fn"
1025 ceph_adm -i "$key_fn" auth add "mds.$name" mon 'allow profile mds' osd 'allow rw tag cephfs *=*' mds 'allow' mgr 'allow profile mds'
1026 if [ "$standby" -eq 1 ]; then
1027 prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name="mds.${name}s" \
1028 "$CEPH_DEV_DIR/mds.${name}s/keyring"
1029 ceph_adm -i "$CEPH_DEV_DIR/mds.${name}s/keyring" auth add "mds.${name}s" \
1030 mon 'allow profile mds' osd 'allow *' mds 'allow' mgr 'allow profile mds'
1031 fi
1032 fi
1033
1034 run 'mds' $name $CEPH_BIN/ceph-mds -i $name $ARGS $CMDS_ARGS
1035 if [ "$standby" -eq 1 ]; then
1036 run 'mds' $name $CEPH_BIN/ceph-mds -i ${name}s $ARGS $CMDS_ARGS
1037 fi
1038
1039 #valgrind --tool=massif $CEPH_BIN/ceph-mds $ARGS --mds_log_max_segments 2 --mds_thrash_fragments 0 --mds_thrash_exports 0 > m #--debug_ms 20
1040 #$CEPH_BIN/ceph-mds -d $ARGS --mds_thrash_fragments 0 --mds_thrash_exports 0 #--debug_ms 20
1041 #ceph_adm mds set max_mds 2
1042 done
1043
1044 if [ $new -eq 1 ]; then
1045 if [ "$CEPH_NUM_FS" -gt "0" ] ; then
1046 sleep 5 # time for MDS to come up as standby to avoid health warnings on fs creation
1047 if [ "$CEPH_NUM_FS" -gt "1" ] ; then
1048 ceph_adm fs flag set enable_multiple true --yes-i-really-mean-it
1049 fi
1050
1051 # wait for volume module to load
1052 while ! ceph_adm fs volume ls ; do sleep 1 ; done
1053 local fs=0
1054 for name in a b c d e f g h i j k l m n o p
1055 do
1056 ceph_adm fs volume create ${name}
1057 ceph_adm fs authorize ${name} "client.fs_${name}" / rwp >> "$keyring_fn"
1058 fs=$(($fs + 1))
1059 [ $fs -eq $CEPH_NUM_FS ] && break
1060 done
1061 fi
1062 fi
1063
1064 }
1065
1066 # Ganesha Daemons requires nfs-ganesha nfs-ganesha-ceph nfs-ganesha-rados-grace
1067 # nfs-ganesha-rados-urls (version 3.3 and above) packages installed. On
1068 # Fedora>=31 these packages can be installed directly with 'dnf'. For CentOS>=8
1069 # the packages are available at
1070 # https://wiki.centos.org/SpecialInterestGroup/Storage
1071 # Similarly for Ubuntu>=16.04 follow the instructions on
1072 # https://launchpad.net/~nfs-ganesha
1073
1074 start_ganesha() {
1075 cluster_id="vstart"
1076 GANESHA_PORT=$(($CEPH_PORT + 4000))
1077 local ganesha=0
1078 test_user="ganesha-$cluster_id"
1079 pool_name="nfs-ganesha"
1080 namespace=$cluster_id
1081 url="rados://$pool_name/$namespace/conf-nfs.$test_user"
1082
1083 prun ceph_adm auth get-or-create client.$test_user \
1084 mon "allow r" \
1085 osd "allow rw pool=$pool_name namespace=$namespace, allow rw tag cephfs data=a" \
1086 mds "allow rw path=/" \
1087 >> "$keyring_fn"
1088
1089 ceph_adm mgr module enable test_orchestrator
1090 ceph_adm orch set backend test_orchestrator
1091 ceph_adm test_orchestrator load_data -i $CEPH_ROOT/src/pybind/mgr/test_orchestrator/dummy_data.json
1092 prun ceph_adm nfs cluster create cephfs $cluster_id
1093 prun ceph_adm nfs export create cephfs "a" $cluster_id "/cephfs"
1094
1095 for name in a b c d e f g h i j k l m n o p
1096 do
1097 [ $ganesha -eq $GANESHA_DAEMON_NUM ] && break
1098
1099 port=$(($GANESHA_PORT + ganesha))
1100 ganesha=$(($ganesha + 1))
1101 ganesha_dir="$CEPH_DEV_DIR/ganesha.$name"
1102 prun rm -rf $ganesha_dir
1103 prun mkdir -p $ganesha_dir
1104
1105 echo "NFS_CORE_PARAM {
1106 Enable_NLM = false;
1107 Enable_RQUOTA = false;
1108 Protocols = 4;
1109 NFS_Port = $port;
1110 }
1111
1112 MDCACHE {
1113 Dir_Chunk = 0;
1114 }
1115
1116 NFSv4 {
1117 RecoveryBackend = rados_cluster;
1118 Minor_Versions = 1, 2;
1119 }
1120
1121 %url $url
1122
1123 RADOS_KV {
1124 pool = $pool_name;
1125 namespace = $namespace;
1126 UserId = $test_user;
1127 nodeid = $name;
1128 }
1129
1130 RADOS_URLS {
1131 Userid = $test_user;
1132 watch_url = \"$url\";
1133 }" > "$ganesha_dir/ganesha-$name.conf"
1134 wconf <<EOF
1135 [ganesha.$name]
1136 host = $HOSTNAME
1137 ip = $IP
1138 port = $port
1139 ganesha data = $ganesha_dir
1140 pid file = $ganesha_dir/ganesha-$name.pid
1141 EOF
1142
1143 prun env CEPH_CONF="${conf_fn}" ganesha-rados-grace --userid $test_user -p $pool_name -n $namespace add $name
1144 prun env CEPH_CONF="${conf_fn}" ganesha-rados-grace --userid $test_user -p $pool_name -n $namespace
1145
1146 prun env CEPH_CONF="${conf_fn}" ganesha.nfsd -L "$CEPH_OUT_DIR/ganesha-$name.log" -f "$ganesha_dir/ganesha-$name.conf" -p "$CEPH_OUT_DIR/ganesha-$name.pid" -N NIV_DEBUG
1147
1148 # Wait few seconds for grace period to be removed
1149 sleep 2
1150
1151 prun env CEPH_CONF="${conf_fn}" ganesha-rados-grace --userid $test_user -p $pool_name -n $namespace
1152
1153 if $with_mgr_dashboard; then
1154 $CEPH_BIN/rados -p $pool_name put "conf-$name" "$ganesha_dir/ganesha-$name.conf"
1155 fi
1156
1157 echo "$test_user ganesha daemon $name started on port: $port"
1158 done
1159
1160 if $with_mgr_dashboard; then
1161 ceph_adm dashboard set-ganesha-clusters-rados-pool-namespace $pool_name
1162 fi
1163 }
1164
1165 if [ "$debug" -eq 0 ]; then
1166 CMONDEBUG='
1167 debug mon = 10
1168 debug ms = 1'
1169 else
1170 debug echo "** going verbose **"
1171 CMONDEBUG='
1172 debug mon = 20
1173 debug paxos = 20
1174 debug auth = 20
1175 debug mgrc = 20
1176 debug ms = 1'
1177 fi
1178
1179 if [ -n "$MON_ADDR" ]; then
1180 CMON_ARGS=" -m "$MON_ADDR
1181 COSD_ARGS=" -m "$MON_ADDR
1182 CMDS_ARGS=" -m "$MON_ADDR
1183 fi
1184
1185 if [ -z "$CEPH_PORT" ]; then
1186 while [ true ]
1187 do
1188 CEPH_PORT="$(echo $(( RANDOM % 1000 + 40000 )))"
1189 ss -a -n | egrep "\<LISTEN\>.+:${CEPH_PORT}\s+" 1>/dev/null 2>&1 || break
1190 done
1191 fi
1192
1193 [ -z "$INIT_CEPH" ] && INIT_CEPH=$CEPH_BIN/init-ceph
1194
1195 # sudo if btrfs
1196 [ -d $CEPH_DEV_DIR/osd0/. ] && [ -e $CEPH_DEV_DIR/sudo ] && SUDO="sudo"
1197
1198 if [ $inc_osd_num -eq 0 ]; then
1199 prun $SUDO rm -f core*
1200 fi
1201
1202 [ -d $CEPH_ASOK_DIR ] || mkdir -p $CEPH_ASOK_DIR
1203 [ -d $CEPH_OUT_DIR ] || mkdir -p $CEPH_OUT_DIR
1204 [ -d $CEPH_DEV_DIR ] || mkdir -p $CEPH_DEV_DIR
1205 if [ $inc_osd_num -eq 0 ]; then
1206 $SUDO rm -rf $CEPH_OUT_DIR/*
1207 fi
1208 [ -d gmon ] && $SUDO rm -rf gmon/*
1209
1210 [ "$cephx" -eq 1 ] && [ "$new" -eq 1 ] && [ -e $keyring_fn ] && rm $keyring_fn
1211
1212
1213 # figure machine's ip
1214 HOSTNAME=`hostname -s`
1215 if [ -n "$ip" ]; then
1216 IP="$ip"
1217 else
1218 echo hostname $HOSTNAME
1219 if [ -x "$(which ip 2>/dev/null)" ]; then
1220 IP_CMD="ip addr"
1221 else
1222 IP_CMD="ifconfig"
1223 fi
1224 # filter out IPv4 and localhost addresses
1225 IP="$($IP_CMD | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p' | head -n1)"
1226 # if nothing left, try using localhost address, it might work
1227 if [ -z "$IP" ]; then IP="127.0.0.1"; fi
1228 fi
1229 echo "ip $IP"
1230 echo "port $CEPH_PORT"
1231
1232
1233 [ -z $CEPH_ADM ] && CEPH_ADM=$CEPH_BIN/ceph
1234
1235 ceph_adm() {
1236 if [ "$cephx" -eq 1 ]; then
1237 prun $SUDO "$CEPH_ADM" -c "$conf_fn" -k "$keyring_fn" "$@"
1238 else
1239 prun $SUDO "$CEPH_ADM" -c "$conf_fn" "$@"
1240 fi
1241 }
1242
1243 if [ $inc_osd_num -gt 0 ]; then
1244 start_osd
1245 exit
1246 fi
1247
1248 if [ "$new" -eq 1 ]; then
1249 prepare_conf
1250 fi
1251
1252 if [ $CEPH_NUM_MON -gt 0 ]; then
1253 start_mon
1254
1255 debug echo Populating config ...
1256 cat <<EOF | $CEPH_BIN/ceph -c $conf_fn config assimilate-conf -i -
1257 [global]
1258 osd_pool_default_size = $OSD_POOL_DEFAULT_SIZE
1259 osd_pool_default_min_size = 1
1260
1261 [mon]
1262 mon_osd_reporter_subtree_level = osd
1263 mon_data_avail_warn = 2
1264 mon_data_avail_crit = 1
1265 mon_allow_pool_delete = true
1266
1267 [osd]
1268 osd_scrub_load_threshold = 2000
1269 osd_debug_op_order = true
1270 osd_debug_misdirected_ops = true
1271 osd_copyfrom_max_chunk = 524288
1272
1273 [mds]
1274 mds_debug_frag = true
1275 mds_debug_auth_pins = true
1276 mds_debug_subtrees = true
1277
1278 [mgr]
1279 mgr/telemetry/nag = false
1280 mgr/telemetry/enable = false
1281
1282 EOF
1283
1284 if [ "$debug" -ne 0 ]; then
1285 debug echo Setting debug configs ...
1286 cat <<EOF | $CEPH_BIN/ceph -c $conf_fn config assimilate-conf -i -
1287 [mgr]
1288 debug_ms = 1
1289 debug_mgr = 20
1290 debug_monc = 20
1291 debug_mon = 20
1292
1293 [osd]
1294 debug_ms = 1
1295 debug_osd = 25
1296 debug_objecter = 20
1297 debug_monc = 20
1298 debug_mgrc = 20
1299 debug_journal = 20
1300 debug_filestore = 20
1301 debug_bluestore = 20
1302 debug_bluefs = 20
1303 debug_rocksdb = 20
1304 debug_bdev = 20
1305 debug_reserver = 10
1306 debug_objclass = 20
1307
1308 [mds]
1309 debug_ms = 1
1310 debug_mds = 20
1311 debug_monc = 20
1312 debug_mgrc = 20
1313 mds_debug_scatterstat = true
1314 mds_verify_scatter = true
1315 EOF
1316 fi
1317 if [ "$cephadm" -gt 0 ]; then
1318 debug echo Setting mon public_network ...
1319 public_network=$(ip route list | grep -w "$IP" | awk '{print $1}')
1320 ceph_adm config set mon public_network $public_network
1321 fi
1322 fi
1323
1324 if [ $CEPH_NUM_MGR -gt 0 ]; then
1325 start_mgr
1326 fi
1327
1328 # osd
1329 if [ $CEPH_NUM_OSD -gt 0 ]; then
1330 start_osd
1331 fi
1332
1333 # mds
1334 if [ "$smallmds" -eq 1 ]; then
1335 wconf <<EOF
1336 [mds]
1337 mds log max segments = 2
1338 # Default 'mds cache memory limit' is 1GiB, and here we set it to 100MiB.
1339 mds cache memory limit = 100M
1340 EOF
1341 fi
1342
1343 if [ $CEPH_NUM_MDS -gt 0 ]; then
1344 start_mds
1345 # key with access to all FS
1346 ceph_adm fs authorize \* "client.fs" / rwp >> "$keyring_fn"
1347 fi
1348
1349 # Don't set max_mds until all the daemons are started, otherwise
1350 # the intended standbys might end up in active roles.
1351 if [ "$CEPH_MAX_MDS" -gt 1 ]; then
1352 sleep 5 # wait for daemons to make it into FSMap before increasing max_mds
1353 fi
1354 fs=0
1355 for name in a b c d e f g h i j k l m n o p
1356 do
1357 [ $fs -eq $CEPH_NUM_FS ] && break
1358 fs=$(($fs + 1))
1359 if [ "$CEPH_MAX_MDS" -gt 1 ]; then
1360 ceph_adm fs set "${name}" max_mds "$CEPH_MAX_MDS"
1361 fi
1362 done
1363
1364 # mgr
1365
1366 if [ "$ec" -eq 1 ]; then
1367 ceph_adm <<EOF
1368 osd erasure-code-profile set ec-profile m=2 k=2
1369 osd pool create ec erasure ec-profile
1370 EOF
1371 fi
1372
1373 # Ganesha Daemons
1374 if [ $GANESHA_DAEMON_NUM -gt 0 ]; then
1375 pseudo_path="/cephfs"
1376 if [ "$cephadm" -gt 0 ]; then
1377 cluster_id="vstart"
1378 prun ceph_adm nfs cluster create cephfs $cluster_id
1379 prun ceph_adm nfs export create cephfs "a" $cluster_id $pseudo_path
1380 port="2049"
1381 else
1382 start_ganesha
1383 port="<ganesha-port-num>"
1384 fi
1385 echo "Mount using: mount -t nfs -o port=$port $IP:$pseudo_path mountpoint"
1386 fi
1387
1388 do_cache() {
1389 while [ -n "$*" ]; do
1390 p="$1"
1391 shift
1392 debug echo "creating cache for pool $p ..."
1393 ceph_adm <<EOF
1394 osd pool create ${p}-cache
1395 osd tier add $p ${p}-cache
1396 osd tier cache-mode ${p}-cache writeback
1397 osd tier set-overlay $p ${p}-cache
1398 EOF
1399 done
1400 }
1401 do_cache $cache
1402
1403 do_hitsets() {
1404 while [ -n "$*" ]; do
1405 pool="$1"
1406 type="$2"
1407 shift
1408 shift
1409 debug echo "setting hit_set on pool $pool type $type ..."
1410 ceph_adm <<EOF
1411 osd pool set $pool hit_set_type $type
1412 osd pool set $pool hit_set_count 8
1413 osd pool set $pool hit_set_period 30
1414 EOF
1415 done
1416 }
1417 do_hitsets $hitset
1418
1419 do_rgw_create_users()
1420 {
1421 # Create S3 user
1422 local akey='0555b35654ad1656d804'
1423 local skey='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=='
1424 debug echo "setting up user testid"
1425 $CEPH_BIN/radosgw-admin user create --uid testid --access-key $akey --secret $skey --display-name 'M. Tester' --email tester@ceph.com -c $conf_fn > /dev/null
1426
1427 # Create S3-test users
1428 # See: https://github.com/ceph/s3-tests
1429 debug echo "setting up s3-test users"
1430 $CEPH_BIN/radosgw-admin user create \
1431 --uid 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef \
1432 --access-key ABCDEFGHIJKLMNOPQRST \
1433 --secret abcdefghijklmnopqrstuvwxyzabcdefghijklmn \
1434 --display-name youruseridhere \
1435 --email s3@example.com -c $conf_fn > /dev/null
1436 $CEPH_BIN/radosgw-admin user create \
1437 --uid 56789abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234 \
1438 --access-key NOPQRSTUVWXYZABCDEFG \
1439 --secret nopqrstuvwxyzabcdefghijklmnabcdefghijklm \
1440 --display-name john.doe \
1441 --email john.doe@example.com -c $conf_fn > /dev/null
1442 $CEPH_BIN/radosgw-admin user create \
1443 --tenant testx \
1444 --uid 9876543210abcdef0123456789abcdef0123456789abcdef0123456789abcdef \
1445 --access-key HIJKLMNOPQRSTUVWXYZA \
1446 --secret opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab \
1447 --display-name tenanteduser \
1448 --email tenanteduser@example.com -c $conf_fn > /dev/null
1449
1450 # Create Swift user
1451 debug echo "setting up user tester"
1452 $CEPH_BIN/radosgw-admin user create -c $conf_fn --subuser=test:tester --display-name=Tester-Subuser --key-type=swift --secret=testing --access=full > /dev/null
1453
1454 echo ""
1455 echo "S3 User Info:"
1456 echo " access key: $akey"
1457 echo " secret key: $skey"
1458 echo ""
1459 echo "Swift User Info:"
1460 echo " account : test"
1461 echo " user : tester"
1462 echo " password : testing"
1463 echo ""
1464 }
1465
1466 do_rgw()
1467 {
1468 if [ "$new" -eq 1 ]; then
1469 do_rgw_create_users
1470 if [ -n "$rgw_compression" ]; then
1471 debug echo "setting compression type=$rgw_compression"
1472 $CEPH_BIN/radosgw-admin zone placement modify -c $conf_fn --rgw-zone=default --placement-id=default-placement --compression=$rgw_compression > /dev/null
1473 fi
1474 fi
1475 # Start server
1476 RGWDEBUG=""
1477 if [ "$debug" -ne 0 ]; then
1478 RGWDEBUG="--debug-rgw=20 --debug-ms=1"
1479 fi
1480
1481 local CEPH_RGW_PORT_NUM="${CEPH_RGW_PORT}"
1482 local CEPH_RGW_HTTPS="${CEPH_RGW_PORT: -1}"
1483 if [[ "${CEPH_RGW_HTTPS}" = "s" ]]; then
1484 CEPH_RGW_PORT_NUM="${CEPH_RGW_PORT::-1}"
1485 else
1486 CEPH_RGW_HTTPS=""
1487 fi
1488 RGWSUDO=
1489 [ $CEPH_RGW_PORT_NUM -lt 1024 ] && RGWSUDO=sudo
1490
1491 current_port=$CEPH_RGW_PORT
1492 for n in $(seq 1 $CEPH_NUM_RGW); do
1493 rgw_name="client.rgw.${current_port}"
1494
1495 ceph_adm auth get-or-create $rgw_name \
1496 mon 'allow rw' \
1497 osd 'allow rwx' \
1498 mgr 'allow rw' \
1499 >> "$keyring_fn"
1500
1501 debug echo start rgw on http${CEPH_RGW_HTTPS}://localhost:${current_port}
1502 run 'rgw' $current_port $RGWSUDO $CEPH_BIN/radosgw -c $conf_fn \
1503 --log-file=${CEPH_OUT_DIR}/radosgw.${current_port}.log \
1504 --admin-socket=${CEPH_OUT_DIR}/radosgw.${current_port}.asok \
1505 --pid-file=${CEPH_OUT_DIR}/radosgw.${current_port}.pid \
1506 ${RGWDEBUG} \
1507 -n ${rgw_name} \
1508 "--rgw_frontends=${rgw_frontend} port=${current_port}${CEPH_RGW_HTTPS}"
1509
1510 i=$(($i + 1))
1511 [ $i -eq $CEPH_NUM_RGW ] && break
1512
1513 current_port=$((current_port+1))
1514 done
1515 }
1516 if [ "$CEPH_NUM_RGW" -gt 0 ]; then
1517 do_rgw
1518 fi
1519
1520 debug echo "vstart cluster complete. Use stop.sh to stop. See out/* (e.g. 'tail -f out/????') for debug output."
1521
1522 echo ""
1523 if [ "$new" -eq 1 ]; then
1524 if $with_mgr_dashboard; then
1525 echo "dashboard urls: $DASH_URLS"
1526 echo " w/ user/pass: admin / admin"
1527 fi
1528 echo "restful urls: $RESTFUL_URLS"
1529 echo " w/ user/pass: admin / $RESTFUL_SECRET"
1530 echo ""
1531 fi
1532 echo ""
1533 # add header to the environment file
1534 {
1535 echo "#"
1536 echo "# source this file into your shell to set up the environment."
1537 echo "# For example:"
1538 echo "# $ . $CEPH_DIR/vstart_environment.sh"
1539 echo "#"
1540 } > $CEPH_DIR/vstart_environment.sh
1541 {
1542 echo "export PYTHONPATH=$PYBIND:$CYTHON_PYTHONPATH:$CEPH_PYTHON_COMMON\$PYTHONPATH"
1543 echo "export LD_LIBRARY_PATH=$CEPH_LIB:\$LD_LIBRARY_PATH"
1544
1545 if [ "$CEPH_DIR" != "$PWD" ]; then
1546 echo "export CEPH_CONF=$conf_fn"
1547 echo "export CEPH_KEYRING=$keyring_fn"
1548 fi
1549
1550 if [ -n "$CEPHFS_SHELL" ]; then
1551 echo "alias cephfs-shell=$CEPHFS_SHELL"
1552 fi
1553 } | tee -a $CEPH_DIR/vstart_environment.sh
1554
1555 echo "CEPH_DEV=1"
1556
1557 # always keep this section at the very bottom of this file
1558 STRAY_CONF_PATH="/etc/ceph/ceph.conf"
1559 if [ -f "$STRAY_CONF_PATH" -a -n "$conf_fn" -a ! "$conf_fn" -ef "$STRAY_CONF_PATH" ]; then
1560 echo ""
1561 echo ""
1562 echo "WARNING:"
1563 echo " Please remove stray $STRAY_CONF_PATH if not needed."
1564 echo " Your conf files $conf_fn and $STRAY_CONF_PATH may not be in sync"
1565 echo " and may lead to undesired results."
1566 echo ""
1567 echo "NOTE:"
1568 echo " Remember to restart cluster after removing $STRAY_CONF_PATH"
1569 fi
Ceph