ceph/src/vstart.sh

   1 #!/usr/bin/env bash
   2 # -*- mode:sh; tab-width:4; sh-basic-offset:4; indent-tabs-mode:nil -*-
   3 # vim: softtabstop=4 shiftwidth=4 expandtab
   4
   5 # abort on failure
   6 set -e
   7
   8 quoted_print() {
   9     for s in "$@"; do
  10         if [[ "$s" =~ \  ]]; then
  11             printf -- "'%s' " "$s"
  12         else
  13             printf -- "$s "
  14         fi
  15     done
  16     printf '\n'
  17 }
  18
  19 debug() {
  20   "$@" >&2
  21 }
  22
  23 prunb() {
  24     debug quoted_print "$@" '&'
  25     "$@" &
  26 }
  27
  28 prun() {
  29     debug quoted_print "$@"
  30     "$@"
  31 }
  32
  33
  34 if [ -n "$VSTART_DEST" ]; then
  35     SRC_PATH=`dirname $0`
  36     SRC_PATH=`(cd $SRC_PATH; pwd)`
  37
  38     CEPH_DIR=$SRC_PATH
  39     CEPH_BIN=${PWD}/bin
  40     CEPH_LIB=${PWD}/lib
  41
  42     CEPH_CONF_PATH=$VSTART_DEST
  43     CEPH_DEV_DIR=$VSTART_DEST/dev
  44     CEPH_OUT_DIR=$VSTART_DEST/out
  45     CEPH_ASOK_DIR=$VSTART_DEST/out
  46 fi
  47
  48 get_cmake_variable() {
  49     local variable=$1
  50     grep "${variable}:" CMakeCache.txt | cut -d "=" -f 2
  51 }
  52
  53 # for running out of the CMake build directory
  54 if [ -e CMakeCache.txt ]; then
  55     # Out of tree build, learn source location from CMakeCache.txt
  56     CEPH_ROOT=$(get_cmake_variable ceph_SOURCE_DIR)
  57     CEPH_BUILD_DIR=`pwd`
  58     [ -z "$MGR_PYTHON_PATH" ] && MGR_PYTHON_PATH=$CEPH_ROOT/src/pybind/mgr
  59 fi
  60
  61 # use CEPH_BUILD_ROOT to vstart from a 'make install'
  62 if [ -n "$CEPH_BUILD_ROOT" ]; then
  63     [ -z "$CEPH_BIN" ] && CEPH_BIN=$CEPH_BUILD_ROOT/bin
  64     [ -z "$CEPH_LIB" ] && CEPH_LIB=$CEPH_BUILD_ROOT/lib
  65     [ -z "$EC_PATH" ] && EC_PATH=$CEPH_LIB/erasure-code
  66     [ -z "$OBJCLASS_PATH" ] && OBJCLASS_PATH=$CEPH_LIB/rados-classes
  67     # make install should install python extensions into PYTHONPATH
  68 elif [ -n "$CEPH_ROOT" ]; then
  69     [ -z "$CEPHFS_SHELL" ] && CEPHFS_SHELL=$CEPH_ROOT/src/tools/cephfs/cephfs-shell
  70     [ -z "$PYBIND" ] && PYBIND=$CEPH_ROOT/src/pybind
  71     [ -z "$CEPH_BIN" ] && CEPH_BIN=$CEPH_BUILD_DIR/bin
  72     [ -z "$CEPH_ADM" ] && CEPH_ADM=$CEPH_BIN/ceph
  73     [ -z "$INIT_CEPH" ] && INIT_CEPH=$CEPH_BIN/init-ceph
  74     [ -z "$CEPH_LIB" ] && CEPH_LIB=$CEPH_BUILD_DIR/lib
  75     [ -z "$OBJCLASS_PATH" ] && OBJCLASS_PATH=$CEPH_LIB
  76     [ -z "$EC_PATH" ] && EC_PATH=$CEPH_LIB
  77     [ -z "$CEPH_PYTHON_COMMON" ] && CEPH_PYTHON_COMMON=$CEPH_ROOT/src/python-common
  78 fi
  79
  80 if [ -z "${CEPH_VSTART_WRAPPER}" ]; then
  81     PATH=$(pwd):$PATH
  82 fi
  83
  84 [ -z "$PYBIND" ] && PYBIND=./pybind
  85
  86 [ -n "$CEPH_PYTHON_COMMON" ] && CEPH_PYTHON_COMMON="$CEPH_PYTHON_COMMON:"
  87 CYTHON_PYTHONPATH="$CEPH_LIB/cython_modules/lib.3"
  88 export PYTHONPATH=$PYBIND:$CYTHON_PYTHONPATH:$CEPH_PYTHON_COMMON$PYTHONPATH
  89
  90 export LD_LIBRARY_PATH=$CEPH_LIB:$LD_LIBRARY_PATH
  91 export DYLD_LIBRARY_PATH=$CEPH_LIB:$DYLD_LIBRARY_PATH
  92 # Suppress logging for regular use that indicated that we are using a
  93 # development version. vstart.sh is only used during testing and
  94 # development
  95 export CEPH_DEV=1
  96
  97 [ -z "$CEPH_NUM_MON" ] && CEPH_NUM_MON="$MON"
  98 [ -z "$CEPH_NUM_OSD" ] && CEPH_NUM_OSD="$OSD"
  99 [ -z "$CEPH_NUM_MDS" ] && CEPH_NUM_MDS="$MDS"
 100 [ -z "$CEPH_NUM_MGR" ] && CEPH_NUM_MGR="$MGR"
 101 [ -z "$CEPH_NUM_FS"  ] && CEPH_NUM_FS="$FS"
 102 [ -z "$CEPH_NUM_RGW" ] && CEPH_NUM_RGW="$RGW"
 103 [ -z "$GANESHA_DAEMON_NUM" ] && GANESHA_DAEMON_NUM="$GANESHA"
 104
 105 # if none of the CEPH_NUM_* number is specified, kill the existing
 106 # cluster.
 107 if [ -z "$CEPH_NUM_MON" -a \
 108      -z "$CEPH_NUM_OSD" -a \
 109      -z "$CEPH_NUM_MDS" -a \
 110      -z "$CEPH_NUM_MGR" -a \
 111      -z "$GANESHA_DAEMON_NUM" ]; then
 112     kill_all=1
 113 else
 114     kill_all=0
 115 fi
 116
 117 [ -z "$CEPH_NUM_MON" ] && CEPH_NUM_MON=3
 118 [ -z "$CEPH_NUM_OSD" ] && CEPH_NUM_OSD=3
 119 [ -z "$CEPH_NUM_MDS" ] && CEPH_NUM_MDS=3
 120 [ -z "$CEPH_NUM_MGR" ] && CEPH_NUM_MGR=1
 121 [ -z "$CEPH_NUM_FS"  ] && CEPH_NUM_FS=1
 122 [ -z "$CEPH_MAX_MDS" ] && CEPH_MAX_MDS=1
 123 [ -z "$CEPH_NUM_RGW" ] && CEPH_NUM_RGW=0
 124 [ -z "$GANESHA_DAEMON_NUM" ] && GANESHA_DAEMON_NUM=0
 125
 126 [ -z "$CEPH_DIR" ] && CEPH_DIR="$PWD"
 127 [ -z "$CEPH_DEV_DIR" ] && CEPH_DEV_DIR="$CEPH_DIR/dev"
 128 [ -z "$CEPH_OUT_DIR" ] && CEPH_OUT_DIR="$CEPH_DIR/out"
 129 [ -z "$CEPH_RGW_PORT" ] && CEPH_RGW_PORT=8000
 130 [ -z "$CEPH_CONF_PATH" ] && CEPH_CONF_PATH=$CEPH_DIR
 131
 132 if [ $CEPH_NUM_OSD -gt 3 ]; then
 133     OSD_POOL_DEFAULT_SIZE=3
 134 else
 135     OSD_POOL_DEFAULT_SIZE=$CEPH_NUM_OSD
 136 fi
 137
 138 extra_conf=""
 139 new=0
 140 standby=0
 141 debug=0
 142 ip=""
 143 nodaemon=0
 144 redirect=0
 145 smallmds=0
 146 short=0
 147 ec=0
 148 cephadm=0
 149 parallel=true
 150 hitset=""
 151 overwrite_conf=0
 152 cephx=1 #turn cephx on by default
 153 gssapi_authx=0
 154 cache=""
 155 if [ `uname` = FreeBSD ]; then
 156     objectstore="filestore"
 157 else
 158     objectstore="bluestore"
 159 fi
 160 ceph_osd=ceph-osd
 161 rgw_frontend="beast"
 162 rgw_compression=""
 163 lockdep=${LOCKDEP:-1}
 164 spdk_enabled=0 #disable SPDK by default
 165 pci_id=""
 166
 167 with_mgr_dashboard=true
 168 if [[ "$(get_cmake_variable WITH_MGR_DASHBOARD_FRONTEND)" != "ON" ]] ||
 169    [[ "$(get_cmake_variable WITH_RBD)" != "ON" ]]; then
 170     debug echo "ceph-mgr dashboard not built - disabling."
 171     with_mgr_dashboard=false
 172 fi
 173
 174 filestore_path=
 175 kstore_path=
 176 bluestore_dev=
 177
 178 VSTART_SEC="client.vstart.sh"
 179
 180 MON_ADDR=""
 181 DASH_URLS=""
 182 RESTFUL_URLS=""
 183
 184 conf_fn="$CEPH_CONF_PATH/ceph.conf"
 185 keyring_fn="$CEPH_CONF_PATH/keyring"
 186 osdmap_fn="/tmp/ceph_osdmap.$$"
 187 monmap_fn="/tmp/ceph_monmap.$$"
 188 inc_osd_num=0
 189
 190 msgr="21"
 191
 192 usage="usage: $0 [option]... \nex: MON=3 OSD=1 MDS=1 MGR=1 RGW=1 GANESHA=1 $0 -n -d\n"
 193 usage=$usage"options:\n"
 194 usage=$usage"\t-d, --debug\n"
 195 usage=$usage"\t-s, --standby_mds: Generate standby-replay MDS for each active\n"
 196 usage=$usage"\t-l, --localhost: use localhost instead of hostname\n"
 197 usage=$usage"\t-i <ip>: bind to specific ip\n"
 198 usage=$usage"\t-n, --new\n"
 199 usage=$usage"\t--valgrind[_{osd,mds,mon,rgw}] 'toolname args...'\n"
 200 usage=$usage"\t--nodaemon: use ceph-run as wrapper for mon/osd/mds\n"
 201 usage=$usage"\t--redirect-output: only useful with nodaemon, directs output to log file\n"
 202 usage=$usage"\t--smallmds: limit mds cache memory limit\n"
 203 usage=$usage"\t-m ip:port\t\tspecify monitor address\n"
 204 usage=$usage"\t-k keep old configuration files (default)\n"
 205 usage=$usage"\t-x enable cephx (on by default)\n"
 206 usage=$usage"\t-X disable cephx\n"
 207 usage=$usage"\t-g --gssapi enable Kerberos/GSSApi authentication\n"
 208 usage=$usage"\t-G disable Kerberos/GSSApi authentication\n"
 209 usage=$usage"\t--hitset <pool> <hit_set_type>: enable hitset tracking\n"
 210 usage=$usage"\t-e : create an erasure pool\n";
 211 usage=$usage"\t-o config\t\t add extra config parameters to all sections\n"
 212 usage=$usage"\t--rgw_port specify ceph rgw http listen port\n"
 213 usage=$usage"\t--rgw_frontend specify the rgw frontend configuration\n"
 214 usage=$usage"\t--rgw_compression specify the rgw compression plugin\n"
 215 usage=$usage"\t-b, --bluestore use bluestore as the osd objectstore backend (default)\n"
 216 usage=$usage"\t-f, --filestore use filestore as the osd objectstore backend\n"
 217 usage=$usage"\t-K, --kstore use kstore as the osd objectstore backend\n"
 218 usage=$usage"\t--memstore use memstore as the osd objectstore backend\n"
 219 usage=$usage"\t--cache <pool>: enable cache tiering on pool\n"
 220 usage=$usage"\t--short: short object names only; necessary for ext4 dev\n"
 221 usage=$usage"\t--nolockdep disable lockdep\n"
 222 usage=$usage"\t--multimds <count> allow multimds with maximum active count\n"
 223 usage=$usage"\t--without-dashboard: do not run using mgr dashboard\n"
 224 usage=$usage"\t--bluestore-spdk <vendor>:<device>: enable SPDK and specify the PCI-ID of the NVME device\n"
 225 usage=$usage"\t--msgr1: use msgr1 only\n"
 226 usage=$usage"\t--msgr2: use msgr2 only\n"
 227 usage=$usage"\t--msgr21: use msgr2 and msgr1\n"
 228 usage=$usage"\t--crimson: use crimson-osd instead of ceph-osd\n"
 229 usage=$usage"\t--osd-args: specify any extra osd specific options\n"
 230 usage=$usage"\t--bluestore-devs: comma-separated list of blockdevs to use for bluestore\n"
 231 usage=$usage"\t--inc-osd: append some more osds into existing vcluster\n"
 232 usage=$usage"\t--cephadm: enable cephadm orchestrator with ~/.ssh/id_rsa[.pub]\n"
 233 usage=$usage"\t--no-parallel: dont start all OSDs in parallel\n"
 234
 235 usage_exit() {
 236     printf "$usage"
 237     exit
 238 }
 239
 240 while [ $# -ge 1 ]; do
 241 case $1 in
 242     -d | --debug )
 243         debug=1
 244         ;;
 245     -s | --standby_mds)
 246         standby=1
 247         ;;
 248     -l | --localhost )
 249         ip="127.0.0.1"
 250         ;;
 251     -i )
 252         [ -z "$2" ] && usage_exit
 253         ip="$2"
 254         shift
 255         ;;
 256     -e )
 257         ec=1
 258         ;;
 259     --new | -n )
 260         new=1
 261         ;;
 262     --inc-osd )
 263         new=0
 264         kill_all=0
 265         inc_osd_num=$2
 266         if [ "$inc_osd_num" == "" ]; then
 267             inc_osd_num=1
 268         else
 269             shift
 270         fi
 271         ;;
 272     --short )
 273         short=1
 274         ;;
 275     --crimson )
 276         ceph_osd=crimson-osd
 277         ;;
 278     --osd-args )
 279         extra_osd_args="$2"
 280         shift
 281         ;;
 282     --msgr1 )
 283         msgr="1"
 284         ;;
 285     --msgr2 )
 286         msgr="2"
 287         ;;
 288     --msgr21 )
 289         msgr="21"
 290         ;;
 291     --cephadm )
 292         cephadm=1
 293         ;;
 294     --no-parallel )
 295         parallel=false
 296         ;;
 297     --valgrind )
 298         [ -z "$2" ] && usage_exit
 299         valgrind=$2
 300         shift
 301         ;;
 302     --valgrind_args )
 303         valgrind_args="$2"
 304         shift
 305         ;;
 306     --valgrind_mds )
 307         [ -z "$2" ] && usage_exit
 308         valgrind_mds=$2
 309         shift
 310         ;;
 311     --valgrind_osd )
 312         [ -z "$2" ] && usage_exit
 313         valgrind_osd=$2
 314         shift
 315         ;;
 316     --valgrind_mon )
 317         [ -z "$2" ] && usage_exit
 318         valgrind_mon=$2
 319         shift
 320         ;;
 321     --valgrind_mgr )
 322         [ -z "$2" ] && usage_exit
 323         valgrind_mgr=$2
 324         shift
 325         ;;
 326     --valgrind_rgw )
 327         [ -z "$2" ] && usage_exit
 328         valgrind_rgw=$2
 329         shift
 330         ;;
 331     --nodaemon )
 332         nodaemon=1
 333         ;;
 334     --redirect-output)
 335         redirect=1
 336         ;;
 337     --smallmds )
 338         smallmds=1
 339         ;;
 340     --rgw_port )
 341         CEPH_RGW_PORT=$2
 342         shift
 343         ;;
 344     --rgw_frontend )
 345         rgw_frontend=$2
 346         shift
 347         ;;
 348     --rgw_compression )
 349         rgw_compression=$2
 350         shift
 351         ;;
 352     --kstore_path )
 353         kstore_path=$2
 354         shift
 355         ;;
 356     --filestore_path )
 357         filestore_path=$2
 358         shift
 359         ;;
 360     -m )
 361         [ -z "$2" ] && usage_exit
 362         MON_ADDR=$2
 363         shift
 364         ;;
 365     -x )
 366         cephx=1 # this is on be default, flag exists for historical consistency
 367         ;;
 368     -X )
 369         cephx=0
 370         ;;
 371
 372     -g | --gssapi)
 373         gssapi_authx=1
 374         ;;
 375     -G)
 376         gssapi_authx=0
 377         ;;
 378
 379     -k )
 380         if [ ! -r $conf_fn ]; then
 381             echo "cannot use old configuration: $conf_fn not readable." >&2
 382             exit
 383         fi
 384         new=0
 385         ;;
 386     --memstore )
 387         objectstore="memstore"
 388         ;;
 389     -b | --bluestore )
 390         objectstore="bluestore"
 391         ;;
 392     -f | --filestore )
 393         objectstore="filestore"
 394         ;;
 395     -K | --kstore )
 396         objectstore="kstore"
 397         ;;
 398     --hitset )
 399         hitset="$hitset $2 $3"
 400         shift
 401         shift
 402         ;;
 403     -o )
 404         extra_conf="$extra_conf $2
 405 "
 406         shift
 407         ;;
 408     --cache )
 409         if [ -z "$cache" ]; then
 410             cache="$2"
 411         else
 412             cache="$cache $2"
 413         fi
 414         shift
 415         ;;
 416     --nolockdep )
 417         lockdep=0
 418         ;;
 419     --multimds)
 420         CEPH_MAX_MDS="$2"
 421         shift
 422         ;;
 423     --without-dashboard)
 424         with_mgr_dashboard=false
 425         ;;
 426     --bluestore-spdk )
 427         [ -z "$2" ] && usage_exit
 428         pci_id="$2"
 429         spdk_enabled=1
 430         shift
 431         ;;
 432     --bluestore-devs )
 433         IFS=',' read -r -a bluestore_dev <<< "$2"
 434         for dev in "${bluestore_dev[@]}"; do
 435             if [ ! -b $dev -o ! -w $dev ]; then
 436                 echo "All --bluestore-devs must refer to writable block devices"
 437                 exit 1
 438             fi
 439         done
 440         shift
 441         ;;
 442     * )
 443         usage_exit
 444 esac
 445 shift
 446 done
 447
 448 if [ $kill_all -eq 1 ]; then
 449     $SUDO $INIT_CEPH stop
 450 fi
 451
 452 if [ "$new" -eq 0 ]; then
 453     if [ -z "$CEPH_ASOK_DIR" ]; then
 454         CEPH_ASOK_DIR=`dirname $($CEPH_BIN/ceph-conf  -c $conf_fn --show-config-value admin_socket)`
 455     fi
 456     mkdir -p $CEPH_ASOK_DIR
 457     MON=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_mon 2>/dev/null` && \
 458         CEPH_NUM_MON="$MON"
 459     OSD=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_osd 2>/dev/null` && \
 460         CEPH_NUM_OSD="$OSD"
 461     MDS=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_mds 2>/dev/null` && \
 462         CEPH_NUM_MDS="$MDS"
 463     MGR=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_mgr 2>/dev/null` && \
 464         CEPH_NUM_MGR="$MGR"
 465     RGW=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_rgw 2>/dev/null` && \
 466         CEPH_NUM_RGW="$RGW"
 467     GANESHA=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_ganesha 2>/dev/null` && \
 468         GANESHA_DAEMON_NUM="$GANESHA"
 469 else
 470     # only delete if -n
 471     if [ -e "$conf_fn" ]; then
 472         asok_dir=`dirname $($CEPH_BIN/ceph-conf  -c $conf_fn --show-config-value admin_socket)`
 473         rm -- "$conf_fn"
 474         if [ $asok_dir != /var/run/ceph ]; then
 475             [ -d $asok_dir ] && rm -f $asok_dir/* && rmdir $asok_dir
 476         fi
 477     fi
 478     if [ -z "$CEPH_ASOK_DIR" ]; then
 479         CEPH_ASOK_DIR=`mktemp -u -d "${TMPDIR:-/tmp}/ceph-asok.XXXXXX"`
 480     fi
 481 fi
 482
 483 ARGS="-c $conf_fn"
 484
 485 run() {
 486     type=$1
 487     shift
 488     num=$1
 489     shift
 490     eval "valg=\$valgrind_$type"
 491     [ -z "$valg" ] && valg="$valgrind"
 492
 493     if [ -n "$valg" ]; then
 494         prunb valgrind --tool="$valg" $valgrind_args "$@" -f
 495         sleep 1
 496     else
 497         if [ "$nodaemon" -eq 0 ]; then
 498             prun "$@"
 499         elif [ "$redirect" -eq 0 ]; then
 500             prunb ${CEPH_ROOT}/src/ceph-run "$@" -f
 501         else
 502             ( prunb ${CEPH_ROOT}/src/ceph-run "$@" -f ) >$CEPH_OUT_DIR/$type.$num.stdout 2>&1
 503         fi
 504     fi
 505 }
 506
 507 wconf() {
 508     if [ "$new" -eq 1 -o "$overwrite_conf" -eq 1 ]; then
 509         cat >> "$conf_fn"
 510     fi
 511 }
 512
 513 get_pci_selector() {
 514     which_pci=$1
 515     lspci -mm -n -D -d $pci_id | cut -d ' ' -f 1 | sed -n $which_pci'p'
 516 }
 517
 518 get_pci_selector_num() {
 519     lspci -mm -n -D -d $pci_id | cut -d' ' -f 1 | wc -l
 520 }
 521
 522 do_rgw_conf() {
 523
 524     if [ $CEPH_NUM_RGW -eq 0 ]; then
 525         return 0
 526     fi
 527
 528     # setup each rgw on a sequential port, starting at $CEPH_RGW_PORT.
 529     # individual rgw's ids will be their ports.
 530     current_port=$CEPH_RGW_PORT
 531     for n in $(seq 1 $CEPH_NUM_RGW); do
 532         wconf << EOF
 533 [client.rgw.${current_port}]
 534         rgw frontends = $rgw_frontend port=${current_port}
 535         admin socket = ${CEPH_OUT_DIR}/radosgw.${current_port}.asok
 536 EOF
 537         current_port=$((current_port + 1))
 538 done
 539
 540 }
 541
 542 prepare_conf() {
 543     local DAEMONOPTS="
 544         log file = $CEPH_OUT_DIR/\$name.log
 545         admin socket = $CEPH_ASOK_DIR/\$name.asok
 546         chdir = \"\"
 547         pid file = $CEPH_OUT_DIR/\$name.pid
 548         heartbeat file = $CEPH_OUT_DIR/\$name.heartbeat
 549 "
 550
 551     local mgr_modules="restful iostat"
 552     if $with_mgr_dashboard; then
 553         mgr_modules="dashboard $mgr_modules"
 554     fi
 555
 556     local msgr_conf=''
 557     if [ $msgr -eq 21 ]; then
 558         msgr_conf="
 559         ms bind msgr2 = true
 560         ms bind msgr1 = true
 561 ";
 562     fi
 563     if [ $msgr -eq 2 ]; then
 564         msgr_conf="
 565         ms bind msgr2 = true
 566         ms bind msgr1 = false
 567 ";
 568     fi
 569     if [ $msgr -eq 1 ]; then
 570         msgr_conf="
 571         ms bind msgr2 = false
 572         ms bind msgr1 = true
 573 ";
 574     fi
 575
 576     wconf <<EOF
 577 ; generated by vstart.sh on `date`
 578 [$VSTART_SEC]
 579         num mon = $CEPH_NUM_MON
 580         num osd = $CEPH_NUM_OSD
 581         num mds = $CEPH_NUM_MDS
 582         num mgr = $CEPH_NUM_MGR
 583         num rgw = $CEPH_NUM_RGW
 584         num ganesha = $GANESHA_DAEMON_NUM
 585
 586 [global]
 587         fsid = $(uuidgen)
 588         osd failsafe full ratio = .99
 589         mon osd full ratio = .99
 590         mon osd nearfull ratio = .99
 591         mon osd backfillfull ratio = .99
 592         mon_max_pg_per_osd = ${MON_MAX_PG_PER_OSD:-1000}
 593         erasure code dir = $EC_PATH
 594         plugin dir = $CEPH_LIB
 595         filestore fd cache size = 32
 596         run dir = $CEPH_OUT_DIR
 597         crash dir = $CEPH_OUT_DIR
 598         enable experimental unrecoverable data corrupting features = *
 599         osd_crush_chooseleaf_type = 0
 600         debug asok assert abort = true
 601 $msgr_conf
 602 $extra_conf
 603 EOF
 604     if [ "$lockdep" -eq 1 ] ; then
 605         wconf <<EOF
 606         lockdep = true
 607 EOF
 608     fi
 609     if [ "$cephx" -eq 1 ] ; then
 610         wconf <<EOF
 611         auth cluster required = cephx
 612         auth service required = cephx
 613         auth client required = cephx
 614 EOF
 615     elif [ "$gssapi_authx" -eq 1 ] ; then
 616         wconf <<EOF
 617         auth cluster required = gss
 618         auth service required = gss
 619         auth client required = gss
 620         gss ktab client file = $CEPH_DEV_DIR/gss_\$name.keytab
 621 EOF
 622     else
 623         wconf <<EOF
 624         auth cluster required = none
 625         auth service required = none
 626         auth client required = none
 627 EOF
 628     fi
 629     if [ "$short" -eq 1 ]; then
 630         COSDSHORT="        osd max object name len = 460
 631         osd max object namespace len = 64"
 632     fi
 633     if [ "$objectstore" == "bluestore" ]; then
 634         if [ "$spdk_enabled" -eq 1 ]; then
 635             if [ "$(get_pci_selector_num)" -eq 0 ]; then
 636                 echo "Not find the specified NVME device, please check." >&2
 637                 exit
 638             fi
 639             if [ $(get_pci_selector_num) -lt $CEPH_NUM_OSD ]; then
 640                 echo "OSD number ($CEPH_NUM_OSD) is greater than NVME SSD number ($(get_pci_selector_num)), please check." >&2
 641                 exit
 642             fi
 643             BLUESTORE_OPTS="        bluestore_block_db_path = \"\"
 644         bluestore_block_db_size = 0
 645         bluestore_block_db_create = false
 646         bluestore_block_wal_path = \"\"
 647         bluestore_block_wal_size = 0
 648         bluestore_block_wal_create = false
 649         bluestore_spdk_mem = 2048"
 650         else
 651             BLUESTORE_OPTS="        bluestore block db path = $CEPH_DEV_DIR/osd\$id/block.db.file
 652         bluestore block db size = 1073741824
 653         bluestore block db create = true
 654         bluestore block wal path = $CEPH_DEV_DIR/osd\$id/block.wal.file
 655         bluestore block wal size = 1048576000
 656         bluestore block wal create = true"
 657         fi
 658     fi
 659     wconf <<EOF
 660 [client]
 661         keyring = $keyring_fn
 662         log file = $CEPH_OUT_DIR/\$name.\$pid.log
 663         admin socket = $CEPH_ASOK_DIR/\$name.\$pid.asok
 664
 665         ; needed for s3tests
 666         rgw crypt s3 kms backend = testing
 667         rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo=
 668         rgw crypt require ssl = false
 669         ; uncomment the following to set LC days as the value in seconds;
 670         ; needed for passing lc time based s3-tests (can be verbose)
 671         ; rgw lc debug interval = 10
 672         ; The following settings are for SSE-KMS with Vault
 673         ;rgw crypt s3 kms backend = vault
 674         ;rgw crypt vault auth = token
 675         ;rgw crypt vault token file = $CEPH_CONF_PATH/vault.token
 676         ;rgw crypt vault addr = http://127.0.0.1:8200
 677         ;rgw crypt vault secret engine = kv
 678         ;rgw crypt vault prefix = /v1/kv/data
 679         ;rgw crypt vault secret engine = transit
 680         ;rgw crypt vault prefix = /v1/transit/export/encryption-key/
 681
 682 $extra_conf
 683 EOF
 684 wconf <<EOF
 685 [cephfs-shell]
 686         debug shell = true
 687
 688 $extra_conf
 689 EOF
 690
 691         do_rgw_conf
 692         wconf << EOF
 693 [mds]
 694 $DAEMONOPTS
 695         mds data = $CEPH_DEV_DIR/mds.\$id
 696         mds root ino uid = `id -u`
 697         mds root ino gid = `id -g`
 698 $extra_conf
 699 [mgr]
 700         mgr data = $CEPH_DEV_DIR/mgr.\$id
 701         mgr module path = $MGR_PYTHON_PATH
 702         cephadm path = $CEPH_ROOT/src/cephadm/cephadm
 703 $DAEMONOPTS
 704 $extra_conf
 705 [osd]
 706 $DAEMONOPTS
 707         osd_check_max_object_name_len_on_startup = false
 708         osd data = $CEPH_DEV_DIR/osd\$id
 709         osd journal = $CEPH_DEV_DIR/osd\$id/journal
 710         osd journal size = 100
 711         osd class tmp = out
 712         osd class dir = $OBJCLASS_PATH
 713         osd class load list = *
 714         osd class default list = *
 715         osd fast shutdown = false
 716
 717         filestore wbthrottle xfs ios start flusher = 10
 718         filestore wbthrottle xfs ios hard limit = 20
 719         filestore wbthrottle xfs inodes hard limit = 30
 720         filestore wbthrottle btrfs ios start flusher = 10
 721         filestore wbthrottle btrfs ios hard limit = 20
 722         filestore wbthrottle btrfs inodes hard limit = 30
 723         bluestore fsck on mount = true
 724         bluestore block create = true
 725 $BLUESTORE_OPTS
 726
 727         ; kstore
 728         kstore fsck on mount = true
 729         osd objectstore = $objectstore
 730 $COSDSHORT
 731 $extra_conf
 732 [mon]
 733         mgr initial modules = $mgr_modules
 734 $DAEMONOPTS
 735 $CMONDEBUG
 736 $extra_conf
 737         mon cluster log file = $CEPH_OUT_DIR/cluster.mon.\$id.log
 738         osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd
 739 EOF
 740 }
 741
 742 start_mon() {
 743     local MONS=""
 744     local count=0
 745     for f in a b c d e f g h i j k l m n o p q r s t u v w x y z
 746     do
 747         [ $count -eq $CEPH_NUM_MON ] && break;
 748         count=$(($count + 1))
 749         if [ -z "$MONS" ]; then
 750             MONS="$f"
 751         else
 752             MONS="$MONS $f"
 753         fi
 754     done
 755
 756     if [ "$new" -eq 1 ]; then
 757         if [ `echo $IP | grep '^127\\.'` ]; then
 758             echo
 759             echo "NOTE: hostname resolves to loopback; remote hosts will not be able to"
 760             echo "  connect.  either adjust /etc/hosts, or edit this script to use your"
 761             echo "  machine's real IP."
 762             echo
 763         fi
 764
 765         prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name=mon. "$keyring_fn" --cap mon 'allow *'
 766         prun $SUDO "$CEPH_BIN/ceph-authtool" --gen-key --name=client.admin \
 767              --cap mon 'allow *' \
 768              --cap osd 'allow *' \
 769              --cap mds 'allow *' \
 770              --cap mgr 'allow *' \
 771              "$keyring_fn"
 772
 773         # build a fresh fs monmap, mon fs
 774         local params=()
 775         local count=0
 776         local mon_host=""
 777         for f in $MONS
 778         do
 779             if [ $msgr -eq 1 ]; then
 780                 A="v1:$IP:$(($CEPH_PORT+$count+1))"
 781             fi
 782             if [ $msgr -eq 2 ]; then
 783                 A="v2:$IP:$(($CEPH_PORT+$count+1))"
 784             fi
 785             if [ $msgr -eq 21 ]; then
 786                 A="[v2:$IP:$(($CEPH_PORT+$count)),v1:$IP:$(($CEPH_PORT+$count+1))]"
 787             fi
 788             params+=("--addv" "$f" "$A")
 789             mon_host="$mon_host $A"
 790             wconf <<EOF
 791 [mon.$f]
 792         host = $HOSTNAME
 793         mon data = $CEPH_DEV_DIR/mon.$f
 794 EOF
 795             count=$(($count + 2))
 796         done
 797         wconf <<EOF
 798 [global]
 799         mon host = $mon_host
 800 EOF
 801         prun "$CEPH_BIN/monmaptool" --create --clobber "${params[@]}" --print "$monmap_fn"
 802
 803         for f in $MONS
 804         do
 805             prun rm -rf -- "$CEPH_DEV_DIR/mon.$f"
 806             prun mkdir -p "$CEPH_DEV_DIR/mon.$f"
 807             prun "$CEPH_BIN/ceph-mon" --mkfs -c "$conf_fn" -i "$f" --monmap="$monmap_fn" --keyring="$keyring_fn"
 808         done
 809
 810         prun rm -- "$monmap_fn"
 811     fi
 812
 813     # start monitors
 814     for f in $MONS
 815     do
 816         run 'mon' $f $CEPH_BIN/ceph-mon -i $f $ARGS $CMON_ARGS
 817     done
 818 }
 819
 820 start_osd() {
 821     if [ $inc_osd_num -gt 0 ]; then
 822         old_maxosd=$($CEPH_BIN/ceph osd getmaxosd | sed -e 's/max_osd = //' -e 's/ in epoch.*//')
 823         start=$old_maxosd
 824         end=$(($start-1+$inc_osd_num))
 825         overwrite_conf=1 # fake wconf
 826     else
 827         start=0
 828         end=$(($CEPH_NUM_OSD-1))
 829     fi
 830     local osds_wait
 831     for osd in `seq $start $end`
 832     do
 833         local extra_seastar_args
 834         if [ "$ceph_osd" == "crimson-osd" ]; then
 835             # designate a single CPU node $osd for osd.$osd
 836             extra_seastar_args="--smp 1 --cpuset $osd"
 837             if [ "$debug" -ne 0 ]; then
 838                 extra_seastar_args+=" --debug"
 839             fi
 840         fi
 841         if [ "$new" -eq 1 -o $inc_osd_num -gt 0 ]; then
 842             wconf <<EOF
 843 [osd.$osd]
 844         host = $HOSTNAME
 845 EOF
 846             if [ "$spdk_enabled" -eq 1 ]; then
 847                 wconf <<EOF
 848         bluestore_block_path = spdk:$(get_pci_selector $((osd+1)))
 849 EOF
 850             fi
 851
 852             rm -rf $CEPH_DEV_DIR/osd$osd || true
 853             if command -v btrfs > /dev/null; then
 854                 for f in $CEPH_DEV_DIR/osd$osd/*; do btrfs sub delete $f &> /dev/null || true; done
 855             fi
 856             if [ -n "$filestore_path" ]; then
 857                 ln -s $filestore_path $CEPH_DEV_DIR/osd$osd
 858             elif [ -n "$kstore_path" ]; then
 859                 ln -s $kstore_path $CEPH_DEV_DIR/osd$osd
 860             else
 861                 mkdir -p $CEPH_DEV_DIR/osd$osd
 862                 if [ -n "${bluestore_dev[$osd]}" ]; then
 863                     dd if=/dev/zero of=${bluestore_dev[$osd]} bs=1M count=1
 864                     ln -s ${bluestore_dev[$osd]} $CEPH_DEV_DIR/osd$osd/block
 865                     wconf <<EOF
 866         bluestore fsck on mount = false
 867 EOF
 868                 fi
 869             fi
 870
 871             local uuid=`uuidgen`
 872             echo "add osd$osd $uuid"
 873             OSD_SECRET=$($CEPH_BIN/ceph-authtool --gen-print-key)
 874             echo "{\"cephx_secret\": \"$OSD_SECRET\"}" > $CEPH_DEV_DIR/osd$osd/new.json
 875             ceph_adm osd new $uuid -i $CEPH_DEV_DIR/osd$osd/new.json
 876             rm $CEPH_DEV_DIR/osd$osd/new.json
 877             $SUDO $CEPH_BIN/$ceph_osd $extra_osd_args -i $osd $ARGS --mkfs --key $OSD_SECRET --osd-uuid $uuid $extra_seastar_args
 878
 879             local key_fn=$CEPH_DEV_DIR/osd$osd/keyring
 880             cat > $key_fn<<EOF
 881 [osd.$osd]
 882         key = $OSD_SECRET
 883 EOF
 884         fi
 885         echo start osd.$osd
 886         local osd_pid
 887         run 'osd' $osd $SUDO $CEPH_BIN/$ceph_osd \
 888             $extra_seastar_args $extra_osd_args \
 889             -i $osd $ARGS $COSD_ARGS &
 890         osd_pid=$!
 891         if $parallel; then
 892             osds_wait=$osd_pid
 893         else
 894             wait $osd_pid
 895         fi
 896     done
 897     if $parallel; then
 898         for p in $osds_wait; do
 899             wait $p
 900         done
 901         debug echo OSDs started
 902     fi
 903     if [ $inc_osd_num -gt 0 ]; then
 904         # update num osd
 905         new_maxosd=$($CEPH_BIN/ceph osd getmaxosd | sed -e 's/max_osd = //' -e 's/ in epoch.*//')
 906         sed -i "s/num osd = .*/num osd = $new_maxosd/g" $conf_fn
 907     fi
 908 }
 909
 910 start_mgr() {
 911     local mgr=0
 912     local ssl=${DASHBOARD_SSL:-1}
 913     # avoid monitors on nearby ports (which test/*.sh use extensively)
 914     MGR_PORT=$(($CEPH_PORT + 1000))
 915     PROMETHEUS_PORT=9283
 916     for name in x y z a b c d e f g h i j k l m n o p
 917     do
 918         [ $mgr -eq $CEPH_NUM_MGR ] && break
 919         mgr=$(($mgr + 1))
 920         if [ "$new" -eq 1 ]; then
 921             mkdir -p $CEPH_DEV_DIR/mgr.$name
 922             key_fn=$CEPH_DEV_DIR/mgr.$name/keyring
 923             $SUDO $CEPH_BIN/ceph-authtool --create-keyring --gen-key --name=mgr.$name $key_fn
 924             ceph_adm -i $key_fn auth add mgr.$name mon 'allow profile mgr' mds 'allow *' osd 'allow *'
 925
 926             wconf <<EOF
 927 [mgr.$name]
 928         host = $HOSTNAME
 929 EOF
 930
 931             if $with_mgr_dashboard ; then
 932                 local port_option="ssl_server_port"
 933                 local http_proto="https"
 934                 if [ "$ssl" == "0" ]; then
 935                     port_option="server_port"
 936                     http_proto="http"
 937                     ceph_adm config set mgr mgr/dashboard/ssl false --force
 938                 fi
 939                 ceph_adm config set mgr mgr/dashboard/$name/$port_option $MGR_PORT --force
 940                 if [ $mgr -eq 1 ]; then
 941                     DASH_URLS="$http_proto://$IP:$MGR_PORT"
 942                 else
 943                     DASH_URLS+=", $http_proto://$IP:$MGR_PORT"
 944                 fi
 945             fi
 946             MGR_PORT=$(($MGR_PORT + 1000))
 947             ceph_adm config set mgr mgr/prometheus/$name/server_port $PROMETHEUS_PORT --force
 948             PROMETHEUS_PORT=$(($PROMETHEUS_PORT + 1000))
 949
 950             ceph_adm config set mgr mgr/restful/$name/server_port $MGR_PORT --force
 951             if [ $mgr -eq 1 ]; then
 952                 RESTFUL_URLS="https://$IP:$MGR_PORT"
 953             else
 954                 RESTFUL_URLS+=", https://$IP:$MGR_PORT"
 955             fi
 956             MGR_PORT=$(($MGR_PORT + 1000))
 957         fi
 958
 959         debug echo "Starting mgr.${name}"
 960         run 'mgr' $name $CEPH_BIN/ceph-mgr -i $name $ARGS
 961     done
 962
 963     if [ "$new" -eq 1 ]; then
 964         # setting login credentials for dashboard
 965         if $with_mgr_dashboard; then
 966             while ! ceph_adm -h | grep -c -q ^dashboard ; do
 967                 debug echo 'waiting for mgr dashboard module to start'
 968                 sleep 1
 969             done
 970             ceph_adm dashboard ac-user-create --force-password admin admin administrator
 971             if [ "$ssl" != "0" ]; then
 972                 if ! ceph_adm dashboard create-self-signed-cert;  then
 973                     debug echo dashboard module not working correctly!
 974                 fi
 975             fi
 976         fi
 977
 978         while ! ceph_adm -h | grep -c -q ^restful ; do
 979             debug echo 'waiting for mgr restful module to start'
 980             sleep 1
 981         done
 982         if ceph_adm restful create-self-signed-cert; then
 983             SF=`mktemp`
 984             ceph_adm restful create-key admin -o $SF
 985             RESTFUL_SECRET=`cat $SF`
 986             rm $SF
 987         else
 988             debug echo MGR Restful is not working, perhaps the package is not installed?
 989         fi
 990     fi
 991
 992     if [ "$cephadm" -eq 1 ]; then
 993         debug echo Enabling cephadm orchestrator
 994         ceph_adm config-key set mgr/cephadm/ssh_identity_key -i ~/.ssh/id_rsa
 995         ceph_adm config-key set mgr/cephadm/ssh_identity_pub -i ~/.ssh/id_rsa.pub
 996         ceph_adm mgr module enable cephadm
 997         ceph_adm orch set backend cephadm
 998         ceph_adm orch host add $HOSTNAME
 999         ceph_adm orch apply crash '*'
1000         ceph_adm config set mgr mgr/cephadm/allow_ptrace true
1001     fi
1002 }
1003
1004 start_mds() {
1005     local mds=0
1006     for name in a b c d e f g h i j k l m n o p
1007     do
1008         [ $mds -eq $CEPH_NUM_MDS ] && break
1009         mds=$(($mds + 1))
1010
1011         if [ "$new" -eq 1 ]; then
1012             prun mkdir -p "$CEPH_DEV_DIR/mds.$name"
1013             key_fn=$CEPH_DEV_DIR/mds.$name/keyring
1014             wconf <<EOF
1015 [mds.$name]
1016         host = $HOSTNAME
1017 EOF
1018             if [ "$standby" -eq 1 ]; then
1019                 mkdir -p $CEPH_DEV_DIR/mds.${name}s
1020                 wconf <<EOF
1021         mds standby for rank = $mds
1022 [mds.${name}s]
1023         mds standby replay = true
1024         mds standby for name = ${name}
1025 EOF
1026             fi
1027             prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name="mds.$name" "$key_fn"
1028             ceph_adm -i "$key_fn" auth add "mds.$name" mon 'allow profile mds' osd 'allow rw tag cephfs *=*' mds 'allow' mgr 'allow profile mds'
1029             if [ "$standby" -eq 1 ]; then
1030                 prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name="mds.${name}s" \
1031                      "$CEPH_DEV_DIR/mds.${name}s/keyring"
1032                 ceph_adm -i "$CEPH_DEV_DIR/mds.${name}s/keyring" auth add "mds.${name}s" \
1033                              mon 'allow profile mds' osd 'allow *' mds 'allow' mgr 'allow profile mds'
1034             fi
1035         fi
1036
1037         run 'mds' $name $CEPH_BIN/ceph-mds -i $name $ARGS $CMDS_ARGS
1038         if [ "$standby" -eq 1 ]; then
1039             run 'mds' $name $CEPH_BIN/ceph-mds -i ${name}s $ARGS $CMDS_ARGS
1040         fi
1041
1042         #valgrind --tool=massif $CEPH_BIN/ceph-mds $ARGS --mds_log_max_segments 2 --mds_thrash_fragments 0 --mds_thrash_exports 0 > m  #--debug_ms 20
1043         #$CEPH_BIN/ceph-mds -d $ARGS --mds_thrash_fragments 0 --mds_thrash_exports 0 #--debug_ms 20
1044         #ceph_adm mds set max_mds 2
1045     done
1046
1047     if [ $new -eq 1 ]; then
1048         if [ "$CEPH_NUM_FS" -gt "0" ] ; then
1049             sleep 5 # time for MDS to come up as standby to avoid health warnings on fs creation
1050             if [ "$CEPH_NUM_FS" -gt "1" ] ; then
1051                 ceph_adm fs flag set enable_multiple true --yes-i-really-mean-it
1052             fi
1053
1054             # wait for volume module to load
1055             while ! ceph_adm fs volume ls ; do sleep 1 ; done
1056             local fs=0
1057             for name in a b c d e f g h i j k l m n o p
1058             do
1059                 ceph_adm fs volume create ${name}
1060                 ceph_adm fs authorize ${name} "client.fs_${name}" / rwp >> "$keyring_fn"
1061                 fs=$(($fs + 1))
1062                 [ $fs -eq $CEPH_NUM_FS ] && break
1063             done
1064         fi
1065     fi
1066
1067 }
1068
1069 # Ganesha Daemons requires nfs-ganesha nfs-ganesha-ceph nfs-ganesha-rados-grace
1070 # (version 2.7.6-2 and above) packages installed. On Fedora>=30 these packages
1071 # can be installed directly with 'dnf'. For CentOS>=8 the packages need to be
1072 # downloaded first from  https://download.nfs-ganesha.org/2.7/2.7.6/CentOS/ and
1073 # then install it. Similarly for Ubuntu 16.04 follow the instructions on
1074 # https://launchpad.net/~nfs-ganesha/+archive/ubuntu/nfs-ganesha-2.7
1075
1076 start_ganesha() {
1077     GANESHA_PORT=$(($CEPH_PORT + 4000))
1078     local ganesha=0
1079
1080     for name in a b c d e f g h i j k l m n o p
1081     do
1082         [ $ganesha -eq $GANESHA_DAEMON_NUM ] && break
1083
1084         port=$(($GANESHA_PORT + ganesha))
1085         ganesha=$(($ganesha + 1))
1086         ganesha_dir="$CEPH_DEV_DIR/ganesha.$name"
1087
1088         echo "Starting ganesha.$name on port: $port"
1089
1090         prun rm -rf $ganesha_dir
1091         prun mkdir -p $ganesha_dir
1092
1093         echo "NFS_CORE_PARAM {
1094         Enable_NLM = false;
1095         Enable_RQUOTA = false;
1096         Protocols = 4;
1097         NFS_Port = $port;
1098 }
1099
1100 CACHEINODE {
1101         Dir_Chunk = 0;
1102         NParts = 1;
1103         Cache_Size = 1;
1104 }
1105
1106 NFSv4 {
1107         RecoveryBackend = 'rados_cluster';
1108         Minor_Versions = 1, 2;
1109 }
1110
1111 EXPORT {
1112         Export_Id = 100;
1113         Transports = TCP;
1114         Path = /;
1115         Pseudo = /ceph/;
1116         Protocols = 4;
1117         Access_Type = RW;
1118         Attr_Expiration_Time = 0;
1119         Squash = None;
1120         FSAL {
1121             Name = CEPH;
1122         }
1123 }
1124
1125 CEPH {
1126         Ceph_Conf = $conf_fn;
1127 }
1128
1129 RADOS_KV {
1130         Ceph_Conf = $conf_fn;
1131         pool = 'nfs-ganesha';
1132         namespace = 'ganesha';
1133         UserId = 'admin';
1134         nodeid = $name;
1135 }" > "$ganesha_dir/ganesha.conf"
1136
1137
1138         wconf <<EOF
1139 [ganesha.$name]
1140         host = $HOSTNAME
1141         ip = $IP
1142         port = $port
1143         ganesha data = $ganesha_dir
1144         pid file = $ganesha_dir/ganesha.pid
1145 EOF
1146
1147         if !($CEPH_BIN/rados lspools | grep "nfs-ganesha"); then
1148             prun ceph_adm osd pool create nfs-ganesha
1149             prun ceph_adm osd pool application enable nfs-ganesha nfs
1150         fi
1151
1152         prun ganesha-rados-grace -p nfs-ganesha -n ganesha add $name
1153         prun ganesha-rados-grace -p nfs-ganesha -n ganesha
1154
1155         prun /usr/bin/ganesha.nfsd -L "$ganesha_dir/ganesha.log" -f "$ganesha_dir/ganesha.conf" -p "$ganesha_dir/ganesha.pid" -N NIV_DEBUG
1156
1157         # Wait few seconds for grace period to be removed
1158         sleep 2
1159         prun ganesha-rados-grace -p nfs-ganesha -n ganesha
1160
1161         if $with_mgr_dashboard; then
1162             $CEPH_BIN/rados -p nfs-ganesha put "conf-$name" "$ganesha_dir/ganesha.conf"
1163         fi
1164     done
1165
1166     if $with_mgr_dashboard; then
1167         ceph_adm dashboard set-ganesha-clusters-rados-pool-namespace nfs-ganesha
1168     fi
1169 }
1170
1171 if [ "$debug" -eq 0 ]; then
1172     CMONDEBUG='
1173         debug mon = 10
1174         debug ms = 1'
1175 else
1176     debug echo "** going verbose **"
1177     CMONDEBUG='
1178         debug mon = 20
1179         debug paxos = 20
1180         debug auth = 20
1181         debug mgrc = 20
1182         debug ms = 1'
1183 fi
1184
1185 if [ -n "$MON_ADDR" ]; then
1186     CMON_ARGS=" -m "$MON_ADDR
1187     COSD_ARGS=" -m "$MON_ADDR
1188     CMDS_ARGS=" -m "$MON_ADDR
1189 fi
1190
1191 if [ -z "$CEPH_PORT" ]; then
1192     while [ true ]
1193     do
1194         CEPH_PORT="$(echo $(( RANDOM % 1000 + 40000 )))"
1195         ss -a -n | egrep "\<LISTEN\>.+:${CEPH_PORT}\s+" 1>/dev/null 2>&1 || break
1196     done
1197 fi
1198
1199 [ -z "$INIT_CEPH" ] && INIT_CEPH=$CEPH_BIN/init-ceph
1200
1201 # sudo if btrfs
1202 [ -d $CEPH_DEV_DIR/osd0/. ] && [ -e $CEPH_DEV_DIR/sudo ] && SUDO="sudo"
1203
1204 if [ $inc_osd_num -eq 0 ]; then
1205     prun $SUDO rm -f core*
1206 fi
1207
1208 [ -d $CEPH_ASOK_DIR ] || mkdir -p $CEPH_ASOK_DIR
1209 [ -d $CEPH_OUT_DIR  ] || mkdir -p $CEPH_OUT_DIR
1210 [ -d $CEPH_DEV_DIR  ] || mkdir -p $CEPH_DEV_DIR
1211 if [ $inc_osd_num -eq 0 ]; then
1212     $SUDO rm -rf $CEPH_OUT_DIR/*
1213 fi
1214 [ -d gmon ] && $SUDO rm -rf gmon/*
1215
1216 [ "$cephx" -eq 1 ] && [ "$new" -eq 1 ] && [ -e $keyring_fn ] && rm $keyring_fn
1217
1218
1219 # figure machine's ip
1220 HOSTNAME=`hostname -s`
1221 if [ -n "$ip" ]; then
1222     IP="$ip"
1223 else
1224     echo hostname $HOSTNAME
1225     if [ -x "$(which ip 2>/dev/null)" ]; then
1226         IP_CMD="ip addr"
1227     else
1228         IP_CMD="ifconfig"
1229     fi
1230     # filter out IPv4 and localhost addresses
1231     IP="$($IP_CMD | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p' | head -n1)"
1232     # if nothing left, try using localhost address, it might work
1233     if [ -z "$IP" ]; then IP="127.0.0.1"; fi
1234 fi
1235 echo "ip $IP"
1236 echo "port $CEPH_PORT"
1237
1238
1239 [ -z $CEPH_ADM ] && CEPH_ADM=$CEPH_BIN/ceph
1240
1241 ceph_adm() {
1242     if [ "$cephx" -eq 1 ]; then
1243         prun $SUDO "$CEPH_ADM" -c "$conf_fn" -k "$keyring_fn" "$@"
1244     else
1245         prun $SUDO "$CEPH_ADM" -c "$conf_fn" "$@"
1246     fi
1247 }
1248
1249 if [ $inc_osd_num -gt 0 ]; then
1250     start_osd
1251     exit
1252 fi
1253
1254 if [ "$new" -eq 1 ]; then
1255     prepare_conf
1256 fi
1257
1258 if [ $CEPH_NUM_MON -gt 0 ]; then
1259     start_mon
1260
1261     debug echo Populating config ...
1262     cat <<EOF | $CEPH_BIN/ceph -c $conf_fn config assimilate-conf -i -
1263 [global]
1264 osd_pool_default_size = $OSD_POOL_DEFAULT_SIZE
1265 osd_pool_default_min_size = 1
1266
1267 [mon]
1268 mon_osd_reporter_subtree_level = osd
1269 mon_data_avail_warn = 2
1270 mon_data_avail_crit = 1
1271 mon_allow_pool_delete = true
1272
1273 [osd]
1274 osd_scrub_load_threshold = 2000
1275 osd_debug_op_order = true
1276 osd_debug_misdirected_ops = true
1277 osd_copyfrom_max_chunk = 524288
1278
1279 [mds]
1280 mds_debug_frag = true
1281 mds_debug_auth_pins = true
1282 mds_debug_subtrees = true
1283
1284 [mgr]
1285 mgr/telemetry/nag = false
1286 mgr/telemetry/enable = false
1287
1288 EOF
1289
1290     if [ "$debug" -ne 0 ]; then
1291         debug echo Setting debug configs ...
1292         cat <<EOF | $CEPH_BIN/ceph -c $conf_fn config assimilate-conf -i -
1293 [mgr]
1294 debug_ms = 1
1295 debug_mgr = 20
1296 debug_monc = 20
1297 debug_mon = 20
1298
1299 [osd]
1300 debug_ms = 1
1301 debug_osd = 25
1302 debug_objecter = 20
1303 debug_monc = 20
1304 debug_mgrc = 20
1305 debug_journal = 20
1306 debug_filestore = 20
1307 debug_bluestore = 20
1308 debug_bluefs = 20
1309 debug_rocksdb = 20
1310 debug_bdev = 20
1311 debug_reserver = 10
1312 debug_objclass = 20
1313
1314 [mds]
1315 debug_ms = 1
1316 debug_mds = 20
1317 debug_monc = 20
1318 debug_mgrc = 20
1319 mds_debug_scatterstat = true
1320 mds_verify_scatter = true
1321 EOF
1322     fi
1323 fi
1324
1325 if [ $CEPH_NUM_MGR -gt 0 ]; then
1326     start_mgr
1327 fi
1328
1329 # osd
1330 if [ $CEPH_NUM_OSD -gt 0 ]; then
1331     start_osd
1332 fi
1333
1334 # mds
1335 if [ "$smallmds" -eq 1 ]; then
1336     wconf <<EOF
1337 [mds]
1338         mds log max segments = 2
1339         # Default 'mds cache memory limit' is 1GiB, and here we set it to 100MiB.
1340         mds cache memory limit = 100M
1341 EOF
1342 fi
1343
1344 if [ $CEPH_NUM_MDS -gt 0 ]; then
1345     start_mds
1346     # key with access to all FS
1347     ceph_adm fs authorize \* "client.fs" / rwp >> "$keyring_fn"
1348 fi
1349
1350 # Don't set max_mds until all the daemons are started, otherwise
1351 # the intended standbys might end up in active roles.
1352 if [ "$CEPH_MAX_MDS" -gt 1 ]; then
1353     sleep 5  # wait for daemons to make it into FSMap before increasing max_mds
1354 fi
1355 fs=0
1356 for name in a b c d e f g h i j k l m n o p
1357 do
1358     [ $fs -eq $CEPH_NUM_FS ] && break
1359     fs=$(($fs + 1))
1360     if [ "$CEPH_MAX_MDS" -gt 1 ]; then
1361         ceph_adm fs set "cephfs_${name}" max_mds "$CEPH_MAX_MDS"
1362     fi
1363 done
1364
1365 # mgr
1366
1367 if [ "$ec" -eq 1 ]; then
1368     ceph_adm <<EOF
1369 osd erasure-code-profile set ec-profile m=2 k=2
1370 osd pool create ec erasure ec-profile
1371 EOF
1372 fi
1373
1374 # Ganesha Daemons
1375 if [ $GANESHA_DAEMON_NUM -gt 0 ]; then
1376     start_ganesha
1377 fi
1378
1379 do_cache() {
1380     while [ -n "$*" ]; do
1381         p="$1"
1382         shift
1383         debug echo "creating cache for pool $p ..."
1384         ceph_adm <<EOF
1385 osd pool create ${p}-cache
1386 osd tier add $p ${p}-cache
1387 osd tier cache-mode ${p}-cache writeback
1388 osd tier set-overlay $p ${p}-cache
1389 EOF
1390     done
1391 }
1392 do_cache $cache
1393
1394 do_hitsets() {
1395     while [ -n "$*" ]; do
1396         pool="$1"
1397         type="$2"
1398         shift
1399         shift
1400         debug echo "setting hit_set on pool $pool type $type ..."
1401         ceph_adm <<EOF
1402 osd pool set $pool hit_set_type $type
1403 osd pool set $pool hit_set_count 8
1404 osd pool set $pool hit_set_period 30
1405 EOF
1406     done
1407 }
1408 do_hitsets $hitset
1409
1410 do_rgw_create_users()
1411 {
1412     # Create S3 user
1413     local akey='0555b35654ad1656d804'
1414     local skey='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=='
1415     debug echo "setting up user testid"
1416     $CEPH_BIN/radosgw-admin user create --uid testid --access-key $akey --secret $skey --display-name 'M. Tester' --email tester@ceph.com -c $conf_fn > /dev/null
1417
1418     # Create S3-test users
1419     # See: https://github.com/ceph/s3-tests
1420     debug echo "setting up s3-test users"
1421     $CEPH_BIN/radosgw-admin user create \
1422         --uid 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef \
1423         --access-key ABCDEFGHIJKLMNOPQRST \
1424         --secret abcdefghijklmnopqrstuvwxyzabcdefghijklmn \
1425         --display-name youruseridhere \
1426         --email s3@example.com -c $conf_fn > /dev/null
1427     $CEPH_BIN/radosgw-admin user create \
1428         --uid 56789abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234 \
1429         --access-key NOPQRSTUVWXYZABCDEFG \
1430         --secret nopqrstuvwxyzabcdefghijklmnabcdefghijklm \
1431         --display-name john.doe \
1432         --email john.doe@example.com -c $conf_fn > /dev/null
1433     $CEPH_BIN/radosgw-admin user create \
1434         --tenant testx \
1435         --uid 9876543210abcdef0123456789abcdef0123456789abcdef0123456789abcdef \
1436         --access-key HIJKLMNOPQRSTUVWXYZA \
1437         --secret opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab \
1438         --display-name tenanteduser \
1439         --email tenanteduser@example.com -c $conf_fn > /dev/null
1440
1441     # Create Swift user
1442     debug echo "setting up user tester"
1443     $CEPH_BIN/radosgw-admin user create -c $conf_fn --subuser=test:tester --display-name=Tester-Subuser --key-type=swift --secret=testing --access=full > /dev/null
1444
1445     echo ""
1446     echo "S3 User Info:"
1447     echo "  access key:  $akey"
1448     echo "  secret key:  $skey"
1449     echo ""
1450     echo "Swift User Info:"
1451     echo "  account   : test"
1452     echo "  user      : tester"
1453     echo "  password  : testing"
1454     echo ""
1455 }
1456
1457 do_rgw()
1458 {
1459     if [ "$new" -eq 1 ]; then
1460         do_rgw_create_users
1461         if [ -n "$rgw_compression" ]; then
1462             debug echo "setting compression type=$rgw_compression"
1463             $CEPH_BIN/radosgw-admin zone placement modify -c $conf_fn --rgw-zone=default --placement-id=default-placement --compression=$rgw_compression > /dev/null
1464         fi
1465     fi
1466     # Start server
1467     RGWDEBUG=""
1468     if [ "$debug" -ne 0 ]; then
1469         RGWDEBUG="--debug-rgw=20 --debug-ms=1"
1470     fi
1471
1472     local CEPH_RGW_PORT_NUM="${CEPH_RGW_PORT}"
1473     local CEPH_RGW_HTTPS="${CEPH_RGW_PORT: -1}"
1474     if [[ "${CEPH_RGW_HTTPS}" = "s" ]]; then
1475         CEPH_RGW_PORT_NUM="${CEPH_RGW_PORT::-1}"
1476     else
1477         CEPH_RGW_HTTPS=""
1478     fi
1479     RGWSUDO=
1480     [ $CEPH_RGW_PORT_NUM -lt 1024 ] && RGWSUDO=sudo
1481
1482     current_port=$CEPH_RGW_PORT
1483     for n in $(seq 1 $CEPH_NUM_RGW); do
1484         rgw_name="client.rgw.${current_port}"
1485
1486         ceph_adm auth get-or-create $rgw_name \
1487             mon 'allow rw' \
1488             osd 'allow rwx' \
1489             mgr 'allow rw' \
1490             >> "$keyring_fn"
1491
1492         debug echo start rgw on http${CEPH_RGW_HTTPS}://localhost:${current_port}
1493         run 'rgw' $current_port $RGWSUDO $CEPH_BIN/radosgw -c $conf_fn \
1494             --log-file=${CEPH_OUT_DIR}/radosgw.${current_port}.log \
1495             --admin-socket=${CEPH_OUT_DIR}/radosgw.${current_port}.asok \
1496             --pid-file=${CEPH_OUT_DIR}/radosgw.${current_port}.pid \
1497             ${RGWDEBUG} \
1498             -n ${rgw_name} \
1499             "--rgw_frontends=${rgw_frontend} port=${current_port}${CEPH_RGW_HTTPS}"
1500
1501         i=$(($i + 1))
1502         [ $i -eq $CEPH_NUM_RGW ] && break
1503
1504         current_port=$((current_port+1))
1505     done
1506 }
1507 if [ "$CEPH_NUM_RGW" -gt 0 ]; then
1508     do_rgw
1509 fi
1510
1511 debug echo "vstart cluster complete. Use stop.sh to stop. See out/* (e.g. 'tail -f out/????') for debug output."
1512
1513 echo ""
1514 if [ "$new" -eq 1 ]; then
1515     if $with_mgr_dashboard; then
1516         echo "dashboard urls: $DASH_URLS"
1517         echo "  w/ user/pass: admin / admin"
1518     fi
1519     echo "restful urls: $RESTFUL_URLS"
1520     echo "  w/ user/pass: admin / $RESTFUL_SECRET"
1521     echo ""
1522 fi
1523 echo ""
1524 # add header to the environment file
1525 {
1526     echo "#"
1527     echo "# source this file into your shell to set up the environment."
1528     echo "# For example:"
1529     echo "# $ . $CEPH_DIR/vstart_environment.sh"
1530     echo "#"
1531 } > $CEPH_DIR/vstart_environment.sh
1532 {
1533     echo "export PYTHONPATH=$PYBIND:$CYTHON_PYTHONPATH:$CEPH_PYTHON_COMMON\$PYTHONPATH"
1534     echo "export LD_LIBRARY_PATH=$CEPH_LIB:\$LD_LIBRARY_PATH"
1535
1536     if [ "$CEPH_DIR" != "$PWD" ]; then
1537         echo "export CEPH_CONF=$conf_fn"
1538         echo "export CEPH_KEYRING=$keyring_fn"
1539     fi
1540
1541     if [ -n "$CEPHFS_SHELL" ]; then
1542         echo "alias cephfs-shell=$CEPHFS_SHELL"
1543     fi
1544 } | tee -a $CEPH_DIR/vstart_environment.sh
1545
1546 echo "CEPH_DEV=1"
1547
1548 # always keep this section at the very bottom of this file
1549 STRAY_CONF_PATH="/etc/ceph/ceph.conf"
1550 if [ -f "$STRAY_CONF_PATH" -a -n "$conf_fn" -a ! "$conf_fn" -ef "$STRAY_CONF_PATH" ]; then
1551     echo ""
1552     echo ""
1553     echo "WARNING:"
1554     echo "    Please remove stray $STRAY_CONF_PATH if not needed."
1555     echo "    Your conf files $conf_fn and $STRAY_CONF_PATH may not be in sync"
1556     echo "    and may lead to undesired results."
1557     echo ""
1558     echo "NOTE:"
1559     echo "    Remember to restart cluster after removing $STRAY_CONF_PATH"
1560 fi