]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | #!/bin/sh |
2 | ||
3 | # abort on failure | |
4 | set -e | |
5 | ||
6 | if [ -n "$VSTART_DEST" ]; then | |
7 | SRC_PATH=`dirname $0` | |
8 | SRC_PATH=`(cd $SRC_PATH; pwd)` | |
9 | ||
10 | CEPH_DIR=$SRC_PATH | |
11 | CEPH_BIN=${PWD}/bin | |
12 | CEPH_LIB=${PWD}/lib | |
13 | ||
14 | CEPH_CONF_PATH=$VSTART_DEST | |
15 | CEPH_DEV_DIR=$VSTART_DEST/dev | |
16 | CEPH_OUT_DIR=$VSTART_DEST/out | |
17 | fi | |
18 | ||
19 | # for running out of the CMake build directory | |
20 | if [ -e CMakeCache.txt ]; then | |
21 | # Out of tree build, learn source location from CMakeCache.txt | |
22 | CEPH_ROOT=`grep ceph_SOURCE_DIR CMakeCache.txt | cut -d "=" -f 2` | |
23 | CEPH_BUILD_DIR=`pwd` | |
24 | [ -z "$MGR_PYTHON_PATH" ] && MGR_PYTHON_PATH=$CEPH_ROOT/src/pybind/mgr | |
25 | fi | |
26 | ||
27 | # use CEPH_BUILD_ROOT to vstart from a 'make install' | |
28 | if [ -n "$CEPH_BUILD_ROOT" ]; then | |
29 | [ -z "$CEPH_BIN" ] && CEPH_BIN=$CEPH_BUILD_ROOT/bin | |
30 | [ -z "$CEPH_LIB" ] && CEPH_LIB=$CEPH_BUILD_ROOT/lib | |
31 | [ -z "$EC_PATH" ] && EC_PATH=$CEPH_LIB/erasure-code | |
32 | [ -z "$OBJCLASS_PATH" ] && OBJCLASS_PATH=$CEPH_LIB/rados-classes | |
33 | elif [ -n "$CEPH_ROOT" ]; then | |
34 | [ -z "$PYBIND" ] && PYBIND=$CEPH_ROOT/src/pybind | |
35 | [ -z "$CEPH_BIN" ] && CEPH_BIN=$CEPH_BUILD_DIR/bin | |
36 | [ -z "$CEPH_ADM" ] && CEPH_ADM=$CEPH_BIN/ceph | |
37 | [ -z "$INIT_CEPH" ] && INIT_CEPH=$CEPH_BUILD_DIR/bin/init-ceph | |
38 | [ -z "$CEPH_LIB" ] && CEPH_LIB=$CEPH_BUILD_DIR/lib | |
39 | [ -z "$OBJCLASS_PATH" ] && OBJCLASS_PATH=$CEPH_LIB | |
40 | [ -z "$EC_PATH" ] && EC_PATH=$CEPH_LIB | |
41 | fi | |
42 | ||
43 | if [ -z "${CEPH_VSTART_WRAPPER}" ]; then | |
44 | PATH=$(pwd):$PATH | |
45 | fi | |
46 | ||
47 | [ -z "$PYBIND" ] && PYBIND=./pybind | |
48 | ||
49 | export PYTHONPATH=$PYBIND:$CEPH_LIB/cython_modules/lib.2:$PYTHONPATH | |
50 | export LD_LIBRARY_PATH=$CEPH_LIB:$LD_LIBRARY_PATH | |
51 | export DYLD_LIBRARY_PATH=$CEPH_LIB:$DYLD_LIBRARY_PATH | |
52 | ||
53 | [ -z "$CEPH_NUM_MON" ] && CEPH_NUM_MON="$MON" | |
54 | [ -z "$CEPH_NUM_OSD" ] && CEPH_NUM_OSD="$OSD" | |
55 | [ -z "$CEPH_NUM_MDS" ] && CEPH_NUM_MDS="$MDS" | |
56 | [ -z "$CEPH_NUM_MGR" ] && CEPH_NUM_MGR="$MGR" | |
57 | [ -z "$CEPH_NUM_FS" ] && CEPH_NUM_FS="$FS" | |
58 | [ -z "$CEPH_NUM_RGW" ] && CEPH_NUM_RGW="$RGW" | |
59 | ||
60 | # if none of the CEPH_NUM_* number is specified, kill the existing | |
61 | # cluster. | |
62 | if [ -z "$CEPH_NUM_MON" -a \ | |
63 | -z "$CEPH_NUM_OSD" -a \ | |
64 | -z "$CEPH_NUM_MDS" -a \ | |
65 | -z "$CEPH_NUM_MGR" ]; then | |
66 | kill_all=1 | |
67 | else | |
68 | kill_all=0 | |
69 | fi | |
70 | ||
71 | [ -z "$CEPH_NUM_MON" ] && CEPH_NUM_MON=3 | |
72 | [ -z "$CEPH_NUM_OSD" ] && CEPH_NUM_OSD=3 | |
73 | [ -z "$CEPH_NUM_MDS" ] && CEPH_NUM_MDS=3 | |
74 | [ -z "$CEPH_NUM_MGR" ] && CEPH_NUM_MGR=1 | |
75 | [ -z "$CEPH_NUM_FS" ] && CEPH_NUM_FS=1 | |
76 | [ -z "$CEPH_MAX_MDS" ] && CEPH_MAX_MDS=1 | |
77 | [ -z "$CEPH_NUM_RGW" ] && CEPH_NUM_RGW=0 | |
78 | ||
79 | [ -z "$CEPH_DIR" ] && CEPH_DIR="$PWD" | |
80 | [ -z "$CEPH_DEV_DIR" ] && CEPH_DEV_DIR="$CEPH_DIR/dev" | |
81 | [ -z "$CEPH_OUT_DIR" ] && CEPH_OUT_DIR="$CEPH_DIR/out" | |
82 | [ -z "$CEPH_RGW_PORT" ] && CEPH_RGW_PORT=8000 | |
83 | [ -z "$CEPH_CONF_PATH" ] && CEPH_CONF_PATH=$CEPH_DIR | |
84 | ||
85 | if [ $CEPH_NUM_OSD -gt 3 ]; then | |
86 | OSD_POOL_DEFAULT_SIZE=3 | |
87 | else | |
88 | OSD_POOL_DEFAULT_SIZE=$CEPH_NUM_OSD | |
89 | fi | |
90 | ||
91 | extra_conf="" | |
92 | new=0 | |
93 | standby=0 | |
94 | debug=0 | |
95 | ip="" | |
96 | nodaemon=0 | |
97 | smallmds=0 | |
98 | short=0 | |
99 | ec=0 | |
100 | hitset="" | |
101 | overwrite_conf=1 | |
102 | cephx=1 #turn cephx on by default | |
103 | cache="" | |
104 | memstore=0 | |
105 | bluestore=0 | |
106 | rgw_frontend="civetweb" | |
107 | lockdep=${LOCKDEP:-1} | |
108 | ||
109 | VSTART_SEC="client.vstart.sh" | |
110 | ||
111 | MON_ADDR="" | |
112 | ||
113 | conf_fn="$CEPH_CONF_PATH/ceph.conf" | |
114 | keyring_fn="$CEPH_CONF_PATH/keyring" | |
115 | osdmap_fn="/tmp/ceph_osdmap.$$" | |
116 | monmap_fn="/tmp/ceph_monmap.$$" | |
117 | ||
118 | usage="usage: $0 [option]... \nex: $0 -n -d --mon_num 3 --osd_num 3 --mds_num 1 --rgw_num 1\n" | |
119 | usage=$usage"options:\n" | |
120 | usage=$usage"\t-d, --debug\n" | |
121 | usage=$usage"\t-s, --standby_mds: Generate standby-replay MDS for each active\n" | |
122 | usage=$usage"\t-l, --localhost: use localhost instead of hostname\n" | |
123 | usage=$usage"\t-i <ip>: bind to specific ip\n" | |
124 | usage=$usage"\t-n, --new\n" | |
125 | usage=$usage"\t-N, --not-new: reuse existing cluster config (default)\n" | |
126 | usage=$usage"\t--valgrind[_{osd,mds,mon,rgw}] 'toolname args...'\n" | |
127 | usage=$usage"\t--nodaemon: use ceph-run as wrapper for mon/osd/mds\n" | |
128 | usage=$usage"\t--smallmds: limit mds cache size\n" | |
129 | usage=$usage"\t-m ip:port\t\tspecify monitor address\n" | |
130 | usage=$usage"\t-k keep old configuration files\n" | |
131 | usage=$usage"\t-x enable cephx (on by default)\n" | |
132 | usage=$usage"\t-X disable cephx\n" | |
133 | usage=$usage"\t--hitset <pool> <hit_set_type>: enable hitset tracking\n" | |
134 | usage=$usage"\t-e : create an erasure pool\n"; | |
135 | usage=$usage"\t-o config\t\t add extra config parameters to all sections\n" | |
136 | usage=$usage"\t--mon_num specify ceph monitor count\n" | |
137 | usage=$usage"\t--osd_num specify ceph osd count\n" | |
138 | usage=$usage"\t--mds_num specify ceph mds count\n" | |
139 | usage=$usage"\t--rgw_num specify ceph rgw count\n" | |
140 | usage=$usage"\t--mgr_num specify ceph mgr count\n" | |
141 | usage=$usage"\t--rgw_port specify ceph rgw http listen port\n" | |
142 | usage=$usage"\t--rgw_frontend specify the rgw frontend configuration\n" | |
143 | usage=$usage"\t-b, --bluestore use bluestore as the osd objectstore backend\n" | |
144 | usage=$usage"\t--memstore use memstore as the osd objectstore backend\n" | |
145 | usage=$usage"\t--cache <pool>: enable cache tiering on pool\n" | |
146 | usage=$usage"\t--short: short object names only; necessary for ext4 dev\n" | |
147 | usage=$usage"\t--nolockdep disable lockdep\n" | |
148 | usage=$usage"\t--multimds <count> allow multimds with maximum active count\n" | |
149 | ||
150 | usage_exit() { | |
151 | printf "$usage" | |
152 | exit | |
153 | } | |
154 | ||
155 | while [ $# -ge 1 ]; do | |
156 | case $1 in | |
157 | -d | --debug ) | |
158 | debug=1 | |
159 | ;; | |
160 | -s | --standby_mds) | |
161 | standby=1 | |
162 | ;; | |
163 | -l | --localhost ) | |
164 | ip="127.0.0.1" | |
165 | ;; | |
166 | -i ) | |
167 | [ -z "$2" ] && usage_exit | |
168 | ip="$2" | |
169 | shift | |
170 | ;; | |
171 | -e ) | |
172 | ec=1 | |
173 | ;; | |
174 | --new | -n ) | |
175 | new=1 | |
176 | ;; | |
177 | --not-new | -N ) | |
178 | new=0 | |
179 | ;; | |
180 | --short ) | |
181 | short=1 | |
182 | ;; | |
183 | --valgrind ) | |
184 | [ -z "$2" ] && usage_exit | |
185 | valgrind=$2 | |
186 | shift | |
187 | ;; | |
188 | --valgrind_args ) | |
189 | valgrind_args="$2" | |
190 | shift | |
191 | ;; | |
192 | --valgrind_mds ) | |
193 | [ -z "$2" ] && usage_exit | |
194 | valgrind_mds=$2 | |
195 | shift | |
196 | ;; | |
197 | --valgrind_osd ) | |
198 | [ -z "$2" ] && usage_exit | |
199 | valgrind_osd=$2 | |
200 | shift | |
201 | ;; | |
202 | --valgrind_mon ) | |
203 | [ -z "$2" ] && usage_exit | |
204 | valgrind_mon=$2 | |
205 | shift | |
206 | ;; | |
207 | --valgrind_mgr ) | |
208 | [ -z "$2" ] && usage_exit | |
209 | valgrind_mgr=$2 | |
210 | shift | |
211 | ;; | |
212 | --valgrind_rgw ) | |
213 | [ -z "$2" ] && usage_exit | |
214 | valgrind_rgw=$2 | |
215 | shift | |
216 | ;; | |
217 | --nodaemon ) | |
218 | nodaemon=1 | |
219 | ;; | |
220 | --smallmds ) | |
221 | smallmds=1 | |
222 | ;; | |
223 | --mon_num ) | |
224 | echo "mon_num:$2" | |
225 | CEPH_NUM_MON="$2" | |
226 | shift | |
227 | ;; | |
228 | --osd_num ) | |
229 | CEPH_NUM_OSD=$2 | |
230 | shift | |
231 | ;; | |
232 | --mds_num ) | |
233 | CEPH_NUM_MDS=$2 | |
234 | shift | |
235 | ;; | |
236 | --rgw_num ) | |
237 | CEPH_NUM_RGW=$2 | |
238 | shift | |
239 | ;; | |
240 | --mgr_num ) | |
241 | CEPH_NUM_MGR=$2 | |
242 | shift | |
243 | ;; | |
244 | --rgw_port ) | |
245 | CEPH_RGW_PORT=$2 | |
246 | shift | |
247 | ;; | |
248 | --rgw_frontend ) | |
249 | rgw_frontend=$2 | |
250 | shift | |
251 | ;; | |
252 | -m ) | |
253 | [ -z "$2" ] && usage_exit | |
254 | MON_ADDR=$2 | |
255 | shift | |
256 | ;; | |
257 | -x ) | |
258 | cephx=1 # this is on be default, flag exists for historical consistency | |
259 | ;; | |
260 | -X ) | |
261 | cephx=0 | |
262 | ;; | |
263 | -k ) | |
264 | if [ ! -r $conf_fn ]; then | |
265 | echo "cannot use old configuration: $conf_fn not readable." >&2 | |
266 | exit | |
267 | fi | |
268 | overwrite_conf=0 | |
269 | ;; | |
270 | --memstore ) | |
271 | memstore=1 | |
272 | ;; | |
273 | -b | --bluestore ) | |
274 | bluestore=1 | |
275 | ;; | |
276 | --hitset ) | |
277 | hitset="$hitset $2 $3" | |
278 | shift | |
279 | shift | |
280 | ;; | |
281 | -o ) | |
282 | extra_conf="$extra_conf $2 | |
283 | " | |
284 | shift | |
285 | ;; | |
286 | --cache ) | |
287 | if [ -z "$cache" ]; then | |
288 | cache="$2" | |
289 | else | |
290 | cache="$cache $2" | |
291 | fi | |
292 | shift | |
293 | ;; | |
294 | --nolockdep ) | |
295 | lockdep=0 | |
296 | ;; | |
297 | --multimds) | |
298 | CEPH_MAX_MDS="$2" | |
299 | shift | |
300 | ;; | |
301 | * ) | |
302 | usage_exit | |
303 | esac | |
304 | shift | |
305 | done | |
306 | ||
307 | if [ $kill_all -eq 1 ]; then | |
308 | $SUDO $INIT_CEPH stop | |
309 | fi | |
310 | ||
311 | if [ "$overwrite_conf" -eq 0 ]; then | |
312 | MON=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC num_mon 2>/dev/null` && \ | |
313 | CEPH_NUM_MON="$MON" | |
314 | OSD=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC num_osd 2>/dev/null` && \ | |
315 | CEPH_NUM_OSD="$OSD" | |
316 | MDS=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC num_mds 2>/dev/null` && \ | |
317 | CEPH_NUM_MDS="$MDS" | |
318 | MGR=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC num_mgr 2>/dev/null` && \ | |
319 | CEPH_NUM_MGR="$MGR" | |
320 | RGW=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC num_rgw 2>/dev/null` && \ | |
321 | CEPH_NUM_RGW="$RGW" | |
322 | else | |
323 | if [ "$new" -ne 0 ]; then | |
324 | # only delete if -n | |
325 | [ -e "$conf_fn" ] && rm -- "$conf_fn" | |
326 | else | |
327 | # -k is implied... (doesn't make sense otherwise) | |
328 | overwrite_conf=0 | |
329 | fi | |
330 | fi | |
331 | ||
332 | ARGS="-c $conf_fn" | |
333 | ||
334 | prunb() { | |
335 | echo "$* &" | |
336 | "$@" & | |
337 | } | |
338 | ||
339 | prun() { | |
340 | echo "$*" | |
341 | "$@" | |
342 | } | |
343 | ||
344 | run() { | |
345 | type=$1 | |
346 | shift | |
347 | eval "valg=\$valgrind_$type" | |
348 | [ -z "$valg" ] && valg="$valgrind" | |
349 | ||
350 | if [ -n "$valg" ]; then | |
351 | prunb valgrind --tool="$valg" $valgrind_args "$@" -f | |
352 | sleep 1 | |
353 | else | |
354 | if [ "$nodaemon" -eq 0 ]; then | |
355 | prun "$@" | |
356 | else | |
357 | prunb ./ceph-run "$@" -f | |
358 | fi | |
359 | fi | |
360 | } | |
361 | ||
362 | wconf() { | |
363 | if [ "$overwrite_conf" -eq 1 ]; then | |
364 | cat >> "$conf_fn" | |
365 | fi | |
366 | } | |
367 | ||
368 | prepare_conf() { | |
369 | local DAEMONOPTS=" | |
370 | log file = $CEPH_OUT_DIR/\$name.log | |
371 | admin socket = $CEPH_OUT_DIR/\$name.asok | |
372 | chdir = \"\" | |
373 | pid file = $CEPH_OUT_DIR/\$name.pid | |
374 | heartbeat file = $CEPH_OUT_DIR/\$name.heartbeat | |
375 | " | |
376 | ||
377 | wconf <<EOF | |
378 | ; generated by vstart.sh on `date` | |
379 | [$VSTART_SEC] | |
380 | num mon = $CEPH_NUM_MON | |
381 | num osd = $CEPH_NUM_OSD | |
382 | num mds = $CEPH_NUM_MDS | |
383 | num mgr = $CEPH_NUM_MGR | |
384 | num rgw = $CEPH_NUM_RGW | |
385 | ||
386 | [global] | |
387 | fsid = $(uuidgen) | |
388 | osd pg bits = 3 | |
389 | osd pgp bits = 5 ; (invalid, but ceph should cope!) | |
390 | osd pool default size = $OSD_POOL_DEFAULT_SIZE | |
391 | osd crush chooseleaf type = 0 | |
392 | osd pool default min size = 1 | |
393 | osd failsafe full ratio = .99 | |
394 | mon osd reporter subtree level = osd | |
395 | mon osd full ratio = .99 | |
396 | mon data avail warn = 10 | |
397 | mon data avail crit = 1 | |
398 | erasure code dir = $EC_PATH | |
399 | plugin dir = $CEPH_LIB | |
400 | osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 ruleset-failure-domain=osd | |
401 | rgw frontends = $rgw_frontend port=$CEPH_RGW_PORT | |
402 | filestore fd cache size = 32 | |
403 | run dir = $CEPH_OUT_DIR | |
404 | enable experimental unrecoverable data corrupting features = * | |
405 | EOF | |
406 | if [ "$lockdep" -eq 1 ] ; then | |
407 | wconf <<EOF | |
408 | lockdep = true | |
409 | EOF | |
410 | fi | |
411 | if [ "$cephx" -eq 1 ] ; then | |
412 | wconf <<EOF | |
413 | auth cluster required = cephx | |
414 | auth service required = cephx | |
415 | auth client required = cephx | |
416 | EOF | |
417 | else | |
418 | wconf <<EOF | |
419 | auth cluster required = none | |
420 | auth service required = none | |
421 | auth client required = none | |
422 | EOF | |
423 | fi | |
424 | if [ "$short" -eq 1 ]; then | |
425 | COSDSHORT=" osd max object name len = 460 | |
426 | osd max object namespace len = 64" | |
427 | fi | |
428 | wconf <<EOF | |
429 | [client] | |
430 | keyring = $keyring_fn | |
431 | log file = $CEPH_OUT_DIR/\$name.\$pid.log | |
432 | admin socket = $CEPH_OUT_DIR/\$name.\$pid.asok | |
433 | ||
434 | [mds] | |
435 | $DAEMONOPTS | |
436 | $CMDSDEBUG | |
437 | mds debug frag = true | |
438 | mds debug auth pins = true | |
439 | mds debug subtrees = true | |
440 | mds data = $CEPH_DEV_DIR/mds.\$id | |
441 | mds root ino uid = `id -u` | |
442 | mds root ino gid = `id -g` | |
443 | $extra_conf | |
444 | [mgr] | |
445 | mgr modules = rest fsstatus | |
446 | mgr data = $CEPH_DEV_DIR/mgr.\$id | |
447 | mgr module path = $MGR_PYTHON_PATH | |
448 | mon reweight min pgs per osd = 4 | |
449 | $DAEMONOPTS | |
450 | $CMGRDEBUG | |
451 | $extra_conf | |
452 | [osd] | |
453 | $DAEMONOPTS | |
454 | osd_check_max_object_name_len_on_startup = false | |
455 | osd data = $CEPH_DEV_DIR/osd\$id | |
456 | osd journal = $CEPH_DEV_DIR/osd\$id/journal | |
457 | osd journal size = 100 | |
458 | osd class tmp = out | |
459 | osd class dir = $OBJCLASS_PATH | |
460 | osd class load list = * | |
461 | osd class default list = * | |
462 | osd scrub load threshold = 2000.0 | |
463 | osd debug op order = true | |
464 | osd debug misdirected ops = true | |
465 | filestore wbthrottle xfs ios start flusher = 10 | |
466 | filestore wbthrottle xfs ios hard limit = 20 | |
467 | filestore wbthrottle xfs inodes hard limit = 30 | |
468 | filestore wbthrottle btrfs ios start flusher = 10 | |
469 | filestore wbthrottle btrfs ios hard limit = 20 | |
470 | filestore wbthrottle btrfs inodes hard limit = 30 | |
471 | osd copyfrom max chunk = 524288 | |
472 | bluestore fsck on mount = true | |
473 | bluestore block create = true | |
474 | bluestore block db size = 67108864 | |
475 | bluestore block db create = true | |
476 | bluestore block wal size = 1048576000 | |
477 | bluestore block wal create = true | |
478 | $COSDDEBUG | |
479 | $COSDMEMSTORE | |
480 | $COSDSHORT | |
481 | $extra_conf | |
482 | [mon] | |
483 | mon pg warn min per osd = 3 | |
484 | mon osd allow primary affinity = true | |
485 | mon osd allow pg upmap = true | |
486 | mon reweight min pgs per osd = 4 | |
487 | mon osd prime pg temp = true | |
488 | crushtool = $CEPH_BIN/crushtool | |
489 | mon allow pool delete = true | |
490 | $DAEMONOPTS | |
491 | $CMONDEBUG | |
492 | $extra_conf | |
493 | mon cluster log file = $CEPH_OUT_DIR/cluster.mon.\$id.log | |
494 | [global] | |
495 | $extra_conf | |
496 | EOF | |
497 | } | |
498 | ||
499 | start_mon() { | |
500 | local MONS="" | |
501 | local count=0 | |
502 | for f in a b c d e f g h i j k l m n o p q r s t u v w x y z | |
503 | do | |
504 | [ $count -eq $CEPH_NUM_MON ] && break; | |
505 | count=$(($count + 1)) | |
506 | if [ -z "$MONS" ]; | |
507 | then | |
508 | MONS="$f" | |
509 | else | |
510 | MONS="$MONS $f" | |
511 | fi | |
512 | done | |
513 | ||
514 | if [ "$new" -eq 1 ]; then | |
515 | if [ `echo $IP | grep '^127\\.'` ] | |
516 | then | |
517 | echo | |
518 | echo "NOTE: hostname resolves to loopback; remote hosts will not be able to" | |
519 | echo " connect. either adjust /etc/hosts, or edit this script to use your" | |
520 | echo " machine's real IP." | |
521 | echo | |
522 | fi | |
523 | ||
524 | prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name=mon. "$keyring_fn" --cap mon 'allow *' | |
525 | prun $SUDO "$CEPH_BIN/ceph-authtool" --gen-key --name=client.admin --set-uid=0 \ | |
526 | --cap mon 'allow *' \ | |
527 | --cap osd 'allow *' \ | |
528 | --cap mds 'allow *' \ | |
529 | --cap mgr 'allow *' \ | |
530 | "$keyring_fn" | |
531 | ||
532 | # build a fresh fs monmap, mon fs | |
533 | local str="" | |
534 | local count=0 | |
535 | for f in $MONS | |
536 | do | |
537 | str="$str --add $f $IP:$(($CEPH_PORT+$count))" | |
538 | wconf <<EOF | |
539 | [mon.$f] | |
540 | host = $HOSTNAME | |
541 | mon data = $CEPH_DEV_DIR/mon.$f | |
542 | mon addr = $IP:$(($CEPH_PORT+$count)) | |
543 | EOF | |
544 | count=$(($count + 1)) | |
545 | done | |
546 | prun "$CEPH_BIN/monmaptool" --create --clobber $str --print "$monmap_fn" | |
547 | ||
548 | for f in $MONS | |
549 | do | |
550 | prun rm -rf -- "$CEPH_DEV_DIR/mon.$f" | |
551 | prun mkdir -p "$CEPH_DEV_DIR/mon.$f" | |
552 | prun "$CEPH_BIN/ceph-mon" --mkfs -c "$conf_fn" -i "$f" --monmap="$monmap_fn" --keyring="$keyring_fn" | |
553 | done | |
554 | ||
555 | prun rm -- "$monmap_fn" | |
556 | fi | |
557 | ||
558 | # start monitors | |
559 | for f in $MONS | |
560 | do | |
561 | run 'mon' $CEPH_BIN/ceph-mon -i $f $ARGS $CMON_ARGS | |
562 | done | |
563 | } | |
564 | ||
565 | start_osd() { | |
566 | for osd in `seq 0 $((CEPH_NUM_OSD-1))` | |
567 | do | |
568 | if [ "$new" -eq 1 ]; then | |
569 | wconf <<EOF | |
570 | [osd.$osd] | |
571 | host = $HOSTNAME | |
572 | EOF | |
573 | ||
574 | rm -rf $CEPH_DEV_DIR/osd$osd || true | |
575 | if command -v btrfs > /dev/null; then | |
576 | for f in $CEPH_DEV_DIR/osd$osd/*; do btrfs sub delete $f &> /dev/null || true; done | |
577 | fi | |
578 | mkdir -p $CEPH_DEV_DIR/osd$osd | |
579 | ||
580 | local uuid=`uuidgen` | |
581 | echo "add osd$osd $uuid" | |
582 | ceph_adm osd create $uuid | |
583 | ceph_adm osd crush add osd.$osd 1.0 host=$HOSTNAME root=default | |
584 | $SUDO $CEPH_BIN/ceph-osd -i $osd $ARGS --mkfs --mkkey --osd-uuid $uuid | |
585 | ||
586 | local key_fn=$CEPH_DEV_DIR/osd$osd/keyring | |
587 | echo adding osd$osd key to auth repository | |
588 | ceph_adm -i "$key_fn" auth add osd.$osd osd "allow *" mon "allow profile osd" mgr "allow profile osd" | |
589 | fi | |
590 | echo start osd$osd | |
591 | run 'osd' $SUDO $CEPH_BIN/ceph-osd -i $osd $ARGS $COSD_ARGS | |
592 | done | |
593 | } | |
594 | ||
595 | start_mgr() { | |
596 | local mgr=0 | |
597 | for name in x y z a b c d e f g h i j k l m n o p | |
598 | do | |
599 | [ $mgr -eq $CEPH_NUM_MGR ] && break | |
600 | mgr=$(($mgr + 1)) | |
601 | if [ "$new" -eq 1 ]; then | |
602 | mkdir -p $CEPH_DEV_DIR/mgr.$name | |
603 | key_fn=$CEPH_DEV_DIR/mgr.$name/keyring | |
604 | $SUDO $CEPH_BIN/ceph-authtool --create-keyring --gen-key --name=mgr.$name $key_fn | |
605 | ceph_adm -i $key_fn auth add mgr.$name mon 'allow profile mgr' mds 'allow *' osd 'allow *' | |
606 | fi | |
607 | ||
608 | wconf <<EOF | |
609 | [mgr.$name] | |
610 | host = $HOSTNAME | |
611 | EOF | |
612 | ||
613 | echo "Starting mgr.${name}" | |
614 | run 'mgr' $CEPH_BIN/ceph-mgr -i $name $ARGS | |
615 | done | |
616 | } | |
617 | ||
618 | start_mds() { | |
619 | if [ $new -eq 1 ]; then | |
620 | if [ "$CEPH_NUM_FS" -gt "0" ] ; then | |
621 | if [ "$CEPH_NUM_FS" -gt "1" ] ; then | |
622 | ceph_adm fs flag set enable_multiple true --yes-i-really-mean-it | |
623 | fi | |
624 | ||
625 | local fs=0 | |
626 | for name in a b c d e f g h i j k l m n o p | |
627 | do | |
628 | ceph_adm osd pool create "cephfs_data_${name}" 8 | |
629 | ceph_adm osd pool create "cephfs_metadata_${name}" 8 | |
630 | ceph_adm fs new "cephfs_${name}" "cephfs_metadata_${name}" "cephfs_data_${name}" | |
631 | fs=$(($fs + 1)) | |
632 | [ $fs -eq $CEPH_NUM_FS ] && break | |
633 | done | |
634 | fi | |
635 | fi | |
636 | ||
637 | local mds=0 | |
638 | for name in a b c d e f g h i j k l m n o p | |
639 | do | |
640 | [ $mds -eq $CEPH_NUM_MDS ] && break | |
641 | mds=$(($mds + 1)) | |
642 | ||
643 | if [ "$new" -eq 1 ]; then | |
644 | prun mkdir -p "$CEPH_DEV_DIR/mds.$name" | |
645 | key_fn=$CEPH_DEV_DIR/mds.$name/keyring | |
646 | wconf <<EOF | |
647 | [mds.$name] | |
648 | host = $HOSTNAME | |
649 | EOF | |
650 | if [ "$standby" -eq 1 ]; then | |
651 | mkdir -p $CEPH_DEV_DIR/mds.${name}s | |
652 | wconf <<EOF | |
653 | mds standby for rank = $mds | |
654 | [mds.${name}s] | |
655 | mds standby replay = true | |
656 | mds standby for name = ${name} | |
657 | EOF | |
658 | fi | |
659 | prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name="mds.$name" "$key_fn" | |
660 | ceph_adm -i "$key_fn" auth add "mds.$name" mon 'allow profile mds' osd 'allow *' mds 'allow' mgr 'allow profile mds' | |
661 | if [ "$standby" -eq 1 ]; then | |
662 | prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name="mds.${name}s" \ | |
663 | "$CEPH_DEV_DIR/mds.${name}s/keyring" | |
664 | ceph_adm -i "$CEPH_DEV_DIR/mds.${name}s/keyring" auth add "mds.${name}s" \ | |
665 | mon 'allow profile mds' osd 'allow *' mds 'allow' mgr 'allow profile mds' | |
666 | fi | |
667 | fi | |
668 | ||
669 | run 'mds' $CEPH_BIN/ceph-mds -i $name $ARGS $CMDS_ARGS | |
670 | if [ "$standby" -eq 1 ]; then | |
671 | run 'mds' $CEPH_BIN/ceph-mds -i ${name}s $ARGS $CMDS_ARGS | |
672 | fi | |
673 | ||
674 | #valgrind --tool=massif $CEPH_BIN/ceph-mds $ARGS --mds_log_max_segments 2 --mds_thrash_fragments 0 --mds_thrash_exports 0 > m #--debug_ms 20 | |
675 | #$CEPH_BIN/ceph-mds -d $ARGS --mds_thrash_fragments 0 --mds_thrash_exports 0 #--debug_ms 20 | |
676 | #ceph_adm mds set max_mds 2 | |
677 | done | |
678 | } | |
679 | ||
680 | if [ "$debug" -eq 0 ]; then | |
681 | CMONDEBUG=' | |
682 | debug mon = 10 | |
683 | debug ms = 1' | |
684 | COSDDEBUG=' | |
685 | debug ms = 1' | |
686 | CMDSDEBUG=' | |
687 | debug ms = 1' | |
688 | else | |
689 | echo "** going verbose **" | |
690 | CMONDEBUG=' | |
691 | debug mon = 20 | |
692 | debug paxos = 20 | |
693 | debug auth = 20 | |
694 | debug mgrc = 20 | |
695 | debug ms = 1' | |
696 | COSDDEBUG=' | |
697 | debug ms = 1 | |
698 | debug osd = 25 | |
699 | debug objecter = 20 | |
700 | debug monc = 20 | |
701 | debug mgrc = 20 | |
702 | debug journal = 20 | |
703 | debug filestore = 20 | |
704 | debug bluestore = 30 | |
705 | debug bluefs = 20 | |
706 | debug rocksdb = 10 | |
707 | debug bdev = 20 | |
708 | debug rgw = 20 | |
709 | debug objclass = 20' | |
710 | CMDSDEBUG=' | |
711 | debug ms = 1 | |
712 | debug mds = 20 | |
713 | debug auth = 20 | |
714 | debug monc = 20 | |
715 | debug mgrc = 20 | |
716 | mds debug scatterstat = true | |
717 | mds verify scatter = true | |
718 | mds log max segments = 2' | |
719 | CMGRDEBUG=' | |
720 | debug ms = 1 | |
721 | debug monc = 20 | |
722 | debug mgr = 20' | |
723 | fi | |
724 | ||
725 | if [ -n "$MON_ADDR" ]; then | |
726 | CMON_ARGS=" -m "$MON_ADDR | |
727 | COSD_ARGS=" -m "$MON_ADDR | |
728 | CMDS_ARGS=" -m "$MON_ADDR | |
729 | fi | |
730 | ||
731 | if [ "$memstore" -eq 1 ]; then | |
732 | COSDMEMSTORE=' | |
733 | osd objectstore = memstore' | |
734 | fi | |
735 | if [ "$bluestore" -eq 1 ]; then | |
736 | COSDMEMSTORE=' | |
737 | osd objectstore = bluestore' | |
738 | fi | |
739 | ||
740 | if [ -z "$CEPH_PORT" ]; then | |
741 | CEPH_PORT=6789 | |
742 | [ -e ".ceph_port" ] && CEPH_PORT=`cat .ceph_port` | |
743 | fi | |
744 | ||
745 | [ -z "$INIT_CEPH" ] && INIT_CEPH=$CEPH_BIN/init-ceph | |
746 | ||
747 | # sudo if btrfs | |
748 | test -d $CEPH_DEV_DIR/osd0/. && test -e $CEPH_DEV_DIR/sudo && SUDO="sudo" | |
749 | ||
750 | prun $SUDO rm -f core* | |
751 | ||
752 | test -d $CEPH_OUT_DIR || mkdir $CEPH_OUT_DIR | |
753 | test -d $CEPH_DEV_DIR || mkdir $CEPH_DEV_DIR | |
754 | $SUDO rm -rf $CEPH_OUT_DIR/* | |
755 | test -d gmon && $SUDO rm -rf gmon/* | |
756 | ||
757 | [ "$cephx" -eq 1 ] && [ "$new" -eq 1 ] && test -e $keyring_fn && rm $keyring_fn | |
758 | ||
759 | ||
760 | # figure machine's ip | |
761 | HOSTNAME=`hostname -s` | |
762 | if [ -n "$ip" ]; then | |
763 | IP="$ip" | |
764 | else | |
765 | echo hostname $HOSTNAME | |
766 | if [ -x "$(which ip 2>/dev/null)" ]; then | |
767 | IP_CMD="ip addr" | |
768 | else | |
769 | IP_CMD="ifconfig" | |
770 | fi | |
771 | # filter out IPv6 and localhost addresses | |
772 | IP="$($IP_CMD | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p' | head -n1)" | |
773 | # if nothing left, try using localhost address, it might work | |
774 | if [ -z "$IP" ]; then IP="127.0.0.1"; fi | |
775 | fi | |
776 | echo "ip $IP" | |
777 | echo "port $CEPH_PORT" | |
778 | ||
779 | ||
780 | [ -z $CEPH_ADM ] && CEPH_ADM=$CEPH_BIN/ceph | |
781 | ||
782 | ceph_adm() { | |
783 | if [ "$cephx" -eq 1 ]; then | |
784 | prun $SUDO "$CEPH_ADM" -c "$conf_fn" -k "$keyring_fn" "$@" | |
785 | else | |
786 | prun $SUDO "$CEPH_ADM" -c "$conf_fn" "$@" | |
787 | fi | |
788 | } | |
789 | ||
790 | if [ "$new" -eq 1 ]; then | |
791 | prepare_conf | |
792 | fi | |
793 | ||
794 | if [ $CEPH_NUM_MON -gt 0 ]; then | |
795 | start_mon | |
796 | fi | |
797 | ||
798 | # osd | |
799 | if [ $CEPH_NUM_OSD -gt 0 ]; then | |
800 | start_osd | |
801 | fi | |
802 | ||
803 | # mds | |
804 | if [ "$smallmds" -eq 1 ]; then | |
805 | wconf <<EOF | |
806 | [mds] | |
807 | mds log max segments = 2 | |
808 | mds cache size = 10000 | |
809 | EOF | |
810 | fi | |
811 | ||
812 | if [ $CEPH_NUM_MDS -gt 0 ]; then | |
813 | start_mds | |
814 | fi | |
815 | ||
816 | # Don't set max_mds until all the daemons are started, otherwise | |
817 | # the intended standbys might end up in active roles. | |
818 | if [ "$CEPH_MAX_MDS" -gt 1 ]; then | |
819 | sleep 5 # wait for daemons to make it into FSMap before increasing max_mds | |
820 | fi | |
821 | fs=0 | |
822 | for name in a b c d e f g h i j k l m n o p | |
823 | do | |
824 | [ $fs -eq $CEPH_NUM_FS ] && break | |
825 | fs=$(($fs + 1)) | |
826 | if [ "$CEPH_MAX_MDS" -gt 1 ]; then | |
827 | ceph_adm fs set "cephfs_${name}" allow_multimds true --yes-i-really-mean-it | |
828 | ceph_adm fs set "cephfs_${name}" max_mds "$CEPH_MAX_MDS" | |
829 | fi | |
830 | done | |
831 | ||
832 | # mgr | |
833 | ||
834 | if [ $CEPH_NUM_MGR -gt 0 ]; then | |
835 | start_mgr | |
836 | fi | |
837 | ||
838 | if [ "$ec" -eq 1 ]; then | |
839 | ceph_adm <<EOF | |
840 | osd erasure-code-profile set ec-profile m=2 k=2 | |
841 | osd pool create ec 8 8 erasure ec-profile | |
842 | EOF | |
843 | fi | |
844 | ||
845 | do_cache() { | |
846 | while [ -n "$*" ]; do | |
847 | p="$1" | |
848 | shift | |
849 | echo "creating cache for pool $p ..." | |
850 | ceph_adm <<EOF | |
851 | osd pool create ${p}-cache 8 | |
852 | osd tier add $p ${p}-cache | |
853 | osd tier cache-mode ${p}-cache writeback | |
854 | osd tier set-overlay $p ${p}-cache | |
855 | EOF | |
856 | done | |
857 | } | |
858 | do_cache $cache | |
859 | ||
860 | do_hitsets() { | |
861 | while [ -n "$*" ]; do | |
862 | pool="$1" | |
863 | type="$2" | |
864 | shift | |
865 | shift | |
866 | echo "setting hit_set on pool $pool type $type ..." | |
867 | ceph_adm <<EOF | |
868 | osd pool set $pool hit_set_type $type | |
869 | osd pool set $pool hit_set_count 8 | |
870 | osd pool set $pool hit_set_period 30 | |
871 | EOF | |
872 | done | |
873 | } | |
874 | do_hitsets $hitset | |
875 | ||
876 | do_rgw() | |
877 | { | |
878 | # Create S3 user | |
879 | local akey='0555b35654ad1656d804' | |
880 | local skey='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==' | |
881 | echo "setting up user testid" | |
882 | $CEPH_BIN/radosgw-admin user create --uid testid --access-key $akey --secret $skey --display-name 'M. Tester' --email tester@ceph.com -c $conf_fn > /dev/null | |
883 | ||
884 | # Create S3-test users | |
885 | # See: https://github.com/ceph/s3-tests | |
886 | echo "setting up s3-test users" | |
887 | $CEPH_BIN/radosgw-admin user create \ | |
888 | --uid 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef \ | |
889 | --access-key ABCDEFGHIJKLMNOPQRST \ | |
890 | --secret abcdefghijklmnopqrstuvwxyzabcdefghijklmn \ | |
891 | --display-name youruseridhere \ | |
892 | --email s3@example.com -c $conf_fn > /dev/null | |
893 | $CEPH_BIN/radosgw-admin user create \ | |
894 | --uid 56789abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234 \ | |
895 | --access-key NOPQRSTUVWXYZABCDEFG \ | |
896 | --secret nopqrstuvwxyzabcdefghijklmnabcdefghijklm \ | |
897 | --display-name john.doe \ | |
898 | --email john.doe@example.com -c $conf_fn > /dev/null | |
899 | ||
900 | # Create Swift user | |
901 | echo "setting up user tester" | |
902 | $CEPH_BIN/radosgw-admin user create -c $conf_fn --subuser=test:tester --display-name=Tester-Subuser --key-type=swift --secret=testing --access=full > /dev/null | |
903 | ||
904 | echo "" | |
905 | echo "S3 User Info:" | |
906 | echo " access key: $akey" | |
907 | echo " secret key: $skey" | |
908 | echo "" | |
909 | echo "Swift User Info:" | |
910 | echo " account : test" | |
911 | echo " user : tester" | |
912 | echo " password : testing" | |
913 | echo "" | |
914 | ||
915 | # Start server | |
916 | echo start rgw on http://localhost:$CEPH_RGW_PORT | |
917 | RGWDEBUG="" | |
918 | if [ "$debug" -ne 0 ]; then | |
919 | RGWDEBUG="--debug-rgw=20" | |
920 | fi | |
921 | ||
922 | RGWSUDO= | |
923 | [ $CEPH_RGW_PORT -lt 1024 ] && RGWSUDO=sudo | |
924 | n=$(($CEPH_NUM_RGW - 1)) | |
925 | for rgw in `seq 0 $n`; do | |
926 | run 'rgw' $RGWSUDO $CEPH_BIN/radosgw -c $conf_fn --log-file=${CEPH_OUT_DIR}/rgw.$rgw.log ${RGWDEBUG} --debug-ms=1 | |
927 | done | |
928 | } | |
929 | if [ "$CEPH_NUM_RGW" -gt 0 ]; then | |
930 | do_rgw | |
931 | fi | |
932 | ||
933 | echo "started. stop.sh to stop. see out/* (e.g. 'tail -f out/????') for debug output." | |
934 | ||
935 | echo "" | |
936 | echo "export PYTHONPATH=./pybind:$PYTHONPATH" | |
937 | echo "export LD_LIBRARY_PATH=$CEPH_LIB" | |
938 | ||
939 | if [ "$CEPH_DIR" != "$PWD" ]; then | |
940 | echo "export CEPH_CONF=$conf_fn" | |
941 | echo "export CEPH_KEYRING=$keyring_fn" | |
942 | fi |