]>
Commit | Line | Data |
---|---|---|
31f18b77 | 1 | #!/bin/bash |
7c673cae FG |
2 | |
3 | # abort on failure | |
4 | set -e | |
5 | ||
6 | if [ -n "$VSTART_DEST" ]; then | |
7 | SRC_PATH=`dirname $0` | |
8 | SRC_PATH=`(cd $SRC_PATH; pwd)` | |
9 | ||
10 | CEPH_DIR=$SRC_PATH | |
11 | CEPH_BIN=${PWD}/bin | |
12 | CEPH_LIB=${PWD}/lib | |
13 | ||
14 | CEPH_CONF_PATH=$VSTART_DEST | |
15 | CEPH_DEV_DIR=$VSTART_DEST/dev | |
16 | CEPH_OUT_DIR=$VSTART_DEST/out | |
17 | fi | |
18 | ||
19 | # for running out of the CMake build directory | |
20 | if [ -e CMakeCache.txt ]; then | |
21 | # Out of tree build, learn source location from CMakeCache.txt | |
22 | CEPH_ROOT=`grep ceph_SOURCE_DIR CMakeCache.txt | cut -d "=" -f 2` | |
23 | CEPH_BUILD_DIR=`pwd` | |
24 | [ -z "$MGR_PYTHON_PATH" ] && MGR_PYTHON_PATH=$CEPH_ROOT/src/pybind/mgr | |
25 | fi | |
26 | ||
27 | # use CEPH_BUILD_ROOT to vstart from a 'make install' | |
28 | if [ -n "$CEPH_BUILD_ROOT" ]; then | |
29 | [ -z "$CEPH_BIN" ] && CEPH_BIN=$CEPH_BUILD_ROOT/bin | |
30 | [ -z "$CEPH_LIB" ] && CEPH_LIB=$CEPH_BUILD_ROOT/lib | |
31 | [ -z "$EC_PATH" ] && EC_PATH=$CEPH_LIB/erasure-code | |
32 | [ -z "$OBJCLASS_PATH" ] && OBJCLASS_PATH=$CEPH_LIB/rados-classes | |
33 | elif [ -n "$CEPH_ROOT" ]; then | |
34 | [ -z "$PYBIND" ] && PYBIND=$CEPH_ROOT/src/pybind | |
35 | [ -z "$CEPH_BIN" ] && CEPH_BIN=$CEPH_BUILD_DIR/bin | |
36 | [ -z "$CEPH_ADM" ] && CEPH_ADM=$CEPH_BIN/ceph | |
37 | [ -z "$INIT_CEPH" ] && INIT_CEPH=$CEPH_BUILD_DIR/bin/init-ceph | |
38 | [ -z "$CEPH_LIB" ] && CEPH_LIB=$CEPH_BUILD_DIR/lib | |
39 | [ -z "$OBJCLASS_PATH" ] && OBJCLASS_PATH=$CEPH_LIB | |
40 | [ -z "$EC_PATH" ] && EC_PATH=$CEPH_LIB | |
41 | fi | |
42 | ||
43 | if [ -z "${CEPH_VSTART_WRAPPER}" ]; then | |
44 | PATH=$(pwd):$PATH | |
45 | fi | |
46 | ||
47 | [ -z "$PYBIND" ] && PYBIND=./pybind | |
48 | ||
49 | export PYTHONPATH=$PYBIND:$CEPH_LIB/cython_modules/lib.2:$PYTHONPATH | |
50 | export LD_LIBRARY_PATH=$CEPH_LIB:$LD_LIBRARY_PATH | |
51 | export DYLD_LIBRARY_PATH=$CEPH_LIB:$DYLD_LIBRARY_PATH | |
31f18b77 FG |
52 | # Suppress logging for regular use that indicated that we are using a |
53 | # development version. vstart.sh is only used during testing and | |
54 | # development | |
55 | export CEPH_DEV=1 | |
7c673cae FG |
56 | |
57 | [ -z "$CEPH_NUM_MON" ] && CEPH_NUM_MON="$MON" | |
58 | [ -z "$CEPH_NUM_OSD" ] && CEPH_NUM_OSD="$OSD" | |
59 | [ -z "$CEPH_NUM_MDS" ] && CEPH_NUM_MDS="$MDS" | |
60 | [ -z "$CEPH_NUM_MGR" ] && CEPH_NUM_MGR="$MGR" | |
61 | [ -z "$CEPH_NUM_FS" ] && CEPH_NUM_FS="$FS" | |
62 | [ -z "$CEPH_NUM_RGW" ] && CEPH_NUM_RGW="$RGW" | |
63 | ||
64 | # if none of the CEPH_NUM_* number is specified, kill the existing | |
65 | # cluster. | |
66 | if [ -z "$CEPH_NUM_MON" -a \ | |
67 | -z "$CEPH_NUM_OSD" -a \ | |
68 | -z "$CEPH_NUM_MDS" -a \ | |
69 | -z "$CEPH_NUM_MGR" ]; then | |
70 | kill_all=1 | |
71 | else | |
72 | kill_all=0 | |
73 | fi | |
74 | ||
75 | [ -z "$CEPH_NUM_MON" ] && CEPH_NUM_MON=3 | |
76 | [ -z "$CEPH_NUM_OSD" ] && CEPH_NUM_OSD=3 | |
77 | [ -z "$CEPH_NUM_MDS" ] && CEPH_NUM_MDS=3 | |
78 | [ -z "$CEPH_NUM_MGR" ] && CEPH_NUM_MGR=1 | |
79 | [ -z "$CEPH_NUM_FS" ] && CEPH_NUM_FS=1 | |
80 | [ -z "$CEPH_MAX_MDS" ] && CEPH_MAX_MDS=1 | |
81 | [ -z "$CEPH_NUM_RGW" ] && CEPH_NUM_RGW=0 | |
82 | ||
83 | [ -z "$CEPH_DIR" ] && CEPH_DIR="$PWD" | |
84 | [ -z "$CEPH_DEV_DIR" ] && CEPH_DEV_DIR="$CEPH_DIR/dev" | |
85 | [ -z "$CEPH_OUT_DIR" ] && CEPH_OUT_DIR="$CEPH_DIR/out" | |
86 | [ -z "$CEPH_RGW_PORT" ] && CEPH_RGW_PORT=8000 | |
87 | [ -z "$CEPH_CONF_PATH" ] && CEPH_CONF_PATH=$CEPH_DIR | |
88 | ||
89 | if [ $CEPH_NUM_OSD -gt 3 ]; then | |
90 | OSD_POOL_DEFAULT_SIZE=3 | |
91 | else | |
92 | OSD_POOL_DEFAULT_SIZE=$CEPH_NUM_OSD | |
93 | fi | |
94 | ||
95 | extra_conf="" | |
96 | new=0 | |
97 | standby=0 | |
98 | debug=0 | |
99 | ip="" | |
100 | nodaemon=0 | |
101 | smallmds=0 | |
102 | short=0 | |
103 | ec=0 | |
104 | hitset="" | |
105 | overwrite_conf=1 | |
106 | cephx=1 #turn cephx on by default | |
107 | cache="" | |
108 | memstore=0 | |
109 | bluestore=0 | |
110 | rgw_frontend="civetweb" | |
111 | lockdep=${LOCKDEP:-1} | |
112 | ||
31f18b77 FG |
113 | filestore_path= |
114 | ||
7c673cae FG |
115 | VSTART_SEC="client.vstart.sh" |
116 | ||
117 | MON_ADDR="" | |
31f18b77 FG |
118 | DASH_URLS="" |
119 | RESTFUL_URLS="" | |
7c673cae FG |
120 | |
121 | conf_fn="$CEPH_CONF_PATH/ceph.conf" | |
122 | keyring_fn="$CEPH_CONF_PATH/keyring" | |
123 | osdmap_fn="/tmp/ceph_osdmap.$$" | |
124 | monmap_fn="/tmp/ceph_monmap.$$" | |
125 | ||
126 | usage="usage: $0 [option]... \nex: $0 -n -d --mon_num 3 --osd_num 3 --mds_num 1 --rgw_num 1\n" | |
127 | usage=$usage"options:\n" | |
128 | usage=$usage"\t-d, --debug\n" | |
129 | usage=$usage"\t-s, --standby_mds: Generate standby-replay MDS for each active\n" | |
130 | usage=$usage"\t-l, --localhost: use localhost instead of hostname\n" | |
131 | usage=$usage"\t-i <ip>: bind to specific ip\n" | |
132 | usage=$usage"\t-n, --new\n" | |
133 | usage=$usage"\t-N, --not-new: reuse existing cluster config (default)\n" | |
134 | usage=$usage"\t--valgrind[_{osd,mds,mon,rgw}] 'toolname args...'\n" | |
135 | usage=$usage"\t--nodaemon: use ceph-run as wrapper for mon/osd/mds\n" | |
136 | usage=$usage"\t--smallmds: limit mds cache size\n" | |
137 | usage=$usage"\t-m ip:port\t\tspecify monitor address\n" | |
138 | usage=$usage"\t-k keep old configuration files\n" | |
139 | usage=$usage"\t-x enable cephx (on by default)\n" | |
140 | usage=$usage"\t-X disable cephx\n" | |
141 | usage=$usage"\t--hitset <pool> <hit_set_type>: enable hitset tracking\n" | |
142 | usage=$usage"\t-e : create an erasure pool\n"; | |
143 | usage=$usage"\t-o config\t\t add extra config parameters to all sections\n" | |
144 | usage=$usage"\t--mon_num specify ceph monitor count\n" | |
145 | usage=$usage"\t--osd_num specify ceph osd count\n" | |
146 | usage=$usage"\t--mds_num specify ceph mds count\n" | |
147 | usage=$usage"\t--rgw_num specify ceph rgw count\n" | |
148 | usage=$usage"\t--mgr_num specify ceph mgr count\n" | |
149 | usage=$usage"\t--rgw_port specify ceph rgw http listen port\n" | |
150 | usage=$usage"\t--rgw_frontend specify the rgw frontend configuration\n" | |
151 | usage=$usage"\t-b, --bluestore use bluestore as the osd objectstore backend\n" | |
152 | usage=$usage"\t--memstore use memstore as the osd objectstore backend\n" | |
153 | usage=$usage"\t--cache <pool>: enable cache tiering on pool\n" | |
154 | usage=$usage"\t--short: short object names only; necessary for ext4 dev\n" | |
155 | usage=$usage"\t--nolockdep disable lockdep\n" | |
156 | usage=$usage"\t--multimds <count> allow multimds with maximum active count\n" | |
157 | ||
158 | usage_exit() { | |
159 | printf "$usage" | |
160 | exit | |
161 | } | |
162 | ||
163 | while [ $# -ge 1 ]; do | |
164 | case $1 in | |
165 | -d | --debug ) | |
166 | debug=1 | |
167 | ;; | |
168 | -s | --standby_mds) | |
169 | standby=1 | |
170 | ;; | |
171 | -l | --localhost ) | |
172 | ip="127.0.0.1" | |
173 | ;; | |
174 | -i ) | |
175 | [ -z "$2" ] && usage_exit | |
176 | ip="$2" | |
177 | shift | |
178 | ;; | |
179 | -e ) | |
180 | ec=1 | |
181 | ;; | |
182 | --new | -n ) | |
183 | new=1 | |
184 | ;; | |
185 | --not-new | -N ) | |
186 | new=0 | |
187 | ;; | |
188 | --short ) | |
189 | short=1 | |
190 | ;; | |
191 | --valgrind ) | |
192 | [ -z "$2" ] && usage_exit | |
193 | valgrind=$2 | |
194 | shift | |
195 | ;; | |
196 | --valgrind_args ) | |
197 | valgrind_args="$2" | |
198 | shift | |
199 | ;; | |
200 | --valgrind_mds ) | |
201 | [ -z "$2" ] && usage_exit | |
202 | valgrind_mds=$2 | |
203 | shift | |
204 | ;; | |
205 | --valgrind_osd ) | |
206 | [ -z "$2" ] && usage_exit | |
207 | valgrind_osd=$2 | |
208 | shift | |
209 | ;; | |
210 | --valgrind_mon ) | |
211 | [ -z "$2" ] && usage_exit | |
212 | valgrind_mon=$2 | |
213 | shift | |
214 | ;; | |
215 | --valgrind_mgr ) | |
216 | [ -z "$2" ] && usage_exit | |
217 | valgrind_mgr=$2 | |
218 | shift | |
219 | ;; | |
220 | --valgrind_rgw ) | |
221 | [ -z "$2" ] && usage_exit | |
222 | valgrind_rgw=$2 | |
223 | shift | |
224 | ;; | |
225 | --nodaemon ) | |
226 | nodaemon=1 | |
227 | ;; | |
228 | --smallmds ) | |
229 | smallmds=1 | |
230 | ;; | |
231 | --mon_num ) | |
232 | echo "mon_num:$2" | |
233 | CEPH_NUM_MON="$2" | |
234 | shift | |
235 | ;; | |
236 | --osd_num ) | |
237 | CEPH_NUM_OSD=$2 | |
238 | shift | |
239 | ;; | |
240 | --mds_num ) | |
241 | CEPH_NUM_MDS=$2 | |
242 | shift | |
243 | ;; | |
244 | --rgw_num ) | |
245 | CEPH_NUM_RGW=$2 | |
246 | shift | |
247 | ;; | |
248 | --mgr_num ) | |
249 | CEPH_NUM_MGR=$2 | |
250 | shift | |
251 | ;; | |
252 | --rgw_port ) | |
253 | CEPH_RGW_PORT=$2 | |
254 | shift | |
255 | ;; | |
256 | --rgw_frontend ) | |
257 | rgw_frontend=$2 | |
258 | shift | |
259 | ;; | |
31f18b77 FG |
260 | --filestore_path ) |
261 | filestore_path=$2 | |
262 | shift | |
263 | ;; | |
7c673cae FG |
264 | -m ) |
265 | [ -z "$2" ] && usage_exit | |
266 | MON_ADDR=$2 | |
267 | shift | |
268 | ;; | |
269 | -x ) | |
270 | cephx=1 # this is on be default, flag exists for historical consistency | |
271 | ;; | |
272 | -X ) | |
273 | cephx=0 | |
274 | ;; | |
275 | -k ) | |
276 | if [ ! -r $conf_fn ]; then | |
277 | echo "cannot use old configuration: $conf_fn not readable." >&2 | |
278 | exit | |
279 | fi | |
280 | overwrite_conf=0 | |
281 | ;; | |
282 | --memstore ) | |
283 | memstore=1 | |
284 | ;; | |
285 | -b | --bluestore ) | |
286 | bluestore=1 | |
287 | ;; | |
288 | --hitset ) | |
289 | hitset="$hitset $2 $3" | |
290 | shift | |
291 | shift | |
292 | ;; | |
293 | -o ) | |
294 | extra_conf="$extra_conf $2 | |
295 | " | |
296 | shift | |
297 | ;; | |
298 | --cache ) | |
299 | if [ -z "$cache" ]; then | |
300 | cache="$2" | |
301 | else | |
302 | cache="$cache $2" | |
303 | fi | |
304 | shift | |
305 | ;; | |
306 | --nolockdep ) | |
307 | lockdep=0 | |
308 | ;; | |
309 | --multimds) | |
310 | CEPH_MAX_MDS="$2" | |
311 | shift | |
312 | ;; | |
313 | * ) | |
314 | usage_exit | |
315 | esac | |
316 | shift | |
317 | done | |
318 | ||
319 | if [ $kill_all -eq 1 ]; then | |
320 | $SUDO $INIT_CEPH stop | |
321 | fi | |
322 | ||
323 | if [ "$overwrite_conf" -eq 0 ]; then | |
324 | MON=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC num_mon 2>/dev/null` && \ | |
325 | CEPH_NUM_MON="$MON" | |
326 | OSD=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC num_osd 2>/dev/null` && \ | |
327 | CEPH_NUM_OSD="$OSD" | |
328 | MDS=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC num_mds 2>/dev/null` && \ | |
329 | CEPH_NUM_MDS="$MDS" | |
330 | MGR=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC num_mgr 2>/dev/null` && \ | |
331 | CEPH_NUM_MGR="$MGR" | |
332 | RGW=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC num_rgw 2>/dev/null` && \ | |
333 | CEPH_NUM_RGW="$RGW" | |
334 | else | |
335 | if [ "$new" -ne 0 ]; then | |
336 | # only delete if -n | |
337 | [ -e "$conf_fn" ] && rm -- "$conf_fn" | |
338 | else | |
339 | # -k is implied... (doesn't make sense otherwise) | |
340 | overwrite_conf=0 | |
341 | fi | |
342 | fi | |
343 | ||
344 | ARGS="-c $conf_fn" | |
345 | ||
346 | prunb() { | |
347 | echo "$* &" | |
348 | "$@" & | |
349 | } | |
350 | ||
351 | prun() { | |
352 | echo "$*" | |
353 | "$@" | |
354 | } | |
355 | ||
356 | run() { | |
357 | type=$1 | |
358 | shift | |
359 | eval "valg=\$valgrind_$type" | |
360 | [ -z "$valg" ] && valg="$valgrind" | |
361 | ||
362 | if [ -n "$valg" ]; then | |
363 | prunb valgrind --tool="$valg" $valgrind_args "$@" -f | |
364 | sleep 1 | |
365 | else | |
366 | if [ "$nodaemon" -eq 0 ]; then | |
367 | prun "$@" | |
368 | else | |
369 | prunb ./ceph-run "$@" -f | |
370 | fi | |
371 | fi | |
372 | } | |
373 | ||
374 | wconf() { | |
375 | if [ "$overwrite_conf" -eq 1 ]; then | |
376 | cat >> "$conf_fn" | |
377 | fi | |
378 | } | |
379 | ||
380 | prepare_conf() { | |
381 | local DAEMONOPTS=" | |
382 | log file = $CEPH_OUT_DIR/\$name.log | |
383 | admin socket = $CEPH_OUT_DIR/\$name.asok | |
384 | chdir = \"\" | |
385 | pid file = $CEPH_OUT_DIR/\$name.pid | |
386 | heartbeat file = $CEPH_OUT_DIR/\$name.heartbeat | |
387 | " | |
388 | ||
389 | wconf <<EOF | |
390 | ; generated by vstart.sh on `date` | |
391 | [$VSTART_SEC] | |
392 | num mon = $CEPH_NUM_MON | |
393 | num osd = $CEPH_NUM_OSD | |
394 | num mds = $CEPH_NUM_MDS | |
395 | num mgr = $CEPH_NUM_MGR | |
396 | num rgw = $CEPH_NUM_RGW | |
397 | ||
398 | [global] | |
399 | fsid = $(uuidgen) | |
400 | osd pg bits = 3 | |
401 | osd pgp bits = 5 ; (invalid, but ceph should cope!) | |
402 | osd pool default size = $OSD_POOL_DEFAULT_SIZE | |
403 | osd crush chooseleaf type = 0 | |
404 | osd pool default min size = 1 | |
405 | osd failsafe full ratio = .99 | |
406 | mon osd reporter subtree level = osd | |
407 | mon osd full ratio = .99 | |
408 | mon data avail warn = 10 | |
409 | mon data avail crit = 1 | |
410 | erasure code dir = $EC_PATH | |
411 | plugin dir = $CEPH_LIB | |
412 | osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 ruleset-failure-domain=osd | |
413 | rgw frontends = $rgw_frontend port=$CEPH_RGW_PORT | |
31f18b77 FG |
414 | ; needed for s3tests |
415 | rgw enable static website = 1 | |
416 | rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo= | |
417 | rgw crypt require ssl = false | |
418 | rgw lc debug interval = 10 | |
7c673cae FG |
419 | filestore fd cache size = 32 |
420 | run dir = $CEPH_OUT_DIR | |
421 | enable experimental unrecoverable data corrupting features = * | |
422 | EOF | |
423 | if [ "$lockdep" -eq 1 ] ; then | |
424 | wconf <<EOF | |
425 | lockdep = true | |
426 | EOF | |
427 | fi | |
428 | if [ "$cephx" -eq 1 ] ; then | |
429 | wconf <<EOF | |
430 | auth cluster required = cephx | |
431 | auth service required = cephx | |
432 | auth client required = cephx | |
433 | EOF | |
434 | else | |
435 | wconf <<EOF | |
436 | auth cluster required = none | |
437 | auth service required = none | |
438 | auth client required = none | |
439 | EOF | |
440 | fi | |
441 | if [ "$short" -eq 1 ]; then | |
442 | COSDSHORT=" osd max object name len = 460 | |
443 | osd max object namespace len = 64" | |
444 | fi | |
445 | wconf <<EOF | |
446 | [client] | |
447 | keyring = $keyring_fn | |
448 | log file = $CEPH_OUT_DIR/\$name.\$pid.log | |
449 | admin socket = $CEPH_OUT_DIR/\$name.\$pid.asok | |
450 | ||
451 | [mds] | |
452 | $DAEMONOPTS | |
453 | $CMDSDEBUG | |
454 | mds debug frag = true | |
455 | mds debug auth pins = true | |
456 | mds debug subtrees = true | |
457 | mds data = $CEPH_DEV_DIR/mds.\$id | |
458 | mds root ino uid = `id -u` | |
459 | mds root ino gid = `id -g` | |
460 | $extra_conf | |
461 | [mgr] | |
31f18b77 | 462 | mgr modules = restful fsstatus dashboard |
7c673cae FG |
463 | mgr data = $CEPH_DEV_DIR/mgr.\$id |
464 | mgr module path = $MGR_PYTHON_PATH | |
465 | mon reweight min pgs per osd = 4 | |
31f18b77 | 466 | mon pg warn min per osd = 3 |
7c673cae FG |
467 | $DAEMONOPTS |
468 | $CMGRDEBUG | |
469 | $extra_conf | |
470 | [osd] | |
471 | $DAEMONOPTS | |
472 | osd_check_max_object_name_len_on_startup = false | |
473 | osd data = $CEPH_DEV_DIR/osd\$id | |
474 | osd journal = $CEPH_DEV_DIR/osd\$id/journal | |
475 | osd journal size = 100 | |
476 | osd class tmp = out | |
477 | osd class dir = $OBJCLASS_PATH | |
478 | osd class load list = * | |
479 | osd class default list = * | |
480 | osd scrub load threshold = 2000.0 | |
481 | osd debug op order = true | |
482 | osd debug misdirected ops = true | |
483 | filestore wbthrottle xfs ios start flusher = 10 | |
484 | filestore wbthrottle xfs ios hard limit = 20 | |
485 | filestore wbthrottle xfs inodes hard limit = 30 | |
486 | filestore wbthrottle btrfs ios start flusher = 10 | |
487 | filestore wbthrottle btrfs ios hard limit = 20 | |
488 | filestore wbthrottle btrfs inodes hard limit = 30 | |
489 | osd copyfrom max chunk = 524288 | |
490 | bluestore fsck on mount = true | |
491 | bluestore block create = true | |
492 | bluestore block db size = 67108864 | |
493 | bluestore block db create = true | |
494 | bluestore block wal size = 1048576000 | |
495 | bluestore block wal create = true | |
496 | $COSDDEBUG | |
497 | $COSDMEMSTORE | |
498 | $COSDSHORT | |
499 | $extra_conf | |
500 | [mon] | |
501 | mon pg warn min per osd = 3 | |
502 | mon osd allow primary affinity = true | |
503 | mon osd allow pg upmap = true | |
504 | mon reweight min pgs per osd = 4 | |
505 | mon osd prime pg temp = true | |
506 | crushtool = $CEPH_BIN/crushtool | |
507 | mon allow pool delete = true | |
508 | $DAEMONOPTS | |
509 | $CMONDEBUG | |
510 | $extra_conf | |
511 | mon cluster log file = $CEPH_OUT_DIR/cluster.mon.\$id.log | |
512 | [global] | |
513 | $extra_conf | |
514 | EOF | |
515 | } | |
516 | ||
517 | start_mon() { | |
518 | local MONS="" | |
519 | local count=0 | |
520 | for f in a b c d e f g h i j k l m n o p q r s t u v w x y z | |
521 | do | |
522 | [ $count -eq $CEPH_NUM_MON ] && break; | |
523 | count=$(($count + 1)) | |
524 | if [ -z "$MONS" ]; | |
525 | then | |
526 | MONS="$f" | |
527 | else | |
528 | MONS="$MONS $f" | |
529 | fi | |
530 | done | |
531 | ||
532 | if [ "$new" -eq 1 ]; then | |
533 | if [ `echo $IP | grep '^127\\.'` ] | |
534 | then | |
535 | echo | |
536 | echo "NOTE: hostname resolves to loopback; remote hosts will not be able to" | |
537 | echo " connect. either adjust /etc/hosts, or edit this script to use your" | |
538 | echo " machine's real IP." | |
539 | echo | |
540 | fi | |
541 | ||
542 | prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name=mon. "$keyring_fn" --cap mon 'allow *' | |
543 | prun $SUDO "$CEPH_BIN/ceph-authtool" --gen-key --name=client.admin --set-uid=0 \ | |
544 | --cap mon 'allow *' \ | |
545 | --cap osd 'allow *' \ | |
546 | --cap mds 'allow *' \ | |
547 | --cap mgr 'allow *' \ | |
548 | "$keyring_fn" | |
549 | ||
550 | # build a fresh fs monmap, mon fs | |
551 | local str="" | |
552 | local count=0 | |
553 | for f in $MONS | |
554 | do | |
555 | str="$str --add $f $IP:$(($CEPH_PORT+$count))" | |
556 | wconf <<EOF | |
557 | [mon.$f] | |
558 | host = $HOSTNAME | |
559 | mon data = $CEPH_DEV_DIR/mon.$f | |
560 | mon addr = $IP:$(($CEPH_PORT+$count)) | |
561 | EOF | |
562 | count=$(($count + 1)) | |
563 | done | |
564 | prun "$CEPH_BIN/monmaptool" --create --clobber $str --print "$monmap_fn" | |
565 | ||
566 | for f in $MONS | |
567 | do | |
568 | prun rm -rf -- "$CEPH_DEV_DIR/mon.$f" | |
569 | prun mkdir -p "$CEPH_DEV_DIR/mon.$f" | |
570 | prun "$CEPH_BIN/ceph-mon" --mkfs -c "$conf_fn" -i "$f" --monmap="$monmap_fn" --keyring="$keyring_fn" | |
571 | done | |
572 | ||
573 | prun rm -- "$monmap_fn" | |
574 | fi | |
575 | ||
576 | # start monitors | |
577 | for f in $MONS | |
578 | do | |
579 | run 'mon' $CEPH_BIN/ceph-mon -i $f $ARGS $CMON_ARGS | |
580 | done | |
581 | } | |
582 | ||
583 | start_osd() { | |
584 | for osd in `seq 0 $((CEPH_NUM_OSD-1))` | |
585 | do | |
586 | if [ "$new" -eq 1 ]; then | |
587 | wconf <<EOF | |
588 | [osd.$osd] | |
589 | host = $HOSTNAME | |
590 | EOF | |
591 | ||
592 | rm -rf $CEPH_DEV_DIR/osd$osd || true | |
593 | if command -v btrfs > /dev/null; then | |
594 | for f in $CEPH_DEV_DIR/osd$osd/*; do btrfs sub delete $f &> /dev/null || true; done | |
595 | fi | |
31f18b77 FG |
596 | if [ -n "$filestore_path" ]; then |
597 | ln -s $filestore_path $CEPH_DEV_DIR/osd$osd | |
598 | else | |
599 | mkdir -p $CEPH_DEV_DIR/osd$osd | |
600 | fi | |
7c673cae FG |
601 | |
602 | local uuid=`uuidgen` | |
603 | echo "add osd$osd $uuid" | |
604 | ceph_adm osd create $uuid | |
605 | ceph_adm osd crush add osd.$osd 1.0 host=$HOSTNAME root=default | |
606 | $SUDO $CEPH_BIN/ceph-osd -i $osd $ARGS --mkfs --mkkey --osd-uuid $uuid | |
607 | ||
608 | local key_fn=$CEPH_DEV_DIR/osd$osd/keyring | |
609 | echo adding osd$osd key to auth repository | |
610 | ceph_adm -i "$key_fn" auth add osd.$osd osd "allow *" mon "allow profile osd" mgr "allow profile osd" | |
611 | fi | |
31f18b77 | 612 | echo start osd.$osd |
7c673cae FG |
613 | run 'osd' $SUDO $CEPH_BIN/ceph-osd -i $osd $ARGS $COSD_ARGS |
614 | done | |
615 | } | |
616 | ||
617 | start_mgr() { | |
618 | local mgr=0 | |
31f18b77 FG |
619 | # avoid monitors on nearby ports (which test/*.sh use extensively) |
620 | MGR_PORT=$(($CEPH_PORT + 1000)) | |
7c673cae FG |
621 | for name in x y z a b c d e f g h i j k l m n o p |
622 | do | |
623 | [ $mgr -eq $CEPH_NUM_MGR ] && break | |
624 | mgr=$(($mgr + 1)) | |
625 | if [ "$new" -eq 1 ]; then | |
626 | mkdir -p $CEPH_DEV_DIR/mgr.$name | |
627 | key_fn=$CEPH_DEV_DIR/mgr.$name/keyring | |
628 | $SUDO $CEPH_BIN/ceph-authtool --create-keyring --gen-key --name=mgr.$name $key_fn | |
629 | ceph_adm -i $key_fn auth add mgr.$name mon 'allow profile mgr' mds 'allow *' osd 'allow *' | |
630 | fi | |
631 | ||
632 | wconf <<EOF | |
633 | [mgr.$name] | |
634 | host = $HOSTNAME | |
635 | EOF | |
636 | ||
31f18b77 FG |
637 | ceph_adm config-key put mgr/dashboard/$name/server_addr $IP |
638 | ceph_adm config-key put mgr/dashboard/$name/server_port $MGR_PORT | |
639 | DASH_URLS+="http://$IP:$MGR_PORT/" | |
640 | MGR_PORT=$(($MGR_PORT + 1000)) | |
641 | ||
642 | ceph_adm config-key put mgr/restful/$name/server_addr $IP | |
643 | ceph_adm config-key put mgr/restful/$name/server_port $MGR_PORT | |
644 | ||
645 | RESTFUL_URLS+="https://$IP:$MGR_PORT" | |
646 | MGR_PORT=$(($MGR_PORT + 1000)) | |
647 | ||
7c673cae FG |
648 | echo "Starting mgr.${name}" |
649 | run 'mgr' $CEPH_BIN/ceph-mgr -i $name $ARGS | |
650 | done | |
31f18b77 FG |
651 | |
652 | SF=`mktemp` | |
653 | ceph_adm tell mgr restful create-self-signed-cert | |
654 | ceph_adm tell mgr restful create-key admin -o $SF | |
655 | RESTFUL_SECRET=`cat $SF` | |
656 | rm $SF | |
7c673cae FG |
657 | } |
658 | ||
659 | start_mds() { | |
660 | if [ $new -eq 1 ]; then | |
661 | if [ "$CEPH_NUM_FS" -gt "0" ] ; then | |
662 | if [ "$CEPH_NUM_FS" -gt "1" ] ; then | |
663 | ceph_adm fs flag set enable_multiple true --yes-i-really-mean-it | |
664 | fi | |
665 | ||
666 | local fs=0 | |
667 | for name in a b c d e f g h i j k l m n o p | |
668 | do | |
669 | ceph_adm osd pool create "cephfs_data_${name}" 8 | |
670 | ceph_adm osd pool create "cephfs_metadata_${name}" 8 | |
671 | ceph_adm fs new "cephfs_${name}" "cephfs_metadata_${name}" "cephfs_data_${name}" | |
672 | fs=$(($fs + 1)) | |
673 | [ $fs -eq $CEPH_NUM_FS ] && break | |
674 | done | |
675 | fi | |
676 | fi | |
677 | ||
678 | local mds=0 | |
679 | for name in a b c d e f g h i j k l m n o p | |
680 | do | |
681 | [ $mds -eq $CEPH_NUM_MDS ] && break | |
682 | mds=$(($mds + 1)) | |
683 | ||
684 | if [ "$new" -eq 1 ]; then | |
685 | prun mkdir -p "$CEPH_DEV_DIR/mds.$name" | |
686 | key_fn=$CEPH_DEV_DIR/mds.$name/keyring | |
687 | wconf <<EOF | |
688 | [mds.$name] | |
689 | host = $HOSTNAME | |
690 | EOF | |
691 | if [ "$standby" -eq 1 ]; then | |
692 | mkdir -p $CEPH_DEV_DIR/mds.${name}s | |
693 | wconf <<EOF | |
694 | mds standby for rank = $mds | |
695 | [mds.${name}s] | |
696 | mds standby replay = true | |
697 | mds standby for name = ${name} | |
698 | EOF | |
699 | fi | |
700 | prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name="mds.$name" "$key_fn" | |
701 | ceph_adm -i "$key_fn" auth add "mds.$name" mon 'allow profile mds' osd 'allow *' mds 'allow' mgr 'allow profile mds' | |
702 | if [ "$standby" -eq 1 ]; then | |
703 | prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name="mds.${name}s" \ | |
704 | "$CEPH_DEV_DIR/mds.${name}s/keyring" | |
705 | ceph_adm -i "$CEPH_DEV_DIR/mds.${name}s/keyring" auth add "mds.${name}s" \ | |
706 | mon 'allow profile mds' osd 'allow *' mds 'allow' mgr 'allow profile mds' | |
707 | fi | |
708 | fi | |
709 | ||
710 | run 'mds' $CEPH_BIN/ceph-mds -i $name $ARGS $CMDS_ARGS | |
711 | if [ "$standby" -eq 1 ]; then | |
712 | run 'mds' $CEPH_BIN/ceph-mds -i ${name}s $ARGS $CMDS_ARGS | |
713 | fi | |
714 | ||
715 | #valgrind --tool=massif $CEPH_BIN/ceph-mds $ARGS --mds_log_max_segments 2 --mds_thrash_fragments 0 --mds_thrash_exports 0 > m #--debug_ms 20 | |
716 | #$CEPH_BIN/ceph-mds -d $ARGS --mds_thrash_fragments 0 --mds_thrash_exports 0 #--debug_ms 20 | |
717 | #ceph_adm mds set max_mds 2 | |
718 | done | |
719 | } | |
720 | ||
721 | if [ "$debug" -eq 0 ]; then | |
722 | CMONDEBUG=' | |
723 | debug mon = 10 | |
724 | debug ms = 1' | |
725 | COSDDEBUG=' | |
726 | debug ms = 1' | |
727 | CMDSDEBUG=' | |
728 | debug ms = 1' | |
31f18b77 FG |
729 | CMGRDEBUG=' |
730 | debug ms = 1' | |
7c673cae FG |
731 | else |
732 | echo "** going verbose **" | |
733 | CMONDEBUG=' | |
734 | debug mon = 20 | |
735 | debug paxos = 20 | |
736 | debug auth = 20 | |
737 | debug mgrc = 20 | |
738 | debug ms = 1' | |
739 | COSDDEBUG=' | |
740 | debug ms = 1 | |
741 | debug osd = 25 | |
742 | debug objecter = 20 | |
743 | debug monc = 20 | |
744 | debug mgrc = 20 | |
745 | debug journal = 20 | |
746 | debug filestore = 20 | |
747 | debug bluestore = 30 | |
748 | debug bluefs = 20 | |
749 | debug rocksdb = 10 | |
750 | debug bdev = 20 | |
751 | debug rgw = 20 | |
752 | debug objclass = 20' | |
753 | CMDSDEBUG=' | |
754 | debug ms = 1 | |
755 | debug mds = 20 | |
756 | debug auth = 20 | |
757 | debug monc = 20 | |
758 | debug mgrc = 20 | |
759 | mds debug scatterstat = true | |
760 | mds verify scatter = true | |
761 | mds log max segments = 2' | |
762 | CMGRDEBUG=' | |
763 | debug ms = 1 | |
764 | debug monc = 20 | |
31f18b77 | 765 | debug mon = 20 |
7c673cae FG |
766 | debug mgr = 20' |
767 | fi | |
768 | ||
769 | if [ -n "$MON_ADDR" ]; then | |
770 | CMON_ARGS=" -m "$MON_ADDR | |
771 | COSD_ARGS=" -m "$MON_ADDR | |
772 | CMDS_ARGS=" -m "$MON_ADDR | |
773 | fi | |
774 | ||
775 | if [ "$memstore" -eq 1 ]; then | |
776 | COSDMEMSTORE=' | |
777 | osd objectstore = memstore' | |
778 | fi | |
779 | if [ "$bluestore" -eq 1 ]; then | |
780 | COSDMEMSTORE=' | |
781 | osd objectstore = bluestore' | |
782 | fi | |
783 | ||
784 | if [ -z "$CEPH_PORT" ]; then | |
785 | CEPH_PORT=6789 | |
786 | [ -e ".ceph_port" ] && CEPH_PORT=`cat .ceph_port` | |
787 | fi | |
788 | ||
789 | [ -z "$INIT_CEPH" ] && INIT_CEPH=$CEPH_BIN/init-ceph | |
790 | ||
791 | # sudo if btrfs | |
792 | test -d $CEPH_DEV_DIR/osd0/. && test -e $CEPH_DEV_DIR/sudo && SUDO="sudo" | |
793 | ||
794 | prun $SUDO rm -f core* | |
795 | ||
796 | test -d $CEPH_OUT_DIR || mkdir $CEPH_OUT_DIR | |
797 | test -d $CEPH_DEV_DIR || mkdir $CEPH_DEV_DIR | |
798 | $SUDO rm -rf $CEPH_OUT_DIR/* | |
799 | test -d gmon && $SUDO rm -rf gmon/* | |
800 | ||
801 | [ "$cephx" -eq 1 ] && [ "$new" -eq 1 ] && test -e $keyring_fn && rm $keyring_fn | |
802 | ||
803 | ||
804 | # figure machine's ip | |
805 | HOSTNAME=`hostname -s` | |
806 | if [ -n "$ip" ]; then | |
807 | IP="$ip" | |
808 | else | |
809 | echo hostname $HOSTNAME | |
810 | if [ -x "$(which ip 2>/dev/null)" ]; then | |
811 | IP_CMD="ip addr" | |
812 | else | |
813 | IP_CMD="ifconfig" | |
814 | fi | |
815 | # filter out IPv6 and localhost addresses | |
816 | IP="$($IP_CMD | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p' | head -n1)" | |
817 | # if nothing left, try using localhost address, it might work | |
818 | if [ -z "$IP" ]; then IP="127.0.0.1"; fi | |
819 | fi | |
820 | echo "ip $IP" | |
821 | echo "port $CEPH_PORT" | |
822 | ||
823 | ||
824 | [ -z $CEPH_ADM ] && CEPH_ADM=$CEPH_BIN/ceph | |
825 | ||
826 | ceph_adm() { | |
827 | if [ "$cephx" -eq 1 ]; then | |
828 | prun $SUDO "$CEPH_ADM" -c "$conf_fn" -k "$keyring_fn" "$@" | |
829 | else | |
830 | prun $SUDO "$CEPH_ADM" -c "$conf_fn" "$@" | |
831 | fi | |
832 | } | |
833 | ||
834 | if [ "$new" -eq 1 ]; then | |
835 | prepare_conf | |
836 | fi | |
837 | ||
838 | if [ $CEPH_NUM_MON -gt 0 ]; then | |
839 | start_mon | |
840 | fi | |
841 | ||
842 | # osd | |
843 | if [ $CEPH_NUM_OSD -gt 0 ]; then | |
844 | start_osd | |
845 | fi | |
846 | ||
847 | # mds | |
848 | if [ "$smallmds" -eq 1 ]; then | |
849 | wconf <<EOF | |
850 | [mds] | |
851 | mds log max segments = 2 | |
852 | mds cache size = 10000 | |
853 | EOF | |
854 | fi | |
855 | ||
856 | if [ $CEPH_NUM_MDS -gt 0 ]; then | |
857 | start_mds | |
858 | fi | |
859 | ||
860 | # Don't set max_mds until all the daemons are started, otherwise | |
861 | # the intended standbys might end up in active roles. | |
862 | if [ "$CEPH_MAX_MDS" -gt 1 ]; then | |
863 | sleep 5 # wait for daemons to make it into FSMap before increasing max_mds | |
864 | fi | |
865 | fs=0 | |
866 | for name in a b c d e f g h i j k l m n o p | |
867 | do | |
868 | [ $fs -eq $CEPH_NUM_FS ] && break | |
869 | fs=$(($fs + 1)) | |
870 | if [ "$CEPH_MAX_MDS" -gt 1 ]; then | |
871 | ceph_adm fs set "cephfs_${name}" allow_multimds true --yes-i-really-mean-it | |
872 | ceph_adm fs set "cephfs_${name}" max_mds "$CEPH_MAX_MDS" | |
873 | fi | |
874 | done | |
875 | ||
876 | # mgr | |
877 | ||
878 | if [ $CEPH_NUM_MGR -gt 0 ]; then | |
879 | start_mgr | |
880 | fi | |
881 | ||
882 | if [ "$ec" -eq 1 ]; then | |
883 | ceph_adm <<EOF | |
884 | osd erasure-code-profile set ec-profile m=2 k=2 | |
885 | osd pool create ec 8 8 erasure ec-profile | |
886 | EOF | |
887 | fi | |
888 | ||
889 | do_cache() { | |
890 | while [ -n "$*" ]; do | |
891 | p="$1" | |
892 | shift | |
893 | echo "creating cache for pool $p ..." | |
894 | ceph_adm <<EOF | |
895 | osd pool create ${p}-cache 8 | |
896 | osd tier add $p ${p}-cache | |
897 | osd tier cache-mode ${p}-cache writeback | |
898 | osd tier set-overlay $p ${p}-cache | |
899 | EOF | |
900 | done | |
901 | } | |
902 | do_cache $cache | |
903 | ||
904 | do_hitsets() { | |
905 | while [ -n "$*" ]; do | |
906 | pool="$1" | |
907 | type="$2" | |
908 | shift | |
909 | shift | |
910 | echo "setting hit_set on pool $pool type $type ..." | |
911 | ceph_adm <<EOF | |
912 | osd pool set $pool hit_set_type $type | |
913 | osd pool set $pool hit_set_count 8 | |
914 | osd pool set $pool hit_set_period 30 | |
915 | EOF | |
916 | done | |
917 | } | |
918 | do_hitsets $hitset | |
919 | ||
920 | do_rgw() | |
921 | { | |
922 | # Create S3 user | |
923 | local akey='0555b35654ad1656d804' | |
924 | local skey='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==' | |
925 | echo "setting up user testid" | |
926 | $CEPH_BIN/radosgw-admin user create --uid testid --access-key $akey --secret $skey --display-name 'M. Tester' --email tester@ceph.com -c $conf_fn > /dev/null | |
927 | ||
928 | # Create S3-test users | |
929 | # See: https://github.com/ceph/s3-tests | |
930 | echo "setting up s3-test users" | |
931 | $CEPH_BIN/radosgw-admin user create \ | |
932 | --uid 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef \ | |
933 | --access-key ABCDEFGHIJKLMNOPQRST \ | |
934 | --secret abcdefghijklmnopqrstuvwxyzabcdefghijklmn \ | |
935 | --display-name youruseridhere \ | |
936 | --email s3@example.com -c $conf_fn > /dev/null | |
937 | $CEPH_BIN/radosgw-admin user create \ | |
938 | --uid 56789abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234 \ | |
939 | --access-key NOPQRSTUVWXYZABCDEFG \ | |
940 | --secret nopqrstuvwxyzabcdefghijklmnabcdefghijklm \ | |
941 | --display-name john.doe \ | |
942 | --email john.doe@example.com -c $conf_fn > /dev/null | |
31f18b77 FG |
943 | $CEPH_BIN/radosgw-admin user create \ |
944 | --tenant testx \ | |
945 | --uid 9876543210abcdef0123456789abcdef0123456789abcdef0123456789abcdef \ | |
946 | --access-key HIJKLMNOPQRSTUVWXYZA \ | |
947 | --secret opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab \ | |
948 | --display-name tenanteduser \ | |
949 | --email tenanteduser@example.com -c $conf_fn > /dev/null | |
7c673cae FG |
950 | |
951 | # Create Swift user | |
952 | echo "setting up user tester" | |
953 | $CEPH_BIN/radosgw-admin user create -c $conf_fn --subuser=test:tester --display-name=Tester-Subuser --key-type=swift --secret=testing --access=full > /dev/null | |
954 | ||
955 | echo "" | |
956 | echo "S3 User Info:" | |
957 | echo " access key: $akey" | |
958 | echo " secret key: $skey" | |
959 | echo "" | |
960 | echo "Swift User Info:" | |
961 | echo " account : test" | |
962 | echo " user : tester" | |
963 | echo " password : testing" | |
964 | echo "" | |
965 | ||
966 | # Start server | |
967 | echo start rgw on http://localhost:$CEPH_RGW_PORT | |
968 | RGWDEBUG="" | |
969 | if [ "$debug" -ne 0 ]; then | |
970 | RGWDEBUG="--debug-rgw=20" | |
971 | fi | |
972 | ||
973 | RGWSUDO= | |
974 | [ $CEPH_RGW_PORT -lt 1024 ] && RGWSUDO=sudo | |
975 | n=$(($CEPH_NUM_RGW - 1)) | |
976 | for rgw in `seq 0 $n`; do | |
977 | run 'rgw' $RGWSUDO $CEPH_BIN/radosgw -c $conf_fn --log-file=${CEPH_OUT_DIR}/rgw.$rgw.log ${RGWDEBUG} --debug-ms=1 | |
978 | done | |
979 | } | |
980 | if [ "$CEPH_NUM_RGW" -gt 0 ]; then | |
981 | do_rgw | |
982 | fi | |
983 | ||
984 | echo "started. stop.sh to stop. see out/* (e.g. 'tail -f out/????') for debug output." | |
985 | ||
31f18b77 FG |
986 | echo "" |
987 | echo "dashboard urls: $DASH_URLS" | |
988 | echo " restful urls: $RESTFUL_URLS" | |
989 | echo " w/ user/pass: admin / $RESTFUL_SECRET" | |
7c673cae FG |
990 | echo "" |
991 | echo "export PYTHONPATH=./pybind:$PYTHONPATH" | |
992 | echo "export LD_LIBRARY_PATH=$CEPH_LIB" | |
993 | ||
994 | if [ "$CEPH_DIR" != "$PWD" ]; then | |
995 | echo "export CEPH_CONF=$conf_fn" | |
996 | echo "export CEPH_KEYRING=$keyring_fn" | |
997 | fi | |
31f18b77 FG |
998 | |
999 | echo "CEPH_DEV=1" | |
1000 |