1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
15 #include <sys/types.h>
22 #include "common/config.h"
23 #include "include/ceph_features.h"
25 #include "mon/MonMap.h"
26 #include "mon/Monitor.h"
27 #include "mon/MonitorDBStore.h"
28 #include "mon/MonClient.h"
30 #include "msg/Messenger.h"
32 #include "include/CompatSet.h"
34 #include "common/ceph_argparse.h"
35 #include "common/pick_address.h"
36 #include "common/Throttle.h"
37 #include "common/Timer.h"
38 #include "common/errno.h"
39 #include "common/Preforker.h"
41 #include "global/global_init.h"
42 #include "global/signal_handler.h"
44 #include "perfglue/heap_profiler.h"
46 #include "include/ceph_assert.h"
48 #define dout_subsys ceph_subsys_mon
54 using std::ostringstream
;
58 using ceph::bufferlist
;
61 using ceph::JSONFormatter
;
66 void handle_mon_signal(int signum
)
69 mon
->handle_signal(signum
);
73 int obtain_monmap(MonitorDBStore
&store
, bufferlist
&bl
)
75 dout(10) << __func__
<< dendl
;
77 * the monmap may be in one of three places:
78 * 'mon_sync:temp_newer_monmap' - stashed newer map for bootstrap
79 * 'monmap:<latest_version_no>' - the monmap we'd really like to have
80 * 'mon_sync:latest_monmap' - last monmap backed up for the last sync
81 * 'mkfs:monmap' - a monmap resulting from mkfs
84 if (store
.exists("monmap", "last_committed")) {
85 version_t latest_ver
= store
.get("monmap", "last_committed");
86 if (store
.exists("monmap", latest_ver
)) {
87 int err
= store
.get("monmap", latest_ver
, bl
);
88 ceph_assert(err
== 0);
89 ceph_assert(bl
.length() > 0);
90 dout(10) << __func__
<< " read last committed monmap ver "
91 << latest_ver
<< dendl
;
93 // see if there is stashed newer map (see bootstrap())
94 if (store
.exists("mon_sync", "temp_newer_monmap")) {
96 int err
= store
.get("mon_sync", "temp_newer_monmap", bl2
);
97 ceph_assert(err
== 0);
98 ceph_assert(bl2
.length() > 0);
101 if (b
.get_epoch() > latest_ver
) {
102 dout(10) << __func__
<< " using stashed monmap " << b
.get_epoch()
103 << " instead" << dendl
;
106 dout(10) << __func__
<< " ignoring stashed monmap " << b
.get_epoch()
114 if (store
.exists("mon_sync", "in_sync")
115 || store
.exists("mon_sync", "force_sync")) {
116 dout(10) << __func__
<< " detected aborted sync" << dendl
;
117 if (store
.exists("mon_sync", "latest_monmap")) {
118 int err
= store
.get("mon_sync", "latest_monmap", bl
);
119 ceph_assert(err
== 0);
120 ceph_assert(bl
.length() > 0);
121 dout(10) << __func__
<< " read backup monmap" << dendl
;
126 if (store
.exists("mon_sync", "temp_newer_monmap")) {
127 dout(10) << __func__
<< " found temp_newer_monmap" << dendl
;
128 int err
= store
.get("mon_sync", "temp_newer_monmap", bl
);
129 ceph_assert(err
== 0);
130 ceph_assert(bl
.length() > 0);
134 if (store
.exists("mkfs", "monmap")) {
135 dout(10) << __func__
<< " found mkfs monmap" << dendl
;
136 int err
= store
.get("mkfs", "monmap", bl
);
137 ceph_assert(err
== 0);
138 ceph_assert(bl
.length() > 0);
142 derr
<< __func__
<< " unable to find a monmap" << dendl
;
146 int check_mon_data_exists()
148 string mon_data
= g_conf()->mon_data
;
150 if (::stat(mon_data
.c_str(), &buf
)) {
151 if (errno
!= ENOENT
) {
152 derr
<< "stat(" << mon_data
<< ") " << cpp_strerror(errno
) << dendl
;
159 /** Check whether **mon data** is empty.
161 * Being empty means mkfs has not been run and there's no monitor setup
162 * at **g_conf()->mon_data**.
164 * If the directory g_conf()->mon_data is not empty we will return -ENOTEMPTY.
165 * Otherwise we will return 0. Any other negative returns will represent
166 * a failure to be handled by the caller.
168 * @return **0** on success, -ENOTEMPTY if not empty or **-errno** otherwise.
170 int check_mon_data_empty()
172 string mon_data
= g_conf()->mon_data
;
174 DIR *dir
= ::opendir(mon_data
.c_str());
176 derr
<< "opendir(" << mon_data
<< ") " << cpp_strerror(errno
) << dendl
;
180 struct dirent
*de
= nullptr;
182 while ((de
= ::readdir(dir
))) {
183 if (string(".") != de
->d_name
&&
184 string("..") != de
->d_name
&&
185 string("kv_backend") != de
->d_name
) {
191 derr
<< "readdir(" << mon_data
<< ") " << cpp_strerror(errno
) << dendl
;
202 cout
<< "usage: ceph-mon -i <ID> [flags]\n"
203 << " --debug_mon n\n"
204 << " debug monitor level (e.g. 10)\n"
206 << " build fresh monitor fs\n"
208 << " force a sync from another mon by wiping local data (BE CAREFUL)\n"
209 << " --yes-i-really-mean-it\n"
210 << " mandatory safeguard for --force-sync\n"
212 << " compact the monitor store\n"
213 << " --osdmap <filename>\n"
214 << " only used when --mkfs is provided: load the osdmap from <filename>\n"
215 << " --inject-monmap <filename>\n"
216 << " write the <filename> monmap to the local monitor store and exit\n"
217 << " --extract-monmap <filename>\n"
218 << " extract the monmap from the local monitor store and exit\n"
219 << " --mon-data <directory>\n"
220 << " where the mon store and keyring are located\n"
221 << " --set-crush-location <bucket>=<foo>"
222 << " sets monitor's crush bucket location (only for stretch mode)"
224 generic_server_usage();
227 entity_addrvec_t
make_mon_addrs(entity_addr_t a
)
229 entity_addrvec_t addrs
;
230 if (a
.get_port() == 0) {
231 a
.set_type(entity_addr_t::TYPE_MSGR2
);
232 a
.set_port(CEPH_MON_PORT_IANA
);
233 addrs
.v
.push_back(a
);
234 a
.set_type(entity_addr_t::TYPE_LEGACY
);
235 a
.set_port(CEPH_MON_PORT_LEGACY
);
236 addrs
.v
.push_back(a
);
237 } else if (a
.get_port() == CEPH_MON_PORT_LEGACY
) {
238 a
.set_type(entity_addr_t::TYPE_LEGACY
);
239 addrs
.v
.push_back(a
);
240 } else if (a
.get_type() == entity_addr_t::TYPE_ANY
) {
241 a
.set_type(entity_addr_t::TYPE_MSGR2
);
242 addrs
.v
.push_back(a
);
244 addrs
.v
.push_back(a
);
249 int main(int argc
, const char **argv
)
251 // reset our process name, in case we did a respawn, so that it's not
253 ceph_pthread_setname(pthread_self(), "ceph-mon");
258 bool compact
= false;
259 bool force_sync
= false;
260 bool yes_really
= false;
261 std::string osdmapfn
, inject_monmap
, extract_monmap
, crush_loc
;
263 vector
<const char*> args
;
264 argv_to_vec(argc
, argv
, args
);
266 cerr
<< argv
[0] << ": -h or --help for usage" << std::endl
;
269 if (ceph_argparse_need_usage(args
)) {
274 // We need to specify some default values that may be overridden by the
275 // user, that are specific to the monitor. The options we are overriding
276 // are also used on the OSD (or in any other component that uses leveldb),
277 // so changing the global defaults is not an option.
278 // This is not the prettiest way of doing this, especially since it has us
279 // having a different place defining default values, but it's not horribly
280 // wrong enough to prevent us from doing it :)
282 // NOTE: user-defined options will take precedence over ours.
284 // leveldb_write_buffer_size = 32*1024*1024 = 33554432 // 32MB
285 // leveldb_cache_size = 512*1024*1204 = 536870912 // 512MB
286 // leveldb_block_size = 64*1024 = 65536 // 64KB
287 // leveldb_compression = false
289 map
<string
,string
> defaults
= {
290 { "leveldb_write_buffer_size", "33554432" },
291 { "leveldb_cache_size", "536870912" },
292 { "leveldb_block_size", "65536" },
293 { "leveldb_compression", "false"},
294 { "leveldb_log", "" },
295 { "keyring", "$mon_data/keyring" },
300 vector
<const char*> args_copy
= args
;
302 for (std::vector
<const char*>::iterator i
= args_copy
.begin();
303 i
!= args_copy
.end(); ) {
304 if (ceph_argparse_double_dash(args_copy
, i
)) {
306 } else if (ceph_argparse_flag(args_copy
, i
, "--mkfs", (char*)NULL
)) {
307 flags
|= CINIT_FLAG_NO_DAEMON_ACTIONS
;
308 } else if (ceph_argparse_witharg(args_copy
, i
, &val
, "--inject_monmap", (char*)NULL
)) {
309 flags
|= CINIT_FLAG_NO_DAEMON_ACTIONS
;
310 } else if (ceph_argparse_witharg(args_copy
, i
, &val
, "--extract-monmap", (char*)NULL
)) {
311 flags
|= CINIT_FLAG_NO_DAEMON_ACTIONS
;
318 // don't try to get config from mon cluster during startup
319 flags
|= CINIT_FLAG_NO_MON_CONFIG
;
321 auto cct
= global_init(&defaults
, args
,
322 CEPH_ENTITY_TYPE_MON
, CODE_ENVIRONMENT_DAEMON
,
324 ceph_heap_profiler_init();
327 for (std::vector
<const char*>::iterator i
= args
.begin(); i
!= args
.end(); ) {
328 if (ceph_argparse_double_dash(args
, i
)) {
330 } else if (ceph_argparse_flag(args
, i
, "--mkfs", (char*)NULL
)) {
332 } else if (ceph_argparse_flag(args
, i
, "--compact", (char*)NULL
)) {
334 } else if (ceph_argparse_flag(args
, i
, "--force-sync", (char*)NULL
)) {
336 } else if (ceph_argparse_flag(args
, i
, "--yes-i-really-mean-it", (char*)NULL
)) {
338 } else if (ceph_argparse_witharg(args
, i
, &val
, "--osdmap", (char*)NULL
)) {
340 } else if (ceph_argparse_witharg(args
, i
, &val
, "--inject_monmap", (char*)NULL
)) {
342 } else if (ceph_argparse_witharg(args
, i
, &val
, "--extract-monmap", (char*)NULL
)) {
343 extract_monmap
= val
;
344 } else if (ceph_argparse_witharg(args
, i
, &val
, "--set-crush-location", (char*)NULL
)) {
351 cerr
<< "too many arguments: " << args
<< std::endl
;
355 if (force_sync
&& !yes_really
) {
356 cerr
<< "are you SURE you want to force a sync? this will erase local data and may\n"
357 << "break your mon cluster. pass --yes-i-really-mean-it if you do." << std::endl
;
361 if (g_conf()->mon_data
.empty()) {
362 cerr
<< "must specify '--mon-data=foo' data path" << std::endl
;
366 if (g_conf()->name
.get_id().empty()) {
367 cerr
<< "must specify id (--id <id> or --name mon.<id>)" << std::endl
;
374 int err
= check_mon_data_exists();
375 if (err
== -ENOENT
) {
376 if (::mkdir(g_conf()->mon_data
.c_str(), 0755)) {
377 derr
<< "mkdir(" << g_conf()->mon_data
<< ") : "
378 << cpp_strerror(errno
) << dendl
;
381 } else if (err
< 0) {
382 derr
<< "error opening '" << g_conf()->mon_data
<< "': "
383 << cpp_strerror(-err
) << dendl
;
387 err
= check_mon_data_empty();
388 if (err
== -ENOTEMPTY
) {
389 // Mon may exist. Let the user know and exit gracefully.
390 derr
<< "'" << g_conf()->mon_data
<< "' already exists and is not empty"
391 << ": monitor may already exist" << dendl
;
393 } else if (err
< 0) {
394 derr
<< "error checking if '" << g_conf()->mon_data
<< "' is empty: "
395 << cpp_strerror(-err
) << dendl
;
399 // resolve public_network -> public_addr
400 pick_addresses(g_ceph_context
, CEPH_PICK_ADDRESS_PUBLIC
);
402 dout(10) << "public_network " << g_conf()->public_network
<< dendl
;
403 dout(10) << "public_addr " << g_conf()->public_addr
<< dendl
;
404 dout(10) << "public_addrv " << g_conf()->public_addrv
<< dendl
;
406 common_init_finish(g_ceph_context
);
408 bufferlist monmapbl
, osdmapbl
;
412 // load or generate monmap
413 const auto monmap_fn
= g_conf().get_val
<string
>("monmap");
414 if (monmap_fn
.length()) {
415 int err
= monmapbl
.read_file(monmap_fn
.c_str(), &error
);
417 derr
<< argv
[0] << ": error reading " << monmap_fn
<< ": " << error
<< dendl
;
421 monmap
.decode(monmapbl
);
423 // always mark seed/mkfs monmap as epoch 0
425 } catch (const ceph::buffer::error
& e
) {
426 derr
<< argv
[0] << ": error decoding monmap " << monmap_fn
<< ": " << e
.what() << dendl
;
430 dout(1) << "imported monmap:\n";
431 monmap
.print(*_dout
);
436 int err
= monmap
.build_initial(g_ceph_context
, true, oss
);
438 derr
<< oss
.str() << dendl
;
440 derr
<< argv
[0] << ": warning: no initial monitors; must use admin socket to feed hints" << dendl
;
443 dout(1) << "initial generated monmap:\n";
444 monmap
.print(*_dout
);
447 // am i part of the initial quorum?
448 if (monmap
.contains(g_conf()->name
.get_id())) {
449 // hmm, make sure the ip listed exists on the current host?
451 } else if (!g_conf()->public_addrv
.empty()) {
452 entity_addrvec_t av
= g_conf()->public_addrv
;
454 if (monmap
.contains(av
, &name
)) {
455 monmap
.rename(name
, g_conf()->name
.get_id());
456 dout(0) << argv
[0] << ": renaming mon." << name
<< " " << av
457 << " to mon." << g_conf()->name
.get_id() << dendl
;
459 } else if (!g_conf()->public_addr
.is_blank_ip()) {
460 entity_addrvec_t av
= make_mon_addrs(g_conf()->public_addr
);
462 if (monmap
.contains(av
, &name
)) {
463 monmap
.rename(name
, g_conf()->name
.get_id());
464 dout(0) << argv
[0] << ": renaming mon." << name
<< " " << av
465 << " to mon." << g_conf()->name
.get_id() << dendl
;
468 // is a local address listed without a name? if so, name myself.
469 list
<entity_addr_t
> ls
;
470 monmap
.list_addrs(ls
);
471 dout(0) << " monmap addrs are " << ls
<< ", checking if any are local"
475 if (have_local_addr(g_ceph_context
, ls
, &local
)) {
476 dout(0) << " have local addr " << local
<< dendl
;
478 local
.set_type(entity_addr_t::TYPE_MSGR2
);
479 if (!monmap
.get_addr_name(local
, name
)) {
480 local
.set_type(entity_addr_t::TYPE_LEGACY
);
481 if (!monmap
.get_addr_name(local
, name
)) {
482 dout(0) << "no local addresses appear in bootstrap monmap"
486 if (name
.compare(0, 7, "noname-") == 0) {
487 dout(0) << argv
[0] << ": mon." << name
<< " " << local
488 << " is local, renaming to mon." << g_conf()->name
.get_id()
490 monmap
.rename(name
, g_conf()->name
.get_id());
491 } else if (name
.size()) {
492 dout(0) << argv
[0] << ": mon." << name
<< " " << local
493 << " is local, but not 'noname-' + something; "
494 << "not assuming it's me" << dendl
;
497 dout(0) << " no local addrs match monmap" << dendl
;
502 const auto fsid
= g_conf().get_val
<uuid_d
>("fsid");
503 if (!fsid
.is_zero()) {
505 dout(0) << argv
[0] << ": set fsid to " << fsid
<< dendl
;
508 if (monmap
.fsid
.is_zero()) {
509 derr
<< argv
[0] << ": generated monmap has no fsid; use '--fsid <uuid>'" << dendl
;
513 //monmap.print(cout);
516 if (osdmapfn
.length()) {
517 err
= osdmapbl
.read_file(osdmapfn
.c_str(), &error
);
519 derr
<< argv
[0] << ": error reading " << osdmapfn
<< ": "
526 MonitorDBStore
store(g_conf()->mon_data
);
528 int r
= store
.create_and_open(oss
);
530 derr
<< oss
.str() << dendl
;
532 derr
<< argv
[0] << ": error opening mon data directory at '"
533 << g_conf()->mon_data
<< "': " << cpp_strerror(r
) << dendl
;
538 Monitor
mon(g_ceph_context
, g_conf()->name
.get_id(), &store
, 0, 0, &monmap
);
539 r
= mon
.mkfs(osdmapbl
);
541 derr
<< argv
[0] << ": error creating monfs: " << cpp_strerror(r
) << dendl
;
545 dout(0) << argv
[0] << ": created monfs at " << g_conf()->mon_data
546 << " for " << g_conf()->name
<< dendl
;
550 err
= check_mon_data_exists();
551 if (err
< 0 && err
== -ENOENT
) {
552 derr
<< "monitor data directory at '" << g_conf()->mon_data
<< "'"
553 << " does not exist: have you run 'mkfs'?" << dendl
;
555 } else if (err
< 0) {
556 derr
<< "error accessing monitor data directory at '"
557 << g_conf()->mon_data
<< "': " << cpp_strerror(-err
) << dendl
;
561 err
= check_mon_data_empty();
563 derr
<< "monitor data directory at '" << g_conf()->mon_data
564 << "' is empty: have you run 'mkfs'?" << dendl
;
566 } else if (err
< 0 && err
!= -ENOTEMPTY
) {
567 // we don't want an empty data dir by now
568 derr
<< "error accessing '" << g_conf()->mon_data
<< "': "
569 << cpp_strerror(-err
) << dendl
;
574 // check fs stats. don't start if it's critically close to full.
575 ceph_data_stats_t stats
;
576 int err
= get_fs_stats(stats
, g_conf()->mon_data
.c_str());
578 derr
<< "error checking monitor data's fs stats: " << cpp_strerror(err
)
582 if (stats
.avail_percent
<= g_conf()->mon_data_avail_crit
) {
583 derr
<< "error: monitor data filesystem reached concerning levels of"
584 << " available storage space (available: "
585 << stats
.avail_percent
<< "% " << byte_u_t(stats
.byte_avail
)
586 << ")\nyou may adjust 'mon data avail crit' to a lower value"
587 << " to make this go away (default: " << g_conf()->mon_data_avail_crit
593 // we fork early to prevent leveldb's environment static state from
596 if (!(flags
& CINIT_FLAG_NO_DAEMON_ACTIONS
)) {
597 if (global_init_prefork(g_ceph_context
) >= 0) {
599 err
= prefork
.prefork(err_msg
);
601 derr
<< err_msg
<< dendl
;
604 if (prefork
.is_parent()) {
605 err
= prefork
.parent_wait(err_msg
);
607 derr
<< err_msg
<< dendl
;
611 global_init_postfork_start(g_ceph_context
);
613 common_init_finish(g_ceph_context
);
614 global_init_chdir(g_ceph_context
);
615 if (global_init_preload_erasure_code(g_ceph_context
) < 0)
619 // set up signal handlers, now that we've daemonized/forked.
620 init_async_signal_handler();
621 register_async_signal_handler(SIGHUP
, sighup_handler
);
623 MonitorDBStore
*store
= new MonitorDBStore(g_conf()->mon_data
);
625 // make sure we aren't upgrading too fast
628 int r
= store
->read_meta("min_mon_release", &val
);
629 if (r
>= 0 && val
.size()) {
630 ceph_release_t from_release
= ceph_release_from_name(val
);
632 if (!can_upgrade_from(from_release
, "min_mon_release", err
)) {
633 derr
<< err
.str() << dendl
;
641 err
= store
->open(oss
);
643 derr
<< oss
.str() << dendl
;
645 derr
<< "error opening mon data directory at '"
646 << g_conf()->mon_data
<< "': " << cpp_strerror(err
) << dendl
;
652 err
= store
->get(Monitor::MONITOR_NAME
, "magic", magicbl
);
653 if (err
|| !magicbl
.length()) {
654 derr
<< "unable to read magic from mon data" << dendl
;
657 string
magic(magicbl
.c_str(), magicbl
.length()-1); // ignore trailing \n
658 if (strcmp(magic
.c_str(), CEPH_MON_ONDISK_MAGIC
)) {
659 derr
<< "mon fs magic '" << magic
<< "' != current '" << CEPH_MON_ONDISK_MAGIC
<< "'" << dendl
;
663 err
= Monitor::check_features(store
);
665 derr
<< "error checking features: " << cpp_strerror(err
) << dendl
;
669 // inject new monmap?
670 if (!inject_monmap
.empty()) {
673 int r
= bl
.read_file(inject_monmap
.c_str(), &error
);
675 derr
<< "unable to read monmap from " << inject_monmap
<< ": "
681 version_t v
= store
->get("monmap", "last_committed");
682 dout(0) << "last committed monmap epoch is " << v
<< ", injected map will be " << (v
+1)
689 if (tmp
.get_epoch() != v
) {
690 dout(0) << "changing monmap epoch from " << tmp
.get_epoch()
691 << " to " << v
<< dendl
;
695 tmp
.encode(mapbl
, CEPH_FEATURES_ALL
);
698 encode(mapbl
, final
);
700 auto t(std::make_shared
<MonitorDBStore::Transaction
>());
702 t
->put("monmap", v
, mapbl
);
703 t
->put("monmap", "latest", final
);
704 t
->put("monmap", "last_committed", v
);
705 store
->apply_transaction(t
);
707 dout(0) << "done." << dendl
;
714 // note that even if we don't find a viable monmap, we should go ahead
715 // and try to build it up in the next if-else block.
717 int err
= obtain_monmap(*store
, mapbl
);
720 monmap
.decode(mapbl
);
721 } catch (const ceph::buffer::error
& e
) {
722 derr
<< "can't decode monmap: " << e
.what() << dendl
;
725 derr
<< "unable to obtain a monmap: " << cpp_strerror(err
) << dendl
;
728 dout(10) << __func__
<< " monmap:\n";
729 JSONFormatter
jf(true);
730 jf
.dump_object("monmap", monmap
);
734 if (!extract_monmap
.empty()) {
735 int r
= mapbl
.write_file(extract_monmap
.c_str());
738 derr
<< "error writing monmap to " << extract_monmap
<< ": " << cpp_strerror(r
) << dendl
;
741 derr
<< "wrote monmap to " << extract_monmap
<< dendl
;
746 // this is what i will bind to
747 entity_addrvec_t ipaddrs
;
749 if (monmap
.contains(g_conf()->name
.get_id())) {
750 ipaddrs
= monmap
.get_addrs(g_conf()->name
.get_id());
752 // print helpful warning if the conf file doesn't match
753 std::vector
<std::string
> my_sections
= g_conf().get_my_sections();
754 std::string mon_addr_str
;
755 if (g_conf().get_val_from_conf_file(my_sections
, "mon addr",
756 mon_addr_str
, true) == 0) {
757 entity_addr_t conf_addr
;
758 if (conf_addr
.parse(mon_addr_str
.c_str())) {
759 entity_addrvec_t conf_addrs
= make_mon_addrs(conf_addr
);
760 if (ipaddrs
!= conf_addrs
) {
761 derr
<< "WARNING: 'mon addr' config option " << conf_addrs
762 << " does not match monmap file" << std::endl
763 << " continuing with monmap configuration" << dendl
;
766 derr
<< "WARNING: invalid 'mon addr' config option" << std::endl
767 << " continuing with monmap configuration" << dendl
;
770 dout(0) << g_conf()->name
<< " does not exist in monmap, will attempt to join an existing cluster" << dendl
;
772 pick_addresses(g_ceph_context
, CEPH_PICK_ADDRESS_PUBLIC
);
773 if (!g_conf()->public_addrv
.empty()) {
774 ipaddrs
= g_conf()->public_addrv
;
775 dout(0) << "using public_addrv " << ipaddrs
<< dendl
;
776 } else if (!g_conf()->public_addr
.is_blank_ip()) {
777 ipaddrs
= make_mon_addrs(g_conf()->public_addr
);
778 dout(0) << "using public_addr " << g_conf()->public_addr
<< " -> "
783 int err
= tmpmap
.build_initial(g_ceph_context
, true, oss
);
785 derr
<< oss
.str() << dendl
;
787 derr
<< argv
[0] << ": error generating initial monmap: "
788 << cpp_strerror(err
) << dendl
;
791 if (tmpmap
.contains(g_conf()->name
.get_id())) {
792 ipaddrs
= tmpmap
.get_addrs(g_conf()->name
.get_id());
794 derr
<< "no public_addr or public_network specified, and "
795 << g_conf()->name
<< " not present in monmap or ceph.conf" << dendl
;
802 int rank
= monmap
.get_rank(g_conf()->name
.get_id());
803 std::string public_msgr_type
= g_conf()->ms_public_type
.empty() ? g_conf().get_val
<std::string
>("ms_type") : g_conf()->ms_public_type
;
804 Messenger
*msgr
= Messenger::create(g_ceph_context
, public_msgr_type
,
805 entity_name_t::MON(rank
), "mon", 0);
808 msgr
->set_cluster_protocol(CEPH_MON_PROTOCOL
);
809 msgr
->set_default_send_priority(CEPH_MSG_PRIO_HIGH
);
811 msgr
->set_default_policy(Messenger::Policy::stateless_server(0));
812 msgr
->set_policy(entity_name_t::TYPE_MON
,
813 Messenger::Policy::lossless_peer_reuse(
814 CEPH_FEATURE_SERVER_LUMINOUS
));
815 msgr
->set_policy(entity_name_t::TYPE_OSD
,
816 Messenger::Policy::stateless_server(
817 CEPH_FEATURE_SERVER_LUMINOUS
));
818 msgr
->set_policy(entity_name_t::TYPE_CLIENT
,
819 Messenger::Policy::stateless_server(0));
820 msgr
->set_policy(entity_name_t::TYPE_MDS
,
821 Messenger::Policy::stateless_server(0));
823 // throttle client traffic
824 Throttle
*client_throttler
= new Throttle(g_ceph_context
, "mon_client_bytes",
825 g_conf()->mon_client_bytes
);
826 msgr
->set_policy_throttlers(entity_name_t::TYPE_CLIENT
,
827 client_throttler
, NULL
);
829 // throttle daemon traffic
830 // NOTE: actual usage on the leader may multiply by the number of
831 // monitors if they forward large update messages from daemons.
832 Throttle
*daemon_throttler
= new Throttle(g_ceph_context
, "mon_daemon_bytes",
833 g_conf()->mon_daemon_bytes
);
834 msgr
->set_policy_throttlers(entity_name_t::TYPE_OSD
, daemon_throttler
,
836 msgr
->set_policy_throttlers(entity_name_t::TYPE_MDS
, daemon_throttler
,
839 entity_addrvec_t bind_addrs
= ipaddrs
;
840 entity_addrvec_t public_addrs
= ipaddrs
;
842 // check if the public_bind_addr option is set
843 if (!g_conf()->public_bind_addr
.is_blank_ip()) {
844 bind_addrs
= make_mon_addrs(g_conf()->public_bind_addr
);
847 dout(0) << "starting " << g_conf()->name
<< " rank " << rank
848 << " at public addrs " << public_addrs
849 << " at bind addrs " << bind_addrs
850 << " mon_data " << g_conf()->mon_data
851 << " fsid " << monmap
.get_fsid()
854 Messenger
*mgr_msgr
= Messenger::create(g_ceph_context
, public_msgr_type
,
855 entity_name_t::MON(rank
), "mon-mgrc",
856 Messenger::get_pid_nonce());
858 derr
<< "unable to create mgr_msgr" << dendl
;
862 mon
= new Monitor(g_ceph_context
, g_conf()->name
.get_id(), store
,
863 msgr
, mgr_msgr
, &monmap
);
865 mon
->orig_argc
= argc
;
866 mon
->orig_argv
= argv
;
869 derr
<< "flagging a forced sync ..." << dendl
;
871 JSONFormatter
jf(true);
872 mon
->sync_force(&jf
);
878 err
= mon
->preinit();
880 derr
<< "failed to initialize" << dendl
;
884 if (compact
|| g_conf()->mon_compact_on_start
) {
885 derr
<< "compacting monitor store ..." << dendl
;
886 mon
->store
->compact();
887 derr
<< "done compacting" << dendl
;
891 err
= msgr
->bindv(bind_addrs
);
893 derr
<< "unable to bind monitor to " << bind_addrs
<< dendl
;
897 // if the public and bind addr are different set the msgr addr
898 // to the public one, now that the bind is complete.
899 if (public_addrs
!= bind_addrs
) {
900 msgr
->set_addrs(public_addrs
);
903 if (g_conf()->daemonize
) {
904 global_init_postfork_finish(g_ceph_context
);
911 mon
->set_mon_crush_location(crush_loc
);
914 register_async_signal_handler_oneshot(SIGINT
, handle_mon_signal
);
915 register_async_signal_handler_oneshot(SIGTERM
, handle_mon_signal
);
917 if (g_conf()->inject_early_sigterm
)
918 kill(getpid(), SIGTERM
);
925 unregister_async_signal_handler(SIGHUP
, sighup_handler
);
926 unregister_async_signal_handler(SIGINT
, handle_mon_signal
);
927 unregister_async_signal_handler(SIGTERM
, handle_mon_signal
);
928 shutdown_async_signal_handler();
934 delete client_throttler
;
935 delete daemon_throttler
;
937 // cd on exit, so that gmon.out (if any) goes into a separate directory for each node.
939 snprintf(s
, sizeof(s
), "gmon/%d", getpid());
940 if ((mkdir(s
, 0755) == 0) && (chdir(s
) == 0)) {
941 dout(0) << "ceph-mon: gmon.out should be in " << s
<< dendl
;
944 prefork
.signal_exit(0);