]> git.proxmox.com Git - ceph.git/blame - ceph/src/mon/MonMap.cc
bump version to 16.2.6-pve2
[ceph.git] / ceph / src / mon / MonMap.cc
CommitLineData
11fdf7f2
TL
1// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2// vim: ts=8 sw=2 smarttab
7c673cae
FG
3
4#include "MonMap.h"
5
6#include <algorithm>
7#include <sys/types.h>
8#include <sys/stat.h>
9#include <fcntl.h>
10
11fdf7f2
TL
11#ifdef WITH_SEASTAR
12#include <seastar/core/fstream.hh>
13#include <seastar/core/reactor.hh>
14#include <seastar/net/dns.hh>
15#include "crimson/common/config_proxy.h"
16#endif
17
7c673cae
FG
18#include "common/Formatter.h"
19
20#include "include/ceph_features.h"
21#include "include/addr_parsing.h"
22#include "common/ceph_argparse.h"
23#include "common/dns_resolve.h"
24#include "common/errno.h"
7c673cae 25#include "common/dout.h"
11fdf7f2 26#include "common/Clock.h"
f67539c2 27#include "mon/health_check.h"
7c673cae 28
9f95a23c
TL
29using std::list;
30using std::map;
31using std::ostream;
32using std::set;
33using std::string;
34using std::vector;
35
36using ceph::DNSResolver;
7c673cae
FG
37using ceph::Formatter;
38
9f95a23c 39void mon_info_t::encode(ceph::buffer::list& bl, uint64_t features) const
7c673cae 40{
f67539c2
TL
41 uint8_t v = 5;
42 uint8_t min_v = 1;
43 if (!crush_loc.empty()) {
44 // we added crush_loc in version 5, but need to let old clients decode it
45 // so just leave the min_v at version 4. Monitors are protected
46 // from misunderstandings about location because setting it is blocked
47 // on FEATURE_PINGING
48 min_v = 4;
49 }
11fdf7f2
TL
50 if (!HAVE_FEATURE(features, SERVER_NAUTILUS)) {
51 v = 2;
52 }
f67539c2 53 ENCODE_START(v, min_v, bl);
11fdf7f2
TL
54 encode(name, bl);
55 if (v < 3) {
f67539c2 56 ceph_assert(min_v == 1);
eafe8130
TL
57 auto a = public_addrs.legacy_addr();
58 if (a != entity_addr_t()) {
59 encode(a, bl, features);
60 } else {
61 // note: we don't have a legacy addr here, so lie so that it looks
62 // like one, just so that old clients get a valid-looking map.
63 // they won't be able to talk to the v2 mons, but that's better
64 // than nothing.
65 encode(public_addrs.as_legacy_addr(), bl, features);
66 }
11fdf7f2
TL
67 } else {
68 encode(public_addrs, bl, features);
69 }
70 encode(priority, bl);
9f95a23c 71 encode(weight, bl);
f67539c2 72 encode(crush_loc, bl);
7c673cae
FG
73 ENCODE_FINISH(bl);
74}
75
9f95a23c 76void mon_info_t::decode(ceph::buffer::list::const_iterator& p)
7c673cae 77{
f67539c2 78 DECODE_START(5, p);
11fdf7f2
TL
79 decode(name, p);
80 decode(public_addrs, p);
224ce89b 81 if (struct_v >= 2) {
11fdf7f2 82 decode(priority, p);
224ce89b 83 }
9f95a23c
TL
84 if (struct_v >= 4) {
85 decode(weight, p);
86 }
f67539c2
TL
87 if (struct_v >= 5) {
88 decode(crush_loc, p);
89 }
7c673cae
FG
90 DECODE_FINISH(p);
91}
92
93void mon_info_t::print(ostream& out) const
94{
95 out << "mon." << name
11fdf7f2 96 << " addrs " << public_addrs
9f95a23c 97 << " priority " << priority
f67539c2
TL
98 << " weight " << weight
99 << " crush location " << crush_loc;
7c673cae
FG
100}
101
7c673cae
FG
102namespace {
103 struct rank_cmp {
104 bool operator()(const mon_info_t &a, const mon_info_t &b) const {
11fdf7f2 105 if (a.public_addrs.legacy_or_front_addr() == b.public_addrs.legacy_or_front_addr())
7c673cae 106 return a.name < b.name;
11fdf7f2 107 return a.public_addrs.legacy_or_front_addr() < b.public_addrs.legacy_or_front_addr();
7c673cae
FG
108 }
109 };
110}
111
11fdf7f2
TL
112void MonMap::calc_legacy_ranks()
113{
7c673cae 114 ranks.resize(mon_info.size());
7c673cae
FG
115
116 // Used to order entries according to public_addr, because that's
117 // how the ranks are expected to be ordered by. We may expand this
118 // later on, according to some other criteria, by specifying a
119 // different comparator.
120 //
121 // Please note that we use a 'set' here instead of resorting to
122 // std::sort() because we need more info than that's available in
123 // the vector. The vector will thus be ordered by, e.g., public_addr
124 // while only containing the names of each individual monitor.
125 // The only way of achieving this with std::sort() would be to first
126 // insert every mon_info_t entry into a vector 'foo', std::sort() 'foo'
127 // with custom comparison functions, and then copy each invidual entry
128 // to a new vector. Unless there's a simpler way, we don't think the
129 // added complexity makes up for the additional memory usage of a 'set'.
130 set<mon_info_t, rank_cmp> tmp;
131
9f95a23c 132 for (auto p = mon_info.begin(); p != mon_info.end(); ++p) {
7c673cae
FG
133 mon_info_t &m = p->second;
134 tmp.insert(m);
7c673cae
FG
135 }
136
137 // map the set to the actual ranks etc
138 unsigned i = 0;
9f95a23c 139 for (auto p = tmp.begin(); p != tmp.end(); ++p, ++i) {
7c673cae
FG
140 ranks[i] = p->name;
141 }
142}
143
9f95a23c 144void MonMap::encode(ceph::buffer::list& blist, uint64_t con_features) const
7c673cae 145{
7c673cae 146 if ((con_features & CEPH_FEATURE_MONNAMES) == 0) {
11fdf7f2 147 using ceph::encode;
7c673cae 148 __u16 v = 1;
11fdf7f2 149 encode(v, blist);
9f95a23c 150 ceph::encode_raw(fsid, blist);
11fdf7f2
TL
151 encode(epoch, blist);
152 vector<entity_inst_t> mon_inst(ranks.size());
153 for (unsigned n = 0; n < ranks.size(); n++) {
154 mon_inst[n].name = entity_name_t::MON(n);
155 mon_inst[n].addr = get_addrs(n).legacy_addr();
156 }
157 encode(mon_inst, blist, con_features);
158 encode(last_changed, blist);
159 encode(created, blist);
7c673cae
FG
160 return;
161 }
162
11fdf7f2
TL
163 map<string,entity_addr_t> legacy_mon_addr;
164 if (!HAVE_FEATURE(con_features, MONENC) ||
165 !HAVE_FEATURE(con_features, SERVER_NAUTILUS)) {
166 for (auto& [name, info] : mon_info) {
167 legacy_mon_addr[name] = info.public_addrs.legacy_addr();
168 }
169 }
170
171 if (!HAVE_FEATURE(con_features, MONENC)) {
172 /* we keep the mon_addr map when encoding to ensure compatibility
173 * with clients and other monitors that do not yet support the 'mons'
174 * map. This map keeps its original behavior, containing a mapping of
175 * monitor id (i.e., 'foo' in 'mon.foo') to the monitor's public
176 * address -- which is obtained from the public address of each entry
177 * in the 'mons' map.
178 */
179 using ceph::encode;
7c673cae 180 __u16 v = 2;
11fdf7f2 181 encode(v, blist);
9f95a23c 182 ceph::encode_raw(fsid, blist);
11fdf7f2
TL
183 encode(epoch, blist);
184 encode(legacy_mon_addr, blist, con_features);
185 encode(last_changed, blist);
186 encode(created, blist);
187 return;
188 }
189
190 if (!HAVE_FEATURE(con_features, SERVER_NAUTILUS)) {
191 ENCODE_START(5, 3, blist);
9f95a23c 192 ceph::encode_raw(fsid, blist);
11fdf7f2
TL
193 encode(epoch, blist);
194 encode(legacy_mon_addr, blist, con_features);
195 encode(last_changed, blist);
196 encode(created, blist);
197 encode(persistent_features, blist);
198 encode(optional_features, blist);
199 encode(mon_info, blist, con_features);
200 ENCODE_FINISH(blist);
201 return;
202 }
203
f67539c2 204 ENCODE_START(9, 6, blist);
9f95a23c 205 ceph::encode_raw(fsid, blist);
11fdf7f2
TL
206 encode(epoch, blist);
207 encode(last_changed, blist);
208 encode(created, blist);
209 encode(persistent_features, blist);
210 encode(optional_features, blist);
211 encode(mon_info, blist, con_features);
212 encode(ranks, blist);
213 encode(min_mon_release, blist);
f67539c2
TL
214 encode(removed_ranks, blist);
215 uint8_t t = strategy;
216 encode(t, blist);
217 encode(disallowed_leaders, blist);
218 encode(stretch_mode_enabled, blist);
219 encode(tiebreaker_mon, blist);
220 encode(stretch_marked_down_mons, blist);
7c673cae
FG
221 ENCODE_FINISH(blist);
222}
223
9f95a23c 224void MonMap::decode(ceph::buffer::list::const_iterator& p)
7c673cae
FG
225{
226 map<string,entity_addr_t> mon_addr;
f67539c2 227 DECODE_START_LEGACY_COMPAT_LEN_16(9, 3, 3, p);
9f95a23c 228 ceph::decode_raw(fsid, p);
11fdf7f2 229 decode(epoch, p);
7c673cae
FG
230 if (struct_v == 1) {
231 vector<entity_inst_t> mon_inst;
11fdf7f2 232 decode(mon_inst, p);
7c673cae
FG
233 for (unsigned i = 0; i < mon_inst.size(); i++) {
234 char n[2];
235 n[0] = '0' + i;
236 n[1] = 0;
237 string name = n;
238 mon_addr[name] = mon_inst[i].addr;
239 }
11fdf7f2
TL
240 } else if (struct_v < 6) {
241 decode(mon_addr, p);
7c673cae 242 }
11fdf7f2
TL
243 decode(last_changed, p);
244 decode(created, p);
7c673cae 245 if (struct_v >= 4) {
11fdf7f2
TL
246 decode(persistent_features, p);
247 decode(optional_features, p);
7c673cae 248 }
11fdf7f2
TL
249 if (struct_v < 5) {
250 // generate mon_info from legacy mon_addr
251 for (auto& [name, addr] : mon_addr) {
252 mon_info_t &m = mon_info[name];
253 m.name = name;
254 m.public_addrs = entity_addrvec_t(addr);
255 }
256 } else {
257 decode(mon_info, p);
258 }
259 if (struct_v < 6) {
260 calc_legacy_ranks();
261 } else {
262 decode(ranks, p);
263 }
264 if (struct_v >= 7) {
265 decode(min_mon_release, p);
7c673cae 266 } else {
11fdf7f2 267 min_mon_release = infer_ceph_release_from_mon_features(persistent_features);
7c673cae 268 }
f67539c2
TL
269 if (struct_v >= 8) {
270 decode(removed_ranks, p);
271 uint8_t t;
272 decode(t, p);
273 strategy = static_cast<election_strategy>(t);
274 decode(disallowed_leaders, p);
275 }
276 if (struct_v >= 9) {
277 decode(stretch_mode_enabled, p);
278 decode(tiebreaker_mon, p);
279 decode(stretch_marked_down_mons, p);
280 } else {
281 stretch_mode_enabled = false;
282 tiebreaker_mon = "";
283 stretch_marked_down_mons.clear();
284 }
11fdf7f2 285 calc_addr_mons();
7c673cae 286 DECODE_FINISH(p);
7c673cae
FG
287}
288
289void MonMap::generate_test_instances(list<MonMap*>& o)
290{
291 o.push_back(new MonMap);
292 o.push_back(new MonMap);
293 o.back()->epoch = 1;
294 o.back()->last_changed = utime_t(123, 456);
295 o.back()->created = utime_t(789, 101112);
11fdf7f2 296 o.back()->add("one", entity_addrvec_t());
7c673cae
FG
297
298 MonMap *m = new MonMap;
299 {
300 m->epoch = 1;
301 m->last_changed = utime_t(123, 456);
302
11fdf7f2
TL
303 entity_addrvec_t empty_addr_one = entity_addrvec_t(entity_addr_t());
304 empty_addr_one.v[0].set_nonce(1);
7c673cae 305 m->add("empty_addr_one", empty_addr_one);
11fdf7f2
TL
306 entity_addrvec_t empty_addr_two = entity_addrvec_t(entity_addr_t());
307 empty_addr_two.v[0].set_nonce(2);
308 m->add("empty_addr_two", empty_addr_two);
7c673cae
FG
309
310 const char *local_pub_addr_s = "127.0.1.2";
311
312 const char *end_p = local_pub_addr_s + strlen(local_pub_addr_s);
11fdf7f2 313 entity_addrvec_t local_pub_addr;
7c673cae
FG
314 local_pub_addr.parse(local_pub_addr_s, &end_p);
315
9f95a23c 316 m->add(mon_info_t("filled_pub_addr", entity_addrvec_t(local_pub_addr), 1, 1));
7c673cae 317
11fdf7f2 318 m->add("empty_addr_zero", entity_addrvec_t());
7c673cae
FG
319 }
320 o.push_back(m);
321}
322
323// read from/write to a file
324int MonMap::write(const char *fn)
325{
326 // encode
9f95a23c 327 ceph::buffer::list bl;
7c673cae
FG
328 encode(bl, CEPH_FEATURES_ALL);
329
330 return bl.write_file(fn);
331}
332
333int MonMap::read(const char *fn)
334{
335 // read
9f95a23c 336 ceph::buffer::list bl;
7c673cae
FG
337 std::string error;
338 int r = bl.read_file(fn, &error);
339 if (r < 0)
340 return r;
341 decode(bl);
342 return 0;
343}
344
345void MonMap::print_summary(ostream& out) const
346{
347 out << "e" << epoch << ": "
348 << mon_info.size() << " mons at {";
349 // the map that we used to print, as it was, no longer
350 // maps strings to the monitor's public address, but to
351 // mon_info_t instead. As such, print the map in a way
352 // that keeps the expected format.
353 bool has_printed = false;
9f95a23c 354 for (auto p = mon_info.begin(); p != mon_info.end(); ++p) {
7c673cae
FG
355 if (has_printed)
356 out << ",";
11fdf7f2 357 out << p->first << "=" << p->second.public_addrs;
7c673cae
FG
358 has_printed = true;
359 }
360 out << "}";
361}
362
363void MonMap::print(ostream& out) const
364{
365 out << "epoch " << epoch << "\n";
366 out << "fsid " << fsid << "\n";
367 out << "last_changed " << last_changed << "\n";
368 out << "created " << created << "\n";
f67539c2 369 out << "min_mon_release " << to_integer<unsigned>(min_mon_release)
9f95a23c 370 << " (" << min_mon_release << ")\n";
f67539c2
TL
371 out << "election_strategy: " << strategy << "\n";
372 if (disallowed_leaders.size()) {
373 out << "disallowed_leaders " << disallowed_leaders << "\n";
374 }
7c673cae 375 unsigned i = 0;
9f95a23c 376 for (auto p = ranks.begin(); p != ranks.end(); ++p) {
f67539c2
TL
377 const auto &mi = mon_info.find(*p);
378 ceph_assert(mi != mon_info.end());
379 out << i++ << ": " << mi->second.public_addrs << " mon." << *p;
380 if (!mi->second.crush_loc.empty()) {
381 out << "; crush_location " << mi->second.crush_loc;
382 }
383 out << "\n";
7c673cae
FG
384 }
385}
386
387void MonMap::dump(Formatter *f) const
388{
389 f->dump_unsigned("epoch", epoch);
390 f->dump_stream("fsid") << fsid;
9f95a23c
TL
391 last_changed.gmtime(f->dump_stream("modified"));
392 created.gmtime(f->dump_stream("created"));
f67539c2
TL
393 f->dump_unsigned("min_mon_release", to_integer<unsigned>(min_mon_release));
394 f->dump_string("min_mon_release_name", to_string(min_mon_release));
395 f->dump_int ("election_strategy", strategy);
396 f->dump_stream("disallowed_leaders: ") << disallowed_leaders;
397 f->dump_bool("stretch_mode", stretch_mode_enabled);
7c673cae
FG
398 f->open_object_section("features");
399 persistent_features.dump(f, "persistent");
400 optional_features.dump(f, "optional");
401 f->close_section();
402 f->open_array_section("mons");
403 int i = 0;
9f95a23c 404 for (auto p = ranks.begin(); p != ranks.end(); ++p, ++i) {
7c673cae
FG
405 f->open_object_section("mon");
406 f->dump_int("rank", i);
407 f->dump_string("name", *p);
11fdf7f2
TL
408 f->dump_object("public_addrs", get_addrs(*p));
409 // compat: make these look like pre-nautilus entity_addr_t
410 f->dump_stream("addr") << get_addrs(*p).get_legacy_str();
411 f->dump_stream("public_addr") << get_addrs(*p).get_legacy_str();
9f95a23c
TL
412 f->dump_unsigned("priority", get_priority(*p));
413 f->dump_unsigned("weight", get_weight(*p));
f67539c2
TL
414 const auto &mi = mon_info.find(*p);
415 // we don't need to assert this validity as all the get_* functions did
416 f->dump_stream("crush_location") << mi->second.crush_loc;
7c673cae
FG
417 f->close_section();
418 }
419 f->close_section();
420}
421
9f95a23c
TL
422void MonMap::dump_summary(Formatter *f) const
423{
424 f->dump_unsigned("epoch", epoch);
f67539c2 425 f->dump_string("min_mon_release_name", to_string(min_mon_release));
9f95a23c
TL
426 f->dump_unsigned("num_mons", ranks.size());
427}
428
11fdf7f2
TL
429// an ambiguous mon addr may be legacy or may be msgr2--we aren' sure.
430// when that happens we need to try them both (unless we can
431// reasonably infer from the port number which it is).
432void MonMap::_add_ambiguous_addr(const string& name,
9f95a23c
TL
433 entity_addr_t addr,
434 int priority,
435 int weight,
436 bool for_mkfs)
11fdf7f2
TL
437{
438 if (addr.get_type() != entity_addr_t::TYPE_ANY) {
439 // a v1: or v2: prefix was specified
440 if (addr.get_port() == 0) {
441 // use default port
9f95a23c 442 if (addr.get_type() == entity_addr_t::TYPE_LEGACY) {
11fdf7f2
TL
443 addr.set_port(CEPH_MON_PORT_LEGACY);
444 } else if (addr.get_type() == entity_addr_t::TYPE_MSGR2) {
445 addr.set_port(CEPH_MON_PORT_IANA);
446 } else {
447 // wth
448 return;
449 }
450 if (!contains(addr)) {
9f95a23c 451 add(name, entity_addrvec_t(addr), priority, weight);
11fdf7f2
TL
452 }
453 } else {
454 if (!contains(addr)) {
9f95a23c 455 add(name, entity_addrvec_t(addr), priority, weight);
11fdf7f2
TL
456 }
457 }
458 } else {
459 // no v1: or v2: prefix specified
460 if (addr.get_port() == CEPH_MON_PORT_LEGACY) {
461 // legacy port implies legacy addr
462 addr.set_type(entity_addr_t::TYPE_LEGACY);
463 if (!contains(addr)) {
464 if (!for_mkfs) {
9f95a23c 465 add(name + "-legacy", entity_addrvec_t(addr), priority, weight);
11fdf7f2 466 } else {
9f95a23c 467 add(name, entity_addrvec_t(addr), priority, weight);
11fdf7f2
TL
468 }
469 }
470 } else if (addr.get_port() == CEPH_MON_PORT_IANA) {
471 // iana port implies msgr2 addr
472 addr.set_type(entity_addr_t::TYPE_MSGR2);
473 if (!contains(addr)) {
9f95a23c 474 add(name, entity_addrvec_t(addr), priority, weight);
11fdf7f2
TL
475 }
476 } else if (addr.get_port() == 0) {
477 // no port; include both msgr2 and legacy ports
478 if (!for_mkfs) {
479 addr.set_type(entity_addr_t::TYPE_MSGR2);
480 addr.set_port(CEPH_MON_PORT_IANA);
481 if (!contains(addr)) {
9f95a23c 482 add(name, entity_addrvec_t(addr), priority, weight);
11fdf7f2
TL
483 }
484 addr.set_type(entity_addr_t::TYPE_LEGACY);
485 addr.set_port(CEPH_MON_PORT_LEGACY);
486 if (!contains(addr)) {
9f95a23c 487 add(name + "-legacy", entity_addrvec_t(addr), priority, weight);
11fdf7f2
TL
488 }
489 } else {
490 entity_addrvec_t av;
491 addr.set_type(entity_addr_t::TYPE_MSGR2);
492 addr.set_port(CEPH_MON_PORT_IANA);
493 av.v.push_back(addr);
494 addr.set_type(entity_addr_t::TYPE_LEGACY);
495 addr.set_port(CEPH_MON_PORT_LEGACY);
496 av.v.push_back(addr);
497 if (!contains(av)) {
9f95a23c 498 add(name, av, priority, weight);
11fdf7f2
TL
499 }
500 }
501 } else {
502 addr.set_type(entity_addr_t::TYPE_MSGR2);
503 if (!contains(addr)) {
9f95a23c 504 add(name, entity_addrvec_t(addr), priority, weight);
11fdf7f2
TL
505 }
506 if (!for_mkfs) {
507 // try legacy on same port too
508 addr.set_type(entity_addr_t::TYPE_LEGACY);
509 if (!contains(addr)) {
9f95a23c 510 add(name + "-legacy", entity_addrvec_t(addr), priority, weight);
11fdf7f2
TL
511 }
512 }
513 }
514 }
515}
7c673cae 516
f91f0fd5
TL
517void MonMap::init_with_addrs(const std::vector<entity_addrvec_t>& addrs,
518 bool for_mkfs,
519 std::string_view prefix)
520{
521 char id = 'a';
522 for (auto& addr : addrs) {
523 string name{prefix};
524 name += id++;
525 if (addr.v.size() == 1) {
526 _add_ambiguous_addr(name, addr.front(), 0, 0, for_mkfs);
527 } else {
528 // they specified an addrvec, so let's assume they also specified
529 // the addr *type* and *port*. (we could possibly improve this?)
530 add(name, addr, 0);
531 }
532 }
533}
534
11fdf7f2
TL
535int MonMap::init_with_ips(const std::string& ips,
536 bool for_mkfs,
f91f0fd5 537 std::string_view prefix)
7c673cae 538{
11fdf7f2
TL
539 vector<entity_addrvec_t> addrs;
540 if (!parse_ip_port_vec(
541 ips.c_str(), addrs,
542 entity_addr_t::TYPE_ANY)) {
543 return -EINVAL;
544 }
545 if (addrs.empty())
546 return -ENOENT;
f91f0fd5 547 init_with_addrs(addrs, for_mkfs, prefix);
11fdf7f2
TL
548 return 0;
549}
7c673cae 550
11fdf7f2
TL
551int MonMap::init_with_hosts(const std::string& hostlist,
552 bool for_mkfs,
f91f0fd5 553 std::string_view prefix)
11fdf7f2 554{
7c673cae 555 // maybe they passed us a DNS-resolvable name
11fdf7f2 556 char *hosts = resolve_addrs(hostlist.c_str());
7c673cae
FG
557 if (!hosts)
558 return -EINVAL;
11fdf7f2
TL
559
560 vector<entity_addrvec_t> addrs;
561 bool success = parse_ip_port_vec(
562 hosts, addrs,
92f5a8d4 563 entity_addr_t::TYPE_ANY);
7c673cae
FG
564 free(hosts);
565 if (!success)
566 return -EINVAL;
7c673cae
FG
567 if (addrs.empty())
568 return -ENOENT;
f91f0fd5 569 init_with_addrs(addrs, for_mkfs, prefix);
11fdf7f2 570 calc_legacy_ranks();
7c673cae
FG
571 return 0;
572}
573
574void MonMap::set_initial_members(CephContext *cct,
575 list<std::string>& initial_members,
11fdf7f2
TL
576 string my_name,
577 const entity_addrvec_t& my_addrs,
578 set<entity_addrvec_t> *removed)
7c673cae
FG
579{
580 // remove non-initial members
581 unsigned i = 0;
582 while (i < size()) {
583 string n = get_name(i);
11fdf7f2
TL
584 if (std::find(initial_members.begin(), initial_members.end(), n)
585 != initial_members.end()) {
586 lgeneric_dout(cct, 1) << " keeping " << n << " " << get_addrs(i) << dendl;
7c673cae
FG
587 i++;
588 continue;
589 }
590
11fdf7f2
TL
591 lgeneric_dout(cct, 1) << " removing " << get_name(i) << " " << get_addrs(i)
592 << dendl;
593 if (removed) {
594 removed->insert(get_addrs(i));
595 }
7c673cae 596 remove(n);
11fdf7f2 597 ceph_assert(!contains(n));
7c673cae
FG
598 }
599
600 // add missing initial members
11fdf7f2
TL
601 for (auto& p : initial_members) {
602 if (!contains(p)) {
603 if (p == my_name) {
604 lgeneric_dout(cct, 1) << " adding self " << p << " " << my_addrs
605 << dendl;
606 add(p, my_addrs);
7c673cae
FG
607 } else {
608 entity_addr_t a;
609 a.set_type(entity_addr_t::TYPE_LEGACY);
610 a.set_family(AF_INET);
611 for (int n=1; ; n++) {
612 a.set_nonce(n);
613 if (!contains(a))
614 break;
615 }
11fdf7f2
TL
616 lgeneric_dout(cct, 1) << " adding " << p << " " << a << dendl;
617 add(p, entity_addrvec_t(a));
7c673cae 618 }
11fdf7f2 619 ceph_assert(contains(p));
7c673cae
FG
620 }
621 }
11fdf7f2 622 calc_legacy_ranks();
7c673cae
FG
623}
624
11fdf7f2
TL
625int MonMap::init_with_config_file(const ConfigProxy& conf,
626 std::ostream& errout)
7c673cae 627{
11fdf7f2
TL
628 std::vector<std::string> sections;
629 int ret = conf.get_all_sections(sections);
7c673cae
FG
630 if (ret) {
631 errout << "Unable to find any monitors in the configuration "
632 << "file, because there was an error listing the sections. error "
633 << ret << std::endl;
634 return -ENOENT;
635 }
11fdf7f2
TL
636 std::vector<std::string> mon_names;
637 for (const auto& section : sections) {
638 if (section.substr(0, 4) == "mon." && section.size() > 4) {
639 mon_names.push_back(section.substr(4));
7c673cae
FG
640 }
641 }
642
643 // Find an address for each monitor in the config file.
11fdf7f2
TL
644 for (const auto& mon_name : mon_names) {
645 std::vector<std::string> sections;
7c673cae
FG
646 std::string m_name("mon");
647 m_name += ".";
11fdf7f2 648 m_name += mon_name;
7c673cae
FG
649 sections.push_back(m_name);
650 sections.push_back("mon");
651 sections.push_back("global");
652 std::string val;
11fdf7f2 653 int res = conf.get_val_from_conf_file(sections, "mon addr", val, true);
7c673cae 654 if (res) {
11fdf7f2
TL
655 errout << "failed to get an address for mon." << mon_name
656 << ": error " << res << std::endl;
7c673cae
FG
657 continue;
658 }
11fdf7f2
TL
659 // the 'mon addr' field is a legacy field, so assume anything
660 // there on a weird port is a v1 address, and do not handle
661 // addrvecs.
7c673cae 662 entity_addr_t addr;
11fdf7f2
TL
663 if (!addr.parse(val.c_str(), nullptr, entity_addr_t::TYPE_LEGACY)) {
664 errout << "unable to parse address for mon." << mon_name
665 << ": addr='" << val << "'" << std::endl;
7c673cae
FG
666 continue;
667 }
11fdf7f2
TL
668 if (addr.get_port() == 0) {
669 addr.set_port(CEPH_MON_PORT_LEGACY);
670 }
224ce89b 671 uint16_t priority = 0;
11fdf7f2 672 if (!conf.get_val_from_conf_file(sections, "mon priority", val, false)) {
224ce89b
WB
673 try {
674 priority = std::stoul(val);
675 } catch (std::logic_error&) {
11fdf7f2 676 errout << "unable to parse priority for mon." << mon_name
224ce89b
WB
677 << ": priority='" << val << "'" << std::endl;
678 continue;
679 }
680 }
9f95a23c
TL
681 uint16_t weight = 0;
682 if (!conf.get_val_from_conf_file(sections, "mon weight", val, false)) {
683 try {
684 weight = std::stoul(val);
685 } catch (std::logic_error&) {
686 errout << "unable to parse weight for mon." << mon_name
687 << ": weight='" << val << "'"
688 << std::endl;
689 continue;
690 }
691 }
11fdf7f2 692
9f95a23c 693 // make sure this mon isn't already in the map
7c673cae
FG
694 if (contains(addr))
695 remove(get_name(addr));
11fdf7f2
TL
696 if (contains(mon_name))
697 remove(mon_name);
9f95a23c 698 _add_ambiguous_addr(mon_name, addr, priority, weight, false);
11fdf7f2
TL
699 }
700 return 0;
701}
702
f67539c2
TL
703void MonMap::check_health(health_check_map_t *checks) const
704{
705 if (stretch_mode_enabled) {
706 list<string> detail;
707 for (auto& p : mon_info) {
708 if (p.second.crush_loc.empty()) {
709 ostringstream ss;
710 ss << "mon " << p.first << " has no location set while in stretch mode";
711 detail.push_back(ss.str());
712 }
713 }
714 if (!detail.empty()) {
715 ostringstream ss;
716 ss << detail.size() << " monitor(s) have no location set while in stretch mode"
717 << "; this may cause issues with failover, OSD connections, netsplit handling, etc";
718 auto& d = checks->add("MON_LOCATION_NOT_SET", HEALTH_WARN,
719 ss.str(), detail.size());
720 d.detail.swap(detail);
721 }
722 }
723}
724
11fdf7f2
TL
725#ifdef WITH_SEASTAR
726
727using namespace seastar;
728
f67539c2 729seastar::future<> MonMap::read_monmap(const std::string& monmap)
11fdf7f2
TL
730{
731 return open_file_dma(monmap, open_flags::ro).then([this] (file f) {
732 return f.size().then([this, f = std::move(f)](size_t s) {
733 return do_with(make_file_input_stream(f), [this, s](input_stream<char>& in) {
734 return in.read_exactly(s).then([this](temporary_buffer<char> buf) {
9f95a23c
TL
735 ceph::buffer::list bl;
736 bl.push_back(ceph::buffer::ptr_node::create(
737 ceph::buffer::create(std::move(buf))));
11fdf7f2
TL
738 decode(bl);
739 });
740 });
741 });
742 });
743}
7c673cae 744
f67539c2 745seastar::future<> MonMap::init_with_dns_srv(bool for_mkfs, const std::string& name)
11fdf7f2
TL
746{
747 string domain;
748 string service = name;
749 // check if domain is also provided and extract it from srv_name
750 size_t idx = name.find("_");
751 if (idx != name.npos) {
752 domain = name.substr(idx + 1);
753 service = name.substr(0, idx);
7c673cae 754 }
9f95a23c
TL
755 return seastar::net::dns::get_srv_records(
756 seastar::net::dns_resolver::srv_proto::tcp,
757 service, domain).then([this](seastar::net::dns_resolver::srv_records records) {
11fdf7f2 758 return parallel_for_each(records, [this](auto record) {
9f95a23c
TL
759 return seastar::net::dns::resolve_name(record.target).then(
760 [record,this](seastar::net::inet_address a) {
11fdf7f2
TL
761 // the resolved address does not contain ceph specific info like nonce
762 // nonce or msgr proto (legacy, msgr2), so set entity_addr_t manually
763 entity_addr_t addr;
764 addr.set_type(entity_addr_t::TYPE_ANY);
765 addr.set_family(int(a.in_family()));
766 addr.set_port(record.port);
767 switch (a.in_family()) {
9f95a23c 768 case seastar::net::inet_address::family::INET:
11fdf7f2
TL
769 addr.in4_addr().sin_addr = a;
770 break;
9f95a23c 771 case seastar::net::inet_address::family::INET6:
11fdf7f2
TL
772 addr.in6_addr().sin6_addr = a;
773 break;
774 }
9f95a23c
TL
775 _add_ambiguous_addr(record.target,
776 addr,
777 record.priority,
778 record.weight,
779 false);
11fdf7f2
TL
780 });
781 });
782 }).handle_exception_type([](const std::system_error& e) {
783 // ignore DNS failures
784 return seastar::make_ready_future<>();
785 });
786}
7c673cae 787
9f95a23c 788seastar::future<> MonMap::build_monmap(const crimson::common::ConfigProxy& conf,
11fdf7f2
TL
789 bool for_mkfs)
790{
791 // -m foo?
792 if (const auto mon_host = conf.get_val<std::string>("mon_host");
793 !mon_host.empty()) {
794 if (auto ret = init_with_ips(mon_host, for_mkfs, "noname-"); ret == 0) {
795 return make_ready_future<>();
7c673cae 796 }
11fdf7f2
TL
797 // TODO: resolve_addrs() is a blocking call
798 if (auto ret = init_with_hosts(mon_host, for_mkfs, "noname-"); ret == 0) {
799 return make_ready_future<>();
800 } else {
801 throw std::runtime_error(cpp_strerror(ret));
802 }
803 }
7c673cae 804
11fdf7f2
TL
805 // What monitors are in the config file?
806 ostringstream errout;
807 if (auto ret = init_with_config_file(conf, errout); ret < 0) {
808 throw std::runtime_error(errout.str());
809 }
810 if (size() > 0) {
811 return make_ready_future<>();
812 }
813 // no info found from conf options lets try use DNS SRV records
814 const string srv_name = conf.get_val<std::string>("mon_dns_srv_name");
815 return init_with_dns_srv(for_mkfs, srv_name).then([this] {
816 if (size() == 0) {
817 throw std::runtime_error("no monitors specified to connect to.");
818 }
819 });
820}
7c673cae 821
f67539c2 822seastar::future<> MonMap::build_initial(const crimson::common::ConfigProxy& conf, bool for_mkfs)
11fdf7f2
TL
823{
824 // file?
825 if (const auto monmap = conf.get_val<std::string>("monmap");
826 !monmap.empty()) {
827 return read_monmap(monmap);
828 } else {
829 // fsid from conf?
830 if (const auto new_fsid = conf.get_val<uuid_d>("fsid");
831 !new_fsid.is_zero()) {
832 fsid = new_fsid;
7c673cae 833 }
11fdf7f2
TL
834 return build_monmap(conf, for_mkfs).then([this] {
835 created = ceph_clock_now();
836 last_changed = created;
837 calc_legacy_ranks();
838 });
839 }
840}
841
842#else // WITH_SEASTAR
843
844int MonMap::init_with_monmap(const std::string& monmap, std::ostream& errout)
845{
846 int r;
847 try {
848 r = read(monmap.c_str());
9f95a23c 849 } catch (ceph::buffer::error&) {
11fdf7f2
TL
850 r = -EINVAL;
851 }
852 if (r >= 0)
853 return 0;
854 errout << "unable to read/decode monmap from " << monmap
855 << ": " << cpp_strerror(-r) << std::endl;
856 return r;
857}
858
859int MonMap::init_with_dns_srv(CephContext* cct,
860 std::string srv_name,
861 bool for_mkfs,
862 std::ostream& errout)
863{
864 string domain;
865 // check if domain is also provided and extract it from srv_name
866 size_t idx = srv_name.find("_");
867 if (idx != string::npos) {
868 domain = srv_name.substr(idx + 1);
869 srv_name = srv_name.substr(0, idx);
870 }
871
872 map<string, DNSResolver::Record> records;
873 if (DNSResolver::get_instance()->resolve_srv_hosts(cct, srv_name,
874 DNSResolver::SRV_Protocol::TCP, domain, &records) != 0) {
875
876 errout << "unable to get monitor info from DNS SRV with service name: "
877 << "ceph-mon" << std::endl;
878 return -1;
879 } else {
880 for (auto& record : records) {
881 record.second.addr.set_type(entity_addr_t::TYPE_ANY);
9f95a23c
TL
882 _add_ambiguous_addr(record.first,
883 record.second.addr,
884 record.second.priority,
885 record.second.weight,
886 false);
7c673cae 887 }
11fdf7f2
TL
888 return 0;
889 }
890}
891
892int MonMap::build_initial(CephContext *cct, bool for_mkfs, ostream& errout)
893{
894 const auto& conf = cct->_conf;
f91f0fd5
TL
895
896 // mon_host_override?
897 auto mon_host_override = conf.get_val<std::string>("mon_host_override");
898 if (!mon_host_override.empty()) {
899 lgeneric_dout(cct, 1) << "Using mon_host_override " << mon_host_override << dendl;
900 auto ret = init_with_ips(mon_host_override, for_mkfs, "noname-");
901 if (ret == -EINVAL) {
902 ret = init_with_hosts(mon_host_override, for_mkfs, "noname-");
903 }
904 if (ret < 0) {
905 errout << "unable to parse addrs in '" << mon_host_override << "'"
906 << std::endl;
907 }
908 return ret;
909 }
910
911 // cct?
912 auto addrs = cct->get_mon_addrs();
913 if (addrs != nullptr && (addrs->size() > 0)) {
914 init_with_addrs(*addrs, for_mkfs, "noname-");
915 return 0;
916 }
917
11fdf7f2
TL
918 // file?
919 if (const auto monmap = conf.get_val<std::string>("monmap");
920 !monmap.empty()) {
921 return init_with_monmap(monmap, errout);
7c673cae
FG
922 }
923
11fdf7f2
TL
924 // fsid from conf?
925 if (const auto new_fsid = conf.get_val<uuid_d>("fsid");
926 !new_fsid.is_zero()) {
927 fsid = new_fsid;
928 }
929 // -m foo?
930 if (const auto mon_host = conf.get_val<std::string>("mon_host");
931 !mon_host.empty()) {
932 auto ret = init_with_ips(mon_host, for_mkfs, "noname-");
933 if (ret == -EINVAL) {
934 ret = init_with_hosts(mon_host, for_mkfs, "noname-");
935 }
936 if (ret < 0) {
937 errout << "unable to parse addrs in '" << mon_host << "'"
938 << std::endl;
939 return ret;
940 }
941 }
942 if (size() == 0) {
943 // What monitors are in the config file?
944 if (auto ret = init_with_config_file(conf, errout); ret < 0) {
945 return ret;
946 }
947 }
948 if (size() == 0) {
949 // no info found from conf options lets try use DNS SRV records
950 string srv_name = conf.get_val<std::string>("mon_dns_srv_name");
951 if (auto ret = init_with_dns_srv(cct, srv_name, for_mkfs, errout); ret < 0) {
952 return -ENOENT;
953 }
954 }
7c673cae
FG
955 if (size() == 0) {
956 errout << "no monitors specified to connect to." << std::endl;
957 return -ENOENT;
958 }
f67539c2 959 strategy = static_cast<election_strategy>(conf.get_val<uint64_t>("mon_election_default_strategy"));
7c673cae
FG
960 created = ceph_clock_now();
961 last_changed = created;
11fdf7f2 962 calc_legacy_ranks();
7c673cae
FG
963 return 0;
964}
11fdf7f2 965#endif // WITH_SEASTAR