]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | /* | |
4 | * Ceph - scalable distributed file system | |
5 | * | |
6 | * Copyright (C) 2016 John Spray <john.spray@redhat.com> | |
7 | * | |
8 | * This is free software; you can redistribute it and/or | |
9 | * modify it under the terms of the GNU Lesser General Public | |
10 | * License version 2.1, as published by the Free Software | |
11 | * Foundation. See file COPYING. | |
12 | */ | |
13 | ||
14 | #include <Python.h> | |
15 | ||
16 | #include "osdc/Objecter.h" | |
17 | #include "client/Client.h" | |
18 | #include "common/errno.h" | |
19 | #include "mon/MonClient.h" | |
20 | #include "include/stringify.h" | |
21 | #include "global/global_context.h" | |
22 | #include "global/signal_handler.h" | |
23 | ||
24 | #include "mgr/MgrContext.h" | |
25 | ||
7c673cae | 26 | #include "DaemonServer.h" |
7c673cae FG |
27 | #include "messages/MMgrDigest.h" |
28 | #include "messages/MCommand.h" | |
29 | #include "messages/MCommandReply.h" | |
30 | #include "messages/MLog.h" | |
224ce89b | 31 | #include "messages/MServiceMap.h" |
11fdf7f2 | 32 | #include "PyModule.h" |
7c673cae FG |
33 | #include "Mgr.h" |
34 | ||
35 | #define dout_context g_ceph_context | |
36 | #define dout_subsys ceph_subsys_mgr | |
37 | #undef dout_prefix | |
38 | #define dout_prefix *_dout << "mgr " << __func__ << " " | |
39 | ||
40 | ||
224ce89b | 41 | Mgr::Mgr(MonClient *monc_, const MgrMap& mgrmap, |
3efd9988 | 42 | PyModuleRegistry *py_module_registry_, |
224ce89b | 43 | Messenger *clientm_, Objecter *objecter_, |
7c673cae FG |
44 | Client* client_, LogChannelRef clog_, LogChannelRef audit_clog_) : |
45 | monc(monc_), | |
46 | objecter(objecter_), | |
47 | client(client_), | |
48 | client_messenger(clientm_), | |
49 | lock("Mgr::lock"), | |
7c673cae | 50 | finisher(g_ceph_context, "Mgr", "mgr-fin"), |
224ce89b | 51 | digest_received(false), |
3efd9988 | 52 | py_module_registry(py_module_registry_), |
224ce89b | 53 | cluster_state(monc, nullptr, mgrmap), |
3efd9988 | 54 | server(monc, finisher, daemon_state, cluster_state, *py_module_registry, |
7c673cae | 55 | clog_, audit_clog_), |
3efd9988 FG |
56 | clog(clog_), |
57 | audit_clog(audit_clog_), | |
7c673cae FG |
58 | initialized(false), |
59 | initializing(false) | |
60 | { | |
61 | cluster_state.set_objecter(objecter); | |
62 | } | |
63 | ||
64 | ||
65 | Mgr::~Mgr() | |
66 | { | |
67 | } | |
68 | ||
b32b8144 | 69 | void MetadataUpdate::finish(int r) |
7c673cae | 70 | { |
b32b8144 FG |
71 | daemon_state.clear_updating(key); |
72 | if (r == 0) { | |
11fdf7f2 TL |
73 | if (key.first == "mds" || key.first == "osd" || |
74 | key.first == "mgr" || key.first == "mon") { | |
b32b8144 FG |
75 | json_spirit::mValue json_result; |
76 | bool read_ok = json_spirit::read( | |
77 | outbl.to_str(), json_result); | |
78 | if (!read_ok) { | |
79 | dout(1) << "mon returned invalid JSON for " | |
80 | << key.first << "." << key.second << dendl; | |
81 | return; | |
82 | } | |
11fdf7f2 TL |
83 | if (json_result.type() != json_spirit::obj_type) { |
84 | dout(1) << "mon returned valid JSON " | |
85 | << key.first << "." << key.second | |
86 | << " but not an object: '" << outbl.to_str() << "'" << dendl; | |
87 | return; | |
88 | } | |
b32b8144 FG |
89 | dout(4) << "mon returned valid metadata JSON for " |
90 | << key.first << "." << key.second << dendl; | |
7c673cae | 91 | |
b32b8144 | 92 | json_spirit::mObject daemon_meta = json_result.get_obj(); |
7c673cae | 93 | |
11fdf7f2 TL |
94 | // Skip daemon who doesn't have hostname yet |
95 | if (daemon_meta.count("hostname") == 0) { | |
96 | dout(1) << "Skipping incomplete metadata entry for " | |
97 | << key.first << "." << key.second << dendl; | |
98 | return; | |
99 | } | |
100 | ||
b32b8144 FG |
101 | // Apply any defaults |
102 | for (const auto &i : defaults) { | |
103 | if (daemon_meta.find(i.first) == daemon_meta.end()) { | |
104 | daemon_meta[i.first] = i.second; | |
7c673cae | 105 | } |
b32b8144 | 106 | } |
7c673cae | 107 | |
b32b8144 FG |
108 | DaemonStatePtr state; |
109 | if (daemon_state.exists(key)) { | |
110 | state = daemon_state.get(key); | |
11fdf7f2 | 111 | if (key.first == "mds" || key.first == "mgr" || key.first == "mon") { |
b32b8144 FG |
112 | daemon_meta.erase("name"); |
113 | } else if (key.first == "osd") { | |
114 | daemon_meta.erase("id"); | |
115 | } | |
116 | daemon_meta.erase("hostname"); | |
11fdf7f2 | 117 | map<string,string> m; |
b32b8144 | 118 | for (const auto &i : daemon_meta) { |
11fdf7f2 TL |
119 | m[i.first] = i.second.get_str(); |
120 | } | |
121 | ||
122 | daemon_state.update_metadata(state, m); | |
b32b8144 FG |
123 | } else { |
124 | state = std::make_shared<DaemonState>(daemon_state.types); | |
125 | state->key = key; | |
126 | state->hostname = daemon_meta.at("hostname").get_str(); | |
7c673cae | 127 | |
11fdf7f2 | 128 | if (key.first == "mds" || key.first == "mgr" || key.first == "mon") { |
7c673cae | 129 | daemon_meta.erase("name"); |
b32b8144 FG |
130 | } else if (key.first == "osd") { |
131 | daemon_meta.erase("id"); | |
7c673cae | 132 | } |
b32b8144 FG |
133 | daemon_meta.erase("hostname"); |
134 | ||
11fdf7f2 | 135 | map<string,string> m; |
b32b8144 | 136 | for (const auto &i : daemon_meta) { |
11fdf7f2 | 137 | m[i.first] = i.second.get_str(); |
b32b8144 | 138 | } |
11fdf7f2 | 139 | state->set_metadata(m); |
b32b8144 FG |
140 | |
141 | daemon_state.insert(state); | |
7c673cae FG |
142 | } |
143 | } else { | |
b32b8144 | 144 | ceph_abort(); |
7c673cae | 145 | } |
b32b8144 FG |
146 | } else { |
147 | dout(1) << "mon failed to return metadata for " | |
148 | << key.first << "." << key.second << ": " | |
149 | << cpp_strerror(r) << dendl; | |
7c673cae | 150 | } |
b32b8144 | 151 | } |
7c673cae | 152 | |
224ce89b | 153 | void Mgr::background_init(Context *completion) |
7c673cae | 154 | { |
11fdf7f2 TL |
155 | std::lock_guard l(lock); |
156 | ceph_assert(!initializing); | |
157 | ceph_assert(!initialized); | |
7c673cae FG |
158 | initializing = true; |
159 | ||
160 | finisher.start(); | |
161 | ||
224ce89b | 162 | finisher.queue(new FunctionContext([this, completion](int r){ |
7c673cae | 163 | init(); |
224ce89b | 164 | completion->complete(0); |
7c673cae FG |
165 | })); |
166 | } | |
167 | ||
11fdf7f2 TL |
168 | std::map<std::string, std::string> Mgr::load_store() |
169 | { | |
170 | ceph_assert(lock.is_locked_by_me()); | |
171 | ||
172 | dout(10) << "listing keys" << dendl; | |
173 | JSONCommand cmd; | |
174 | cmd.run(monc, "{\"prefix\": \"config-key ls\"}"); | |
175 | lock.Unlock(); | |
176 | cmd.wait(); | |
177 | lock.Lock(); | |
178 | ceph_assert(cmd.r == 0); | |
179 | ||
180 | std::map<std::string, std::string> loaded; | |
181 | ||
182 | for (auto &key_str : cmd.json_result.get_array()) { | |
183 | std::string const key = key_str.get_str(); | |
184 | ||
185 | dout(20) << "saw key '" << key << "'" << dendl; | |
186 | ||
187 | const std::string config_prefix = PyModule::config_prefix; | |
188 | const std::string device_prefix = "device/"; | |
189 | ||
190 | if (key.substr(0, config_prefix.size()) == config_prefix || | |
191 | key.substr(0, device_prefix.size()) == device_prefix) { | |
192 | dout(20) << "fetching '" << key << "'" << dendl; | |
193 | Command get_cmd; | |
194 | std::ostringstream cmd_json; | |
195 | cmd_json << "{\"prefix\": \"config-key get\", \"key\": \"" << key << "\"}"; | |
196 | get_cmd.run(monc, cmd_json.str()); | |
197 | lock.Unlock(); | |
198 | get_cmd.wait(); | |
199 | lock.Lock(); | |
200 | if (get_cmd.r == 0) { // tolerate racing config-key change | |
201 | if (key.substr(0, device_prefix.size()) == device_prefix) { | |
202 | // device/ | |
203 | string devid = key.substr(device_prefix.size()); | |
204 | map<string,string> meta; | |
205 | ostringstream ss; | |
206 | string val = get_cmd.outbl.to_str(); | |
207 | int r = get_json_str_map(val, ss, &meta, false); | |
208 | if (r < 0) { | |
209 | derr << __func__ << " failed to parse " << val << ": " << ss.str() | |
210 | << dendl; | |
211 | } else { | |
212 | daemon_state.with_device_create( | |
213 | devid, [&meta] (DeviceState& dev) { | |
214 | dev.set_metadata(std::move(meta)); | |
215 | }); | |
216 | } | |
217 | } else { | |
218 | // config/ | |
219 | loaded[key] = get_cmd.outbl.to_str(); | |
220 | } | |
221 | } | |
222 | } | |
223 | } | |
224 | ||
225 | return loaded; | |
226 | } | |
227 | ||
7c673cae FG |
228 | void Mgr::init() |
229 | { | |
11fdf7f2 TL |
230 | std::lock_guard l(lock); |
231 | ceph_assert(initializing); | |
232 | ceph_assert(!initialized); | |
7c673cae FG |
233 | |
234 | // Start communicating with daemons to learn statistics etc | |
11fdf7f2 | 235 | int r = server.init(monc->get_global_id(), client_messenger->get_myaddrs()); |
7c673cae | 236 | if (r < 0) { |
94b18763 FG |
237 | derr << "Initialize server fail: " << cpp_strerror(r) << dendl; |
238 | // This is typically due to a bind() failure, so let's let | |
239 | // systemd restart us. | |
240 | exit(1); | |
7c673cae | 241 | } |
11fdf7f2 | 242 | dout(4) << "Initialized server at " << server.get_myaddrs() << dendl; |
7c673cae FG |
243 | |
244 | // Preload all daemon metadata (will subsequently keep this | |
245 | // up to date by watching maps, so do the initial load before | |
246 | // we subscribe to any maps) | |
247 | dout(4) << "Loading daemon metadata..." << dendl; | |
248 | load_all_metadata(); | |
249 | ||
250 | // subscribe to all the maps | |
251 | monc->sub_want("log-info", 0, 0); | |
252 | monc->sub_want("mgrdigest", 0, 0); | |
253 | monc->sub_want("fsmap", 0, 0); | |
224ce89b | 254 | monc->sub_want("servicemap", 0, 0); |
7c673cae FG |
255 | |
256 | dout(4) << "waiting for OSDMap..." << dendl; | |
257 | // Subscribe to OSDMap update to pass on to ClusterState | |
258 | objecter->maybe_request_map(); | |
259 | ||
260 | // reset the mon session. we get these maps through subscriptions which | |
261 | // are stateful with the connection, so even if *we* don't have them a | |
262 | // previous incarnation sharing the same MonClient may have. | |
263 | monc->reopen_session(); | |
264 | ||
265 | // Start Objecter and wait for OSD map | |
266 | lock.Unlock(); // Drop lock because OSDMap dispatch calls into my ms_dispatch | |
267 | objecter->wait_for_osd_map(); | |
268 | lock.Lock(); | |
269 | ||
270 | // Populate PGs in ClusterState | |
11fdf7f2 TL |
271 | cluster_state.with_osdmap_and_pgmap([this](const OSDMap &osd_map, |
272 | const PGMap& pg_map) { | |
7c673cae FG |
273 | cluster_state.notify_osdmap(osd_map); |
274 | }); | |
275 | ||
276 | // Wait for FSMap | |
277 | dout(4) << "waiting for FSMap..." << dendl; | |
278 | while (!cluster_state.have_fsmap()) { | |
279 | fs_map_cond.Wait(lock); | |
280 | } | |
281 | ||
282 | dout(4) << "waiting for config-keys..." << dendl; | |
283 | ||
224ce89b WB |
284 | // Wait for MgrDigest... |
285 | dout(4) << "waiting for MgrDigest..." << dendl; | |
286 | while (!digest_received) { | |
287 | digest_cond.Wait(lock); | |
288 | } | |
7c673cae | 289 | |
11fdf7f2 TL |
290 | // Load module KV store |
291 | auto kv_store = load_store(); | |
292 | ||
293 | // Migrate config from KV store on luminous->mimic | |
294 | // drop lock because we do blocking config sets to mon | |
295 | lock.Unlock(); | |
296 | py_module_registry->upgrade_config(monc, kv_store); | |
297 | lock.Lock(); | |
298 | ||
7c673cae | 299 | // assume finisher already initialized in background_init |
3efd9988 | 300 | dout(4) << "starting python modules..." << dendl; |
11fdf7f2 TL |
301 | py_module_registry->active_start(daemon_state, cluster_state, |
302 | kv_store, *monc, clog, audit_clog, *objecter, *client, | |
303 | finisher, server); | |
7c673cae FG |
304 | |
305 | dout(4) << "Complete." << dendl; | |
306 | initializing = false; | |
307 | initialized = true; | |
308 | } | |
309 | ||
310 | void Mgr::load_all_metadata() | |
311 | { | |
11fdf7f2 | 312 | ceph_assert(lock.is_locked_by_me()); |
7c673cae FG |
313 | |
314 | JSONCommand mds_cmd; | |
315 | mds_cmd.run(monc, "{\"prefix\": \"mds metadata\"}"); | |
316 | JSONCommand osd_cmd; | |
317 | osd_cmd.run(monc, "{\"prefix\": \"osd metadata\"}"); | |
318 | JSONCommand mon_cmd; | |
319 | mon_cmd.run(monc, "{\"prefix\": \"mon metadata\"}"); | |
320 | ||
321 | lock.Unlock(); | |
322 | mds_cmd.wait(); | |
323 | osd_cmd.wait(); | |
324 | mon_cmd.wait(); | |
325 | lock.Lock(); | |
326 | ||
11fdf7f2 TL |
327 | ceph_assert(mds_cmd.r == 0); |
328 | ceph_assert(mon_cmd.r == 0); | |
329 | ceph_assert(osd_cmd.r == 0); | |
7c673cae FG |
330 | |
331 | for (auto &metadata_val : mds_cmd.json_result.get_array()) { | |
332 | json_spirit::mObject daemon_meta = metadata_val.get_obj(); | |
333 | if (daemon_meta.count("hostname") == 0) { | |
334 | dout(1) << "Skipping incomplete metadata entry" << dendl; | |
335 | continue; | |
336 | } | |
337 | ||
338 | DaemonStatePtr dm = std::make_shared<DaemonState>(daemon_state.types); | |
224ce89b | 339 | dm->key = DaemonKey("mds", |
7c673cae FG |
340 | daemon_meta.at("name").get_str()); |
341 | dm->hostname = daemon_meta.at("hostname").get_str(); | |
342 | ||
343 | daemon_meta.erase("name"); | |
344 | daemon_meta.erase("hostname"); | |
345 | ||
346 | for (const auto &i : daemon_meta) { | |
347 | dm->metadata[i.first] = i.second.get_str(); | |
348 | } | |
349 | ||
350 | daemon_state.insert(dm); | |
351 | } | |
352 | ||
353 | for (auto &metadata_val : mon_cmd.json_result.get_array()) { | |
354 | json_spirit::mObject daemon_meta = metadata_val.get_obj(); | |
355 | if (daemon_meta.count("hostname") == 0) { | |
356 | dout(1) << "Skipping incomplete metadata entry" << dendl; | |
357 | continue; | |
358 | } | |
359 | ||
360 | DaemonStatePtr dm = std::make_shared<DaemonState>(daemon_state.types); | |
224ce89b | 361 | dm->key = DaemonKey("mon", |
7c673cae FG |
362 | daemon_meta.at("name").get_str()); |
363 | dm->hostname = daemon_meta.at("hostname").get_str(); | |
364 | ||
365 | daemon_meta.erase("name"); | |
366 | daemon_meta.erase("hostname"); | |
367 | ||
11fdf7f2 | 368 | map<string,string> m; |
7c673cae | 369 | for (const auto &i : daemon_meta) { |
11fdf7f2 | 370 | m[i.first] = i.second.get_str(); |
7c673cae | 371 | } |
11fdf7f2 | 372 | dm->set_metadata(m); |
7c673cae FG |
373 | |
374 | daemon_state.insert(dm); | |
375 | } | |
376 | ||
377 | for (auto &osd_metadata_val : osd_cmd.json_result.get_array()) { | |
378 | json_spirit::mObject osd_metadata = osd_metadata_val.get_obj(); | |
379 | if (osd_metadata.count("hostname") == 0) { | |
380 | dout(1) << "Skipping incomplete metadata entry" << dendl; | |
381 | continue; | |
382 | } | |
383 | dout(4) << osd_metadata.at("hostname").get_str() << dendl; | |
384 | ||
385 | DaemonStatePtr dm = std::make_shared<DaemonState>(daemon_state.types); | |
224ce89b | 386 | dm->key = DaemonKey("osd", |
7c673cae FG |
387 | stringify(osd_metadata.at("id").get_int())); |
388 | dm->hostname = osd_metadata.at("hostname").get_str(); | |
389 | ||
390 | osd_metadata.erase("id"); | |
391 | osd_metadata.erase("hostname"); | |
392 | ||
11fdf7f2 | 393 | map<string,string> m; |
7c673cae | 394 | for (const auto &i : osd_metadata) { |
11fdf7f2 | 395 | m[i.first] = i.second.get_str(); |
7c673cae | 396 | } |
11fdf7f2 | 397 | dm->set_metadata(m); |
7c673cae FG |
398 | |
399 | daemon_state.insert(dm); | |
400 | } | |
401 | } | |
402 | ||
7c673cae FG |
403 | |
404 | void Mgr::shutdown() | |
405 | { | |
406 | finisher.queue(new FunctionContext([&](int) { | |
407 | { | |
11fdf7f2 | 408 | std::lock_guard l(lock); |
7c673cae FG |
409 | monc->sub_unwant("log-info"); |
410 | monc->sub_unwant("mgrdigest"); | |
411 | monc->sub_unwant("fsmap"); | |
412 | // First stop the server so that we're not taking any more incoming | |
413 | // requests | |
414 | server.shutdown(); | |
415 | } | |
416 | // after the messenger is stopped, signal modules to shutdown via finisher | |
3efd9988 | 417 | py_module_registry->active_shutdown(); |
7c673cae FG |
418 | })); |
419 | ||
420 | // Then stop the finisher to ensure its enqueued contexts aren't going | |
421 | // to touch references to the things we're about to tear down | |
422 | finisher.wait_for_empty(); | |
423 | finisher.stop(); | |
424 | } | |
425 | ||
426 | void Mgr::handle_osd_map() | |
427 | { | |
11fdf7f2 | 428 | ceph_assert(lock.is_locked_by_me()); |
7c673cae FG |
429 | |
430 | std::set<std::string> names_exist; | |
431 | ||
432 | /** | |
433 | * When we see a new OSD map, inspect the entity addrs to | |
434 | * see if they have changed (service restart), and if so | |
435 | * reload the metadata. | |
436 | */ | |
11fdf7f2 TL |
437 | cluster_state.with_osdmap_and_pgmap([this, &names_exist](const OSDMap &osd_map, |
438 | const PGMap &pg_map) { | |
439 | for (int osd_id = 0; osd_id < osd_map.get_max_osd(); ++osd_id) { | |
7c673cae FG |
440 | if (!osd_map.exists(osd_id)) { |
441 | continue; | |
442 | } | |
443 | ||
444 | // Remember which OSDs exist so that we can cull any that don't | |
445 | names_exist.insert(stringify(osd_id)); | |
446 | ||
447 | // Consider whether to update the daemon metadata (new/restarted daemon) | |
448 | bool update_meta = false; | |
224ce89b | 449 | const auto k = DaemonKey("osd", stringify(osd_id)); |
7c673cae FG |
450 | if (daemon_state.is_updating(k)) { |
451 | continue; | |
452 | } | |
453 | ||
454 | if (daemon_state.exists(k)) { | |
455 | auto metadata = daemon_state.get(k); | |
11fdf7f2 | 456 | std::lock_guard l(metadata->lock); |
7c673cae FG |
457 | auto addr_iter = metadata->metadata.find("front_addr"); |
458 | if (addr_iter != metadata->metadata.end()) { | |
459 | const std::string &metadata_addr = addr_iter->second; | |
11fdf7f2 | 460 | const auto &map_addrs = osd_map.get_addrs(osd_id); |
7c673cae | 461 | |
11fdf7f2 | 462 | if (metadata_addr != stringify(map_addrs)) { |
7c673cae | 463 | dout(4) << "OSD[" << osd_id << "] addr change " << metadata_addr |
11fdf7f2 | 464 | << " != " << stringify(map_addrs) << dendl; |
7c673cae FG |
465 | update_meta = true; |
466 | } else { | |
467 | dout(20) << "OSD[" << osd_id << "] addr unchanged: " | |
468 | << metadata_addr << dendl; | |
469 | } | |
470 | } else { | |
471 | // Awkward case where daemon went into DaemonState because it | |
472 | // sent us a report but its metadata didn't get loaded yet | |
473 | update_meta = true; | |
474 | } | |
475 | } else { | |
476 | update_meta = true; | |
477 | } | |
478 | ||
479 | if (update_meta) { | |
7c673cae FG |
480 | auto c = new MetadataUpdate(daemon_state, k); |
481 | std::ostringstream cmd; | |
482 | cmd << "{\"prefix\": \"osd metadata\", \"id\": " | |
483 | << osd_id << "}"; | |
484 | monc->start_mon_command( | |
485 | {cmd.str()}, | |
486 | {}, &c->outbl, &c->outs, c); | |
487 | } | |
488 | } | |
489 | ||
490 | cluster_state.notify_osdmap(osd_map); | |
491 | }); | |
492 | ||
493 | // TODO: same culling for MonMap | |
224ce89b | 494 | daemon_state.cull("osd", names_exist); |
7c673cae FG |
495 | } |
496 | ||
497 | void Mgr::handle_log(MLog *m) | |
498 | { | |
499 | for (const auto &e : m->entries) { | |
3efd9988 | 500 | py_module_registry->notify_all(e); |
7c673cae FG |
501 | } |
502 | ||
503 | m->put(); | |
504 | } | |
505 | ||
224ce89b WB |
506 | void Mgr::handle_service_map(MServiceMap *m) |
507 | { | |
508 | dout(10) << "e" << m->service_map.epoch << dendl; | |
509 | cluster_state.set_service_map(m->service_map); | |
510 | server.got_service_map(); | |
511 | } | |
512 | ||
11fdf7f2 TL |
513 | void Mgr::handle_mon_map() |
514 | { | |
515 | dout(20) << __func__ << dendl; | |
516 | assert(lock.is_locked_by_me()); | |
517 | std::set<std::string> names_exist; | |
518 | cluster_state.with_monmap([&] (auto &monmap) { | |
519 | for (unsigned int i = 0; i < monmap.size(); i++) { | |
520 | names_exist.insert(monmap.get_name(i)); | |
521 | } | |
522 | }); | |
523 | daemon_state.cull("mon", names_exist); | |
524 | } | |
525 | ||
7c673cae FG |
526 | bool Mgr::ms_dispatch(Message *m) |
527 | { | |
528 | dout(4) << *m << dendl; | |
11fdf7f2 | 529 | std::lock_guard l(lock); |
7c673cae FG |
530 | |
531 | switch (m->get_type()) { | |
532 | case MSG_MGR_DIGEST: | |
533 | handle_mgr_digest(static_cast<MMgrDigest*>(m)); | |
534 | break; | |
535 | case CEPH_MSG_MON_MAP: | |
3efd9988 | 536 | py_module_registry->notify_all("mon_map", ""); |
11fdf7f2 | 537 | handle_mon_map(); |
7c673cae FG |
538 | m->put(); |
539 | break; | |
540 | case CEPH_MSG_FS_MAP: | |
3efd9988 | 541 | py_module_registry->notify_all("fs_map", ""); |
7c673cae FG |
542 | handle_fs_map((MFSMap*)m); |
543 | return false; // I shall let this pass through for Client | |
544 | break; | |
545 | case CEPH_MSG_OSD_MAP: | |
546 | handle_osd_map(); | |
547 | ||
3efd9988 | 548 | py_module_registry->notify_all("osd_map", ""); |
7c673cae FG |
549 | |
550 | // Continuous subscribe, so that we can generate notifications | |
551 | // for our MgrPyModules | |
552 | objecter->maybe_request_map(); | |
553 | m->put(); | |
554 | break; | |
224ce89b | 555 | case MSG_SERVICE_MAP: |
11fdf7f2 | 556 | handle_service_map(static_cast<MServiceMap*>(m)); |
3efd9988 | 557 | py_module_registry->notify_all("service_map", ""); |
224ce89b WB |
558 | m->put(); |
559 | break; | |
7c673cae FG |
560 | case MSG_LOG: |
561 | handle_log(static_cast<MLog *>(m)); | |
562 | break; | |
563 | ||
564 | default: | |
565 | return false; | |
566 | } | |
567 | return true; | |
568 | } | |
569 | ||
570 | ||
571 | void Mgr::handle_fs_map(MFSMap* m) | |
572 | { | |
11fdf7f2 | 573 | ceph_assert(lock.is_locked_by_me()); |
7c673cae FG |
574 | |
575 | std::set<std::string> names_exist; | |
576 | ||
577 | const FSMap &new_fsmap = m->get_fsmap(); | |
578 | ||
579 | fs_map_cond.Signal(); | |
580 | ||
581 | // TODO: callers (e.g. from python land) are potentially going to see | |
582 | // the new fsmap before we've bothered populating all the resulting | |
583 | // daemon_state. Maybe we should block python land while we're making | |
584 | // this kind of update? | |
585 | ||
586 | cluster_state.set_fsmap(new_fsmap); | |
587 | ||
588 | auto mds_info = new_fsmap.get_mds_info(); | |
589 | for (const auto &i : mds_info) { | |
590 | const auto &info = i.second; | |
591 | ||
592 | if (!new_fsmap.gid_exists(i.first)){ | |
593 | continue; | |
594 | } | |
595 | ||
596 | // Remember which MDS exists so that we can cull any that don't | |
597 | names_exist.insert(info.name); | |
598 | ||
224ce89b | 599 | const auto k = DaemonKey("mds", info.name); |
7c673cae FG |
600 | if (daemon_state.is_updating(k)) { |
601 | continue; | |
602 | } | |
603 | ||
604 | bool update = false; | |
605 | if (daemon_state.exists(k)) { | |
606 | auto metadata = daemon_state.get(k); | |
11fdf7f2 | 607 | std::lock_guard l(metadata->lock); |
7c673cae FG |
608 | if (metadata->metadata.empty() || |
609 | metadata->metadata.count("addr") == 0) { | |
610 | update = true; | |
611 | } else { | |
11fdf7f2 TL |
612 | auto metadata_addrs = metadata->metadata.at("addr"); |
613 | const auto map_addrs = info.addrs; | |
614 | update = metadata_addrs != stringify(map_addrs); | |
7c673cae | 615 | if (update) { |
11fdf7f2 TL |
616 | dout(4) << "MDS[" << info.name << "] addr change " << metadata_addrs |
617 | << " != " << stringify(map_addrs) << dendl; | |
7c673cae FG |
618 | } |
619 | } | |
620 | } else { | |
621 | update = true; | |
622 | } | |
623 | ||
624 | if (update) { | |
7c673cae FG |
625 | auto c = new MetadataUpdate(daemon_state, k); |
626 | ||
627 | // Older MDS daemons don't have addr in the metadata, so | |
628 | // fake it if the returned metadata doesn't have the field. | |
11fdf7f2 | 629 | c->set_default("addr", stringify(info.addrs)); |
7c673cae FG |
630 | |
631 | std::ostringstream cmd; | |
632 | cmd << "{\"prefix\": \"mds metadata\", \"who\": \"" | |
633 | << info.name << "\"}"; | |
634 | monc->start_mon_command( | |
635 | {cmd.str()}, | |
636 | {}, &c->outbl, &c->outs, c); | |
637 | } | |
638 | } | |
224ce89b | 639 | daemon_state.cull("mds", names_exist); |
7c673cae FG |
640 | } |
641 | ||
224ce89b WB |
642 | bool Mgr::got_mgr_map(const MgrMap& m) |
643 | { | |
11fdf7f2 | 644 | std::lock_guard l(lock); |
224ce89b WB |
645 | dout(10) << m << dendl; |
646 | ||
647 | set<string> old_modules; | |
648 | cluster_state.with_mgrmap([&](const MgrMap& m) { | |
649 | old_modules = m.modules; | |
650 | }); | |
651 | if (m.modules != old_modules) { | |
652 | derr << "mgrmap module list changed to (" << m.modules << "), respawn" | |
653 | << dendl; | |
654 | return true; | |
655 | } | |
656 | ||
657 | cluster_state.set_mgr_map(m); | |
11fdf7f2 | 658 | server.got_mgr_map(); |
224ce89b WB |
659 | |
660 | return false; | |
661 | } | |
7c673cae FG |
662 | |
663 | void Mgr::handle_mgr_digest(MMgrDigest* m) | |
664 | { | |
665 | dout(10) << m->mon_status_json.length() << dendl; | |
666 | dout(10) << m->health_json.length() << dendl; | |
667 | cluster_state.load_digest(m); | |
3efd9988 FG |
668 | py_module_registry->notify_all("mon_status", ""); |
669 | py_module_registry->notify_all("health", ""); | |
7c673cae FG |
670 | |
671 | // Hack: use this as a tick/opportunity to prompt python-land that | |
672 | // the pgmap might have changed since last time we were here. | |
3efd9988 | 673 | py_module_registry->notify_all("pg_summary", ""); |
7c673cae FG |
674 | dout(10) << "done." << dendl; |
675 | ||
676 | m->put(); | |
224ce89b WB |
677 | |
678 | if (!digest_received) { | |
679 | digest_received = true; | |
680 | digest_cond.Signal(); | |
681 | } | |
7c673cae FG |
682 | } |
683 | ||
3efd9988 FG |
684 | std::map<std::string, std::string> Mgr::get_services() const |
685 | { | |
11fdf7f2 | 686 | std::lock_guard l(lock); |
3efd9988 FG |
687 | |
688 | return py_module_registry->get_services(); | |
689 | } | |
690 |