]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | /* | |
4 | * Ceph - scalable distributed file system | |
5 | * | |
6 | * Copyright (C) 2016 John Spray <john.spray@redhat.com> | |
7 | * | |
8 | * This is free software; you can redistribute it and/or | |
9 | * modify it under the terms of the GNU Lesser General Public | |
10 | * License version 2.1, as published by the Free Software | |
11 | * Foundation. See file COPYING. | |
12 | */ | |
13 | ||
14 | #include <Python.h> | |
7f7e6c64 | 15 | #include <fmt/format.h> |
7c673cae FG |
16 | #include "osdc/Objecter.h" |
17 | #include "client/Client.h" | |
18 | #include "common/errno.h" | |
19 | #include "mon/MonClient.h" | |
20 | #include "include/stringify.h" | |
21 | #include "global/global_context.h" | |
22 | #include "global/signal_handler.h" | |
23 | ||
24 | #include "mgr/MgrContext.h" | |
25 | ||
7c673cae | 26 | #include "DaemonServer.h" |
7c673cae FG |
27 | #include "messages/MMgrDigest.h" |
28 | #include "messages/MCommand.h" | |
29 | #include "messages/MCommandReply.h" | |
30 | #include "messages/MLog.h" | |
224ce89b | 31 | #include "messages/MServiceMap.h" |
11fdf7f2 | 32 | #include "PyModule.h" |
7c673cae FG |
33 | #include "Mgr.h" |
34 | ||
35 | #define dout_context g_ceph_context | |
36 | #define dout_subsys ceph_subsys_mgr | |
37 | #undef dout_prefix | |
38 | #define dout_prefix *_dout << "mgr " << __func__ << " " | |
39 | ||
40 | ||
224ce89b | 41 | Mgr::Mgr(MonClient *monc_, const MgrMap& mgrmap, |
3efd9988 | 42 | PyModuleRegistry *py_module_registry_, |
224ce89b | 43 | Messenger *clientm_, Objecter *objecter_, |
7c673cae FG |
44 | Client* client_, LogChannelRef clog_, LogChannelRef audit_clog_) : |
45 | monc(monc_), | |
46 | objecter(objecter_), | |
47 | client(client_), | |
48 | client_messenger(clientm_), | |
7c673cae | 49 | finisher(g_ceph_context, "Mgr", "mgr-fin"), |
224ce89b | 50 | digest_received(false), |
3efd9988 | 51 | py_module_registry(py_module_registry_), |
224ce89b | 52 | cluster_state(monc, nullptr, mgrmap), |
3efd9988 | 53 | server(monc, finisher, daemon_state, cluster_state, *py_module_registry, |
7c673cae | 54 | clog_, audit_clog_), |
3efd9988 FG |
55 | clog(clog_), |
56 | audit_clog(audit_clog_), | |
7c673cae FG |
57 | initialized(false), |
58 | initializing(false) | |
59 | { | |
60 | cluster_state.set_objecter(objecter); | |
61 | } | |
62 | ||
63 | ||
64 | Mgr::~Mgr() | |
65 | { | |
66 | } | |
67 | ||
b32b8144 | 68 | void MetadataUpdate::finish(int r) |
7c673cae | 69 | { |
b32b8144 FG |
70 | daemon_state.clear_updating(key); |
71 | if (r == 0) { | |
9f95a23c TL |
72 | if (key.type == "mds" || key.type == "osd" || |
73 | key.type == "mgr" || key.type == "mon") { | |
b32b8144 FG |
74 | json_spirit::mValue json_result; |
75 | bool read_ok = json_spirit::read( | |
76 | outbl.to_str(), json_result); | |
77 | if (!read_ok) { | |
9f95a23c | 78 | dout(1) << "mon returned invalid JSON for " << key << dendl; |
b32b8144 FG |
79 | return; |
80 | } | |
11fdf7f2 | 81 | if (json_result.type() != json_spirit::obj_type) { |
9f95a23c | 82 | dout(1) << "mon returned valid JSON " << key |
11fdf7f2 TL |
83 | << " but not an object: '" << outbl.to_str() << "'" << dendl; |
84 | return; | |
85 | } | |
9f95a23c | 86 | dout(4) << "mon returned valid metadata JSON for " << key << dendl; |
7c673cae | 87 | |
b32b8144 | 88 | json_spirit::mObject daemon_meta = json_result.get_obj(); |
7c673cae | 89 | |
11fdf7f2 TL |
90 | // Skip daemon who doesn't have hostname yet |
91 | if (daemon_meta.count("hostname") == 0) { | |
9f95a23c | 92 | dout(1) << "Skipping incomplete metadata entry for " << key << dendl; |
11fdf7f2 TL |
93 | return; |
94 | } | |
95 | ||
b32b8144 FG |
96 | // Apply any defaults |
97 | for (const auto &i : defaults) { | |
98 | if (daemon_meta.find(i.first) == daemon_meta.end()) { | |
99 | daemon_meta[i.first] = i.second; | |
7c673cae | 100 | } |
b32b8144 | 101 | } |
7c673cae | 102 | |
b32b8144 FG |
103 | DaemonStatePtr state; |
104 | if (daemon_state.exists(key)) { | |
105 | state = daemon_state.get(key); | |
9f95a23c TL |
106 | state->hostname = daemon_meta.at("hostname").get_str(); |
107 | ||
108 | if (key.type == "mds" || key.type == "mgr" || key.type == "mon") { | |
b32b8144 | 109 | daemon_meta.erase("name"); |
9f95a23c | 110 | } else if (key.type == "osd") { |
b32b8144 FG |
111 | daemon_meta.erase("id"); |
112 | } | |
113 | daemon_meta.erase("hostname"); | |
11fdf7f2 | 114 | map<string,string> m; |
b32b8144 | 115 | for (const auto &i : daemon_meta) { |
11fdf7f2 TL |
116 | m[i.first] = i.second.get_str(); |
117 | } | |
118 | ||
119 | daemon_state.update_metadata(state, m); | |
b32b8144 FG |
120 | } else { |
121 | state = std::make_shared<DaemonState>(daemon_state.types); | |
122 | state->key = key; | |
123 | state->hostname = daemon_meta.at("hostname").get_str(); | |
7c673cae | 124 | |
9f95a23c | 125 | if (key.type == "mds" || key.type == "mgr" || key.type == "mon") { |
7c673cae | 126 | daemon_meta.erase("name"); |
9f95a23c | 127 | } else if (key.type == "osd") { |
b32b8144 | 128 | daemon_meta.erase("id"); |
7c673cae | 129 | } |
b32b8144 FG |
130 | daemon_meta.erase("hostname"); |
131 | ||
11fdf7f2 | 132 | map<string,string> m; |
b32b8144 | 133 | for (const auto &i : daemon_meta) { |
11fdf7f2 | 134 | m[i.first] = i.second.get_str(); |
b32b8144 | 135 | } |
11fdf7f2 | 136 | state->set_metadata(m); |
b32b8144 FG |
137 | |
138 | daemon_state.insert(state); | |
7c673cae FG |
139 | } |
140 | } else { | |
b32b8144 | 141 | ceph_abort(); |
7c673cae | 142 | } |
b32b8144 | 143 | } else { |
9f95a23c TL |
144 | dout(1) << "mon failed to return metadata for " << key |
145 | << ": " << cpp_strerror(r) << dendl; | |
7c673cae | 146 | } |
b32b8144 | 147 | } |
7c673cae | 148 | |
224ce89b | 149 | void Mgr::background_init(Context *completion) |
7c673cae | 150 | { |
11fdf7f2 TL |
151 | std::lock_guard l(lock); |
152 | ceph_assert(!initializing); | |
153 | ceph_assert(!initialized); | |
7c673cae FG |
154 | initializing = true; |
155 | ||
156 | finisher.start(); | |
157 | ||
9f95a23c | 158 | finisher.queue(new LambdaContext([this, completion](int r){ |
7c673cae | 159 | init(); |
224ce89b | 160 | completion->complete(0); |
7c673cae FG |
161 | })); |
162 | } | |
163 | ||
11fdf7f2 TL |
164 | std::map<std::string, std::string> Mgr::load_store() |
165 | { | |
9f95a23c | 166 | ceph_assert(ceph_mutex_is_locked_by_me(lock)); |
11fdf7f2 TL |
167 | |
168 | dout(10) << "listing keys" << dendl; | |
169 | JSONCommand cmd; | |
170 | cmd.run(monc, "{\"prefix\": \"config-key ls\"}"); | |
9f95a23c | 171 | lock.unlock(); |
11fdf7f2 | 172 | cmd.wait(); |
9f95a23c | 173 | lock.lock(); |
11fdf7f2 TL |
174 | ceph_assert(cmd.r == 0); |
175 | ||
176 | std::map<std::string, std::string> loaded; | |
177 | ||
178 | for (auto &key_str : cmd.json_result.get_array()) { | |
179 | std::string const key = key_str.get_str(); | |
180 | ||
181 | dout(20) << "saw key '" << key << "'" << dendl; | |
182 | ||
183 | const std::string config_prefix = PyModule::config_prefix; | |
184 | const std::string device_prefix = "device/"; | |
185 | ||
186 | if (key.substr(0, config_prefix.size()) == config_prefix || | |
187 | key.substr(0, device_prefix.size()) == device_prefix) { | |
188 | dout(20) << "fetching '" << key << "'" << dendl; | |
189 | Command get_cmd; | |
190 | std::ostringstream cmd_json; | |
191 | cmd_json << "{\"prefix\": \"config-key get\", \"key\": \"" << key << "\"}"; | |
192 | get_cmd.run(monc, cmd_json.str()); | |
9f95a23c | 193 | lock.unlock(); |
11fdf7f2 | 194 | get_cmd.wait(); |
9f95a23c | 195 | lock.lock(); |
11fdf7f2 TL |
196 | if (get_cmd.r == 0) { // tolerate racing config-key change |
197 | if (key.substr(0, device_prefix.size()) == device_prefix) { | |
198 | // device/ | |
199 | string devid = key.substr(device_prefix.size()); | |
200 | map<string,string> meta; | |
201 | ostringstream ss; | |
202 | string val = get_cmd.outbl.to_str(); | |
203 | int r = get_json_str_map(val, ss, &meta, false); | |
204 | if (r < 0) { | |
205 | derr << __func__ << " failed to parse " << val << ": " << ss.str() | |
206 | << dendl; | |
207 | } else { | |
208 | daemon_state.with_device_create( | |
209 | devid, [&meta] (DeviceState& dev) { | |
210 | dev.set_metadata(std::move(meta)); | |
211 | }); | |
212 | } | |
213 | } else { | |
214 | // config/ | |
215 | loaded[key] = get_cmd.outbl.to_str(); | |
216 | } | |
217 | } | |
218 | } | |
219 | } | |
220 | ||
221 | return loaded; | |
222 | } | |
223 | ||
9f95a23c TL |
224 | void Mgr::handle_signal(int signum) |
225 | { | |
226 | ceph_assert(signum == SIGINT || signum == SIGTERM); | |
227 | shutdown(); | |
228 | } | |
229 | ||
230 | static void handle_mgr_signal(int signum) | |
231 | { | |
232 | derr << " *** Got signal " << sig_str(signum) << " ***" << dendl; | |
233 | ||
234 | // The python modules don't reliably shut down, so don't even | |
235 | // try. The mon will blacklist us (and all of our rados/cephfs | |
236 | // clients) anyway. Just exit! | |
237 | ||
238 | _exit(0); // exit with 0 result code, as if we had done an orderly shutdown | |
239 | } | |
240 | ||
7c673cae FG |
241 | void Mgr::init() |
242 | { | |
9f95a23c | 243 | std::unique_lock l(lock); |
11fdf7f2 TL |
244 | ceph_assert(initializing); |
245 | ceph_assert(!initialized); | |
7c673cae | 246 | |
9f95a23c TL |
247 | // Enable signal handlers |
248 | register_async_signal_handler_oneshot(SIGINT, handle_mgr_signal); | |
249 | register_async_signal_handler_oneshot(SIGTERM, handle_mgr_signal); | |
7c673cae FG |
250 | |
251 | // subscribe to all the maps | |
252 | monc->sub_want("log-info", 0, 0); | |
253 | monc->sub_want("mgrdigest", 0, 0); | |
254 | monc->sub_want("fsmap", 0, 0); | |
224ce89b | 255 | monc->sub_want("servicemap", 0, 0); |
7c673cae FG |
256 | |
257 | dout(4) << "waiting for OSDMap..." << dendl; | |
258 | // Subscribe to OSDMap update to pass on to ClusterState | |
259 | objecter->maybe_request_map(); | |
260 | ||
261 | // reset the mon session. we get these maps through subscriptions which | |
262 | // are stateful with the connection, so even if *we* don't have them a | |
263 | // previous incarnation sharing the same MonClient may have. | |
264 | monc->reopen_session(); | |
265 | ||
266 | // Start Objecter and wait for OSD map | |
9f95a23c TL |
267 | lock.unlock(); // Drop lock because OSDMap dispatch calls into my ms_dispatch |
268 | epoch_t e; | |
269 | cluster_state.with_mgrmap([&e](const MgrMap& m) { | |
270 | e = m.last_failure_osd_epoch; | |
271 | }); | |
272 | /* wait for any blacklists to be applied to previous mgr instance */ | |
273 | dout(4) << "Waiting for new OSDMap (e=" << e | |
274 | << ") that may blacklist prior active." << dendl; | |
275 | objecter->wait_for_osd_map(e); | |
276 | lock.lock(); | |
277 | ||
278 | // Start communicating with daemons to learn statistics etc | |
279 | int r = server.init(monc->get_global_id(), client_messenger->get_myaddrs()); | |
280 | if (r < 0) { | |
281 | derr << "Initialize server fail: " << cpp_strerror(r) << dendl; | |
282 | // This is typically due to a bind() failure, so let's let | |
283 | // systemd restart us. | |
284 | exit(1); | |
285 | } | |
286 | dout(4) << "Initialized server at " << server.get_myaddrs() << dendl; | |
287 | ||
288 | // Preload all daemon metadata (will subsequently keep this | |
289 | // up to date by watching maps, so do the initial load before | |
290 | // we subscribe to any maps) | |
291 | dout(4) << "Loading daemon metadata..." << dendl; | |
292 | load_all_metadata(); | |
7c673cae FG |
293 | |
294 | // Populate PGs in ClusterState | |
11fdf7f2 TL |
295 | cluster_state.with_osdmap_and_pgmap([this](const OSDMap &osd_map, |
296 | const PGMap& pg_map) { | |
7c673cae FG |
297 | cluster_state.notify_osdmap(osd_map); |
298 | }); | |
299 | ||
300 | // Wait for FSMap | |
301 | dout(4) << "waiting for FSMap..." << dendl; | |
9f95a23c | 302 | fs_map_cond.wait(l, [this] { return cluster_state.have_fsmap();}); |
7c673cae FG |
303 | |
304 | dout(4) << "waiting for config-keys..." << dendl; | |
305 | ||
224ce89b WB |
306 | // Wait for MgrDigest... |
307 | dout(4) << "waiting for MgrDigest..." << dendl; | |
9f95a23c | 308 | digest_cond.wait(l, [this] { return digest_received; }); |
7c673cae | 309 | |
11fdf7f2 TL |
310 | // Load module KV store |
311 | auto kv_store = load_store(); | |
312 | ||
313 | // Migrate config from KV store on luminous->mimic | |
314 | // drop lock because we do blocking config sets to mon | |
9f95a23c | 315 | lock.unlock(); |
11fdf7f2 | 316 | py_module_registry->upgrade_config(monc, kv_store); |
9f95a23c | 317 | lock.lock(); |
11fdf7f2 | 318 | |
7c673cae | 319 | // assume finisher already initialized in background_init |
3efd9988 | 320 | dout(4) << "starting python modules..." << dendl; |
11fdf7f2 TL |
321 | py_module_registry->active_start(daemon_state, cluster_state, |
322 | kv_store, *monc, clog, audit_clog, *objecter, *client, | |
323 | finisher, server); | |
7c673cae | 324 | |
eafe8130 TL |
325 | cluster_state.final_init(); |
326 | ||
9f95a23c TL |
327 | AdminSocket *admin_socket = g_ceph_context->get_admin_socket(); |
328 | r = admin_socket->register_command( | |
329 | "mgr_status", this, | |
330 | "Dump mgr status"); | |
331 | ceph_assert(r == 0); | |
332 | ||
7c673cae FG |
333 | dout(4) << "Complete." << dendl; |
334 | initializing = false; | |
335 | initialized = true; | |
336 | } | |
337 | ||
338 | void Mgr::load_all_metadata() | |
339 | { | |
9f95a23c | 340 | ceph_assert(ceph_mutex_is_locked_by_me(lock)); |
7c673cae FG |
341 | |
342 | JSONCommand mds_cmd; | |
343 | mds_cmd.run(monc, "{\"prefix\": \"mds metadata\"}"); | |
344 | JSONCommand osd_cmd; | |
345 | osd_cmd.run(monc, "{\"prefix\": \"osd metadata\"}"); | |
346 | JSONCommand mon_cmd; | |
347 | mon_cmd.run(monc, "{\"prefix\": \"mon metadata\"}"); | |
348 | ||
9f95a23c | 349 | lock.unlock(); |
7c673cae FG |
350 | mds_cmd.wait(); |
351 | osd_cmd.wait(); | |
352 | mon_cmd.wait(); | |
9f95a23c | 353 | lock.lock(); |
7c673cae | 354 | |
11fdf7f2 TL |
355 | ceph_assert(mds_cmd.r == 0); |
356 | ceph_assert(mon_cmd.r == 0); | |
357 | ceph_assert(osd_cmd.r == 0); | |
7c673cae FG |
358 | |
359 | for (auto &metadata_val : mds_cmd.json_result.get_array()) { | |
360 | json_spirit::mObject daemon_meta = metadata_val.get_obj(); | |
361 | if (daemon_meta.count("hostname") == 0) { | |
362 | dout(1) << "Skipping incomplete metadata entry" << dendl; | |
363 | continue; | |
364 | } | |
365 | ||
366 | DaemonStatePtr dm = std::make_shared<DaemonState>(daemon_state.types); | |
9f95a23c TL |
367 | dm->key = DaemonKey{"mds", |
368 | daemon_meta.at("name").get_str()}; | |
7c673cae FG |
369 | dm->hostname = daemon_meta.at("hostname").get_str(); |
370 | ||
371 | daemon_meta.erase("name"); | |
372 | daemon_meta.erase("hostname"); | |
373 | ||
374 | for (const auto &i : daemon_meta) { | |
375 | dm->metadata[i.first] = i.second.get_str(); | |
376 | } | |
377 | ||
378 | daemon_state.insert(dm); | |
379 | } | |
380 | ||
381 | for (auto &metadata_val : mon_cmd.json_result.get_array()) { | |
382 | json_spirit::mObject daemon_meta = metadata_val.get_obj(); | |
383 | if (daemon_meta.count("hostname") == 0) { | |
384 | dout(1) << "Skipping incomplete metadata entry" << dendl; | |
385 | continue; | |
386 | } | |
387 | ||
388 | DaemonStatePtr dm = std::make_shared<DaemonState>(daemon_state.types); | |
9f95a23c TL |
389 | dm->key = DaemonKey{"mon", |
390 | daemon_meta.at("name").get_str()}; | |
7c673cae FG |
391 | dm->hostname = daemon_meta.at("hostname").get_str(); |
392 | ||
393 | daemon_meta.erase("name"); | |
394 | daemon_meta.erase("hostname"); | |
395 | ||
11fdf7f2 | 396 | map<string,string> m; |
7c673cae | 397 | for (const auto &i : daemon_meta) { |
11fdf7f2 | 398 | m[i.first] = i.second.get_str(); |
7c673cae | 399 | } |
11fdf7f2 | 400 | dm->set_metadata(m); |
7c673cae FG |
401 | |
402 | daemon_state.insert(dm); | |
403 | } | |
404 | ||
405 | for (auto &osd_metadata_val : osd_cmd.json_result.get_array()) { | |
406 | json_spirit::mObject osd_metadata = osd_metadata_val.get_obj(); | |
407 | if (osd_metadata.count("hostname") == 0) { | |
408 | dout(1) << "Skipping incomplete metadata entry" << dendl; | |
409 | continue; | |
410 | } | |
411 | dout(4) << osd_metadata.at("hostname").get_str() << dendl; | |
412 | ||
413 | DaemonStatePtr dm = std::make_shared<DaemonState>(daemon_state.types); | |
9f95a23c TL |
414 | dm->key = DaemonKey{"osd", |
415 | stringify(osd_metadata.at("id").get_int())}; | |
7c673cae FG |
416 | dm->hostname = osd_metadata.at("hostname").get_str(); |
417 | ||
418 | osd_metadata.erase("id"); | |
419 | osd_metadata.erase("hostname"); | |
420 | ||
11fdf7f2 | 421 | map<string,string> m; |
7c673cae | 422 | for (const auto &i : osd_metadata) { |
11fdf7f2 | 423 | m[i.first] = i.second.get_str(); |
7c673cae | 424 | } |
11fdf7f2 | 425 | dm->set_metadata(m); |
7c673cae FG |
426 | |
427 | daemon_state.insert(dm); | |
428 | } | |
429 | } | |
430 | ||
7c673cae FG |
431 | |
432 | void Mgr::shutdown() | |
433 | { | |
9f95a23c TL |
434 | dout(10) << "mgr shutdown init" << dendl; |
435 | finisher.queue(new LambdaContext([&](int) { | |
7c673cae | 436 | { |
11fdf7f2 | 437 | std::lock_guard l(lock); |
7c673cae FG |
438 | // First stop the server so that we're not taking any more incoming |
439 | // requests | |
440 | server.shutdown(); | |
441 | } | |
442 | // after the messenger is stopped, signal modules to shutdown via finisher | |
3efd9988 | 443 | py_module_registry->active_shutdown(); |
7c673cae FG |
444 | })); |
445 | ||
446 | // Then stop the finisher to ensure its enqueued contexts aren't going | |
447 | // to touch references to the things we're about to tear down | |
448 | finisher.wait_for_empty(); | |
449 | finisher.stop(); | |
450 | } | |
451 | ||
452 | void Mgr::handle_osd_map() | |
453 | { | |
9f95a23c | 454 | ceph_assert(ceph_mutex_is_locked_by_me(lock)); |
7c673cae FG |
455 | |
456 | std::set<std::string> names_exist; | |
457 | ||
458 | /** | |
459 | * When we see a new OSD map, inspect the entity addrs to | |
460 | * see if they have changed (service restart), and if so | |
461 | * reload the metadata. | |
462 | */ | |
11fdf7f2 TL |
463 | cluster_state.with_osdmap_and_pgmap([this, &names_exist](const OSDMap &osd_map, |
464 | const PGMap &pg_map) { | |
465 | for (int osd_id = 0; osd_id < osd_map.get_max_osd(); ++osd_id) { | |
7c673cae FG |
466 | if (!osd_map.exists(osd_id)) { |
467 | continue; | |
468 | } | |
469 | ||
470 | // Remember which OSDs exist so that we can cull any that don't | |
471 | names_exist.insert(stringify(osd_id)); | |
472 | ||
473 | // Consider whether to update the daemon metadata (new/restarted daemon) | |
9f95a23c | 474 | const auto k = DaemonKey{"osd", std::to_string(osd_id)}; |
7c673cae FG |
475 | if (daemon_state.is_updating(k)) { |
476 | continue; | |
477 | } | |
478 | ||
9f95a23c | 479 | bool update_meta = false; |
7c673cae | 480 | if (daemon_state.exists(k)) { |
9f95a23c TL |
481 | if (osd_map.get_up_from(osd_id) == osd_map.get_epoch()) { |
482 | dout(4) << "Mgr::handle_osd_map: osd." << osd_id | |
483 | << " joined cluster at " << "e" << osd_map.get_epoch() | |
484 | << dendl; | |
7c673cae FG |
485 | update_meta = true; |
486 | } | |
487 | } else { | |
488 | update_meta = true; | |
489 | } | |
7c673cae | 490 | if (update_meta) { |
7c673cae FG |
491 | auto c = new MetadataUpdate(daemon_state, k); |
492 | std::ostringstream cmd; | |
493 | cmd << "{\"prefix\": \"osd metadata\", \"id\": " | |
494 | << osd_id << "}"; | |
495 | monc->start_mon_command( | |
496 | {cmd.str()}, | |
497 | {}, &c->outbl, &c->outs, c); | |
498 | } | |
499 | } | |
500 | ||
501 | cluster_state.notify_osdmap(osd_map); | |
502 | }); | |
503 | ||
504 | // TODO: same culling for MonMap | |
224ce89b | 505 | daemon_state.cull("osd", names_exist); |
7c673cae FG |
506 | } |
507 | ||
9f95a23c | 508 | void Mgr::handle_log(ref_t<MLog> m) |
7c673cae FG |
509 | { |
510 | for (const auto &e : m->entries) { | |
3efd9988 | 511 | py_module_registry->notify_all(e); |
7c673cae | 512 | } |
7c673cae FG |
513 | } |
514 | ||
9f95a23c | 515 | void Mgr::handle_service_map(ref_t<MServiceMap> m) |
224ce89b WB |
516 | { |
517 | dout(10) << "e" << m->service_map.epoch << dendl; | |
518 | cluster_state.set_service_map(m->service_map); | |
519 | server.got_service_map(); | |
520 | } | |
521 | ||
11fdf7f2 TL |
522 | void Mgr::handle_mon_map() |
523 | { | |
524 | dout(20) << __func__ << dendl; | |
9f95a23c | 525 | assert(ceph_mutex_is_locked_by_me(lock)); |
11fdf7f2 TL |
526 | std::set<std::string> names_exist; |
527 | cluster_state.with_monmap([&] (auto &monmap) { | |
528 | for (unsigned int i = 0; i < monmap.size(); i++) { | |
529 | names_exist.insert(monmap.get_name(i)); | |
530 | } | |
531 | }); | |
7f7e6c64 TL |
532 | for (const auto& name : names_exist) { |
533 | const auto k = DaemonKey{"mon", name}; | |
534 | if (daemon_state.is_updating(k)) { | |
535 | continue; | |
536 | } | |
537 | auto c = new MetadataUpdate(daemon_state, k); | |
538 | const char* cmd = R"({{"prefix": "mon metadata", "id": "{}"}})"; | |
539 | monc->start_mon_command({fmt::format(cmd, name)}, {}, | |
540 | &c->outbl, &c->outs, c); | |
541 | } | |
11fdf7f2 TL |
542 | daemon_state.cull("mon", names_exist); |
543 | } | |
544 | ||
9f95a23c | 545 | bool Mgr::ms_dispatch2(const ref_t<Message>& m) |
7c673cae | 546 | { |
9f95a23c | 547 | dout(10) << *m << dendl; |
11fdf7f2 | 548 | std::lock_guard l(lock); |
7c673cae FG |
549 | |
550 | switch (m->get_type()) { | |
551 | case MSG_MGR_DIGEST: | |
9f95a23c | 552 | handle_mgr_digest(ref_cast<MMgrDigest>(m)); |
7c673cae FG |
553 | break; |
554 | case CEPH_MSG_MON_MAP: | |
3efd9988 | 555 | py_module_registry->notify_all("mon_map", ""); |
11fdf7f2 | 556 | handle_mon_map(); |
7c673cae FG |
557 | break; |
558 | case CEPH_MSG_FS_MAP: | |
3efd9988 | 559 | py_module_registry->notify_all("fs_map", ""); |
9f95a23c | 560 | handle_fs_map(ref_cast<MFSMap>(m)); |
7c673cae FG |
561 | return false; // I shall let this pass through for Client |
562 | break; | |
563 | case CEPH_MSG_OSD_MAP: | |
564 | handle_osd_map(); | |
565 | ||
3efd9988 | 566 | py_module_registry->notify_all("osd_map", ""); |
7c673cae FG |
567 | |
568 | // Continuous subscribe, so that we can generate notifications | |
569 | // for our MgrPyModules | |
570 | objecter->maybe_request_map(); | |
7c673cae | 571 | break; |
224ce89b | 572 | case MSG_SERVICE_MAP: |
9f95a23c | 573 | handle_service_map(ref_cast<MServiceMap>(m)); |
3efd9988 | 574 | py_module_registry->notify_all("service_map", ""); |
224ce89b | 575 | break; |
7c673cae | 576 | case MSG_LOG: |
9f95a23c | 577 | handle_log(ref_cast<MLog>(m)); |
7c673cae FG |
578 | break; |
579 | ||
580 | default: | |
581 | return false; | |
582 | } | |
583 | return true; | |
584 | } | |
585 | ||
586 | ||
9f95a23c | 587 | void Mgr::handle_fs_map(ref_t<MFSMap> m) |
7c673cae | 588 | { |
9f95a23c | 589 | ceph_assert(ceph_mutex_is_locked_by_me(lock)); |
7c673cae FG |
590 | |
591 | std::set<std::string> names_exist; | |
592 | ||
593 | const FSMap &new_fsmap = m->get_fsmap(); | |
594 | ||
9f95a23c | 595 | fs_map_cond.notify_all(); |
7c673cae FG |
596 | |
597 | // TODO: callers (e.g. from python land) are potentially going to see | |
598 | // the new fsmap before we've bothered populating all the resulting | |
599 | // daemon_state. Maybe we should block python land while we're making | |
600 | // this kind of update? | |
601 | ||
602 | cluster_state.set_fsmap(new_fsmap); | |
603 | ||
604 | auto mds_info = new_fsmap.get_mds_info(); | |
605 | for (const auto &i : mds_info) { | |
606 | const auto &info = i.second; | |
607 | ||
608 | if (!new_fsmap.gid_exists(i.first)){ | |
609 | continue; | |
610 | } | |
611 | ||
612 | // Remember which MDS exists so that we can cull any that don't | |
613 | names_exist.insert(info.name); | |
614 | ||
9f95a23c | 615 | const auto k = DaemonKey{"mds", info.name}; |
7c673cae FG |
616 | if (daemon_state.is_updating(k)) { |
617 | continue; | |
618 | } | |
619 | ||
620 | bool update = false; | |
621 | if (daemon_state.exists(k)) { | |
622 | auto metadata = daemon_state.get(k); | |
11fdf7f2 | 623 | std::lock_guard l(metadata->lock); |
7c673cae FG |
624 | if (metadata->metadata.empty() || |
625 | metadata->metadata.count("addr") == 0) { | |
626 | update = true; | |
627 | } else { | |
11fdf7f2 TL |
628 | auto metadata_addrs = metadata->metadata.at("addr"); |
629 | const auto map_addrs = info.addrs; | |
630 | update = metadata_addrs != stringify(map_addrs); | |
7c673cae | 631 | if (update) { |
11fdf7f2 TL |
632 | dout(4) << "MDS[" << info.name << "] addr change " << metadata_addrs |
633 | << " != " << stringify(map_addrs) << dendl; | |
7c673cae FG |
634 | } |
635 | } | |
636 | } else { | |
637 | update = true; | |
638 | } | |
639 | ||
640 | if (update) { | |
7c673cae FG |
641 | auto c = new MetadataUpdate(daemon_state, k); |
642 | ||
643 | // Older MDS daemons don't have addr in the metadata, so | |
644 | // fake it if the returned metadata doesn't have the field. | |
11fdf7f2 | 645 | c->set_default("addr", stringify(info.addrs)); |
7c673cae FG |
646 | |
647 | std::ostringstream cmd; | |
648 | cmd << "{\"prefix\": \"mds metadata\", \"who\": \"" | |
649 | << info.name << "\"}"; | |
650 | monc->start_mon_command( | |
651 | {cmd.str()}, | |
652 | {}, &c->outbl, &c->outs, c); | |
653 | } | |
654 | } | |
224ce89b | 655 | daemon_state.cull("mds", names_exist); |
7c673cae FG |
656 | } |
657 | ||
224ce89b WB |
658 | bool Mgr::got_mgr_map(const MgrMap& m) |
659 | { | |
11fdf7f2 | 660 | std::lock_guard l(lock); |
224ce89b WB |
661 | dout(10) << m << dendl; |
662 | ||
663 | set<string> old_modules; | |
664 | cluster_state.with_mgrmap([&](const MgrMap& m) { | |
665 | old_modules = m.modules; | |
666 | }); | |
667 | if (m.modules != old_modules) { | |
668 | derr << "mgrmap module list changed to (" << m.modules << "), respawn" | |
669 | << dendl; | |
670 | return true; | |
671 | } | |
672 | ||
673 | cluster_state.set_mgr_map(m); | |
11fdf7f2 | 674 | server.got_mgr_map(); |
224ce89b WB |
675 | |
676 | return false; | |
677 | } | |
7c673cae | 678 | |
9f95a23c | 679 | void Mgr::handle_mgr_digest(ref_t<MMgrDigest> m) |
7c673cae FG |
680 | { |
681 | dout(10) << m->mon_status_json.length() << dendl; | |
682 | dout(10) << m->health_json.length() << dendl; | |
9f95a23c | 683 | cluster_state.load_digest(m.get()); |
3efd9988 FG |
684 | py_module_registry->notify_all("mon_status", ""); |
685 | py_module_registry->notify_all("health", ""); | |
7c673cae FG |
686 | |
687 | // Hack: use this as a tick/opportunity to prompt python-land that | |
688 | // the pgmap might have changed since last time we were here. | |
3efd9988 | 689 | py_module_registry->notify_all("pg_summary", ""); |
7c673cae | 690 | dout(10) << "done." << dendl; |
9f95a23c | 691 | m.reset(); |
224ce89b WB |
692 | |
693 | if (!digest_received) { | |
694 | digest_received = true; | |
9f95a23c | 695 | digest_cond.notify_all(); |
224ce89b | 696 | } |
7c673cae FG |
697 | } |
698 | ||
3efd9988 FG |
699 | std::map<std::string, std::string> Mgr::get_services() const |
700 | { | |
11fdf7f2 | 701 | std::lock_guard l(lock); |
3efd9988 FG |
702 | |
703 | return py_module_registry->get_services(); | |
704 | } | |
705 | ||
9f95a23c TL |
706 | int Mgr::call( |
707 | std::string_view admin_command, | |
708 | const cmdmap_t& cmdmap, | |
709 | Formatter *f, | |
710 | std::ostream& errss, | |
711 | bufferlist& out) | |
712 | { | |
713 | try { | |
714 | if (admin_command == "mgr_status") { | |
715 | f->open_object_section("mgr_status"); | |
716 | cluster_state.with_mgrmap( | |
717 | [f](const MgrMap& mm) { | |
718 | f->dump_unsigned("mgrmap_epoch", mm.get_epoch()); | |
719 | }); | |
720 | f->dump_bool("initialized", initialized); | |
721 | f->close_section(); | |
722 | return 0; | |
723 | } else { | |
724 | return -ENOSYS; | |
725 | } | |
726 | } catch (const TOPNSPC::common::bad_cmd_get& e) { | |
727 | errss << e.what(); | |
728 | return -EINVAL; | |
729 | } | |
730 | return 0; | |
731 | } |