ceph/src/mon/MgrMonitor.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 /*
   4  * Ceph - scalable distributed file system
   5  *
   6  * Copyright (C) 2016 John Spray <john.spray@redhat.com>
   7  *
   8  * This is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License version 2.1, as published by the Free Software
  11  * Foundation.  See file COPYING.
  12  */
  13
  14 #include <boost/tokenizer.hpp>
  15
  16 #include "messages/MMgrBeacon.h"
  17 #include "messages/MMgrMap.h"
  18 #include "messages/MMgrDigest.h"
  19
  20 #include "PGStatService.h"
  21 #include "include/stringify.h"
  22 #include "mgr/MgrContext.h"
  23 #include "mgr/mgr_commands.h"
  24 #include "OSDMonitor.h"
  25
  26 #include "MgrMonitor.h"
  27
  28 #define MGR_METADATA_PREFIX "mgr_metadata"
  29
  30 #define dout_subsys ceph_subsys_mon
  31 #undef dout_prefix
  32 #define dout_prefix _prefix(_dout, mon, map)
  33 static ostream& _prefix(std::ostream *_dout, Monitor *mon,
  34                         const MgrMap& mgrmap) {
  35   return *_dout << "mon." << mon->name << "@" << mon->rank
  36                 << "(" << mon->get_state_name()
  37                 << ").mgr e" << mgrmap.get_epoch() << " ";
  38 }
  39
  40 // Prefix for mon store of active mgr's command descriptions
  41 const static std::string command_descs_prefix = "mgr_command_descs";
  42
  43
  44 version_t MgrMonitor::get_trim_to()
  45 {
  46   int64_t max = g_conf->get_val<int64_t>("mon_max_mgrmap_epochs");
  47   if (map.epoch > max) {
  48     return map.epoch - max;
  49   }
  50   return 0;
  51 }
  52
  53 void MgrMonitor::create_initial()
  54 {
  55   // Take a local copy of initial_modules for tokenizer to iterate over.
  56   auto initial_modules = g_conf->get_val<std::string>("mgr_initial_modules");
  57   boost::tokenizer<> tok(initial_modules);
  58   for (auto& m : tok) {
  59     pending_map.modules.insert(m);
  60   }
  61   pending_command_descs = mgr_commands;
  62   dout(10) << __func__ << " initial modules " << pending_map.modules
  63            << ", " << pending_command_descs.size() << " commands"
  64            << dendl;
  65 }
  66
  67 void MgrMonitor::get_store_prefixes(std::set<string>& s)
  68 {
  69   s.insert(service_name);
  70   s.insert(command_descs_prefix);
  71   s.insert(MGR_METADATA_PREFIX);
  72 }
  73
  74 void MgrMonitor::update_from_paxos(bool *need_bootstrap)
  75 {
  76   version_t version = get_last_committed();
  77   if (version != map.epoch) {
  78     dout(4) << "loading version " << version << dendl;
  79
  80     bufferlist bl;
  81     int err = get_version(version, bl);
  82     assert(err == 0);
  83
  84     bool old_available = map.get_available();
  85     uint64_t old_gid = map.get_active_gid();
  86
  87     bufferlist::iterator p = bl.begin();
  88     map.decode(p);
  89
  90     dout(4) << "active server: " << map.active_addr
  91             << "(" << map.active_gid << ")" << dendl;
  92
  93     ever_had_active_mgr = get_value("ever_had_active_mgr");
  94
  95     load_health();
  96
  97     if (map.available) {
  98       first_seen_inactive = utime_t();
  99     } else {
 100       first_seen_inactive = ceph_clock_now();
 101     }
 102
 103     check_subs();
 104
 105     if (version == 1
 106         || command_descs.empty()
 107         || (map.get_available()
 108             && (!old_available || old_gid != map.get_active_gid()))) {
 109       dout(4) << "mkfs or daemon transitioned to available, loading commands"
 110               << dendl;
 111       bufferlist loaded_commands;
 112       int r = mon->store->get(command_descs_prefix, "", loaded_commands);
 113       if (r < 0) {
 114         derr << "Failed to load mgr commands: " << cpp_strerror(r) << dendl;
 115       } else {
 116         auto p = loaded_commands.begin();
 117         ::decode(command_descs, p);
 118       }
 119     }
 120   }
 121
 122   // feed our pet MgrClient
 123   mon->mgr_client.ms_dispatch(new MMgrMap(map));
 124 }
 125
 126 void MgrMonitor::create_pending()
 127 {
 128   pending_map = map;
 129   pending_map.epoch++;
 130
 131   if (map.get_epoch() == 1 &&
 132       command_descs.empty() &&
 133       pending_command_descs.empty()) {
 134     // we've been through the initial map and we haven't populated the
 135     // command_descs vector. This likely means we came from kraken, where
 136     // we wouldn't populate the vector, nor would we write it to disk, on
 137     // create_initial().
 138     create_initial();
 139   }
 140 }
 141
 142 health_status_t MgrMonitor::should_warn_about_mgr_down()
 143 {
 144   utime_t now = ceph_clock_now();
 145   // we warn if
 146   //   - we've ever had an active mgr, or
 147   //   - we have osds AND we've exceeded the grace period
 148   // which means a new mon cluster and be HEALTH_OK indefinitely as long as
 149   // no OSDs are ever created.
 150   if (ever_had_active_mgr ||
 151       (mon->osdmon()->osdmap.get_num_osds() > 0 &&
 152        now > mon->monmap->created + g_conf->get_val<int64_t>("mon_mgr_mkfs_grace"))) {
 153     health_status_t level = HEALTH_WARN;
 154     if (first_seen_inactive != utime_t() &&
 155         now - first_seen_inactive > g_conf->get_val<int64_t>("mon_mgr_inactive_grace")) {
 156       level = HEALTH_ERR;
 157     }
 158     return level;
 159   }
 160   return HEALTH_OK;
 161 }
 162
 163 void MgrMonitor::encode_pending(MonitorDBStore::TransactionRef t)
 164 {
 165   dout(10) << __func__ << " " << pending_map << dendl;
 166   bufferlist bl;
 167   pending_map.encode(bl, mon->get_quorum_con_features());
 168   put_version(t, pending_map.epoch, bl);
 169   put_last_committed(t, pending_map.epoch);
 170
 171   for (auto& p : pending_metadata) {
 172     dout(10) << __func__ << " set metadata for " << p.first << dendl;
 173     t->put(MGR_METADATA_PREFIX, p.first, p.second);
 174   }
 175   for (auto& name : pending_metadata_rm) {
 176     dout(10) << __func__ << " rm metadata for " << name << dendl;
 177     t->erase(MGR_METADATA_PREFIX, name);
 178   }
 179   pending_metadata.clear();
 180   pending_metadata_rm.clear();
 181
 182   health_check_map_t next;
 183   if (pending_map.active_gid == 0) {
 184     auto level = should_warn_about_mgr_down();
 185     if (level != HEALTH_OK) {
 186       next.add("MGR_DOWN", level, "no active mgr");
 187     } else {
 188       dout(10) << __func__ << " no health warning (never active and new cluster)"
 189                << dendl;
 190     }
 191   } else {
 192     put_value(t, "ever_had_active_mgr", 1);
 193   }
 194   encode_health(next, t);
 195
 196   if (pending_command_descs.size()) {
 197     dout(4) << __func__ << " encoding " << pending_command_descs.size()
 198             << " command_descs" << dendl;
 199     for (auto& p : pending_command_descs) {
 200       p.set_flag(MonCommand::FLAG_MGR);
 201     }
 202     bufferlist bl;
 203     ::encode(pending_command_descs, bl);
 204     t->put(command_descs_prefix, "", bl);
 205     pending_command_descs.clear();
 206   }
 207 }
 208
 209 bool MgrMonitor::check_caps(MonOpRequestRef op, const uuid_d& fsid)
 210 {
 211   // check permissions
 212   MonSession *session = op->get_session();
 213   if (!session)
 214     return false;
 215   if (!session->is_capable("mgr", MON_CAP_X)) {
 216     dout(1) << __func__ << " insufficient caps " << session->caps << dendl;
 217     return false;
 218   }
 219   if (fsid != mon->monmap->fsid) {
 220     dout(1) << __func__ << " op fsid " << fsid
 221             << " != " << mon->monmap->fsid << dendl;
 222     return false;
 223   }
 224   return true;
 225 }
 226
 227 bool MgrMonitor::preprocess_query(MonOpRequestRef op)
 228 {
 229   PaxosServiceMessage *m = static_cast<PaxosServiceMessage*>(op->get_req());
 230   switch (m->get_type()) {
 231     case MSG_MGR_BEACON:
 232       return preprocess_beacon(op);
 233     case MSG_MON_COMMAND:
 234       return preprocess_command(op);
 235     default:
 236       mon->no_reply(op);
 237       derr << "Unhandled message type " << m->get_type() << dendl;
 238       return true;
 239   }
 240 }
 241
 242 bool MgrMonitor::prepare_update(MonOpRequestRef op)
 243 {
 244   PaxosServiceMessage *m = static_cast<PaxosServiceMessage*>(op->get_req());
 245   switch (m->get_type()) {
 246     case MSG_MGR_BEACON:
 247       return prepare_beacon(op);
 248
 249     case MSG_MON_COMMAND:
 250       return prepare_command(op);
 251
 252     default:
 253       mon->no_reply(op);
 254       derr << "Unhandled message type " << m->get_type() << dendl;
 255       return true;
 256   }
 257 }
 258
 259
 260
 261 class C_Updated : public Context {
 262   MgrMonitor *mm;
 263   MonOpRequestRef op;
 264 public:
 265   C_Updated(MgrMonitor *a, MonOpRequestRef c) :
 266     mm(a), op(c) {}
 267   void finish(int r) override {
 268     if (r >= 0) {
 269       // Success
 270     } else if (r == -ECANCELED) {
 271       mm->mon->no_reply(op);
 272     } else {
 273       mm->dispatch(op);        // try again
 274     }
 275   }
 276 };
 277
 278 bool MgrMonitor::preprocess_beacon(MonOpRequestRef op)
 279 {
 280   MMgrBeacon *m = static_cast<MMgrBeacon*>(op->get_req());
 281   dout(4) << "beacon from " << m->get_gid() << dendl;
 282
 283   if (!check_caps(op, m->get_fsid())) {
 284     // drop it on the floor
 285     return true;
 286   }
 287
 288   // always send this to the leader's prepare_beacon()
 289   return false;
 290 }
 291
 292 bool MgrMonitor::prepare_beacon(MonOpRequestRef op)
 293 {
 294   MMgrBeacon *m = static_cast<MMgrBeacon*>(op->get_req());
 295   dout(4) << "beacon from " << m->get_gid() << dendl;
 296
 297   // See if we are seeing same name, new GID for the active daemon
 298   if (m->get_name() == pending_map.active_name
 299       && m->get_gid() != pending_map.active_gid)
 300   {
 301     dout(4) << "Active daemon restart (mgr." << m->get_name() << ")" << dendl;
 302     mon->clog->info() << "Active manager daemon " << m->get_name()
 303                       << " restarted";
 304     drop_active();
 305   }
 306
 307   // See if we are seeing same name, new GID for any standbys
 308   for (const auto &i : pending_map.standbys) {
 309     const StandbyInfo &s = i.second;
 310     if (s.name == m->get_name() && s.gid != m->get_gid()) {
 311       dout(4) << "Standby daemon restart (mgr." << m->get_name() << ")" << dendl;
 312       mon->clog->debug() << "Standby manager daemon " << m->get_name()
 313                          << " restarted";
 314       drop_standby(i.first);
 315       break;
 316     }
 317   }
 318
 319   last_beacon[m->get_gid()] = ceph::coarse_mono_clock::now();
 320
 321   // Track whether we modified pending_map
 322   bool updated = false;
 323
 324   if (pending_map.active_gid == m->get_gid()) {
 325     if (pending_map.services != m->get_services()) {
 326       dout(4) << "updated services from mgr." << m->get_name()
 327               << ": " << m->get_services() << dendl;
 328       pending_map.services = m->get_services();
 329       updated = true;
 330     }
 331
 332     // A beacon from the currently active daemon
 333     if (pending_map.active_addr != m->get_server_addr()) {
 334       dout(4) << "learned address " << m->get_server_addr()
 335               << " (was " << pending_map.active_addr << ")" << dendl;
 336       pending_map.active_addr = m->get_server_addr();
 337       updated = true;
 338     }
 339
 340     if (pending_map.get_available() != m->get_available()) {
 341       dout(4) << "available " << m->get_gid() << dendl;
 342       mon->clog->info() << "Manager daemon " << pending_map.active_name
 343                         << " is now available";
 344
 345       // This beacon should include command descriptions
 346       pending_command_descs = m->get_command_descs();
 347       if (pending_command_descs.empty()) {
 348         // This should not happen, but it also isn't fatal: we just
 349         // won't successfully update our list of commands.
 350         dout(4) << "First available beacon from " << pending_map.active_name
 351                 << "(" << m->get_gid() << ") does not include command descs"
 352                 << dendl;
 353       } else {
 354         dout(4) << "First available beacon from " << pending_map.active_name
 355                 << "(" << m->get_gid() << ") includes "
 356                 << pending_command_descs.size() << " command descs" << dendl;
 357       }
 358
 359       pending_map.available = m->get_available();
 360       updated = true;
 361     }
 362     if (pending_map.available_modules != m->get_available_modules()) {
 363       dout(4) << "available_modules " << m->get_available_modules()
 364               << " (was " << pending_map.available_modules << ")" << dendl;
 365       pending_map.available_modules = m->get_available_modules();
 366       updated = true;
 367     }
 368   } else if (pending_map.active_gid == 0) {
 369     // There is no currently active daemon, select this one.
 370     if (pending_map.standbys.count(m->get_gid())) {
 371       drop_standby(m->get_gid(), false);
 372     }
 373     dout(4) << "selecting new active " << m->get_gid()
 374             << " " << m->get_name()
 375             << " (was " << pending_map.active_gid << " "
 376             << pending_map.active_name << ")" << dendl;
 377     pending_map.active_gid = m->get_gid();
 378     pending_map.active_name = m->get_name();
 379     pending_map.available_modules = m->get_available_modules();
 380     ::encode(m->get_metadata(), pending_metadata[m->get_name()]);
 381     pending_metadata_rm.erase(m->get_name());
 382
 383     mon->clog->info() << "Activating manager daemon "
 384                       << pending_map.active_name;
 385
 386     updated = true;
 387   } else {
 388     if (pending_map.standbys.count(m->get_gid()) > 0) {
 389       dout(10) << "from existing standby " << m->get_gid() << dendl;
 390       if (pending_map.standbys[m->get_gid()].available_modules !=
 391           m->get_available_modules()) {
 392         dout(10) << "existing standby " << m->get_gid() << " available_modules "
 393                  << m->get_available_modules() << " (was "
 394                  << pending_map.standbys[m->get_gid()].available_modules << ")"
 395                  << dendl;
 396         pending_map.standbys[m->get_gid()].available_modules =
 397           m->get_available_modules();
 398         updated = true;
 399       }
 400     } else {
 401       dout(10) << "new standby " << m->get_gid() << dendl;
 402       mon->clog->debug() << "Standby manager daemon " << m->get_name()
 403                          << " started";
 404       pending_map.standbys[m->get_gid()] = {m->get_gid(), m->get_name(),
 405                                             m->get_available_modules()};
 406       ::encode(m->get_metadata(), pending_metadata[m->get_name()]);
 407       pending_metadata_rm.erase(m->get_name());
 408       updated = true;
 409     }
 410   }
 411
 412   if (updated) {
 413     dout(4) << "updating map" << dendl;
 414     wait_for_finished_proposal(op, new C_Updated(this, op));
 415   } else {
 416     dout(10) << "no change" << dendl;
 417   }
 418
 419   return updated;
 420 }
 421
 422 void MgrMonitor::check_subs()
 423 {
 424   const std::string type = "mgrmap";
 425   if (mon->session_map.subs.count(type) == 0)
 426     return;
 427   for (auto sub : *(mon->session_map.subs[type])) {
 428     check_sub(sub);
 429   }
 430 }
 431
 432 void MgrMonitor::check_sub(Subscription *sub)
 433 {
 434   if (sub->type == "mgrmap") {
 435     if (sub->next <= map.get_epoch()) {
 436       dout(20) << "Sending map to subscriber " << sub->session->con
 437                << " " << sub->session->con->get_peer_addr() << dendl;
 438       sub->session->con->send_message(new MMgrMap(map));
 439       if (sub->onetime) {
 440         mon->session_map.remove_sub(sub);
 441       } else {
 442         sub->next = map.get_epoch() + 1;
 443       }
 444     }
 445   } else {
 446     assert(sub->type == "mgrdigest");
 447     if (sub->next == 0) {
 448       // new registration; cancel previous timer
 449       cancel_timer();
 450     }
 451     if (digest_event == nullptr) {
 452       send_digests();
 453     }
 454   }
 455 }
 456
 457 /**
 458  * Handle digest subscriptions separately (outside of check_sub) because
 459  * they are going to be periodic rather than version-driven.
 460  */
 461 void MgrMonitor::send_digests()
 462 {
 463   cancel_timer();
 464
 465   const std::string type = "mgrdigest";
 466   if (mon->session_map.subs.count(type) == 0)
 467     return;
 468
 469   if (!is_active()) {
 470     // if paxos is currently not active, don't send a digest but reenable timer
 471     goto timer;
 472   }
 473   dout(10) << __func__ << dendl;
 474
 475   for (auto sub : *(mon->session_map.subs[type])) {
 476     dout(10) << __func__ << " sending digest to subscriber " << sub->session->con
 477              << " " << sub->session->con->get_peer_addr() << dendl;
 478     MMgrDigest *mdigest = new MMgrDigest;
 479
 480     JSONFormatter f;
 481     mon->get_health_status(true, &f, nullptr, nullptr, nullptr);
 482     f.flush(mdigest->health_json);
 483     f.reset();
 484
 485     std::ostringstream ss;
 486     mon->get_mon_status(&f, ss);
 487     f.flush(mdigest->mon_status_json);
 488     f.reset();
 489
 490     sub->session->con->send_message(mdigest);
 491   }
 492
 493 timer:
 494   digest_event = mon->timer.add_event_after(
 495     g_conf->get_val<int64_t>("mon_mgr_digest_period"),
 496     new C_MonContext(mon, [this](int) {
 497       send_digests();
 498   }));
 499 }
 500
 501 void MgrMonitor::cancel_timer()
 502 {
 503   if (digest_event) {
 504     mon->timer.cancel_event(digest_event);
 505     digest_event = nullptr;
 506   }
 507 }
 508
 509 void MgrMonitor::on_active()
 510 {
 511   if (mon->is_leader()) {
 512     mon->clog->debug() << "mgrmap e" << map.epoch << ": " << map;
 513   }
 514 }
 515
 516 void MgrMonitor::get_health(
 517   list<pair<health_status_t,string> >& summary,
 518   list<pair<health_status_t,string> > *detail,
 519   CephContext *cct) const
 520 {
 521   // start mgr warnings as soon as the mons and osds are all upgraded,
 522   // but before the require_luminous osdmap flag is set.  this way the
 523   // user gets some warning before the osd flag is set and mgr is
 524   // actually *required*.
 525   if (!mon->monmap->get_required_features().contains_all(
 526         ceph::features::mon::FEATURE_LUMINOUS) ||
 527       !HAVE_FEATURE(mon->osdmon()->osdmap.get_up_osd_features(),
 528                     SERVER_LUMINOUS)) {
 529     return;
 530   }
 531
 532   if (map.active_gid == 0) {
 533     auto level = HEALTH_WARN;
 534     // do not escalate to ERR if they are still upgrading to jewel.
 535     if (mon->osdmon()->osdmap.require_osd_release >= CEPH_RELEASE_LUMINOUS) {
 536       utime_t now = ceph_clock_now();
 537       if (first_seen_inactive != utime_t() &&
 538           now - first_seen_inactive > g_conf->get_val<int64_t>("mon_mgr_inactive_grace")) {
 539         level = HEALTH_ERR;
 540       }
 541     }
 542     summary.push_back(make_pair(level, "no active mgr"));
 543   }
 544 }
 545
 546 void MgrMonitor::tick()
 547 {
 548   if (!is_active() || !mon->is_leader())
 549     return;
 550
 551   const auto now = ceph::coarse_mono_clock::now();
 552
 553   const auto mgr_beacon_grace = std::chrono::seconds(
 554       g_conf->get_val<int64_t>("mon_mgr_beacon_grace"));
 555
 556   // Note that this is the mgr daemon's tick period, not ours (the
 557   // beacon is sent with this period).
 558   const auto mgr_tick_period = std::chrono::seconds(
 559       g_conf->get_val<int64_t>("mgr_tick_period"));
 560
 561   if (last_tick != ceph::coarse_mono_clock::time_point::min()
 562       && (now - last_tick > (mgr_beacon_grace - mgr_tick_period))) {
 563     // This case handles either local slowness (calls being delayed
 564     // for whatever reason) or cluster election slowness (a long gap
 565     // between calls while an election happened)
 566     dout(4) << __func__ << ": resetting beacon timeouts due to mon delay "
 567             "(slow election?) of " << now - last_tick << " seconds" << dendl;
 568     for (auto &i : last_beacon) {
 569       i.second = now;
 570     }
 571   }
 572
 573   last_tick = now;
 574
 575   // Populate any missing beacons (i.e. no beacon since MgrMonitor
 576   // instantiation) with the current time, so that they will
 577   // eventually look laggy if they fail to give us a beacon.
 578   if (pending_map.active_gid != 0
 579       && last_beacon.count(pending_map.active_gid) == 0) {
 580     last_beacon[pending_map.active_gid] = now;
 581   }
 582   for (auto s : pending_map.standbys) {
 583     if (last_beacon.count(s.first) == 0) {
 584       last_beacon[s.first] = now;
 585     }
 586   }
 587
 588   // Cull standbys first so that any remaining standbys
 589   // will be eligible to take over from the active if we cull him.
 590   std::list<uint64_t> dead_standbys;
 591   const auto cutoff = now - mgr_beacon_grace;
 592   for (const auto &i : pending_map.standbys) {
 593     auto last_beacon_time = last_beacon.at(i.first);
 594     if (last_beacon_time < cutoff) {
 595       dead_standbys.push_back(i.first);
 596     }
 597   }
 598
 599   bool propose = false;
 600
 601   for (auto i : dead_standbys) {
 602     dout(4) << "Dropping laggy standby " << i << dendl;
 603     drop_standby(i);
 604     propose = true;
 605   }
 606
 607   if (pending_map.active_gid != 0
 608       && last_beacon.at(pending_map.active_gid) < cutoff) {
 609     const std::string old_active_name = pending_map.active_name;
 610     drop_active();
 611     propose = true;
 612     dout(4) << "Dropping active" << pending_map.active_gid << dendl;
 613     if (promote_standby()) {
 614       dout(4) << "Promoted standby " << pending_map.active_gid << dendl;
 615       mon->clog->info() << "Manager daemon " << old_active_name
 616                         << " is unresponsive, replacing it with standby"
 617                         << " daemon " << pending_map.active_name;
 618     } else {
 619       dout(4) << "Active is laggy but have no standbys to replace it" << dendl;
 620       mon->clog->warn() << "Manager daemon " << old_active_name
 621                         << " is unresponsive.  No standby daemons available.";
 622     }
 623   } else if (pending_map.active_gid == 0) {
 624     if (promote_standby()) {
 625       dout(4) << "Promoted standby " << pending_map.active_gid << dendl;
 626       mon->clog->info() << "Activating manager daemon "
 627                       << pending_map.active_name;
 628       propose = true;
 629     }
 630   }
 631
 632   if (!pending_map.available &&
 633       !ever_had_active_mgr &&
 634       should_warn_about_mgr_down() != HEALTH_OK) {
 635     dout(10) << " exceeded mon_mgr_mkfs_grace "
 636              << g_conf->get_val<int64_t>("mon_mgr_mkfs_grace")
 637              << " seconds" << dendl;
 638     propose = true;
 639   }
 640
 641   if (propose) {
 642     propose_pending();
 643   }
 644 }
 645
 646 void MgrMonitor::on_restart()
 647 {
 648   // Clear out the leader-specific state.
 649   last_beacon.clear();
 650   last_tick = ceph::coarse_mono_clock::now();
 651 }
 652
 653
 654 bool MgrMonitor::promote_standby()
 655 {
 656   assert(pending_map.active_gid == 0);
 657   if (pending_map.standbys.size()) {
 658     // Promote a replacement (arbitrary choice of standby)
 659     auto replacement_gid = pending_map.standbys.begin()->first;
 660     pending_map.active_gid = replacement_gid;
 661     pending_map.active_name = pending_map.standbys.at(replacement_gid).name;
 662     pending_map.available = false;
 663     pending_map.active_addr = entity_addr_t();
 664
 665     drop_standby(replacement_gid, false);
 666
 667     return true;
 668   } else {
 669     return false;
 670   }
 671 }
 672
 673 void MgrMonitor::drop_active()
 674 {
 675   if (last_beacon.count(pending_map.active_gid) > 0) {
 676     last_beacon.erase(pending_map.active_gid);
 677   }
 678
 679   pending_metadata_rm.insert(pending_map.active_name);
 680   pending_metadata.erase(pending_map.active_name);
 681   pending_map.active_name = "";
 682   pending_map.active_gid = 0;
 683   pending_map.available = false;
 684   pending_map.active_addr = entity_addr_t();
 685   pending_map.services.clear();
 686
 687   // So that when new active mgr subscribes to mgrdigest, it will
 688   // get an immediate response instead of waiting for next timer
 689   cancel_timer();
 690 }
 691
 692 void MgrMonitor::drop_standby(uint64_t gid, bool drop_meta)
 693 {
 694   if (drop_meta) {
 695     pending_metadata_rm.insert(pending_map.standbys[gid].name);
 696     pending_metadata.erase(pending_map.standbys[gid].name);
 697   }
 698   pending_map.standbys.erase(gid);
 699   if (last_beacon.count(gid) > 0) {
 700     last_beacon.erase(gid);
 701   }
 702 }
 703
 704 bool MgrMonitor::preprocess_command(MonOpRequestRef op)
 705 {
 706   MMonCommand *m = static_cast<MMonCommand*>(op->get_req());
 707   std::stringstream ss;
 708   bufferlist rdata;
 709
 710   std::map<std::string, cmd_vartype> cmdmap;
 711   if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
 712     string rs = ss.str();
 713     mon->reply_command(op, -EINVAL, rs, rdata, get_last_committed());
 714     return true;
 715   }
 716
 717   MonSession *session = m->get_session();
 718   if (!session) {
 719     mon->reply_command(op, -EACCES, "access denied", rdata,
 720                        get_last_committed());
 721     return true;
 722   }
 723
 724   string format;
 725   cmd_getval(g_ceph_context, cmdmap, "format", format, string("json-pretty"));
 726   boost::scoped_ptr<Formatter> f(Formatter::create(format));
 727
 728   string prefix;
 729   cmd_getval(g_ceph_context, cmdmap, "prefix", prefix);
 730   int r = 0;
 731
 732   if (prefix == "mgr dump") {
 733     int64_t epoch = 0;
 734     cmd_getval(g_ceph_context, cmdmap, "epoch", epoch, (int64_t)map.get_epoch());
 735     if (epoch == (int64_t)map.get_epoch()) {
 736       f->dump_object("mgrmap", map);
 737     } else {
 738       bufferlist bl;
 739       int err = get_version(epoch, bl);
 740       if (err == -ENOENT) {
 741         r = -ENOENT;
 742         ss << "there is no map for epoch " << epoch;
 743         goto reply;
 744       }
 745       MgrMap m;
 746       auto p = bl.begin();
 747       m.decode(p);
 748       f->dump_object("mgrmap", m);
 749     }
 750     f->flush(rdata);
 751   } else if (prefix == "mgr module ls") {
 752     f->open_object_section("modules");
 753     {
 754       f->open_array_section("enabled_modules");
 755       for (auto& p : map.modules) {
 756         f->dump_string("module", p);
 757       }
 758       f->close_section();
 759       f->open_array_section("disabled_modules");
 760       for (auto& p : map.available_modules) {
 761         if (map.modules.count(p) == 0) {
 762           f->dump_string("module", p);
 763         }
 764       }
 765       f->close_section();
 766     }
 767     f->close_section();
 768     f->flush(rdata);
 769   } else if (prefix == "mgr services") {
 770     f->open_object_section("services");
 771     for (const auto &i : map.services) {
 772       f->dump_string(i.first.c_str(), i.second);
 773     }
 774     f->close_section();
 775     f->flush(rdata);
 776   } else if (prefix == "mgr metadata") {
 777     string name;
 778     cmd_getval(g_ceph_context, cmdmap, "id", name);
 779     if (name.size() > 0 && !map.have_name(name)) {
 780       ss << "mgr." << name << " does not exist";
 781       r = -ENOENT;
 782       goto reply;
 783     }
 784     string format;
 785     cmd_getval(g_ceph_context, cmdmap, "format", format);
 786     boost::scoped_ptr<Formatter> f(Formatter::create(format, "json-pretty", "json-pretty"));
 787     if (name.size()) {
 788       f->open_object_section("mgr_metadata");
 789       f->dump_string("id", name);
 790       r = dump_metadata(name, f.get(), &ss);
 791       if (r < 0)
 792         goto reply;
 793       f->close_section();
 794     } else {
 795       r = 0;
 796       f->open_array_section("mgr_metadata");
 797       for (auto& i : map.get_all_names()) {
 798         f->open_object_section("mgr");
 799         f->dump_string("id", i);
 800         r = dump_metadata(i, f.get(), NULL);
 801         if (r == -EINVAL || r == -ENOENT) {
 802           // Drop error, continue to get other daemons' metadata
 803           dout(4) << "No metadata for mgr." << i << dendl;
 804           r = 0;
 805         } else if (r < 0) {
 806           // Unexpected error
 807           goto reply;
 808         }
 809         f->close_section();
 810       }
 811       f->close_section();
 812     }
 813     f->flush(rdata);
 814   } else if (prefix == "mgr versions") {
 815     if (!f)
 816       f.reset(Formatter::create("json-pretty"));
 817     count_metadata("ceph_version", f.get());
 818     f->flush(rdata);
 819     r = 0;
 820   } else if (prefix == "mgr count-metadata") {
 821     if (!f)
 822       f.reset(Formatter::create("json-pretty"));
 823     string field;
 824     cmd_getval(g_ceph_context, cmdmap, "property", field);
 825     count_metadata(field, f.get());
 826     f->flush(rdata);
 827     r = 0;
 828   } else {
 829     return false;
 830   }
 831
 832 reply:
 833   string rs;
 834   getline(ss, rs);
 835   mon->reply_command(op, r, rs, rdata, get_last_committed());
 836   return true;
 837 }
 838
 839 bool MgrMonitor::prepare_command(MonOpRequestRef op)
 840 {
 841   MMonCommand *m = static_cast<MMonCommand*>(op->get_req());
 842
 843   std::stringstream ss;
 844   bufferlist rdata;
 845
 846   std::map<std::string, cmd_vartype> cmdmap;
 847   if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
 848     string rs = ss.str();
 849     mon->reply_command(op, -EINVAL, rs, rdata, get_last_committed());
 850     return true;
 851   }
 852
 853   MonSession *session = m->get_session();
 854   if (!session) {
 855     mon->reply_command(op, -EACCES, "access denied", rdata, get_last_committed());
 856     return true;
 857   }
 858
 859   string format;
 860   cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain"));
 861   boost::scoped_ptr<Formatter> f(Formatter::create(format));
 862
 863   string prefix;
 864   cmd_getval(g_ceph_context, cmdmap, "prefix", prefix);
 865
 866   int r = 0;
 867
 868   if (prefix == "mgr fail") {
 869     string who;
 870     cmd_getval(g_ceph_context, cmdmap, "who", who);
 871
 872     std::string err;
 873     uint64_t gid = strict_strtol(who.c_str(), 10, &err);
 874     bool changed = false;
 875     if (!err.empty()) {
 876       // Does not parse as a gid, treat it as a name
 877       if (pending_map.active_name == who) {
 878         drop_active();
 879         changed = true;
 880       } else {
 881         gid = 0;
 882         for (const auto &i : pending_map.standbys) {
 883           if (i.second.name == who) {
 884             gid = i.first;
 885             break;
 886           }
 887         }
 888         if (gid != 0) {
 889           drop_standby(gid);
 890           changed = true;
 891         } else {
 892           ss << "Daemon not found '" << who << "', already failed?";
 893         }
 894       }
 895     } else {
 896       if (pending_map.active_gid == gid) {
 897         drop_active();
 898         changed = true;
 899       } else if (pending_map.standbys.count(gid) > 0) {
 900         drop_standby(gid);
 901         changed = true;
 902       } else {
 903         ss << "Daemon not found '" << gid << "', already failed?";
 904       }
 905     }
 906
 907     if (changed && pending_map.active_gid == 0) {
 908       promote_standby();
 909     }
 910   } else if (prefix == "mgr module enable") {
 911     string module;
 912     cmd_getval(g_ceph_context, cmdmap, "module", module);
 913     if (module.empty()) {
 914       r = -EINVAL;
 915       goto out;
 916     }
 917     string force;
 918     cmd_getval(g_ceph_context, cmdmap, "force", force);
 919     if (!pending_map.all_support_module(module) &&
 920         force != "--force") {
 921       ss << "all mgr daemons do not support module '" << module << "', pass "
 922          << "--force to force enablement";
 923       r = -ENOENT;
 924       goto out;
 925     }
 926     pending_map.modules.insert(module);
 927   } else if (prefix == "mgr module disable") {
 928     string module;
 929     cmd_getval(g_ceph_context, cmdmap, "module", module);
 930     if (module.empty()) {
 931       r = -EINVAL;
 932       goto out;
 933     }
 934     pending_map.modules.erase(module);
 935   } else {
 936     ss << "Command '" << prefix << "' not implemented!";
 937     r = -ENOSYS;
 938   }
 939
 940 out:
 941   dout(4) << __func__ << " done, r=" << r << dendl;
 942   /* Compose response */
 943   string rs;
 944   getline(ss, rs);
 945
 946   if (r >= 0) {
 947     // success.. delay reply
 948     wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, r, rs,
 949                                               get_last_committed() + 1));
 950     return true;
 951   } else {
 952     // reply immediately
 953     mon->reply_command(op, r, rs, rdata, get_last_committed());
 954     return false;
 955   }
 956 }
 957
 958 void MgrMonitor::init()
 959 {
 960   if (digest_event == nullptr) {
 961     send_digests();  // To get it to schedule its own event
 962   }
 963 }
 964
 965 void MgrMonitor::on_shutdown()
 966 {
 967   cancel_timer();
 968 }
 969
 970 int MgrMonitor::load_metadata(const string& name, std::map<string, string>& m,
 971                               ostream *err)
 972 {
 973   bufferlist bl;
 974   int r = mon->store->get(MGR_METADATA_PREFIX, name, bl);
 975   if (r < 0)
 976     return r;
 977   try {
 978     bufferlist::iterator p = bl.begin();
 979     ::decode(m, p);
 980   }
 981   catch (buffer::error& e) {
 982     if (err)
 983       *err << "mgr." << name << " metadata is corrupt";
 984     return -EIO;
 985   }
 986   return 0;
 987 }
 988
 989 void MgrMonitor::count_metadata(const string& field, std::map<string,int> *out)
 990 {
 991   std::set<string> ls = map.get_all_names();
 992   for (auto& name : ls) {
 993     std::map<string,string> meta;
 994     load_metadata(name, meta, nullptr);
 995     auto p = meta.find(field);
 996     if (p == meta.end()) {
 997       (*out)["unknown"]++;
 998     } else {
 999       (*out)[p->second]++;
1000     }
1001   }
1002 }
1003
1004 void MgrMonitor::count_metadata(const string& field, Formatter *f)
1005 {
1006   std::map<string,int> by_val;
1007   count_metadata(field, &by_val);
1008   f->open_object_section(field.c_str());
1009   for (auto& p : by_val) {
1010     f->dump_int(p.first.c_str(), p.second);
1011   }
1012   f->close_section();
1013 }
1014
1015 int MgrMonitor::dump_metadata(const string& name, Formatter *f, ostream *err)
1016 {
1017   std::map<string,string> m;
1018   if (int r = load_metadata(name, m, err))
1019     return r;
1020   for (auto& p : m) {
1021     f->dump_string(p.first.c_str(), p.second);
1022   }
1023   return 0;
1024 }
1025
1026 const std::vector<MonCommand> &MgrMonitor::get_command_descs() const
1027 {
1028   if (command_descs.empty()) {
1029     // must have just upgraded; fallback to static commands
1030     return mgr_commands;
1031   } else {
1032     return command_descs;
1033   }
1034 }