ceph/src/mon/MgrMonitor.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 /*
   4  * Ceph - scalable distributed file system
   5  *
   6  * Copyright (C) 2016 John Spray <john.spray@redhat.com>
   7  *
   8  * This is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License version 2.1, as published by the Free Software
  11  * Foundation.  See file COPYING.
  12  */
  13
  14 #include <boost/tokenizer.hpp>
  15
  16 #include "messages/MMgrBeacon.h"
  17 #include "messages/MMgrMap.h"
  18 #include "messages/MMgrDigest.h"
  19
  20 #include "PGStatService.h"
  21 #include "include/stringify.h"
  22 #include "mgr/MgrContext.h"
  23 #include "mgr/mgr_commands.h"
  24 #include "OSDMonitor.h"
  25
  26 #include "MgrMonitor.h"
  27
  28 #define MGR_METADATA_PREFIX "mgr_metadata"
  29
  30 #define dout_subsys ceph_subsys_mon
  31 #undef dout_prefix
  32 #define dout_prefix _prefix(_dout, mon, map)
  33 static ostream& _prefix(std::ostream *_dout, Monitor *mon,
  34                         const MgrMap& mgrmap) {
  35   return *_dout << "mon." << mon->name << "@" << mon->rank
  36                 << "(" << mon->get_state_name()
  37                 << ").mgr e" << mgrmap.get_epoch() << " ";
  38 }
  39
  40 // Prefix for mon store of active mgr's command descriptions
  41 const static std::string command_descs_prefix = "mgr_command_descs";
  42
  43
  44 version_t MgrMonitor::get_trim_to()
  45 {
  46   int64_t max = g_conf->get_val<int64_t>("mon_max_mgrmap_epochs");
  47   if (map.epoch > max) {
  48     return map.epoch - max;
  49   }
  50   return 0;
  51 }
  52
  53 void MgrMonitor::create_initial()
  54 {
  55   // Take a local copy of initial_modules for tokenizer to iterate over.
  56   auto initial_modules = g_conf->get_val<std::string>("mgr_initial_modules");
  57   boost::tokenizer<> tok(initial_modules);
  58   for (auto& m : tok) {
  59     pending_map.modules.insert(m);
  60   }
  61   pending_command_descs = mgr_commands;
  62   dout(10) << __func__ << " initial modules " << pending_map.modules
  63            << ", " << pending_command_descs.size() << " commands"
  64            << dendl;
  65 }
  66
  67 void MgrMonitor::get_store_prefixes(std::set<string>& s)
  68 {
  69   s.insert(service_name);
  70   s.insert(command_descs_prefix);
  71   s.insert(MGR_METADATA_PREFIX);
  72 }
  73
  74 void MgrMonitor::update_from_paxos(bool *need_bootstrap)
  75 {
  76   version_t version = get_last_committed();
  77   if (version != map.epoch) {
  78     dout(4) << "loading version " << version << dendl;
  79
  80     bufferlist bl;
  81     int err = get_version(version, bl);
  82     assert(err == 0);
  83
  84     bool old_available = map.get_available();
  85     uint64_t old_gid = map.get_active_gid();
  86
  87     bufferlist::iterator p = bl.begin();
  88     map.decode(p);
  89
  90     dout(4) << "active server: " << map.active_addr
  91             << "(" << map.active_gid << ")" << dendl;
  92
  93     ever_had_active_mgr = get_value("ever_had_active_mgr");
  94
  95     load_health();
  96
  97     if (map.available) {
  98       first_seen_inactive = utime_t();
  99     } else {
 100       first_seen_inactive = ceph_clock_now();
 101     }
 102
 103     check_subs();
 104
 105     if (version == 1
 106         || command_descs.empty()
 107         || (map.get_available()
 108             && (!old_available || old_gid != map.get_active_gid()))) {
 109       dout(4) << "mkfs or daemon transitioned to available, loading commands"
 110               << dendl;
 111       bufferlist loaded_commands;
 112       int r = mon->store->get(command_descs_prefix, "", loaded_commands);
 113       if (r < 0) {
 114         derr << "Failed to load mgr commands: " << cpp_strerror(r) << dendl;
 115       } else {
 116         auto p = loaded_commands.begin();
 117         ::decode(command_descs, p);
 118       }
 119     }
 120   }
 121
 122   // feed our pet MgrClient
 123   mon->mgr_client.ms_dispatch(new MMgrMap(map));
 124 }
 125
 126 void MgrMonitor::create_pending()
 127 {
 128   pending_map = map;
 129   pending_map.epoch++;
 130
 131   if (map.get_epoch() == 1 &&
 132       command_descs.empty() &&
 133       pending_command_descs.empty()) {
 134     // we've been through the initial map and we haven't populated the
 135     // command_descs vector. This likely means we came from kraken, where
 136     // we wouldn't populate the vector, nor would we write it to disk, on
 137     // create_initial().
 138     create_initial();
 139   }
 140 }
 141
 142 health_status_t MgrMonitor::should_warn_about_mgr_down()
 143 {
 144   utime_t now = ceph_clock_now();
 145   // we warn if
 146   //   - we've ever had an active mgr, or
 147   //   - we have osds AND we've exceeded the grace period
 148   // which means a new mon cluster and be HEALTH_OK indefinitely as long as
 149   // no OSDs are ever created.
 150   if (ever_had_active_mgr ||
 151       (mon->osdmon()->osdmap.get_num_osds() > 0 &&
 152        now > mon->monmap->created + g_conf->get_val<int64_t>("mon_mgr_mkfs_grace"))) {
 153     health_status_t level = HEALTH_WARN;
 154     if (first_seen_inactive != utime_t() &&
 155         now - first_seen_inactive > g_conf->get_val<int64_t>("mon_mgr_inactive_grace")) {
 156       level = HEALTH_ERR;
 157     }
 158     return level;
 159   }
 160   return HEALTH_OK;
 161 }
 162
 163 void MgrMonitor::encode_pending(MonitorDBStore::TransactionRef t)
 164 {
 165   dout(10) << __func__ << " " << pending_map << dendl;
 166   bufferlist bl;
 167   pending_map.encode(bl, mon->get_quorum_con_features());
 168   put_version(t, pending_map.epoch, bl);
 169   put_last_committed(t, pending_map.epoch);
 170
 171   for (auto& p : pending_metadata) {
 172     dout(10) << __func__ << " set metadata for " << p.first << dendl;
 173     t->put(MGR_METADATA_PREFIX, p.first, p.second);
 174   }
 175   for (auto& name : pending_metadata_rm) {
 176     dout(10) << __func__ << " rm metadata for " << name << dendl;
 177     t->erase(MGR_METADATA_PREFIX, name);
 178   }
 179   pending_metadata.clear();
 180   pending_metadata_rm.clear();
 181
 182   health_check_map_t next;
 183   if (pending_map.active_gid == 0) {
 184     auto level = should_warn_about_mgr_down();
 185     if (level != HEALTH_OK) {
 186       next.add("MGR_DOWN", level, "no active mgr");
 187     } else {
 188       dout(10) << __func__ << " no health warning (never active and new cluster)"
 189                << dendl;
 190     }
 191   } else {
 192     put_value(t, "ever_had_active_mgr", 1);
 193   }
 194   encode_health(next, t);
 195
 196   if (pending_command_descs.size()) {
 197     dout(4) << __func__ << " encoding " << pending_command_descs.size()
 198             << " command_descs" << dendl;
 199     for (auto& p : pending_command_descs) {
 200       p.set_flag(MonCommand::FLAG_MGR);
 201     }
 202     bufferlist bl;
 203     ::encode(pending_command_descs, bl);
 204     t->put(command_descs_prefix, "", bl);
 205     pending_command_descs.clear();
 206   }
 207 }
 208
 209 bool MgrMonitor::check_caps(MonOpRequestRef op, const uuid_d& fsid)
 210 {
 211   // check permissions
 212   MonSession *session = op->get_session();
 213   if (!session)
 214     return false;
 215   if (!session->is_capable("mgr", MON_CAP_X)) {
 216     dout(1) << __func__ << " insufficient caps " << session->caps << dendl;
 217     return false;
 218   }
 219   if (fsid != mon->monmap->fsid) {
 220     dout(1) << __func__ << " op fsid " << fsid
 221             << " != " << mon->monmap->fsid << dendl;
 222     return false;
 223   }
 224   return true;
 225 }
 226
 227 bool MgrMonitor::preprocess_query(MonOpRequestRef op)
 228 {
 229   PaxosServiceMessage *m = static_cast<PaxosServiceMessage*>(op->get_req());
 230   switch (m->get_type()) {
 231     case MSG_MGR_BEACON:
 232       return preprocess_beacon(op);
 233     case MSG_MON_COMMAND:
 234       try {
 235         return preprocess_command(op);
 236       }
 237       catch (const bad_cmd_get& e) {
 238       bufferlist bl;
 239       mon->reply_command(op, -EINVAL, e.what(), bl, get_last_committed());
 240       return true;
 241     }
 242
 243     default:
 244       mon->no_reply(op);
 245       derr << "Unhandled message type " << m->get_type() << dendl;
 246       return true;
 247   }
 248 }
 249
 250 bool MgrMonitor::prepare_update(MonOpRequestRef op)
 251 {
 252   PaxosServiceMessage *m = static_cast<PaxosServiceMessage*>(op->get_req());
 253   switch (m->get_type()) {
 254     case MSG_MGR_BEACON:
 255       return prepare_beacon(op);
 256
 257     case MSG_MON_COMMAND:
 258       try {
 259         return prepare_command(op);
 260       }
 261       catch (const bad_cmd_get& e) {
 262         bufferlist bl;
 263         mon->reply_command(op, -EINVAL, e.what(), bl, get_last_committed());
 264         return true;
 265       }
 266
 267     default:
 268       mon->no_reply(op);
 269       derr << "Unhandled message type " << m->get_type() << dendl;
 270       return true;
 271   }
 272 }
 273
 274
 275
 276 class C_Updated : public Context {
 277   MgrMonitor *mm;
 278   MonOpRequestRef op;
 279 public:
 280   C_Updated(MgrMonitor *a, MonOpRequestRef c) :
 281     mm(a), op(c) {}
 282   void finish(int r) override {
 283     if (r >= 0) {
 284       // Success
 285     } else if (r == -ECANCELED) {
 286       mm->mon->no_reply(op);
 287     } else {
 288       mm->dispatch(op);        // try again
 289     }
 290   }
 291 };
 292
 293 bool MgrMonitor::preprocess_beacon(MonOpRequestRef op)
 294 {
 295   MMgrBeacon *m = static_cast<MMgrBeacon*>(op->get_req());
 296   mon->no_reply(op); // we never reply to beacons
 297   dout(4) << "beacon from " << m->get_gid() << dendl;
 298
 299   if (!check_caps(op, m->get_fsid())) {
 300     // drop it on the floor
 301     return true;
 302   }
 303
 304   // always send this to the leader's prepare_beacon()
 305   return false;
 306 }
 307
 308 bool MgrMonitor::prepare_beacon(MonOpRequestRef op)
 309 {
 310   MMgrBeacon *m = static_cast<MMgrBeacon*>(op->get_req());
 311   dout(4) << "beacon from " << m->get_gid() << dendl;
 312
 313   // See if we are seeing same name, new GID for the active daemon
 314   if (m->get_name() == pending_map.active_name
 315       && m->get_gid() != pending_map.active_gid)
 316   {
 317     dout(4) << "Active daemon restart (mgr." << m->get_name() << ")" << dendl;
 318     mon->clog->info() << "Active manager daemon " << m->get_name()
 319                       << " restarted";
 320     drop_active();
 321   }
 322
 323   // See if we are seeing same name, new GID for any standbys
 324   for (const auto &i : pending_map.standbys) {
 325     const StandbyInfo &s = i.second;
 326     if (s.name == m->get_name() && s.gid != m->get_gid()) {
 327       dout(4) << "Standby daemon restart (mgr." << m->get_name() << ")" << dendl;
 328       mon->clog->debug() << "Standby manager daemon " << m->get_name()
 329                          << " restarted";
 330       drop_standby(i.first);
 331       break;
 332     }
 333   }
 334
 335   last_beacon[m->get_gid()] = ceph::coarse_mono_clock::now();
 336
 337   // Track whether we modified pending_map
 338   bool updated = false;
 339
 340   if (pending_map.active_gid == m->get_gid()) {
 341     if (pending_map.services != m->get_services()) {
 342       dout(4) << "updated services from mgr." << m->get_name()
 343               << ": " << m->get_services() << dendl;
 344       pending_map.services = m->get_services();
 345       updated = true;
 346     }
 347
 348     // A beacon from the currently active daemon
 349     if (pending_map.active_addr != m->get_server_addr()) {
 350       dout(4) << "learned address " << m->get_server_addr()
 351               << " (was " << pending_map.active_addr << ")" << dendl;
 352       pending_map.active_addr = m->get_server_addr();
 353       updated = true;
 354     }
 355
 356     if (pending_map.get_available() != m->get_available()) {
 357       dout(4) << "available " << m->get_gid() << dendl;
 358       mon->clog->info() << "Manager daemon " << pending_map.active_name
 359                         << " is now available";
 360
 361       // This beacon should include command descriptions
 362       pending_command_descs = m->get_command_descs();
 363       if (pending_command_descs.empty()) {
 364         // This should not happen, but it also isn't fatal: we just
 365         // won't successfully update our list of commands.
 366         dout(4) << "First available beacon from " << pending_map.active_name
 367                 << "(" << m->get_gid() << ") does not include command descs"
 368                 << dendl;
 369       } else {
 370         dout(4) << "First available beacon from " << pending_map.active_name
 371                 << "(" << m->get_gid() << ") includes "
 372                 << pending_command_descs.size() << " command descs" << dendl;
 373       }
 374
 375       pending_map.available = m->get_available();
 376       updated = true;
 377     }
 378     if (pending_map.available_modules != m->get_available_modules()) {
 379       dout(4) << "available_modules " << m->get_available_modules()
 380               << " (was " << pending_map.available_modules << ")" << dendl;
 381       pending_map.available_modules = m->get_available_modules();
 382       updated = true;
 383     }
 384   } else if (pending_map.active_gid == 0) {
 385     // There is no currently active daemon, select this one.
 386     if (pending_map.standbys.count(m->get_gid())) {
 387       drop_standby(m->get_gid(), false);
 388     }
 389     dout(4) << "selecting new active " << m->get_gid()
 390             << " " << m->get_name()
 391             << " (was " << pending_map.active_gid << " "
 392             << pending_map.active_name << ")" << dendl;
 393     pending_map.active_gid = m->get_gid();
 394     pending_map.active_name = m->get_name();
 395     pending_map.available_modules = m->get_available_modules();
 396     ::encode(m->get_metadata(), pending_metadata[m->get_name()]);
 397     pending_metadata_rm.erase(m->get_name());
 398
 399     mon->clog->info() << "Activating manager daemon "
 400                       << pending_map.active_name;
 401
 402     updated = true;
 403   } else {
 404     if (pending_map.standbys.count(m->get_gid()) > 0) {
 405       dout(10) << "from existing standby " << m->get_gid() << dendl;
 406       if (pending_map.standbys[m->get_gid()].available_modules !=
 407           m->get_available_modules()) {
 408         dout(10) << "existing standby " << m->get_gid() << " available_modules "
 409                  << m->get_available_modules() << " (was "
 410                  << pending_map.standbys[m->get_gid()].available_modules << ")"
 411                  << dendl;
 412         pending_map.standbys[m->get_gid()].available_modules =
 413           m->get_available_modules();
 414         updated = true;
 415       }
 416     } else {
 417       dout(10) << "new standby " << m->get_gid() << dendl;
 418       mon->clog->debug() << "Standby manager daemon " << m->get_name()
 419                          << " started";
 420       pending_map.standbys[m->get_gid()] = {m->get_gid(), m->get_name(),
 421                                             m->get_available_modules()};
 422       ::encode(m->get_metadata(), pending_metadata[m->get_name()]);
 423       pending_metadata_rm.erase(m->get_name());
 424       updated = true;
 425     }
 426   }
 427
 428   if (updated) {
 429     dout(4) << "updating map" << dendl;
 430     wait_for_finished_proposal(op, new C_Updated(this, op));
 431   } else {
 432     dout(10) << "no change" << dendl;
 433   }
 434
 435   return updated;
 436 }
 437
 438 void MgrMonitor::check_subs()
 439 {
 440   const std::string type = "mgrmap";
 441   if (mon->session_map.subs.count(type) == 0)
 442     return;
 443   for (auto sub : *(mon->session_map.subs[type])) {
 444     check_sub(sub);
 445   }
 446 }
 447
 448 void MgrMonitor::check_sub(Subscription *sub)
 449 {
 450   if (sub->type == "mgrmap") {
 451     if (sub->next <= map.get_epoch()) {
 452       dout(20) << "Sending map to subscriber " << sub->session->con
 453                << " " << sub->session->con->get_peer_addr() << dendl;
 454       sub->session->con->send_message(new MMgrMap(map));
 455       if (sub->onetime) {
 456         mon->session_map.remove_sub(sub);
 457       } else {
 458         sub->next = map.get_epoch() + 1;
 459       }
 460     }
 461   } else {
 462     assert(sub->type == "mgrdigest");
 463     if (sub->next == 0) {
 464       // new registration; cancel previous timer
 465       cancel_timer();
 466     }
 467     if (digest_event == nullptr) {
 468       send_digests();
 469     }
 470   }
 471 }
 472
 473 /**
 474  * Handle digest subscriptions separately (outside of check_sub) because
 475  * they are going to be periodic rather than version-driven.
 476  */
 477 void MgrMonitor::send_digests()
 478 {
 479   cancel_timer();
 480
 481   const std::string type = "mgrdigest";
 482   if (mon->session_map.subs.count(type) == 0)
 483     return;
 484
 485   if (!is_active()) {
 486     // if paxos is currently not active, don't send a digest but reenable timer
 487     goto timer;
 488   }
 489   dout(10) << __func__ << dendl;
 490
 491   for (auto sub : *(mon->session_map.subs[type])) {
 492     dout(10) << __func__ << " sending digest to subscriber " << sub->session->con
 493              << " " << sub->session->con->get_peer_addr() << dendl;
 494     MMgrDigest *mdigest = new MMgrDigest;
 495
 496     JSONFormatter f;
 497     mon->get_health_status(true, &f, nullptr, nullptr, nullptr);
 498     f.flush(mdigest->health_json);
 499     f.reset();
 500
 501     std::ostringstream ss;
 502     mon->get_mon_status(&f, ss);
 503     f.flush(mdigest->mon_status_json);
 504     f.reset();
 505
 506     sub->session->con->send_message(mdigest);
 507   }
 508
 509 timer:
 510   digest_event = mon->timer.add_event_after(
 511     g_conf->get_val<int64_t>("mon_mgr_digest_period"),
 512     new C_MonContext(mon, [this](int) {
 513       send_digests();
 514   }));
 515 }
 516
 517 void MgrMonitor::cancel_timer()
 518 {
 519   if (digest_event) {
 520     mon->timer.cancel_event(digest_event);
 521     digest_event = nullptr;
 522   }
 523 }
 524
 525 void MgrMonitor::on_active()
 526 {
 527   if (mon->is_leader()) {
 528     mon->clog->debug() << "mgrmap e" << map.epoch << ": " << map;
 529   }
 530 }
 531
 532 void MgrMonitor::get_health(
 533   list<pair<health_status_t,string> >& summary,
 534   list<pair<health_status_t,string> > *detail,
 535   CephContext *cct) const
 536 {
 537   // start mgr warnings as soon as the mons and osds are all upgraded,
 538   // but before the require_luminous osdmap flag is set.  this way the
 539   // user gets some warning before the osd flag is set and mgr is
 540   // actually *required*.
 541   if (!mon->monmap->get_required_features().contains_all(
 542         ceph::features::mon::FEATURE_LUMINOUS) ||
 543       !HAVE_FEATURE(mon->osdmon()->osdmap.get_up_osd_features(),
 544                     SERVER_LUMINOUS)) {
 545     return;
 546   }
 547
 548   if (map.active_gid == 0) {
 549     auto level = HEALTH_WARN;
 550     // do not escalate to ERR if they are still upgrading to jewel.
 551     if (mon->osdmon()->osdmap.require_osd_release >= CEPH_RELEASE_LUMINOUS) {
 552       utime_t now = ceph_clock_now();
 553       if (first_seen_inactive != utime_t() &&
 554           now - first_seen_inactive > g_conf->get_val<int64_t>("mon_mgr_inactive_grace")) {
 555         level = HEALTH_ERR;
 556       }
 557     }
 558     summary.push_back(make_pair(level, "no active mgr"));
 559   }
 560 }
 561
 562 void MgrMonitor::tick()
 563 {
 564   if (!is_active() || !mon->is_leader())
 565     return;
 566
 567   const auto now = ceph::coarse_mono_clock::now();
 568
 569   const auto mgr_beacon_grace = std::chrono::seconds(
 570       g_conf->get_val<int64_t>("mon_mgr_beacon_grace"));
 571
 572   // Note that this is the mgr daemon's tick period, not ours (the
 573   // beacon is sent with this period).
 574   const auto mgr_tick_period = std::chrono::seconds(
 575       g_conf->get_val<int64_t>("mgr_tick_period"));
 576
 577   if (last_tick != ceph::coarse_mono_clock::time_point::min()
 578       && (now - last_tick > (mgr_beacon_grace - mgr_tick_period))) {
 579     // This case handles either local slowness (calls being delayed
 580     // for whatever reason) or cluster election slowness (a long gap
 581     // between calls while an election happened)
 582     dout(4) << __func__ << ": resetting beacon timeouts due to mon delay "
 583             "(slow election?) of " << now - last_tick << " seconds" << dendl;
 584     for (auto &i : last_beacon) {
 585       i.second = now;
 586     }
 587   }
 588
 589   last_tick = now;
 590
 591   // Populate any missing beacons (i.e. no beacon since MgrMonitor
 592   // instantiation) with the current time, so that they will
 593   // eventually look laggy if they fail to give us a beacon.
 594   if (pending_map.active_gid != 0
 595       && last_beacon.count(pending_map.active_gid) == 0) {
 596     last_beacon[pending_map.active_gid] = now;
 597   }
 598   for (auto s : pending_map.standbys) {
 599     if (last_beacon.count(s.first) == 0) {
 600       last_beacon[s.first] = now;
 601     }
 602   }
 603
 604   // Cull standbys first so that any remaining standbys
 605   // will be eligible to take over from the active if we cull him.
 606   std::list<uint64_t> dead_standbys;
 607   const auto cutoff = now - mgr_beacon_grace;
 608   for (const auto &i : pending_map.standbys) {
 609     auto last_beacon_time = last_beacon.at(i.first);
 610     if (last_beacon_time < cutoff) {
 611       dead_standbys.push_back(i.first);
 612     }
 613   }
 614
 615   bool propose = false;
 616
 617   for (auto i : dead_standbys) {
 618     dout(4) << "Dropping laggy standby " << i << dendl;
 619     drop_standby(i);
 620     propose = true;
 621   }
 622
 623   if (pending_map.active_gid != 0
 624       && last_beacon.at(pending_map.active_gid) < cutoff) {
 625     const std::string old_active_name = pending_map.active_name;
 626     drop_active();
 627     propose = true;
 628     dout(4) << "Dropping active" << pending_map.active_gid << dendl;
 629     if (promote_standby()) {
 630       dout(4) << "Promoted standby " << pending_map.active_gid << dendl;
 631       mon->clog->info() << "Manager daemon " << old_active_name
 632                         << " is unresponsive, replacing it with standby"
 633                         << " daemon " << pending_map.active_name;
 634     } else {
 635       dout(4) << "Active is laggy but have no standbys to replace it" << dendl;
 636       mon->clog->info() << "Manager daemon " << old_active_name
 637                         << " is unresponsive.  No standby daemons available.";
 638     }
 639   } else if (pending_map.active_gid == 0) {
 640     if (promote_standby()) {
 641       dout(4) << "Promoted standby " << pending_map.active_gid << dendl;
 642       mon->clog->info() << "Activating manager daemon "
 643                       << pending_map.active_name;
 644       propose = true;
 645     }
 646   }
 647
 648   if (!pending_map.available &&
 649       !ever_had_active_mgr &&
 650       should_warn_about_mgr_down() != HEALTH_OK) {
 651     dout(10) << " exceeded mon_mgr_mkfs_grace "
 652              << g_conf->get_val<int64_t>("mon_mgr_mkfs_grace")
 653              << " seconds" << dendl;
 654     propose = true;
 655   }
 656
 657   if (propose) {
 658     propose_pending();
 659   }
 660 }
 661
 662 void MgrMonitor::on_restart()
 663 {
 664   // Clear out the leader-specific state.
 665   last_beacon.clear();
 666   last_tick = ceph::coarse_mono_clock::now();
 667 }
 668
 669
 670 bool MgrMonitor::promote_standby()
 671 {
 672   assert(pending_map.active_gid == 0);
 673   if (pending_map.standbys.size()) {
 674     // Promote a replacement (arbitrary choice of standby)
 675     auto replacement_gid = pending_map.standbys.begin()->first;
 676     pending_map.active_gid = replacement_gid;
 677     pending_map.active_name = pending_map.standbys.at(replacement_gid).name;
 678     pending_map.available = false;
 679     pending_map.active_addr = entity_addr_t();
 680
 681     drop_standby(replacement_gid, false);
 682
 683     return true;
 684   } else {
 685     return false;
 686   }
 687 }
 688
 689 void MgrMonitor::drop_active()
 690 {
 691   if (last_beacon.count(pending_map.active_gid) > 0) {
 692     last_beacon.erase(pending_map.active_gid);
 693   }
 694
 695   pending_metadata_rm.insert(pending_map.active_name);
 696   pending_metadata.erase(pending_map.active_name);
 697   pending_map.active_name = "";
 698   pending_map.active_gid = 0;
 699   pending_map.available = false;
 700   pending_map.active_addr = entity_addr_t();
 701   pending_map.services.clear();
 702
 703   // So that when new active mgr subscribes to mgrdigest, it will
 704   // get an immediate response instead of waiting for next timer
 705   cancel_timer();
 706 }
 707
 708 void MgrMonitor::drop_standby(uint64_t gid, bool drop_meta)
 709 {
 710   if (drop_meta) {
 711     pending_metadata_rm.insert(pending_map.standbys[gid].name);
 712     pending_metadata.erase(pending_map.standbys[gid].name);
 713   }
 714   pending_map.standbys.erase(gid);
 715   if (last_beacon.count(gid) > 0) {
 716     last_beacon.erase(gid);
 717   }
 718 }
 719
 720 bool MgrMonitor::preprocess_command(MonOpRequestRef op)
 721 {
 722   MMonCommand *m = static_cast<MMonCommand*>(op->get_req());
 723   std::stringstream ss;
 724   bufferlist rdata;
 725
 726   std::map<std::string, cmd_vartype> cmdmap;
 727   if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
 728     string rs = ss.str();
 729     mon->reply_command(op, -EINVAL, rs, rdata, get_last_committed());
 730     return true;
 731   }
 732
 733   MonSession *session = m->get_session();
 734   if (!session) {
 735     mon->reply_command(op, -EACCES, "access denied", rdata,
 736                        get_last_committed());
 737     return true;
 738   }
 739
 740   string format;
 741   cmd_getval_throws(g_ceph_context, cmdmap, "format", format, string("json-pretty"));
 742   boost::scoped_ptr<Formatter> f(Formatter::create(format));
 743
 744   string prefix;
 745   cmd_getval_throws(g_ceph_context, cmdmap, "prefix", prefix);
 746   int r = 0;
 747
 748   if (prefix == "mgr dump") {
 749     int64_t epoch = 0;
 750     cmd_getval_throws(g_ceph_context, cmdmap, "epoch", epoch, (int64_t)map.get_epoch());
 751     if (epoch == (int64_t)map.get_epoch()) {
 752       f->dump_object("mgrmap", map);
 753     } else {
 754       bufferlist bl;
 755       int err = get_version(epoch, bl);
 756       if (err == -ENOENT) {
 757         r = -ENOENT;
 758         ss << "there is no map for epoch " << epoch;
 759         goto reply;
 760       }
 761       MgrMap m;
 762       auto p = bl.begin();
 763       m.decode(p);
 764       f->dump_object("mgrmap", m);
 765     }
 766     f->flush(rdata);
 767   } else if (prefix == "mgr module ls") {
 768     f->open_object_section("modules");
 769     {
 770       f->open_array_section("enabled_modules");
 771       for (auto& p : map.modules) {
 772         f->dump_string("module", p);
 773       }
 774       f->close_section();
 775       f->open_array_section("disabled_modules");
 776       for (auto& p : map.available_modules) {
 777         if (map.modules.count(p) == 0) {
 778           f->dump_string("module", p);
 779         }
 780       }
 781       f->close_section();
 782     }
 783     f->close_section();
 784     f->flush(rdata);
 785   } else if (prefix == "mgr services") {
 786     f->open_object_section("services");
 787     for (const auto &i : map.services) {
 788       f->dump_string(i.first.c_str(), i.second);
 789     }
 790     f->close_section();
 791     f->flush(rdata);
 792   } else if (prefix == "mgr metadata") {
 793     string name;
 794     cmd_getval_throws(g_ceph_context, cmdmap, "id", name);
 795     if (name.size() > 0 && !map.have_name(name)) {
 796       ss << "mgr." << name << " does not exist";
 797       r = -ENOENT;
 798       goto reply;
 799     }
 800     string format;
 801     cmd_getval_throws(g_ceph_context, cmdmap, "format", format);
 802     boost::scoped_ptr<Formatter> f(Formatter::create(format, "json-pretty", "json-pretty"));
 803     if (name.size()) {
 804       f->open_object_section("mgr_metadata");
 805       f->dump_string("id", name);
 806       r = dump_metadata(name, f.get(), &ss);
 807       if (r < 0)
 808         goto reply;
 809       f->close_section();
 810     } else {
 811       r = 0;
 812       f->open_array_section("mgr_metadata");
 813       for (auto& i : map.get_all_names()) {
 814         f->open_object_section("mgr");
 815         f->dump_string("id", i);
 816         r = dump_metadata(i, f.get(), NULL);
 817         if (r == -EINVAL || r == -ENOENT) {
 818           // Drop error, continue to get other daemons' metadata
 819           dout(4) << "No metadata for mgr." << i << dendl;
 820           r = 0;
 821         } else if (r < 0) {
 822           // Unexpected error
 823           goto reply;
 824         }
 825         f->close_section();
 826       }
 827       f->close_section();
 828     }
 829     f->flush(rdata);
 830   } else if (prefix == "mgr versions") {
 831     if (!f)
 832       f.reset(Formatter::create("json-pretty"));
 833     count_metadata("ceph_version", f.get());
 834     f->flush(rdata);
 835     r = 0;
 836   } else if (prefix == "mgr count-metadata") {
 837     if (!f)
 838       f.reset(Formatter::create("json-pretty"));
 839     string field;
 840     cmd_getval_throws(g_ceph_context, cmdmap, "property", field);
 841     count_metadata(field, f.get());
 842     f->flush(rdata);
 843     r = 0;
 844   } else {
 845     return false;
 846   }
 847
 848 reply:
 849   string rs;
 850   getline(ss, rs);
 851   mon->reply_command(op, r, rs, rdata, get_last_committed());
 852   return true;
 853 }
 854
 855 bool MgrMonitor::prepare_command(MonOpRequestRef op)
 856 {
 857   MMonCommand *m = static_cast<MMonCommand*>(op->get_req());
 858
 859   std::stringstream ss;
 860   bufferlist rdata;
 861
 862   std::map<std::string, cmd_vartype> cmdmap;
 863   if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
 864     string rs = ss.str();
 865     mon->reply_command(op, -EINVAL, rs, rdata, get_last_committed());
 866     return true;
 867   }
 868
 869   MonSession *session = m->get_session();
 870   if (!session) {
 871     mon->reply_command(op, -EACCES, "access denied", rdata, get_last_committed());
 872     return true;
 873   }
 874
 875   string format;
 876   cmd_getval_throws(g_ceph_context, cmdmap, "format", format, string("plain"));
 877   boost::scoped_ptr<Formatter> f(Formatter::create(format));
 878
 879   string prefix;
 880   cmd_getval_throws(g_ceph_context, cmdmap, "prefix", prefix);
 881
 882   int r = 0;
 883
 884   if (prefix == "mgr fail") {
 885     string who;
 886     cmd_getval_throws(g_ceph_context, cmdmap, "who", who);
 887
 888     std::string err;
 889     uint64_t gid = strict_strtol(who.c_str(), 10, &err);
 890     bool changed = false;
 891     if (!err.empty()) {
 892       // Does not parse as a gid, treat it as a name
 893       if (pending_map.active_name == who) {
 894         drop_active();
 895         changed = true;
 896       } else {
 897         gid = 0;
 898         for (const auto &i : pending_map.standbys) {
 899           if (i.second.name == who) {
 900             gid = i.first;
 901             break;
 902           }
 903         }
 904         if (gid != 0) {
 905           drop_standby(gid);
 906           changed = true;
 907         } else {
 908           ss << "Daemon not found '" << who << "', already failed?";
 909         }
 910       }
 911     } else {
 912       if (pending_map.active_gid == gid) {
 913         drop_active();
 914         changed = true;
 915       } else if (pending_map.standbys.count(gid) > 0) {
 916         drop_standby(gid);
 917         changed = true;
 918       } else {
 919         ss << "Daemon not found '" << gid << "', already failed?";
 920       }
 921     }
 922
 923     if (changed && pending_map.active_gid == 0) {
 924       promote_standby();
 925     }
 926   } else if (prefix == "mgr module enable") {
 927     string module;
 928     cmd_getval_throws(g_ceph_context, cmdmap, "module", module);
 929     if (module.empty()) {
 930       r = -EINVAL;
 931       goto out;
 932     }
 933     string force;
 934     cmd_getval_throws(g_ceph_context, cmdmap, "force", force);
 935     if (!pending_map.all_support_module(module) &&
 936         force != "--force") {
 937       ss << "all mgr daemons do not support module '" << module << "', pass "
 938          << "--force to force enablement";
 939       r = -ENOENT;
 940       goto out;
 941     }
 942     pending_map.modules.insert(module);
 943   } else if (prefix == "mgr module disable") {
 944     string module;
 945     cmd_getval_throws(g_ceph_context, cmdmap, "module", module);
 946     if (module.empty()) {
 947       r = -EINVAL;
 948       goto out;
 949     }
 950     pending_map.modules.erase(module);
 951   } else {
 952     ss << "Command '" << prefix << "' not implemented!";
 953     r = -ENOSYS;
 954   }
 955
 956 out:
 957   dout(4) << __func__ << " done, r=" << r << dendl;
 958   /* Compose response */
 959   string rs;
 960   getline(ss, rs);
 961
 962   if (r >= 0) {
 963     // success.. delay reply
 964     wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, r, rs,
 965                                               get_last_committed() + 1));
 966     return true;
 967   } else {
 968     // reply immediately
 969     mon->reply_command(op, r, rs, rdata, get_last_committed());
 970     return false;
 971   }
 972 }
 973
 974 void MgrMonitor::init()
 975 {
 976   if (digest_event == nullptr) {
 977     send_digests();  // To get it to schedule its own event
 978   }
 979 }
 980
 981 void MgrMonitor::on_shutdown()
 982 {
 983   cancel_timer();
 984 }
 985
 986 int MgrMonitor::load_metadata(const string& name, std::map<string, string>& m,
 987                               ostream *err)
 988 {
 989   bufferlist bl;
 990   int r = mon->store->get(MGR_METADATA_PREFIX, name, bl);
 991   if (r < 0)
 992     return r;
 993   try {
 994     bufferlist::iterator p = bl.begin();
 995     ::decode(m, p);
 996   }
 997   catch (buffer::error& e) {
 998     if (err)
 999       *err << "mgr." << name << " metadata is corrupt";
1000     return -EIO;
1001   }
1002   return 0;
1003 }
1004
1005 void MgrMonitor::count_metadata(const string& field, std::map<string,int> *out)
1006 {
1007   std::set<string> ls = map.get_all_names();
1008   for (auto& name : ls) {
1009     std::map<string,string> meta;
1010     load_metadata(name, meta, nullptr);
1011     auto p = meta.find(field);
1012     if (p == meta.end()) {
1013       (*out)["unknown"]++;
1014     } else {
1015       (*out)[p->second]++;
1016     }
1017   }
1018 }
1019
1020 void MgrMonitor::count_metadata(const string& field, Formatter *f)
1021 {
1022   std::map<string,int> by_val;
1023   count_metadata(field, &by_val);
1024   f->open_object_section(field.c_str());
1025   for (auto& p : by_val) {
1026     f->dump_int(p.first.c_str(), p.second);
1027   }
1028   f->close_section();
1029 }
1030
1031 int MgrMonitor::dump_metadata(const string& name, Formatter *f, ostream *err)
1032 {
1033   std::map<string,string> m;
1034   if (int r = load_metadata(name, m, err))
1035     return r;
1036   for (auto& p : m) {
1037     f->dump_string(p.first.c_str(), p.second);
1038   }
1039   return 0;
1040 }
1041
1042 const std::vector<MonCommand> &MgrMonitor::get_command_descs() const
1043 {
1044   if (command_descs.empty()) {
1045     // must have just upgraded; fallback to static commands
1046     return mgr_commands;
1047   } else {
1048     return command_descs;
1049   }
1050 }