#include "mgr/mgr_commands.h"
#include "OSDMonitor.h"
#include "ConfigMonitor.h"
+#include "HealthMonitor.h"
#include "MgrMonitor.h"
#define dout_subsys ceph_subsys_mon
#undef dout_prefix
#define dout_prefix _prefix(_dout, mon, map)
+using namespace TOPNSPC::common;
static ostream& _prefix(std::ostream *_dout, Monitor *mon,
const MgrMap& mgrmap) {
return *_dout << "mon." << mon->name << "@" << mon->rank
"balancer",
"devicehealth",
"orchestrator_cli",
+ "rbd_support",
"volumes",
}
+ },
+ {
+ CEPH_RELEASE_OCTOPUS, {
+ "crash",
+ "status",
+ "progress",
+ "balancer",
+ "devicehealth",
+ "orchestrator",
+ "rbd_support",
+ "volumes",
+ "pg_autoscaler",
+ "telemetry",
+ }
}
};
void MgrMonitor::prime_mgr_client()
{
dout(10) << __func__ << dendl;
- mon->mgr_client.ms_dispatch(new MMgrMap(map));
+ mon->mgr_client.ms_dispatch2(make_message<MMgrMap>(map));
}
void MgrMonitor::create_pending()
health_status_t MgrMonitor::should_warn_about_mgr_down()
{
utime_t now = ceph_clock_now();
- // we warn if
- // - we've ever had an active mgr, or
- // - we have osds AND we've exceeded the grace period
+ // we warn if we have osds AND we've exceeded the grace period
// which means a new mon cluster and be HEALTH_OK indefinitely as long as
// no OSDs are ever created.
- if (ever_had_active_mgr ||
- (mon->osdmon()->osdmap.get_num_osds() > 0 &&
- now > mon->monmap->created + g_conf().get_val<int64_t>("mon_mgr_mkfs_grace"))) {
+ if (mon->osdmon()->osdmap.get_num_osds() > 0 &&
+ now > mon->monmap->created + g_conf().get_val<int64_t>("mon_mgr_mkfs_grace")) {
health_status_t level = HEALTH_WARN;
if (first_seen_inactive != utime_t() &&
now - first_seen_inactive > g_conf().get_val<int64_t>("mon_mgr_inactive_grace")) {
send_digests();
} else {
cancel_timer();
- wait_for_active_ctx(new C_MonContext(mon, [this](int) {
+ wait_for_active_ctx(new C_MonContext{mon, [this](int) {
send_digests();
- }));
+ }});
}
}
}
if (pending_map.active_gid == 0) {
auto level = should_warn_about_mgr_down();
if (level != HEALTH_OK) {
- next.add("MGR_DOWN", level, "no active mgr");
+ next.add("MGR_DOWN", level, "no active mgr", 0);
} else {
dout(10) << __func__ << " no health warning (never active and new cluster)"
<< dendl;
bool MgrMonitor::preprocess_query(MonOpRequestRef op)
{
- PaxosServiceMessage *m = static_cast<PaxosServiceMessage*>(op->get_req());
+ auto m = op->get_req<PaxosServiceMessage>();
switch (m->get_type()) {
case MSG_MGR_BEACON:
return preprocess_beacon(op);
bool MgrMonitor::prepare_update(MonOpRequestRef op)
{
- PaxosServiceMessage *m = static_cast<PaxosServiceMessage*>(op->get_req());
+ auto m = op->get_req<PaxosServiceMessage>();
switch (m->get_type()) {
case MSG_MGR_BEACON:
return prepare_beacon(op);
bool MgrMonitor::preprocess_beacon(MonOpRequestRef op)
{
- MMgrBeacon *m = static_cast<MMgrBeacon*>(op->get_req());
+ auto m = op->get_req<MMgrBeacon>();
mon->no_reply(op); // we never reply to beacons
dout(4) << "beacon from " << m->get_gid() << dendl;
bool MgrMonitor::prepare_beacon(MonOpRequestRef op)
{
- MMgrBeacon *m = static_cast<MMgrBeacon*>(op->get_req());
+ auto m = op->get_req<MMgrBeacon>();
dout(4) << "beacon from " << m->get_gid() << dendl;
// See if we are seeing same name, new GID for the active daemon
dout(4) << "Active daemon restart (mgr." << m->get_name() << ")" << dendl;
mon->clog->info() << "Active manager daemon " << m->get_name()
<< " restarted";
+ if (!mon->osdmon()->is_writeable()) {
+ dout(1) << __func__ << ": waiting for osdmon writeable to"
+ " blacklist old instance." << dendl;
+ mon->osdmon()->wait_for_writeable(op, new C_RetryMessage(this, op));
+ return false;
+ }
drop_active();
}
pending_map.available_modules = m->get_available_modules();
updated = true;
}
+ const auto& clients = m->get_clients();
+ if (pending_map.clients != clients) {
+ dout(4) << "active's RADOS clients " << clients
+ << " (was " << pending_map.clients << ")" << dendl;
+ pending_map.clients = clients;
+ updated = true;
+ }
} else if (pending_map.active_gid == 0) {
// There is no currently active daemon, select this one.
if (pending_map.standbys.count(m->get_gid())) {
pending_map.active_gid = m->get_gid();
pending_map.active_name = m->get_name();
pending_map.active_change = ceph_clock_now();
+ pending_map.active_mgr_features = m->get_mgr_features();
pending_map.available_modules = m->get_available_modules();
encode(m->get_metadata(), pending_metadata[m->get_name()]);
pending_metadata_rm.erase(m->get_name());
mon->clog->debug() << "Standby manager daemon " << m->get_name()
<< " started";
pending_map.standbys[m->get_gid()] = {m->get_gid(), m->get_name(),
- m->get_available_modules()};
+ m->get_available_modules(),
+ m->get_mgr_features()};
encode(m->get_metadata(), pending_metadata[m->get_name()]);
pending_metadata_rm.erase(m->get_name());
updated = true;
if (sub->next <= map.get_epoch()) {
dout(20) << "Sending map to subscriber " << sub->session->con
<< " " << sub->session->con->get_peer_addr() << dendl;
- sub->session->con->send_message(new MMgrMap(map));
+ sub->session->con->send_message2(make_message<MMgrMap>(map));
if (sub->onetime) {
mon->session_map.remove_sub(sub);
} else {
for (auto sub : *(mon->session_map.subs[type])) {
dout(10) << __func__ << " sending digest to subscriber " << sub->session->con
<< " " << sub->session->con->get_peer_addr() << dendl;
- MMgrDigest *mdigest = new MMgrDigest;
+ auto mdigest = make_message<MMgrDigest>();
JSONFormatter f;
- mon->get_health_status(true, &f, nullptr, nullptr, nullptr);
+ mon->healthmon()->get_health_status(true, &f, nullptr, nullptr, nullptr);
f.flush(mdigest->health_json);
f.reset();
- std::ostringstream ss;
- mon->get_mon_status(&f, ss);
+ mon->get_mon_status(&f);
f.flush(mdigest->mon_status_json);
f.reset();
- sub->session->con->send_message(mdigest);
+ sub->session->con->send_message2(mdigest);
}
timer:
digest_event = mon->timer.add_event_after(
g_conf().get_val<int64_t>("mon_mgr_digest_period"),
- new C_MonContext(mon, [this](int) {
+ new C_MonContext{mon, [this](int) {
send_digests();
- }));
+ }});
}
void MgrMonitor::cancel_timer()
void MgrMonitor::on_active()
{
- if (mon->is_leader()) {
- mon->clog->debug() << "mgrmap e" << map.epoch << ": " << map;
-
- if (HAVE_FEATURE(mon->get_quorum_con_features(), SERVER_NAUTILUS) &&
- pending_map.always_on_modules != always_on_modules) {
- pending_map.always_on_modules = always_on_modules;
- dout(4) << "always on modules changed, pending "
- << pending_map.get_always_on_modules()
- << " != wanted " << always_on_modules << dendl;
- propose_pending();
- }
+ if (!mon->is_leader()) {
+ return;
+ }
+ mon->clog->debug() << "mgrmap e" << map.epoch << ": " << map;
+ if (!HAVE_FEATURE(mon->get_quorum_con_features(), SERVER_NAUTILUS)) {
+ return;
}
+ if (pending_map.always_on_modules == always_on_modules) {
+ return;
+ }
+ dout(4) << "always on modules changed, pending "
+ << pending_map.always_on_modules << " != wanted "
+ << always_on_modules << dendl;
+ pending_map.always_on_modules = always_on_modules;
+ propose_pending();
}
void MgrMonitor::tick()
}
if (pending_map.active_gid != 0
- && last_beacon.at(pending_map.active_gid) < cutoff) {
+ && last_beacon.at(pending_map.active_gid) < cutoff
+ && mon->osdmon()->is_writeable()) {
const std::string old_active_name = pending_map.active_name;
drop_active();
propose = true;
auto replacement_gid = pending_map.standbys.begin()->first;
pending_map.active_gid = replacement_gid;
pending_map.active_name = pending_map.standbys.at(replacement_gid).name;
+ pending_map.available_modules =
+ pending_map.standbys.at(replacement_gid).available_modules;
+ pending_map.active_mgr_features =
+ pending_map.standbys.at(replacement_gid).mgr_features;
pending_map.available = false;
pending_map.active_addrs = entity_addrvec_t();
pending_map.active_change = ceph_clock_now();
void MgrMonitor::drop_active()
{
+ ceph_assert(mon->osdmon()->is_writeable());
+
if (last_beacon.count(pending_map.active_gid) > 0) {
last_beacon.erase(pending_map.active_gid);
}
+ ceph_assert(pending_map.active_gid > 0);
+ auto until = ceph_clock_now();
+ until += g_conf().get_val<double>("mon_mgr_blacklist_interval");
+ dout(5) << "blacklisting previous mgr." << pending_map.active_name << "."
+ << pending_map.active_gid << " ("
+ << pending_map.active_addrs << ")" << dendl;
+ auto blacklist_epoch = mon->osdmon()->blacklist(pending_map.active_addrs, until);
+
+ /* blacklist RADOS clients in use by the mgr */
+ for (const auto& a : pending_map.clients) {
+ mon->osdmon()->blacklist(a, until);
+ }
+ request_proposal(mon->osdmon());
+
pending_metadata_rm.insert(pending_map.active_name);
pending_metadata.erase(pending_map.active_name);
pending_map.active_name = "";
pending_map.active_gid = 0;
pending_map.active_change = ceph_clock_now();
+ pending_map.active_mgr_features = 0;
pending_map.available = false;
pending_map.active_addrs = entity_addrvec_t();
pending_map.services.clear();
+ pending_map.clients.clear();
+ pending_map.last_failure_osd_epoch = blacklist_epoch;
// So that when new active mgr subscribes to mgrdigest, it will
// get an immediate response instead of waiting for next timer
bool MgrMonitor::preprocess_command(MonOpRequestRef op)
{
- MMonCommand *m = static_cast<MMonCommand*>(op->get_req());
+ auto m = op->get_req<MMonCommand>();
std::stringstream ss;
bufferlist rdata;
}
string format;
- cmd_getval(g_ceph_context, cmdmap, "format", format, string("json-pretty"));
- boost::scoped_ptr<Formatter> f(Formatter::create(format));
+ cmd_getval(cmdmap, "format", format);
+ boost::scoped_ptr<Formatter> f(Formatter::create(format, "json-pretty",
+ "json-pretty"));
string prefix;
- cmd_getval(g_ceph_context, cmdmap, "prefix", prefix);
+ cmd_getval(cmdmap, "prefix", prefix);
int r = 0;
if (prefix == "mgr dump") {
int64_t epoch = 0;
- cmd_getval(g_ceph_context, cmdmap, "epoch", epoch, (int64_t)map.get_epoch());
+ cmd_getval(cmdmap, "epoch", epoch, (int64_t)map.get_epoch());
if (epoch == (int64_t)map.get_epoch()) {
f->dump_object("mgrmap", map);
} else {
} else if (prefix == "mgr module ls") {
f->open_object_section("modules");
{
+ f->open_array_section("always_on_modules");
+ for (auto& p : map.get_always_on_modules()) {
+ f->dump_string("module", p);
+ }
+ f->close_section();
f->open_array_section("enabled_modules");
for (auto& p : map.modules) {
if (map.get_always_on_modules().count(p) > 0)
f->flush(rdata);
} else if (prefix == "mgr metadata") {
string name;
- cmd_getval(g_ceph_context, cmdmap, "who", name);
+ cmd_getval(cmdmap, "who", name);
if (name.size() > 0 && !map.have_name(name)) {
ss << "mgr." << name << " does not exist";
r = -ENOENT;
goto reply;
}
string format;
- cmd_getval(g_ceph_context, cmdmap, "format", format);
+ cmd_getval(cmdmap, "format", format);
boost::scoped_ptr<Formatter> f(Formatter::create(format, "json-pretty", "json-pretty"));
if (name.size()) {
f->open_object_section("mgr_metadata");
}
f->flush(rdata);
} else if (prefix == "mgr versions") {
- if (!f)
- f.reset(Formatter::create("json-pretty"));
count_metadata("ceph_version", f.get());
f->flush(rdata);
r = 0;
} else if (prefix == "mgr count-metadata") {
- if (!f)
- f.reset(Formatter::create("json-pretty"));
string field;
- cmd_getval(g_ceph_context, cmdmap, "property", field);
+ cmd_getval(cmdmap, "property", field);
count_metadata(field, f.get());
f->flush(rdata);
r = 0;
bool MgrMonitor::prepare_command(MonOpRequestRef op)
{
- MMonCommand *m = static_cast<MMonCommand*>(op->get_req());
+ auto m = op->get_req<MMonCommand>();
std::stringstream ss;
bufferlist rdata;
}
string format;
- cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain"));
+ cmd_getval(cmdmap, "format", format, string("plain"));
boost::scoped_ptr<Formatter> f(Formatter::create(format));
string prefix;
- cmd_getval(g_ceph_context, cmdmap, "prefix", prefix);
+ cmd_getval(cmdmap, "prefix", prefix);
int r = 0;
if (prefix == "mgr fail") {
string who;
- cmd_getval(g_ceph_context, cmdmap, "who", who);
+ if (!cmd_getval(cmdmap, "who", who)) {
+ if (!map.active_gid) {
+ ss << "Currently no active mgr";
+ goto out;
+ }
+ who = map.active_name;
+ }
std::string err;
uint64_t gid = strict_strtol(who.c_str(), 10, &err);
if (!err.empty()) {
// Does not parse as a gid, treat it as a name
if (pending_map.active_name == who) {
+ if (!mon->osdmon()->is_writeable()) {
+ mon->osdmon()->wait_for_writeable(op, new C_RetryMessage(this, op));
+ return false;
+ }
drop_active();
changed = true;
} else {
}
} else {
if (pending_map.active_gid == gid) {
+ if (!mon->osdmon()->is_writeable()) {
+ mon->osdmon()->wait_for_writeable(op, new C_RetryMessage(this, op));
+ return false;
+ }
drop_active();
changed = true;
} else if (pending_map.standbys.count(gid) > 0) {
}
} else if (prefix == "mgr module enable") {
string module;
- cmd_getval(g_ceph_context, cmdmap, "module", module);
+ cmd_getval(cmdmap, "module", module);
if (module.empty()) {
r = -EINVAL;
goto out;
goto out;
}
string force;
- cmd_getval(g_ceph_context, cmdmap, "force", force);
+ cmd_getval(cmdmap, "force", force);
if (!pending_map.all_support_module(module) &&
force != "--force") {
ss << "all mgr daemons do not support module '" << module << "', pass "
pending_map.modules.insert(module);
} else if (prefix == "mgr module disable") {
string module;
- cmd_getval(g_ceph_context, cmdmap, "module", module);
+ cmd_getval(cmdmap, "module", module);
if (module.empty()) {
r = -EINVAL;
goto out;
projects / ceph.git / blobdiff