*/
namespace TOPNSPC::common {
template<> bool cmd_getval(const cmdmap_t& cmdmap,
- const std::string& k, mds_gid_t &val)
+ std::string_view k, mds_gid_t &val)
{
return cmd_getval(cmdmap, k, (int64_t&)val);
}
template<> bool cmd_getval(const cmdmap_t& cmdmap,
- const std::string& k, mds_rank_t &val)
+ std::string_view k, mds_rank_t &val)
{
return cmd_getval(cmdmap, k, (int64_t&)val);
}
template<> bool cmd_getval(const cmdmap_t& cmdmap,
- const std::string& k, MDSMap::DaemonState &val)
+ std::string_view k, MDSMap::DaemonState &val)
{
return cmd_getval(cmdmap, k, (int64_t&)val);
}
// print map iff 'debug mon = 30' or higher
print_map<30>(pending);
if (!g_conf()->mon_mds_skip_sanity) {
- pending.sanity();
+ pending.sanity(true);
}
// Set 'modified' on maps modified this epoch
health.decode(bl_i);
}
for (const auto &metric : health.metrics) {
+ if (metric.type == MDS_HEALTH_DUMMY) {
+ continue;
+ }
const auto rank = info.rank;
health_check_t *check = &new_checks.get_or_add(
mds_metric_name(metric.type),
mon.send_reply(op, m.detach());
return true;
} else {
- return false; // not booted yet.
+ /* check if we've already recorded its entry in pending */
+ const auto& pending = get_pending_fsmap();
+ if (pending.gid_exists(gid)) {
+ /* MDS is already booted. */
+ goto ignore;
+ } else {
+ return false; // not booted yet.
+ }
}
}
dout(10) << __func__ << ": GID exists in map: " << gid << dendl;
// is there a state change here?
if (info.state != state) {
- // legal state change?
- if ((info.state == MDSMap::STATE_STANDBY ||
- info.state == MDSMap::STATE_STANDBY_REPLAY) && state > 0) {
- dout(10) << "mds_beacon mds can't activate itself (" << ceph_mds_state_name(info.state)
- << " -> " << ceph_mds_state_name(state) << ")" << dendl;
- goto reply;
- }
-
- if ((state == MDSMap::STATE_STANDBY || state == MDSMap::STATE_STANDBY_REPLAY)
- && info.rank != MDS_RANK_NONE)
- {
- dout(4) << "mds_beacon MDS can't go back into standby after taking rank: "
- "held rank " << info.rank << " while requesting state "
- << ceph_mds_state_name(state) << dendl;
- goto reply;
- }
-
_note_beacon(m);
return false;
}
} catch (const bad_cmd_get& e) {
bufferlist bl;
mon.reply_command(op, -EINVAL, e.what(), bl, get_last_committed());
- return true;
+ return false; /* nothing to propose */
}
case MSG_MDS_OFFLOAD_TARGETS:
ceph_abort();
}
- return true;
+ return false; /* nothing to propose! */
}
bool MDSMonitor::prepare_beacon(MonOpRequestRef op)
std::set<mds_metric_t> new_types;
for (const auto &i : new_health) {
+ if (i.type == MDS_HEALTH_DUMMY) {
+ continue;
+ }
new_types.insert(i.type);
}
for (const auto &new_metric: new_health) {
+ if (new_metric.type == MDS_HEALTH_DUMMY) {
+ continue;
+ }
if (old_types.count(new_metric.type) == 0) {
dout(10) << "MDS health message (" << m->get_orig_source()
<< "): " << new_metric.sev << " " << new_metric.message << dendl;
* know which FS it was part of. Nor does this matter. Sending an empty
* MDSMap is sufficient for getting the MDS to respawn.
*/
- wait_for_finished_proposal(op, new LambdaContext([op, this](int r){
- if (r >= 0) {
- auto m = make_message<MMDSMap>(mon.monmap->fsid, MDSMap::create_null_mdsmap());
- mon.send_reply(op, m.detach());
- } else {
- dispatch(op); // try again
- }
- }));
- return true;
+ goto null;
}
const auto& info = pending.get_info_gid(gid);
return true;
}
- if (info.state == MDSMap::STATE_STOPPING &&
- state != MDSMap::STATE_STOPPING &&
- state != MDSMap::STATE_STOPPED) {
- // we can't transition to any other states from STOPPING
- dout(0) << "got beacon for MDS in STATE_STOPPING, ignoring requested state change"
- << dendl;
- _note_beacon(m);
- return true;
+ if (state == MDSMap::STATE_DNE) {
+ dout(1) << __func__ << ": DNE from " << info << dendl;
+ goto evict;
+ }
+
+ // legal state change?
+ if ((info.state == MDSMap::STATE_STANDBY && state != info.state) ||
+ (info.state == MDSMap::STATE_STANDBY_REPLAY && state != info.state && state != MDSMap::STATE_DAMAGED)) {
+ // Standby daemons should never modify their own state.
+ // Except that standby-replay can indicate the rank is damaged due to failure to replay.
+ // Reject any attempts to do so.
+ derr << "standby " << gid << " attempted to change state to "
+ << ceph_mds_state_name(state) << ", rejecting" << dendl;
+ goto evict;
+ } else if (info.state != MDSMap::STATE_STANDBY && state != info.state &&
+ !MDSMap::state_transition_valid(info.state, state)) {
+ // Validate state transitions for daemons that hold a rank
+ derr << "daemon " << gid << " (rank " << info.rank << ") "
+ << "reported invalid state transition "
+ << ceph_mds_state_name(info.state) << " -> "
+ << ceph_mds_state_name(state) << dendl;
+ goto evict;
}
if (info.laggy()) {
pending_daemon_health_rm.insert(erased_gid);
}
}
-
-
} else if (state == MDSMap::STATE_DAMAGED) {
if (!mon.osdmon()->is_writeable()) {
dout(1) << __func__ << ": DAMAGED from rank " << info.rank
return false;
}
+ auto rank = info.rank;
+
// Record this MDS rank as damaged, so that other daemons
// won't try to run it.
- dout(0) << __func__ << ": marking rank "
- << info.rank << " damaged" << dendl;
+ dout(0) << __func__ << ": marking rank " << rank << " damaged" << dendl;
+
+ auto fs = pending.get_filesystem(gid);
+ auto rankgid = fs->mds_map.get_gid(rank);
+ auto rankinfo = pending.get_info_gid(rankgid);
+ auto followergid = fs->mds_map.get_standby_replay(rank);
+
+ ceph_assert(gid == rankgid || gid == followergid);
utime_t until = ceph_clock_now();
until += g_conf().get_val<double>("mon_mds_blocklist_interval");
- const auto blocklist_epoch = mon.osdmon()->blocklist(info.addrs, until);
- request_proposal(mon.osdmon());
- pending.damaged(gid, blocklist_epoch);
- last_beacon.erase(gid);
-
- // Respond to MDS, so that it knows it can continue to shut down
- auto beacon = make_message<MMDSBeacon>(
- mon.monmap->fsid, m->get_global_id(),
- m->get_name(), pending.get_epoch(), state, seq,
- CEPH_FEATURES_SUPPORTED_DEFAULT);
- mon.send_reply(op, beacon.detach());
- } else if (state == MDSMap::STATE_DNE) {
- if (!mon.osdmon()->is_writeable()) {
- dout(1) << __func__ << ": DNE from rank " << info.rank
- << " waiting for osdmon writeable to blocklist it" << dendl;
- mon.osdmon()->wait_for_writeable(op, new C_RetryMessage(this, op));
- return false;
+ const auto blocklist_epoch = mon.osdmon()->blocklist(rankinfo.addrs, until);
+ if (followergid != MDS_GID_NONE) {
+ fail_mds_gid(pending, followergid);
+ last_beacon.erase(followergid);
}
-
- fail_mds_gid(pending, gid);
- ceph_assert(mon.osdmon()->is_writeable());
request_proposal(mon.osdmon());
+ pending.damaged(rankgid, blocklist_epoch);
+ last_beacon.erase(rankgid);
- // Respond to MDS, so that it knows it can continue to shut down
- auto beacon = make_message<MMDSBeacon>(mon.monmap->fsid,
- m->get_global_id(), m->get_name(), pending.get_epoch(), state, seq,
- CEPH_FEATURES_SUPPORTED_DEFAULT);
- mon.send_reply(op, beacon.detach());
- } else if (info.state == MDSMap::STATE_STANDBY && state != info.state) {
- // Standby daemons should never modify their own
- // state. Reject any attempts to do so.
- derr << "standby " << gid << " attempted to change state to "
- << ceph_mds_state_name(state) << ", rejecting" << dendl;
- return true;
- } else if (info.state != MDSMap::STATE_STANDBY && state != info.state &&
- !MDSMap::state_transition_valid(info.state, state)) {
- // Validate state transitions for daemons that hold a rank
- derr << "daemon " << gid << " (rank " << info.rank << ") "
- << "reported invalid state transition "
- << ceph_mds_state_name(info.state) << " -> "
- << ceph_mds_state_name(state) << dendl;
- return true;
+ /* MDS expects beacon reply back */
} else {
if (info.state != MDSMap::STATE_ACTIVE && state == MDSMap::STATE_ACTIVE) {
const auto &fscid = pending.mds_roles.at(gid);
}
}));
+ return true;
+
+evict:
+ if (!mon.osdmon()->is_writeable()) {
+ dout(1) << __func__ << ": waiting for writeable OSDMap to evict" << dendl;
+ mon.osdmon()->wait_for_writeable(op, new C_RetryMessage(this, op));
+ return false;
+ }
+
+ fail_mds_gid(pending, gid);
+ request_proposal(mon.osdmon());
+ dout(5) << __func__ << ": pending map now:" << dendl;
+ print_map(pending);
+
+ goto null;
+
+null:
+ wait_for_finished_proposal(op, new LambdaContext([op, this](int r){
+ if (r >= 0) {
+ auto m = make_message<MMDSMap>(mon.monmap->fsid, MDSMap::create_null_mdsmap());
+ mon.send_reply(op, m.detach());
+ } else {
+ dispatch(op); // try again
+ }
+ }));
+
return true;
}
bool MDSMonitor::prepare_offload_targets(MonOpRequestRef op)
{
auto &pending = get_pending_fsmap_writeable();
+ bool propose = false;
op->mark_mdsmon_event(__func__);
auto m = op->get_req<MMDSLoadTargets>();
if (pending.gid_has_rank(gid)) {
dout(10) << "prepare_offload_targets " << gid << " " << m->targets << dendl;
pending.update_export_targets(gid, m->targets);
+ propose = true;
} else {
dout(10) << "prepare_offload_targets " << gid << " not in map" << dendl;
}
mon.no_reply(op);
- return true;
+ return propose;
}
bool MDSMonitor::should_propose(double& delay)
string prefix;
cmd_getval(cmdmap, "prefix", prefix);
- string format;
- cmd_getval(cmdmap, "format", format, string("plain"));
+ string format = cmd_getval_or<string>(cmdmap, "format", "plain");
std::unique_ptr<Formatter> f(Formatter::create(format));
MonSession *session = op->get_session();
}
}
r = 0;
+ } else if (prefix == "fs lsflags") {
+ string fs_name;
+ cmd_getval(cmdmap, "fs_name", fs_name);
+ const auto &fs = fsmap.get_filesystem(fs_name);
+ if (!fs) {
+ ss << "filesystem '" << fs_name << "' not found";
+ r = -ENOENT;
+ } else {
+ const MDSMap &mds_map = fs->mds_map;
+ if (f) {
+ mds_map.dump_flags_state(f.get());
+ f->flush(ds);
+ }
+ else {
+ mds_map.print_flags(ds);
+ }
+ r = 0;
+ }
}
out:
if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
string rs = ss.str();
mon.reply_command(op, -EINVAL, rs, rdata, get_last_committed());
- return true;
+ return false;
}
string prefix;
MonSession *session = op->get_session();
if (!session) {
mon.reply_command(op, -EACCES, "access denied", rdata, get_last_committed());
- return true;
+ return false;
}
auto &pending = get_pending_fsmap_writeable();
return;
}
}
- dout(10) << __func__ << ": is_mds=" << is_mds << ", fscid= " << fscid << dendl;
+ dout(10) << __func__ << ": is_mds=" << is_mds << ", fscid=" << fscid << dendl;
// Work out the effective latest epoch
const MDSMap *mds_map = nullptr;
if (sub->next > mds_map->epoch) {
return;
}
- auto msg = make_message<MMDSMap>(mon.monmap->fsid, *mds_map,
- mds_map->fs_name);
+ auto msg = make_message<MMDSMap>(mon.monmap->fsid, *mds_map);
sub->session->con->send_message(msg.detach());
if (sub->onetime) {
void MDSMonitor::update_metadata(mds_gid_t gid,
const map<string, string>& metadata)
{
+ dout(20) << __func__ << ": mds." << gid << ": " << metadata << dendl;
if (metadata.empty()) {
+ dout(5) << __func__ << ": mds." << gid << ": no metadata!" << dendl;
return;
}
pending_metadata[gid] = metadata;
bufferlist bl;
encode(pending_metadata, bl);
t->put(MDS_METADATA_PREFIX, "last_metadata", bl);
- paxos.trigger_propose();
}
void MDSMonitor::remove_from_metadata(const FSMap &fsmap, MonitorDBStore::TransactionRef t)
*/
bool MDSMonitor::maybe_resize_cluster(FSMap &fsmap, fs_cluster_id_t fscid)
{
- auto ¤t_mds_map = get_fsmap().get_filesystem(fscid)->mds_map;
auto&& fs = fsmap.get_filesystem(fscid);
auto &mds_map = fs->mds_map;
* current batch of changes in pending. This is important if an MDS is
* becoming active in the next epoch.
*/
- if (!current_mds_map.is_resizeable() ||
+ if (!get_fsmap().filesystem_exists(fscid) ||
+ !get_fsmap().get_filesystem(fscid)->mds_map.is_resizeable() ||
!mds_map.is_resizeable()) {
dout(5) << __func__ << " mds_map is not currently resizeable" << dendl;
return false;
// check beacon timestamps
std::vector<mds_gid_t> to_remove;
+ const bool mon_down = mon.is_mon_down();
+ const auto mds_beacon_mon_down_grace =
+ g_conf().get_val<std::chrono::seconds>("mds_beacon_mon_down_grace");
+ const auto quorum_age = std::chrono::seconds(mon.quorum_age());
+ const bool new_quorum = quorum_age < mds_beacon_mon_down_grace;
for (auto it = last_beacon.begin(); it != last_beacon.end(); ) {
auto& [gid, beacon_info] = *it;
auto since_last = std::chrono::duration<double>(now-beacon_info.stamp);
<< " (gid: " << gid << " addr: " << info.addrs
<< " state: " << ceph_mds_state_name(info.state) << ")"
<< " since " << since_last.count() << dendl;
+ if ((mon_down || new_quorum) && since_last < mds_beacon_mon_down_grace) {
+ /* The MDS may be sending beacons to a monitor not yet in quorum or
+ * temporarily partitioned. Hold off on removal for a little longer...
+ */
+ dout(10) << "deferring removal for mds_beacon_mon_down_grace during MON_DOWN" << dendl;
+ ++it;
+ continue;
+ }
// If the OSDMap is writeable, we can blocklist things, so we can
// try failing any laggy MDS daemons. Consider each one for failure.
if (!info.laggy()) {
derr << "could not get version " << v << dendl;
ceph_abort();
}
- fsmap.decode(bl);
+ try {
+ fsmap.decode(bl);
+ } catch (const ceph::buffer::malformed_input& e) {
+ dout(5) << "flushing old fsmap struct because unable to decode FSMap: " << e.what() << dendl;
+ }
+ /* N.B. FSMap::is_struct_old is also true for undecoded (failed to decode) FSMap */
if (fsmap.is_struct_old()) {
dout(5) << "fsmap struct is too old; proposing to flush out old versions" << dendl;
do_propose = true;
projects / ceph.git / blobdiff