*/
#include <algorithm>
+#include <boost/algorithm/string.hpp>
+#include <locale>
#include <sstream>
-#include "OSDMonitor.h"
-#include "Monitor.h"
-#include "MDSMonitor.h"
-#include "PGMonitor.h"
+#include "mon/OSDMonitor.h"
+#include "mon/Monitor.h"
+#include "mon/MDSMonitor.h"
+#include "mon/PGMonitor.h"
+#include "mon/MgrStatMonitor.h"
+#include "mon/AuthMonitor.h"
+#include "mon/ConfigKeyService.h"
-#include "MonitorDBStore.h"
-#include "Session.h"
+#include "mon/MonitorDBStore.h"
+#include "mon/Session.h"
#include "crush/CrushWrapper.h"
#include "crush/CrushTester.h"
#include "common/cmdparse.h"
#include "include/str_list.h"
#include "include/str_map.h"
+#include "include/scope_guard.h"
+
+#include "auth/cephx/CephxKeyServer.h"
+#include "osd/OSDCap.h"
#include "json_spirit/json_spirit_reader.h"
+#include <boost/algorithm/string/predicate.hpp>
+
#define dout_subsys ceph_subsys_mon
-#define OSD_PG_CREATING_PREFIX "osd_pg_creating"
+static const string OSD_PG_CREATING_PREFIX("osd_pg_creating");
+static const string OSD_METADATA_PREFIX("osd_metadata");
+
+namespace {
+
+const uint32_t MAX_POOL_APPLICATIONS = 4;
+const uint32_t MAX_POOL_APPLICATION_KEYS = 64;
+const uint32_t MAX_POOL_APPLICATION_LENGTH = 128;
+
+bool is_osd_writable(const OSDCapGrant& grant, const std::string* pool_name) {
+ // Note: this doesn't include support for the application tag match
+ if ((grant.spec.allow & OSD_CAP_W) != 0) {
+ auto& match = grant.match;
+ if (match.is_match_all()) {
+ return true;
+ } else if (pool_name != nullptr && match.auid < 0 &&
+ !match.pool_namespace.pool_name.empty() &&
+ match.pool_namespace.pool_name == *pool_name) {
+ return true;
+ }
+ }
+ return false;
+}
+
+bool is_unmanaged_snap_op_permitted(CephContext* cct,
+ const KeyServer& key_server,
+ const EntityName& entity_name,
+ const MonCap& mon_caps,
+ const std::string* pool_name)
+{
+ typedef std::map<std::string, std::string> CommandArgs;
+
+ if (mon_caps.is_capable(cct, CEPH_ENTITY_TYPE_MON,
+ entity_name, "osd",
+ "osd pool op unmanaged-snap",
+ (pool_name == nullptr ?
+ CommandArgs{} /* pool DNE, require unrestricted cap */ :
+ CommandArgs{{"poolname", *pool_name}}),
+ false, true, false)) {
+ return true;
+ }
+
+ AuthCapsInfo caps_info;
+ if (!key_server.get_service_caps(entity_name, CEPH_ENTITY_TYPE_OSD,
+ caps_info)) {
+ dout(10) << "unable to locate OSD cap data for " << entity_name
+ << " in auth db" << dendl;
+ return false;
+ }
+
+ string caps_str;
+ if (caps_info.caps.length() > 0) {
+ auto p = caps_info.caps.begin();
+ try {
+ decode(caps_str, p);
+ } catch (const buffer::error &err) {
+ derr << "corrupt OSD cap data for " << entity_name << " in auth db"
+ << dendl;
+ return false;
+ }
+ }
+
+ OSDCap osd_cap;
+ if (!osd_cap.parse(caps_str, nullptr)) {
+ dout(10) << "unable to parse OSD cap data for " << entity_name
+ << " in auth db" << dendl;
+ return false;
+ }
+
+ // if the entity has write permissions in one or all pools, permit
+ // usage of unmanaged-snapshots
+ if (osd_cap.allow_all()) {
+ return true;
+ }
+
+ for (auto& grant : osd_cap.grants) {
+ if (grant.profile.is_valid()) {
+ for (auto& profile_grant : grant.profile_grants) {
+ if (is_osd_writable(profile_grant, pool_name)) {
+ return true;
+ }
+ }
+ } else if (is_osd_writable(grant, pool_name)) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+} // anonymous namespace
void LastEpochClean::Lec::report(ps_t ps, epoch_t last_epoch_clean)
{
cct(cct),
inc_osd_cache(g_conf->mon_osd_cache_size),
full_osd_cache(g_conf->mon_osd_cache_size),
- last_attempted_minwait_time(utime_t()),
mapper(mn->cct, &mn->cpu_tp),
op_tracker(cct, true, 1)
{}
newmap.decode(bl);
newmap.set_fsid(mon->monmap->fsid);
} else {
- newmap.build_simple(g_ceph_context, 0, mon->monmap->fsid, 0,
- g_conf->osd_pg_bits, g_conf->osd_pgp_bits);
+ newmap.build_simple(g_ceph_context, 0, mon->monmap->fsid, 0);
}
newmap.set_epoch(1);
newmap.created = newmap.modified = ceph_clock_now();
newmap.set_flag(CEPH_OSDMAP_SORTBITWISE);
// new cluster should require latest by default
- newmap.set_flag(CEPH_OSDMAP_REQUIRE_JEWEL);
- newmap.set_flag(CEPH_OSDMAP_REQUIRE_KRAKEN);
- if (!g_conf->mon_debug_no_require_luminous) {
- newmap.set_flag(CEPH_OSDMAP_REQUIRE_LUMINOUS);
+ if (g_conf->mon_debug_no_require_luminous) {
+ newmap.require_osd_release = CEPH_RELEASE_KRAKEN;
+ derr << __func__ << " mon_debug_no_require_luminous=true" << dendl;
+ } else {
+ newmap.require_osd_release = CEPH_RELEASE_LUMINOUS;
+ newmap.flags |=
+ CEPH_OSDMAP_RECOVERY_DELETES |
+ CEPH_OSDMAP_PURGED_SNAPDIRS;
newmap.full_ratio = g_conf->mon_osd_full_ratio;
if (newmap.full_ratio > 1.0) newmap.full_ratio /= 100;
newmap.backfillfull_ratio = g_conf->mon_osd_backfillfull_ratio;
if (newmap.backfillfull_ratio > 1.0) newmap.backfillfull_ratio /= 100;
newmap.nearfull_ratio = g_conf->mon_osd_nearfull_ratio;
if (newmap.nearfull_ratio > 1.0) newmap.nearfull_ratio /= 100;
- newmap.require_min_compat_client = g_conf->mon_osd_initial_require_min_compat_client;
+ int r = ceph_release_from_name(
+ g_conf->mon_osd_initial_require_min_compat_client.c_str());
+ if (r <= 0) {
+ assert(0 == "mon_osd_initial_require_min_compat_client is not valid");
+ }
+ newmap.require_min_compat_client = r;
}
// encode into pending incremental
+ uint64_t features = newmap.get_encoding_features();
newmap.encode(pending_inc.fullmap,
- mon->get_quorum_con_features() | CEPH_FEATURE_RESERVED);
+ features | CEPH_FEATURE_RESERVED);
pending_inc.full_crc = newmap.get_crc();
dout(20) << " full crc " << pending_inc.full_crc << dendl;
}
{
s.insert(service_name);
s.insert(OSD_PG_CREATING_PREFIX);
+ s.insert(OSD_METADATA_PREFIX);
}
void OSDMonitor::update_from_paxos(bool *need_bootstrap)
dout(15) << "update_from_paxos paxos e " << version
<< ", my e " << osdmap.epoch << dendl;
+ if (mapping_job) {
+ if (!mapping_job->is_done()) {
+ dout(1) << __func__ << " mapping job "
+ << mapping_job.get() << " did not complete, "
+ << mapping_job->shards << " left, canceling" << dendl;
+ mapping_job->abort();
+ }
+ mapping_job.reset();
+ }
+
+ load_health();
/*
* We will possibly have a stashed latest that *we* wrote, and we will
}
}
+ // make sure we're using the right pg service.. remove me post-luminous!
+ if (osdmap.require_osd_release >= CEPH_RELEASE_LUMINOUS) {
+ dout(10) << __func__ << " pgservice is mgrstat" << dendl;
+ mon->pgservice = mon->mgrstatmon()->get_pg_stat_service();
+ } else {
+ dout(10) << __func__ << " pgservice is pg" << dendl;
+ mon->pgservice = mon->pgmon()->get_pg_stat_service();
+ }
+
// walk through incrementals
MonitorDBStore::TransactionRef t;
size_t tx_size = 0;
t->erase("mkfs", "osdmap");
}
+ // make sure we're using the right pg service.. remove me post-luminous!
+ if (osdmap.require_osd_release >= CEPH_RELEASE_LUMINOUS) {
+ dout(10) << __func__ << " pgservice is mgrstat" << dendl;
+ mon->pgservice = mon->mgrstatmon()->get_pg_stat_service();
+ } else {
+ dout(10) << __func__ << " pgservice is pg" << dendl;
+ mon->pgservice = mon->pgmon()->get_pg_stat_service();
+ }
+
if (tx_size > g_conf->mon_sync_max_payload_size*2) {
mon->store->apply_transaction(t);
t = MonitorDBStore::TransactionRef();
}
if (mon->monmap->get_required_features().contains_all(
ceph::features::mon::FEATURE_LUMINOUS)) {
- creating_pgs = update_pending_pgs(inc);
for (const auto &osd_state : inc.new_state) {
if (osd_state.second & CEPH_OSD_UP) {
// could be marked up *or* down, but we're too lazy to check which
<< dendl;
mapping_job->abort();
}
- auto fin = new C_UpdateCreatingPGs(this, osdmap.get_epoch());
- mapping_job = mapping.start_update(osdmap, mapper,
- g_conf->mon_osd_mapping_pgs_per_chunk);
- dout(10) << __func__ << " started mapping job " << mapping_job.get()
- << " at " << fin->start << dendl;
- mapping_job->set_finish_event(fin);
+ if (!osdmap.get_pools().empty()) {
+ auto fin = new C_UpdateCreatingPGs(this, osdmap.get_epoch());
+ mapping_job = mapping.start_update(osdmap, mapper,
+ g_conf->mon_osd_mapping_pgs_per_chunk);
+ dout(10) << __func__ << " started mapping job " << mapping_job.get()
+ << " at " << fin->start << dendl;
+ mapping_job->set_finish_event(fin);
+ } else {
+ dout(10) << __func__ << " no pools, no mapping job" << dendl;
+ mapping_job = nullptr;
+ }
}
void OSDMonitor::update_msgr_features()
update_logger();
if (mon->is_leader()) {
- mon->clog->info() << "osdmap " << osdmap;
+ mon->clog->debug() << "osdmap " << osdmap;
} else {
list<MonOpRequestRef> ls;
take_all_failures(ls);
mon->cluster_logger->set(l_cluster_osd_epoch, osdmap.get_epoch());
}
-template <typename F>
-class OSDUtilizationDumper : public CrushTreeDumper::Dumper<F> {
-public:
- typedef CrushTreeDumper::Dumper<F> Parent;
-
- OSDUtilizationDumper(const CrushWrapper *crush, const OSDMap *osdmap_,
- const PGMap *pgm_, bool tree_) :
- Parent(crush),
- osdmap(osdmap_),
- pgm(pgm_),
- tree(tree_),
- average_util(average_utilization()),
- min_var(-1),
- max_var(-1),
- stddev(0),
- sum(0) {
- }
-
-protected:
- void dump_stray(F *f) {
- for (int i = 0; i < osdmap->get_max_osd(); i++) {
- if (osdmap->exists(i) && !this->is_touched(i))
- dump_item(CrushTreeDumper::Item(i, 0, 0), f);
- }
- }
-
- void dump_item(const CrushTreeDumper::Item &qi, F *f) override {
- if (!tree && qi.is_bucket())
- return;
-
- float reweight = qi.is_bucket() ? -1 : osdmap->get_weightf(qi.id);
- int64_t kb = 0, kb_used = 0, kb_avail = 0;
- double util = 0;
- if (get_bucket_utilization(qi.id, &kb, &kb_used, &kb_avail))
- if (kb_used && kb)
- util = 100.0 * (double)kb_used / (double)kb;
-
- double var = 1.0;
- if (average_util)
- var = util / average_util;
-
- size_t num_pgs = qi.is_bucket() ? 0 : pgm->get_num_pg_by_osd(qi.id);
-
- dump_item(qi, reweight, kb, kb_used, kb_avail, util, var, num_pgs, f);
-
- if (!qi.is_bucket() && reweight > 0) {
- if (min_var < 0 || var < min_var)
- min_var = var;
- if (max_var < 0 || var > max_var)
- max_var = var;
-
- double dev = util - average_util;
- dev *= dev;
- stddev += reweight * dev;
- sum += reweight;
- }
- }
-
- virtual void dump_item(const CrushTreeDumper::Item &qi,
- float &reweight,
- int64_t kb,
- int64_t kb_used,
- int64_t kb_avail,
- double& util,
- double& var,
- const size_t num_pgs,
- F *f) = 0;
-
- double dev() {
- return sum > 0 ? sqrt(stddev / sum) : 0;
- }
-
- double average_utilization() {
- int64_t kb = 0, kb_used = 0;
- for (int i = 0; i < osdmap->get_max_osd(); i++) {
- if (!osdmap->exists(i) || osdmap->get_weight(i) == 0)
- continue;
- int64_t kb_i, kb_used_i, kb_avail_i;
- if (get_osd_utilization(i, &kb_i, &kb_used_i, &kb_avail_i)) {
- kb += kb_i;
- kb_used += kb_used_i;
- }
- }
- return kb > 0 ? 100.0 * (double)kb_used / (double)kb : 0;
- }
-
- bool get_osd_utilization(int id, int64_t* kb, int64_t* kb_used,
- int64_t* kb_avail) const {
- typedef ceph::unordered_map<int32_t,osd_stat_t> OsdStat;
- OsdStat::const_iterator p = pgm->osd_stat.find(id);
- if (p == pgm->osd_stat.end())
- return false;
- *kb = p->second.kb;
- *kb_used = p->second.kb_used;
- *kb_avail = p->second.kb_avail;
- return *kb > 0;
- }
-
- bool get_bucket_utilization(int id, int64_t* kb, int64_t* kb_used,
- int64_t* kb_avail) const {
- if (id >= 0) {
- if (osdmap->is_out(id)) {
- *kb = 0;
- *kb_used = 0;
- *kb_avail = 0;
- return true;
- }
- return get_osd_utilization(id, kb, kb_used, kb_avail);
- }
-
- *kb = 0;
- *kb_used = 0;
- *kb_avail = 0;
-
- for (int k = osdmap->crush->get_bucket_size(id) - 1; k >= 0; k--) {
- int item = osdmap->crush->get_bucket_item(id, k);
- int64_t kb_i = 0, kb_used_i = 0, kb_avail_i = 0;
- if (!get_bucket_utilization(item, &kb_i, &kb_used_i, &kb_avail_i))
- return false;
- *kb += kb_i;
- *kb_used += kb_used_i;
- *kb_avail += kb_avail_i;
- }
- return *kb > 0;
- }
-
-protected:
- const OSDMap *osdmap;
- const PGMap *pgm;
- bool tree;
- double average_util;
- double min_var;
- double max_var;
- double stddev;
- double sum;
-};
-
-class OSDUtilizationPlainDumper : public OSDUtilizationDumper<TextTable> {
-public:
- typedef OSDUtilizationDumper<TextTable> Parent;
-
- OSDUtilizationPlainDumper(const CrushWrapper *crush, const OSDMap *osdmap,
- const PGMap *pgm, bool tree) :
- Parent(crush, osdmap, pgm, tree) {}
-
- void dump(TextTable *tbl) {
- tbl->define_column("ID", TextTable::LEFT, TextTable::RIGHT);
- tbl->define_column("WEIGHT", TextTable::LEFT, TextTable::RIGHT);
- tbl->define_column("REWEIGHT", TextTable::LEFT, TextTable::RIGHT);
- tbl->define_column("SIZE", TextTable::LEFT, TextTable::RIGHT);
- tbl->define_column("USE", TextTable::LEFT, TextTable::RIGHT);
- tbl->define_column("AVAIL", TextTable::LEFT, TextTable::RIGHT);
- tbl->define_column("%USE", TextTable::LEFT, TextTable::RIGHT);
- tbl->define_column("VAR", TextTable::LEFT, TextTable::RIGHT);
- tbl->define_column("PGS", TextTable::LEFT, TextTable::RIGHT);
- if (tree)
- tbl->define_column("TYPE NAME", TextTable::LEFT, TextTable::LEFT);
-
- Parent::dump(tbl);
-
- dump_stray(tbl);
-
- *tbl << "" << "" << "TOTAL"
- << si_t(pgm->osd_sum.kb << 10)
- << si_t(pgm->osd_sum.kb_used << 10)
- << si_t(pgm->osd_sum.kb_avail << 10)
- << lowprecision_t(average_util)
- << ""
- << TextTable::endrow;
- }
-
-protected:
- struct lowprecision_t {
- float v;
- explicit lowprecision_t(float _v) : v(_v) {}
- };
- friend std::ostream &operator<<(ostream& out, const lowprecision_t& v);
-
- using OSDUtilizationDumper<TextTable>::dump_item;
- void dump_item(const CrushTreeDumper::Item &qi,
- float &reweight,
- int64_t kb,
- int64_t kb_used,
- int64_t kb_avail,
- double& util,
- double& var,
- const size_t num_pgs,
- TextTable *tbl) override {
- *tbl << qi.id
- << weightf_t(qi.weight)
- << weightf_t(reweight)
- << si_t(kb << 10)
- << si_t(kb_used << 10)
- << si_t(kb_avail << 10)
- << lowprecision_t(util)
- << lowprecision_t(var);
-
- if (qi.is_bucket()) {
- *tbl << "-";
- } else {
- *tbl << num_pgs;
- }
-
- if (tree) {
- ostringstream name;
- for (int k = 0; k < qi.depth; k++)
- name << " ";
- if (qi.is_bucket()) {
- int type = crush->get_bucket_type(qi.id);
- name << crush->get_type_name(type) << " "
- << crush->get_item_name(qi.id);
- } else {
- name << "osd." << qi.id;
- }
- *tbl << name.str();
- }
-
- *tbl << TextTable::endrow;
- }
-
-public:
- string summary() {
- ostringstream out;
- out << "MIN/MAX VAR: " << lowprecision_t(min_var)
- << "/" << lowprecision_t(max_var) << " "
- << "STDDEV: " << lowprecision_t(dev());
- return out.str();
- }
-};
-
-ostream& operator<<(ostream& out,
- const OSDUtilizationPlainDumper::lowprecision_t& v)
-{
- if (v.v < -0.01) {
- return out << "-";
- } else if (v.v < 0.001) {
- return out << "0";
- } else {
- std::streamsize p = out.precision();
- return out << std::fixed << std::setprecision(2) << v.v << std::setprecision(p);
- }
-}
-
-class OSDUtilizationFormatDumper : public OSDUtilizationDumper<Formatter> {
-public:
- typedef OSDUtilizationDumper<Formatter> Parent;
-
- OSDUtilizationFormatDumper(const CrushWrapper *crush, const OSDMap *osdmap,
- const PGMap *pgm, bool tree) :
- Parent(crush, osdmap, pgm, tree) {}
-
- void dump(Formatter *f) {
- f->open_array_section("nodes");
- Parent::dump(f);
- f->close_section();
-
- f->open_array_section("stray");
- dump_stray(f);
- f->close_section();
- }
-
-protected:
- using OSDUtilizationDumper<Formatter>::dump_item;
- void dump_item(const CrushTreeDumper::Item &qi,
- float &reweight,
- int64_t kb,
- int64_t kb_used,
- int64_t kb_avail,
- double& util,
- double& var,
- const size_t num_pgs,
- Formatter *f) override {
- f->open_object_section("item");
- CrushTreeDumper::dump_item_fields(crush, qi, f);
- f->dump_float("reweight", reweight);
- f->dump_int("kb", kb);
- f->dump_int("kb_used", kb_used);
- f->dump_int("kb_avail", kb_avail);
- f->dump_float("utilization", util);
- f->dump_float("var", var);
- f->dump_unsigned("pgs", num_pgs);
- CrushTreeDumper::dump_bucket_children(crush, qi, f);
- f->close_section();
- }
-
-public:
- void summary(Formatter *f) {
- f->open_object_section("summary");
- f->dump_int("total_kb", pgm->osd_sum.kb);
- f->dump_int("total_kb_used", pgm->osd_sum.kb_used);
- f->dump_int("total_kb_avail", pgm->osd_sum.kb_avail);
- f->dump_float("average_utilization", average_util);
- f->dump_float("min_var", min_var);
- f->dump_float("max_var", max_var);
- f->dump_float("dev", dev());
- f->close_section();
- }
-};
-
-void OSDMonitor::print_utilization(ostream &out, Formatter *f, bool tree) const
-{
- const PGMap *pgm = &mon->pgmon()->pg_map;
- const CrushWrapper *crush = osdmap.crush.get();
-
- if (f) {
- f->open_object_section("df");
- OSDUtilizationFormatDumper d(crush, &osdmap, pgm, tree);
- d.dump(f);
- d.summary(f);
- f->close_section();
- f->flush(out);
- } else {
- OSDUtilizationPlainDumper d(crush, &osdmap, pgm, tree);
- TextTable tbl;
- d.dump(&tbl);
- out << tbl
- << d.summary() << "\n";
- }
-}
-
void OSDMonitor::create_pending()
{
pending_inc = OSDMap::Incremental(osdmap.epoch+1);
dout(1) << __func__ << " setting backfillfull_ratio = "
<< pending_inc.new_backfillfull_ratio << dendl;
}
- if (!osdmap.test_flag(CEPH_OSDMAP_REQUIRE_LUMINOUS)) {
+ if (osdmap.get_epoch() > 0 &&
+ osdmap.require_osd_release < CEPH_RELEASE_LUMINOUS) {
// transition full ratios from PGMap to OSDMap (on upgrade)
- PGMap *pg_map = &mon->pgmon()->pg_map;
- if (osdmap.full_ratio != pg_map->full_ratio) {
+ float full_ratio = mon->pgservice->get_full_ratio();
+ float nearfull_ratio = mon->pgservice->get_nearfull_ratio();
+ if (osdmap.full_ratio != full_ratio) {
dout(10) << __func__ << " full_ratio " << osdmap.full_ratio
- << " -> " << pg_map->full_ratio << " (from pgmap)" << dendl;
- pending_inc.new_full_ratio = pg_map->full_ratio;
+ << " -> " << full_ratio << " (from pgmap)" << dendl;
+ pending_inc.new_full_ratio = full_ratio;
}
- if (osdmap.nearfull_ratio != pg_map->nearfull_ratio) {
+ if (osdmap.nearfull_ratio != nearfull_ratio) {
dout(10) << __func__ << " nearfull_ratio " << osdmap.nearfull_ratio
- << " -> " << pg_map->nearfull_ratio << " (from pgmap)" << dendl;
- pending_inc.new_nearfull_ratio = pg_map->nearfull_ratio;
+ << " -> " << nearfull_ratio << " (from pgmap)" << dendl;
+ pending_inc.new_nearfull_ratio = nearfull_ratio;
}
} else {
// safety check (this shouldn't really happen)
<< pending_inc.new_nearfull_ratio << dendl;
}
}
+
+ // Rewrite CRUSH rule IDs if they are using legacy "ruleset"
+ // structure.
+ if (osdmap.crush->has_legacy_rule_ids()) {
+ CrushWrapper newcrush;
+ _get_pending_crush(newcrush);
+
+ // First, for all pools, work out which rule they really used
+ // by resolving ruleset to rule.
+ for (const auto &i : osdmap.get_pools()) {
+ const auto pool_id = i.first;
+ const auto &pool = i.second;
+ int new_rule_id = newcrush.find_rule(pool.crush_rule,
+ pool.type, pool.size);
+
+ dout(1) << __func__ << " rewriting pool "
+ << osdmap.get_pool_name(pool_id) << " crush ruleset "
+ << pool.crush_rule << " -> rule id " << new_rule_id << dendl;
+ if (pending_inc.new_pools.count(pool_id) == 0) {
+ pending_inc.new_pools[pool_id] = pool;
+ }
+ pending_inc.new_pools[pool_id].crush_rule = new_rule_id;
+ }
+
+ // Now, go ahead and renumber all the rules so that their
+ // rule_id field corresponds to their position in the array
+ auto old_to_new = newcrush.renumber_rules();
+ dout(1) << __func__ << " Rewrote " << old_to_new << " crush IDs:" << dendl;
+ for (const auto &i : old_to_new) {
+ dout(1) << __func__ << " " << i.first << " -> " << i.second << dendl;
+ }
+ pending_inc.crush.clear();
+ newcrush.encode(pending_inc.crush, mon->get_quorum_con_features());
+ }
}
creating_pgs_t
-OSDMonitor::update_pending_pgs(const OSDMap::Incremental& inc)
+OSDMonitor::update_pending_pgs(const OSDMap::Incremental& inc,
+ const OSDMap& nextmap)
{
+ dout(10) << __func__ << dendl;
creating_pgs_t pending_creatings;
{
std::lock_guard<std::mutex> l(creating_pgs_lock);
pending_creatings = creating_pgs;
}
- if (pending_creatings.last_scan_epoch > inc.epoch) {
- return pending_creatings;
- }
- for (auto& pg : pending_created_pgs) {
- pending_creatings.created_pools.insert(pg.pool());
- pending_creatings.pgs.erase(pg);
- }
- pending_created_pgs.clear();
- // PAXOS_PGMAP is less than PAXOS_OSDMAP, so PGMonitor::update_from_paxos()
- // should have prepared the latest pgmap if any
- const auto& pgm = mon->pgmon()->pg_map;
- if (pgm.last_pg_scan >= creating_pgs.last_scan_epoch) {
- // TODO: please stop updating pgmap with pgstats once the upgrade is completed
- for (auto& pgid : pgm.creating_pgs) {
- auto st = pgm.pg_stat.find(pgid);
- assert(st != pgm.pg_stat.end());
- auto created = make_pair(st->second.created, st->second.last_scrub_stamp);
- // no need to add the pg, if it already exists in creating_pgs
- pending_creatings.pgs.emplace(pgid, created);
- }
- }
- for (auto old_pool : inc.old_pools) {
- pending_creatings.created_pools.erase(old_pool);
- const auto removed_pool = (uint64_t)old_pool;
- auto first =
- pending_creatings.pgs.lower_bound(pg_t{0, removed_pool});
- auto last =
- pending_creatings.pgs.lower_bound(pg_t{0, removed_pool + 1});
- pending_creatings.pgs.erase(first, last);
- last_epoch_clean.remove_pool(removed_pool);
- }
- scan_for_creating_pgs(osdmap.get_pools(),
- inc.old_pools,
- inc.modified,
- &pending_creatings);
- scan_for_creating_pgs(inc.new_pools,
- inc.old_pools,
- inc.modified,
- &pending_creatings);
- pending_creatings.last_scan_epoch = osdmap.get_epoch();
+ // check for new or old pools
+ if (pending_creatings.last_scan_epoch < inc.epoch) {
+ if (osdmap.get_epoch() &&
+ osdmap.require_osd_release < CEPH_RELEASE_LUMINOUS) {
+ auto added =
+ mon->pgservice->maybe_add_creating_pgs(creating_pgs.last_scan_epoch,
+ osdmap.get_pools(),
+ &pending_creatings);
+ dout(7) << __func__ << " " << added << " pgs added from pgmap" << dendl;
+ }
+ unsigned queued = 0;
+ queued += scan_for_creating_pgs(osdmap.get_pools(),
+ inc.old_pools,
+ inc.modified,
+ &pending_creatings);
+ queued += scan_for_creating_pgs(inc.new_pools,
+ inc.old_pools,
+ inc.modified,
+ &pending_creatings);
+ dout(10) << __func__ << " " << queued << " pools queued" << dendl;
+ for (auto deleted_pool : inc.old_pools) {
+ auto removed = pending_creatings.remove_pool(deleted_pool);
+ dout(10) << __func__ << " " << removed
+ << " pg removed because containing pool deleted: "
+ << deleted_pool << dendl;
+ last_epoch_clean.remove_pool(deleted_pool);
+ }
+ // pgmon updates its creating_pgs in check_osd_map() which is called by
+ // on_active() and check_osd_map() could be delayed if lease expires, so its
+ // creating_pgs could be stale in comparison with the one of osdmon. let's
+ // trim them here. otherwise, they will be added back after being erased.
+ unsigned removed = 0;
+ for (auto& pg : pending_created_pgs) {
+ dout(20) << __func__ << " noting created pg " << pg << dendl;
+ pending_creatings.created_pools.insert(pg.pool());
+ removed += pending_creatings.pgs.erase(pg);
+ }
+ pending_created_pgs.clear();
+ dout(10) << __func__ << " " << removed
+ << " pgs removed because they're created" << dendl;
+ pending_creatings.last_scan_epoch = osdmap.get_epoch();
+ }
+
+ // filter out any pgs that shouldn't exist.
+ {
+ auto i = pending_creatings.pgs.begin();
+ while (i != pending_creatings.pgs.end()) {
+ if (!nextmap.pg_exists(i->first)) {
+ dout(10) << __func__ << " removing pg " << i->first
+ << " which should not exist" << dendl;
+ i = pending_creatings.pgs.erase(i);
+ } else {
+ ++i;
+ }
+ }
+ }
+
+ // process queue
+ unsigned max = MAX(1, g_conf->mon_osd_max_creating_pgs);
+ const auto total = pending_creatings.pgs.size();
+ while (pending_creatings.pgs.size() < max &&
+ !pending_creatings.queue.empty()) {
+ auto p = pending_creatings.queue.begin();
+ int64_t poolid = p->first;
+ dout(10) << __func__ << " pool " << poolid
+ << " created " << p->second.created
+ << " modified " << p->second.modified
+ << " [" << p->second.start << "-" << p->second.end << ")"
+ << dendl;
+ int n = MIN(max - pending_creatings.pgs.size(),
+ p->second.end - p->second.start);
+ ps_t first = p->second.start;
+ ps_t end = first + n;
+ for (ps_t ps = first; ps < end; ++ps) {
+ const pg_t pgid{ps, static_cast<uint64_t>(poolid)};
+ // NOTE: use the *current* epoch as the PG creation epoch so that the
+ // OSD does not have to generate a long set of PastIntervals.
+ pending_creatings.pgs.emplace(pgid, make_pair(inc.epoch,
+ p->second.modified));
+ dout(10) << __func__ << " adding " << pgid << dendl;
+ }
+ p->second.start = end;
+ if (p->second.done()) {
+ dout(10) << __func__ << " done with queue for " << poolid << dendl;
+ pending_creatings.queue.erase(p);
+ } else {
+ dout(10) << __func__ << " pool " << poolid
+ << " now [" << p->second.start << "-" << p->second.end << ")"
+ << dendl;
+ }
+ }
+ dout(10) << __func__ << " queue remaining: " << pending_creatings.queue.size()
+ << " pools" << dendl;
+ dout(10) << __func__
+ << " " << (pending_creatings.pgs.size() - total)
+ << "/" << pending_creatings.pgs.size()
+ << " pgs added from queued pools" << dendl;
return pending_creatings;
}
// check for interesting OSDs
set<int> osds;
- for (map<int32_t,uint8_t>::iterator p = pending_inc.new_state.begin();
+ for (auto p = pending_inc.new_state.begin();
!all && p != pending_inc.new_state.end();
++p) {
if ((p->second & CEPH_OSD_UP) &&
next.deepish_copy_from(osdmap);
next.apply_incremental(pending_inc);
- if (all) {
+ if (next.get_pools().empty()) {
+ dout(10) << __func__ << " no pools, no pg_temp priming" << dendl;
+ } else if (all) {
PrimeTempJob job(next, this);
mapper.queue(&job, g_conf->mon_osd_mapping_pgs_per_chunk);
if (job.wait_for(g_conf->mon_osd_prime_pg_temp_max_time)) {
{
if (mon->monmap->get_required_features().contains_all(
ceph::features::mon::FEATURE_LUMINOUS)) {
+ // TODO: remove this creating_pgs direct access?
if (creating_pgs.pgs.count(pgid)) {
return;
}
} else {
- const auto& pg_map = mon->pgmon()->pg_map;
- if (pg_map.creating_pgs.count(pgid)) {
+ if (mon->pgservice->is_creating_pg(pgid)) {
return;
}
}
int next_up_primary, next_acting_primary;
next.pg_to_up_acting_osds(pgid, &next_up, &next_up_primary,
&next_acting, &next_acting_primary);
- if (acting == next_acting)
+ if (acting == next_acting && next_up != next_acting)
return; // no change since last epoch
if (acting.empty())
if (pool && acting.size() < pool->min_size)
return; // can be no worse off than before
+ if (next_up == next_acting) {
+ acting.clear();
+ dout(20) << __func__ << "next_up === next_acting now, clear pg_temp"
+ << dendl;
+ }
+
dout(20) << __func__ << " " << pgid << " " << up << "/" << acting
<< " -> " << next_up << "/" << next_acting
<< ", priming " << acting
}
mapping_job.reset();
+ // ensure we don't have blank new_state updates. these are interrpeted as
+ // CEPH_OSD_UP (and almost certainly not what we want!).
+ auto p = pending_inc.new_state.begin();
+ while (p != pending_inc.new_state.end()) {
+ if (p->second == 0) {
+ dout(10) << "new_state for osd." << p->first << " is 0, removing" << dendl;
+ p = pending_inc.new_state.erase(p);
+ } else {
+ ++p;
+ }
+ }
+
bufferlist bl;
{
tmp.deepish_copy_from(osdmap);
tmp.apply_incremental(pending_inc);
- if (tmp.test_flag(CEPH_OSDMAP_REQUIRE_LUMINOUS)) {
- // set or clear full/nearfull?
- int full, backfill, nearfull;
- tmp.count_full_nearfull_osds(&full, &backfill, &nearfull);
- if (full > 0) {
- if (!tmp.test_flag(CEPH_OSDMAP_FULL)) {
- dout(10) << __func__ << " setting full flag" << dendl;
- add_flag(CEPH_OSDMAP_FULL);
- remove_flag(CEPH_OSDMAP_NEARFULL);
- }
- } else {
- if (tmp.test_flag(CEPH_OSDMAP_FULL)) {
- dout(10) << __func__ << " clearing full flag" << dendl;
- remove_flag(CEPH_OSDMAP_FULL);
- }
- if (nearfull > 0) {
- if (!tmp.test_flag(CEPH_OSDMAP_NEARFULL)) {
- dout(10) << __func__ << " setting nearfull flag" << dendl;
- add_flag(CEPH_OSDMAP_NEARFULL);
- }
- } else {
- if (tmp.test_flag(CEPH_OSDMAP_NEARFULL)) {
- dout(10) << __func__ << " clearing nearfull flag" << dendl;
- remove_flag(CEPH_OSDMAP_NEARFULL);
- }
- }
+ if (tmp.require_osd_release >= CEPH_RELEASE_LUMINOUS) {
+ // remove any legacy osdmap nearfull/full flags
+ {
+ if (tmp.test_flag(CEPH_OSDMAP_FULL | CEPH_OSDMAP_NEARFULL)) {
+ dout(10) << __func__ << " clearing legacy osdmap nearfull/full flag"
+ << dendl;
+ remove_flag(CEPH_OSDMAP_NEARFULL);
+ remove_flag(CEPH_OSDMAP_FULL);
+ }
+ }
+ // collect which pools are currently affected by
+ // the near/backfill/full osd(s),
+ // and set per-pool near/backfill/full flag instead
+ set<int64_t> full_pool_ids;
+ set<int64_t> backfillfull_pool_ids;
+ set<int64_t> nearfull_pool_ids;
+ tmp.get_full_pools(g_ceph_context,
+ &full_pool_ids,
+ &backfillfull_pool_ids,
+ &nearfull_pool_ids);
+ if (full_pool_ids.empty() ||
+ backfillfull_pool_ids.empty() ||
+ nearfull_pool_ids.empty()) {
+ // normal case - no nearfull, backfillfull or full osds
+ // try cancel any improper nearfull/backfillfull/full pool
+ // flags first
+ for (auto &pool: tmp.get_pools()) {
+ auto p = pool.first;
+ if (tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_NEARFULL) &&
+ nearfull_pool_ids.empty()) {
+ dout(10) << __func__ << " clearing pool '" << tmp.pool_name[p]
+ << "'s nearfull flag" << dendl;
+ if (pending_inc.new_pools.count(p) == 0) {
+ // load original pool info first!
+ pending_inc.new_pools[p] = pool.second;
+ }
+ pending_inc.new_pools[p].flags &= ~pg_pool_t::FLAG_NEARFULL;
+ }
+ if (tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_BACKFILLFULL) &&
+ backfillfull_pool_ids.empty()) {
+ dout(10) << __func__ << " clearing pool '" << tmp.pool_name[p]
+ << "'s backfillfull flag" << dendl;
+ if (pending_inc.new_pools.count(p) == 0) {
+ pending_inc.new_pools[p] = pool.second;
+ }
+ pending_inc.new_pools[p].flags &= ~pg_pool_t::FLAG_BACKFILLFULL;
+ }
+ if (tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_FULL) &&
+ full_pool_ids.empty()) {
+ if (tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_FULL_NO_QUOTA)) {
+ // set by EQUOTA, skipping
+ continue;
+ }
+ dout(10) << __func__ << " clearing pool '" << tmp.pool_name[p]
+ << "'s full flag" << dendl;
+ if (pending_inc.new_pools.count(p) == 0) {
+ pending_inc.new_pools[p] = pool.second;
+ }
+ pending_inc.new_pools[p].flags &= ~pg_pool_t::FLAG_FULL;
+ }
+ }
+ }
+ if (!full_pool_ids.empty()) {
+ dout(10) << __func__ << " marking pool(s) " << full_pool_ids
+ << " as full" << dendl;
+ for (auto &p: full_pool_ids) {
+ if (tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_FULL)) {
+ continue;
+ }
+ if (pending_inc.new_pools.count(p) == 0) {
+ pending_inc.new_pools[p] = tmp.pools[p];
+ }
+ pending_inc.new_pools[p].flags |= pg_pool_t::FLAG_FULL;
+ pending_inc.new_pools[p].flags &= ~pg_pool_t::FLAG_BACKFILLFULL;
+ pending_inc.new_pools[p].flags &= ~pg_pool_t::FLAG_NEARFULL;
+ }
+ // cancel FLAG_FULL for pools which are no longer full too
+ for (auto &pool: tmp.get_pools()) {
+ auto p = pool.first;
+ if (full_pool_ids.count(p)) {
+ // skip pools we have just marked as full above
+ continue;
+ }
+ if (!tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_FULL) ||
+ tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_FULL_NO_QUOTA)) {
+ // don't touch if currently is not full
+ // or is running out of quota (and hence considered as full)
+ continue;
+ }
+ dout(10) << __func__ << " clearing pool '" << tmp.pool_name[p]
+ << "'s full flag" << dendl;
+ if (pending_inc.new_pools.count(p) == 0) {
+ pending_inc.new_pools[p] = pool.second;
+ }
+ pending_inc.new_pools[p].flags &= ~pg_pool_t::FLAG_FULL;
+ }
+ }
+ if (!backfillfull_pool_ids.empty()) {
+ for (auto &p: backfillfull_pool_ids) {
+ if (full_pool_ids.count(p)) {
+ // skip pools we have already considered as full above
+ continue;
+ }
+ if (tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_FULL_NO_QUOTA)) {
+ // make sure FLAG_FULL is truly set, so we are safe not
+ // to set a extra (redundant) FLAG_BACKFILLFULL flag
+ assert(tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_FULL));
+ continue;
+ }
+ if (tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_BACKFILLFULL)) {
+ // don't bother if pool is already marked as backfillfull
+ continue;
+ }
+ dout(10) << __func__ << " marking pool '" << tmp.pool_name[p]
+ << "'s as backfillfull" << dendl;
+ if (pending_inc.new_pools.count(p) == 0) {
+ pending_inc.new_pools[p] = tmp.pools[p];
+ }
+ pending_inc.new_pools[p].flags |= pg_pool_t::FLAG_BACKFILLFULL;
+ pending_inc.new_pools[p].flags &= ~pg_pool_t::FLAG_NEARFULL;
+ }
+ // cancel FLAG_BACKFILLFULL for pools
+ // which are no longer backfillfull too
+ for (auto &pool: tmp.get_pools()) {
+ auto p = pool.first;
+ if (full_pool_ids.count(p) || backfillfull_pool_ids.count(p)) {
+ // skip pools we have just marked as backfillfull/full above
+ continue;
+ }
+ if (!tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_BACKFILLFULL)) {
+ // and don't touch if currently is not backfillfull
+ continue;
+ }
+ dout(10) << __func__ << " clearing pool '" << tmp.pool_name[p]
+ << "'s backfillfull flag" << dendl;
+ if (pending_inc.new_pools.count(p) == 0) {
+ pending_inc.new_pools[p] = pool.second;
+ }
+ pending_inc.new_pools[p].flags &= ~pg_pool_t::FLAG_BACKFILLFULL;
+ }
+ }
+ if (!nearfull_pool_ids.empty()) {
+ for (auto &p: nearfull_pool_ids) {
+ if (full_pool_ids.count(p) || backfillfull_pool_ids.count(p)) {
+ continue;
+ }
+ if (tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_FULL_NO_QUOTA)) {
+ // make sure FLAG_FULL is truly set, so we are safe not
+ // to set a extra (redundant) FLAG_NEARFULL flag
+ assert(tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_FULL));
+ continue;
+ }
+ if (tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_NEARFULL)) {
+ // don't bother if pool is already marked as nearfull
+ continue;
+ }
+ dout(10) << __func__ << " marking pool '" << tmp.pool_name[p]
+ << "'s as nearfull" << dendl;
+ if (pending_inc.new_pools.count(p) == 0) {
+ pending_inc.new_pools[p] = tmp.pools[p];
+ }
+ pending_inc.new_pools[p].flags |= pg_pool_t::FLAG_NEARFULL;
+ }
+ // cancel FLAG_NEARFULL for pools
+ // which are no longer nearfull too
+ for (auto &pool: tmp.get_pools()) {
+ auto p = pool.first;
+ if (full_pool_ids.count(p) ||
+ backfillfull_pool_ids.count(p) ||
+ nearfull_pool_ids.count(p)) {
+ // skip pools we have just marked as
+ // nearfull/backfillfull/full above
+ continue;
+ }
+ if (!tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_NEARFULL)) {
+ // and don't touch if currently is not nearfull
+ continue;
+ }
+ dout(10) << __func__ << " clearing pool '" << tmp.pool_name[p]
+ << "'s nearfull flag" << dendl;
+ if (pending_inc.new_pools.count(p) == 0) {
+ pending_inc.new_pools[p] = pool.second;
+ }
+ pending_inc.new_pools[p].flags &= ~pg_pool_t::FLAG_NEARFULL;
+ }
}
// min_compat_client?
- if (tmp.require_min_compat_client.empty()) {
+ if (tmp.require_min_compat_client == 0) {
auto mv = tmp.get_min_compat_client();
- dout(1) << __func__ << " setting require_min_compat_client to current " << mv
- << dendl;
- mon->clog->info() << "setting require_min_compat_client to currently required "
- << mv;
- pending_inc.new_require_min_compat_client = mv.first;
+ dout(1) << __func__ << " setting require_min_compat_client to currently "
+ << "required " << ceph_release_name(mv) << dendl;
+ mon->clog->info() << "setting require_min_compat_client to currently "
+ << "required " << ceph_release_name(mv);
+ pending_inc.new_require_min_compat_client = mv;
+ }
+
+ if (osdmap.require_osd_release < CEPH_RELEASE_LUMINOUS) {
+ // convert ec profile ruleset-* -> crush-*
+ for (auto& p : tmp.erasure_code_profiles) {
+ bool changed = false;
+ map<string,string> newprofile;
+ for (auto& q : p.second) {
+ if (q.first.find("ruleset-") == 0) {
+ string key = "crush-";
+ key += q.first.substr(8);
+ newprofile[key] = q.second;
+ changed = true;
+ dout(20) << " updating ec profile " << p.first
+ << " key " << q.first << " -> " << key << dendl;
+ } else {
+ newprofile[q.first] = q.second;
+ }
+ }
+ if (changed) {
+ dout(10) << " updated ec profile " << p.first << ": "
+ << newprofile << dendl;
+ pending_inc.new_erasure_code_profiles[p.first] = newprofile;
+ }
+ }
+
+ // auto-enable pool applications upon upgrade
+ // NOTE: this can be removed post-Luminous assuming upgrades need to
+ // proceed through Luminous
+ for (auto &pool_pair : tmp.pools) {
+ int64_t pool_id = pool_pair.first;
+ pg_pool_t pg_pool = pool_pair.second;
+ if (pg_pool.is_tier()) {
+ continue;
+ }
+
+ std::string pool_name = tmp.get_pool_name(pool_id);
+ uint32_t match_count = 0;
+
+ // CephFS
+ const FSMap &pending_fsmap = mon->mdsmon()->get_pending_fsmap();
+ if (pending_fsmap.pool_in_use(pool_id)) {
+ dout(10) << __func__ << " auto-enabling CephFS on pool '"
+ << pool_name << "'" << dendl;
+ pg_pool.application_metadata.insert(
+ {pg_pool_t::APPLICATION_NAME_CEPHFS, {}});
+ ++match_count;
+ }
+
+ // RBD heuristics (default OpenStack pool names from docs and
+ // ceph-ansible)
+ if (boost::algorithm::contains(pool_name, "rbd") ||
+ pool_name == "images" || pool_name == "volumes" ||
+ pool_name == "backups" || pool_name == "vms") {
+ dout(10) << __func__ << " auto-enabling RBD on pool '"
+ << pool_name << "'" << dendl;
+ pg_pool.application_metadata.insert(
+ {pg_pool_t::APPLICATION_NAME_RBD, {}});
+ ++match_count;
+ }
+
+ // RGW heuristics
+ if (boost::algorithm::contains(pool_name, ".rgw") ||
+ boost::algorithm::contains(pool_name, ".log") ||
+ boost::algorithm::contains(pool_name, ".intent-log") ||
+ boost::algorithm::contains(pool_name, ".usage") ||
+ boost::algorithm::contains(pool_name, ".users")) {
+ dout(10) << __func__ << " auto-enabling RGW on pool '"
+ << pool_name << "'" << dendl;
+ pg_pool.application_metadata.insert(
+ {pg_pool_t::APPLICATION_NAME_RGW, {}});
+ ++match_count;
+ }
+
+ // OpenStack gnocchi (from ceph-ansible)
+ if (pool_name == "metrics" && match_count == 0) {
+ dout(10) << __func__ << " auto-enabling OpenStack Gnocchi on pool '"
+ << pool_name << "'" << dendl;
+ pg_pool.application_metadata.insert({"openstack_gnocchi", {}});
+ ++match_count;
+ }
+
+ if (match_count == 1) {
+ pg_pool.last_change = pending_inc.epoch;
+ pending_inc.new_pools[pool_id] = pg_pool;
+ } else if (match_count > 1) {
+ auto pstat = mon->pgservice->get_pool_stat(pool_id);
+ if (pstat != nullptr && pstat->stats.sum.num_objects > 0) {
+ mon->clog->info() << "unable to auto-enable application for pool "
+ << "'" << pool_name << "'";
+ }
+ }
+ }
}
}
}
// tell me about it
- for (map<int32_t,uint8_t>::iterator i = pending_inc.new_state.begin();
+ for (auto i = pending_inc.new_state.begin();
i != pending_inc.new_state.end();
++i) {
int s = i->second ? i->second : CEPH_OSD_UP;
}
}
+ // clean inappropriate pg_upmap/pg_upmap_items (if any)
+ osdmap.maybe_remove_pg_upmaps(cct, osdmap, &pending_inc);
+
// features for osdmap and its incremental
- uint64_t features = mon->get_quorum_con_features();
+ uint64_t features;
// encode full map and determine its crc
OSDMap tmp;
tmp.apply_incremental(pending_inc);
// determine appropriate features
- if (!tmp.test_flag(CEPH_OSDMAP_REQUIRE_LUMINOUS)) {
- dout(10) << __func__ << " encoding without feature SERVER_LUMINOUS"
- << dendl;
- features &= ~CEPH_FEATURE_SERVER_LUMINOUS;
- }
- if (!tmp.test_flag(CEPH_OSDMAP_REQUIRE_JEWEL)) {
- dout(10) << __func__ << " encoding without feature SERVER_JEWEL" << dendl;
- features &= ~CEPH_FEATURE_SERVER_JEWEL;
- }
- if (!tmp.test_flag(CEPH_OSDMAP_REQUIRE_KRAKEN)) {
- dout(10) << __func__ << " encoding without feature SERVER_KRAKEN | "
- << "MSG_ADDR2" << dendl;
- features &= ~(CEPH_FEATURE_SERVER_KRAKEN |
- CEPH_FEATURE_MSG_ADDR2);
- }
- dout(10) << __func__ << " encoding full map with " << features << dendl;
+ features = tmp.get_encoding_features();
+ dout(10) << __func__ << " encoding full map with "
+ << ceph_release_name(tmp.require_osd_release)
+ << " features " << features << dendl;
+
+ // the features should be a subset of the mon quorum's features!
+ assert((features & ~mon->get_quorum_con_features()) == 0);
bufferlist fullbl;
::encode(tmp, fullbl, features | CEPH_FEATURE_RESERVED);
// and pg creating, also!
if (mon->monmap->get_required_features().contains_all(
ceph::features::mon::FEATURE_LUMINOUS)) {
- auto pending_creatings = update_pending_pgs(pending_inc);
- if (!osdmap.test_flag(CEPH_OSDMAP_REQUIRE_LUMINOUS)) {
+ auto pending_creatings = update_pending_pgs(pending_inc, tmp);
+ if (osdmap.get_epoch() &&
+ osdmap.require_osd_release < CEPH_RELEASE_LUMINOUS) {
dout(7) << __func__ << " in the middle of upgrading, "
<< " trimming pending creating_pgs using pgmap" << dendl;
- trim_creating_pgs(&pending_creatings, mon->pgmon()->pg_map);
+ mon->pgservice->maybe_trim_creating_pgs(&pending_creatings);
}
bufferlist creatings_bl;
::encode(pending_creatings, creatings_bl);
t->put(OSD_PG_CREATING_PREFIX, "creating", creatings_bl);
}
+
+ // health
+ health_check_map_t next;
+ tmp.check_health(&next);
+ encode_health(next, t);
}
void OSDMonitor::trim_creating_pgs(creating_pgs_t* creating_pgs,
- const PGMap& pgm)
+ const ceph::unordered_map<pg_t,pg_stat_t>& pg_stat)
{
auto p = creating_pgs->pgs.begin();
while (p != creating_pgs->pgs.end()) {
- auto q = pgm.pg_stat.find(p->first);
- if (q != pgm.pg_stat.end() &&
+ auto q = pg_stat.find(p->first);
+ if (q != pg_stat.end() &&
!(q->second.state & PG_STATE_CREATING)) {
dout(20) << __func__ << " pgmap shows " << p->first << " is created"
<< dendl;
p = creating_pgs->pgs.erase(p);
- creating_pgs->created_pools.insert(q->first.pool());
} else {
++p;
}
return 0;
}
+void OSDMonitor::count_metadata(const string& field, map<string,int> *out)
+{
+ for (int osd = 0; osd < osdmap.get_max_osd(); ++osd) {
+ if (osdmap.is_up(osd)) {
+ map<string,string> meta;
+ load_metadata(osd, meta, nullptr);
+ auto p = meta.find(field);
+ if (p == meta.end()) {
+ (*out)["unknown"]++;
+ } else {
+ (*out)[p->second]++;
+ }
+ }
+ }
+}
+
+void OSDMonitor::count_metadata(const string& field, Formatter *f)
+{
+ map<string,int> by_val;
+ count_metadata(field, &by_val);
+ f->open_object_section(field.c_str());
+ for (auto& p : by_val) {
+ f->dump_int(p.first.c_str(), p.second);
+ }
+ f->close_section();
+}
+
int OSDMonitor::get_osd_objectstore_type(int osd, string *type)
{
map<string, string> metadata;
}
dout(10) << "committed, telling random " << s->inst << " all about it" << dendl;
+
+ // get feature of the peer
+ // use quorum_con_features, if it's an anonymous connection.
+ uint64_t features = s->con_features ? s->con_features :
+ mon->get_quorum_con_features();
// whatev, they'll request more if they need it
- MOSDMap *m = build_incremental(osdmap.get_epoch() - 1, osdmap.get_epoch());
+ MOSDMap *m = build_incremental(osdmap.get_epoch() - 1, osdmap.get_epoch(), features);
s->con->send_message(m);
// NOTE: do *not* record osd has up to this epoch (as we do
// elsewhere) as they may still need to request older values.
version_t OSDMonitor::get_trim_to()
{
- epoch_t floor;
+ if (mon->get_quorum().empty()) {
+ dout(10) << __func__ << ": quorum not formed" << dendl;
+ return 0;
+ }
+ epoch_t floor;
if (mon->monmap->get_required_features().contains_all(
ceph::features::mon::FEATURE_LUMINOUS)) {
{
+ // TODO: Get this hidden in PGStatService
std::lock_guard<std::mutex> l(creating_pgs_lock);
if (!creating_pgs.pgs.empty()) {
return 0;
}
floor = get_min_last_epoch_clean();
} else {
- if (!mon->pgmon()->is_readable())
+ if (!mon->pgservice->is_readable())
return 0;
- if (mon->pgmon()->pg_map.creating_pgs.empty()) {
+ if (mon->pgservice->have_creating_pgs()) {
return 0;
}
- floor = mon->pgmon()->pg_map.get_min_last_epoch_clean();
+ floor = mon->pgservice->get_min_last_epoch_clean();
}
{
dout(10) << " min_last_epoch_clean " << floor << dendl;
return true;
}
- // propose as fast as possible if updating up_thru or pg_temp
- // want to merge OSDMap changes as much as possible
- if ((pending_inc.new_primary_temp.size() == 1
- || pending_inc.new_up_thru.size() == 1)
- && pending_inc.new_state.size() < 2) {
- dout(15) << " propose as fast as possible for up_thru/pg_temp" << dendl;
-
- utime_t now = ceph_clock_now();
- if (now - last_attempted_minwait_time > g_conf->paxos_propose_interval
- && now - paxos->get_last_commit_time() > g_conf->paxos_min_wait) {
- delay = g_conf->paxos_min_wait;
- last_attempted_minwait_time = now;
- return true;
- }
- }
-
return PaxosService::should_propose(delay);
}
{
op->mark_osdmon_event(__func__);
MMonGetOSDMap *m = static_cast<MMonGetOSDMap*>(op->get_req());
+
+ uint64_t features = mon->get_quorum_con_features();
+ if (m->get_session() && m->get_session()->con_features)
+ features = m->get_session()->con_features;
+
dout(10) << __func__ << " " << *m << dendl;
- MOSDMap *reply = new MOSDMap(mon->monmap->fsid);
+ MOSDMap *reply = new MOSDMap(mon->monmap->fsid, features);
epoch_t first = get_first_committed();
epoch_t last = osdmap.get_epoch();
int max = g_conf->osd_map_message_max;
for (epoch_t e = MAX(first, m->get_full_first());
e <= MIN(last, m->get_full_last()) && max > 0;
++e, --max) {
- int r = get_version_full(e, reply->maps[e]);
+ int r = get_version_full(e, features, reply->maps[e]);
assert(r >= 0);
}
for (epoch_t e = MAX(first, m->get_inc_first());
e <= MIN(last, m->get_inc_last()) && max > 0;
++e, --max) {
- int r = get_version(e, reply->incremental_maps[e]);
+ int r = get_version(e, features, reply->incremental_maps[e]);
assert(r >= 0);
}
reply->oldest_map = first;
return false;
didit:
+ mon->no_reply(op);
return true;
}
bool OSDMonitor::can_mark_down(int i)
{
if (osdmap.test_flag(CEPH_OSDMAP_NODOWN)) {
- dout(5) << "can_mark_down NODOWN flag set, will not mark osd." << i << " down" << dendl;
+ dout(5) << __func__ << " NODOWN flag set, will not mark osd." << i
+ << " down" << dendl;
return false;
}
+
+ if (osdmap.is_nodown(i)) {
+ dout(5) << __func__ << " osd." << i << " is marked as nodown, "
+ << "will not mark it down" << dendl;
+ return false;
+ }
+
int num_osds = osdmap.get_num_osds();
if (num_osds == 0) {
- dout(5) << "can_mark_down no osds" << dendl;
+ dout(5) << __func__ << " no osds" << dendl;
return false;
}
int up = osdmap.get_num_up_osds() - pending_inc.get_net_marked_down(&osdmap);
float up_ratio = (float)up / (float)num_osds;
if (up_ratio < g_conf->mon_osd_min_up_ratio) {
- dout(2) << "can_mark_down current up_ratio " << up_ratio << " < min "
+ dout(2) << __func__ << " current up_ratio " << up_ratio << " < min "
<< g_conf->mon_osd_min_up_ratio
<< ", will not mark osd." << i << " down" << dendl;
return false;
bool OSDMonitor::can_mark_up(int i)
{
if (osdmap.test_flag(CEPH_OSDMAP_NOUP)) {
- dout(5) << "can_mark_up NOUP flag set, will not mark osd." << i << " up" << dendl;
+ dout(5) << __func__ << " NOUP flag set, will not mark osd." << i
+ << " up" << dendl;
+ return false;
+ }
+
+ if (osdmap.is_noup(i)) {
+ dout(5) << __func__ << " osd." << i << " is marked as noup, "
+ << "will not mark it up" << dendl;
return false;
}
+
return true;
}
dout(5) << __func__ << " NOOUT flag set, will not mark osds out" << dendl;
return false;
}
+
+ if (osdmap.is_noout(i)) {
+ dout(5) << __func__ << " osd." << i << " is marked as noout, "
+ << "will not mark it out" << dendl;
+ return false;
+ }
+
int num_osds = osdmap.get_num_osds();
if (num_osds == 0) {
dout(5) << __func__ << " no osds" << dendl;
bool OSDMonitor::can_mark_in(int i)
{
if (osdmap.test_flag(CEPH_OSDMAP_NOIN)) {
- dout(5) << "can_mark_in NOIN flag set, will not mark osd." << i << " in" << dendl;
+ dout(5) << __func__ << " NOIN flag set, will not mark osd." << i
+ << " in" << dendl;
+ return false;
+ }
+
+ if (osdmap.is_noin(i)) {
+ dout(5) << __func__ << " osd." << i << " is marked as noin, "
+ << "will not mark it in" << dendl;
return false;
}
+
return true;
}
<< " down" << dendl;
pending_inc.new_state[target_osd] = CEPH_OSD_UP;
- mon->clog->info() << osdmap.get_inst(target_osd) << " failed ("
- << (int)reporters_by_subtree.size() << " reporters from different "
+ mon->clog->info() << "osd." << target_osd << " failed ("
+ << osdmap.crush->get_full_location_ordered_string(
+ target_osd)
+ << ") ("
+ << (int)reporters_by_subtree.size()
+ << " reporters from different "
<< reporter_subtree_level << " after "
<< failed_for << " >= grace " << grace << ")";
return true;
return false;
}
-void OSDMonitor::force_failure(utime_t now, int target_osd)
+void OSDMonitor::force_failure(int target_osd, int by)
{
// already pending failure?
if (pending_inc.new_state.count(target_osd) &&
dout(1) << " we're forcing failure of osd." << target_osd << dendl;
pending_inc.new_state[target_osd] = CEPH_OSD_UP;
- mon->clog->info() << osdmap.get_inst(target_osd) << " failed (forced)";
+ mon->clog->info() << "osd." << target_osd << " failed ("
+ << osdmap.crush->get_full_location_ordered_string(target_osd)
+ << ") (connection refused reported by osd." << by << ")";
return;
}
if (m->is_immediate()) {
mon->clog->debug() << m->get_target() << " reported immediately failed by "
<< m->get_orig_source_inst();
- force_failure(now, target_osd);
+ force_failure(target_osd, reporter);
+ mon->no_reply(op);
return true;
}
mon->clog->debug() << m->get_target() << " reported failed by "
o->mark_event(__func__);
MOSDFailure *m = o->get_req<MOSDFailure>();
send_latest(o, m->get_epoch());
+ mon->no_reply(o);
}
ls.pop_front();
}
goto ignore;
}
- if (osdmap.test_flag(CEPH_OSDMAP_REQUIRE_LUMINOUS) &&
+ if (osdmap.require_osd_release >= CEPH_RELEASE_LUMINOUS &&
!HAVE_FEATURE(m->osd_features, SERVER_LUMINOUS)) {
mon->clog->info() << "disallowing boot of OSD "
<< m->get_orig_source_inst()
goto ignore;
}
- if (osdmap.test_flag(CEPH_OSDMAP_REQUIRE_JEWEL) &&
+ if (osdmap.require_osd_release >= CEPH_RELEASE_JEWEL &&
!(m->osd_features & CEPH_FEATURE_SERVER_JEWEL)) {
mon->clog->info() << "disallowing boot of OSD "
<< m->get_orig_source_inst()
goto ignore;
}
- if (osdmap.test_flag(CEPH_OSDMAP_REQUIRE_KRAKEN) &&
+ if (osdmap.require_osd_release >= CEPH_RELEASE_KRAKEN &&
!HAVE_FEATURE(m->osd_features, SERVER_KRAKEN)) {
mon->clog->info() << "disallowing boot of OSD "
<< m->get_orig_source_inst()
goto ignore;
}
+ if (osdmap.test_flag(CEPH_OSDMAP_RECOVERY_DELETES) &&
+ !(m->osd_features & CEPH_FEATURE_OSD_RECOVERY_DELETES)) {
+ mon->clog->info() << "disallowing boot of OSD "
+ << m->get_orig_source_inst()
+ << " because 'recovery_deletes' osdmap flag is set and OSD lacks the OSD_RECOVERY_DELETES feature";
+ goto ignore;
+ }
+
if (any_of(osdmap.get_pools().begin(),
osdmap.get_pools().end(),
[](const std::pair<int64_t,pg_pool_t>& pool)
// make sure upgrades stop at luminous
if (HAVE_FEATURE(m->osd_features, SERVER_M) &&
- !osdmap.test_flag(CEPH_OSDMAP_REQUIRE_LUMINOUS)) {
+ osdmap.require_osd_release < CEPH_RELEASE_LUMINOUS) {
mon->clog->info() << "disallowing boot of post-luminous OSD "
<< m->get_orig_source_inst()
- << " because require_luminous_osds is not set";
+ << " because require_osd_release < luminous";
goto ignore;
}
// make sure upgrades stop at jewel
if (HAVE_FEATURE(m->osd_features, SERVER_KRAKEN) &&
- !osdmap.test_flag(CEPH_OSDMAP_REQUIRE_JEWEL)) {
+ osdmap.require_osd_release < CEPH_RELEASE_JEWEL) {
mon->clog->info() << "disallowing boot of post-jewel OSD "
<< m->get_orig_source_inst()
- << " because require_jewel_osds is not set";
+ << " because require_osd_release < jewel";
goto ignore;
}
bufferlist osd_metadata;
::encode(m->metadata, osd_metadata);
pending_metadata[from] = osd_metadata;
+ pending_metadata_rm.erase(from);
// adjust last clean unmount epoch?
const osd_info_t& info = osdmap.get_info(from);
auto m = static_cast<MOSDPGCreated*>(op->get_req());
dout(10) << __func__ << " " << *m << dendl;
auto session = m->get_session();
+ mon->no_reply(op);
if (!session) {
dout(10) << __func__ << ": no monitor session!" << dendl;
return true;
goto ignore;
}
+ if (m->forced) {
+ return false;
+ }
+
for (auto p = m->pg_temp.begin(); p != m->pg_temp.end(); ++p) {
dout(20) << " " << p->first
- << (osdmap.pg_temp->count(p->first) ? (*osdmap.pg_temp)[p->first] : empty)
+ << (osdmap.pg_temp->count(p->first) ? osdmap.pg_temp->get(p->first) : empty)
<< " -> " << p->second << dendl;
// does the pool exist?
// an existing pg_primary field to imply a change
if (p->second.size() &&
(osdmap.pg_temp->count(p->first) == 0 ||
- !vectors_equal((*osdmap.pg_temp)[p->first], p->second) ||
+ !vectors_equal(osdmap.pg_temp->get(p->first), p->second) ||
osdmap.primary_temp->count(p->first)))
return false;
}
auto beacon = static_cast<MOSDBeacon*>(op->get_req());
// check caps
auto session = beacon->get_session();
+ mon->no_reply(op);
if (!session) {
dout(10) << __func__ << " no monitor session!" << dendl;
return true;
}
-MOSDMap *OSDMonitor::build_latest_full()
+MOSDMap *OSDMonitor::build_latest_full(uint64_t features)
{
- MOSDMap *r = new MOSDMap(mon->monmap->fsid);
- get_version_full(osdmap.get_epoch(), r->maps[osdmap.get_epoch()]);
+ MOSDMap *r = new MOSDMap(mon->monmap->fsid, features);
+ get_version_full(osdmap.get_epoch(), features, r->maps[osdmap.get_epoch()]);
r->oldest_map = get_first_committed();
r->newest_map = osdmap.get_epoch();
return r;
}
-MOSDMap *OSDMonitor::build_incremental(epoch_t from, epoch_t to)
+MOSDMap *OSDMonitor::build_incremental(epoch_t from, epoch_t to, uint64_t features)
{
- dout(10) << "build_incremental [" << from << ".." << to << "]" << dendl;
- MOSDMap *m = new MOSDMap(mon->monmap->fsid);
+ dout(10) << "build_incremental [" << from << ".." << to << "] with features " << std::hex << features << dendl;
+ MOSDMap *m = new MOSDMap(mon->monmap->fsid, features);
m->oldest_map = get_first_committed();
m->newest_map = osdmap.get_epoch();
for (epoch_t e = to; e >= from && e > 0; e--) {
bufferlist bl;
- int err = get_version(e, bl);
+ int err = get_version(e, features, bl);
if (err == 0) {
assert(bl.length());
// if (get_version(e, bl) > 0) {
} else {
assert(err == -ENOENT);
assert(!bl.length());
- get_version_full(e, bl);
+ get_version_full(e, features, bl);
if (bl.length() > 0) {
//else if (get_version("full", e, bl) > 0) {
dout(20) << "build_incremental full " << e << " "
{
op->mark_osdmon_event(__func__);
dout(5) << "send_full to " << op->get_req()->get_orig_source_inst() << dendl;
- mon->send_reply(op, build_latest_full());
+ mon->send_reply(op, build_latest_full(op->get_session()->con_features));
}
void OSDMonitor::send_incremental(MonOpRequestRef op, epoch_t first)
dout(5) << "send_incremental [" << first << ".." << osdmap.get_epoch() << "]"
<< " to " << session->inst << dendl;
+ // get feature of the peer
+ // use quorum_con_features, if it's an anonymous connection.
+ uint64_t features = session->con_features ? session->con_features :
+ mon->get_quorum_con_features();
+
if (first <= session->osd_epoch) {
- dout(10) << __func__ << session->inst << " should already have epoch "
+ dout(10) << __func__ << " " << session->inst << " should already have epoch "
<< session->osd_epoch << dendl;
first = session->osd_epoch + 1;
}
if (first < get_first_committed()) {
first = get_first_committed();
bufferlist bl;
- int err = get_version_full(first, bl);
+ int err = get_version_full(first, features, bl);
assert(err == 0);
assert(bl.length());
dout(20) << "send_incremental starting with base full "
<< first << " " << bl.length() << " bytes" << dendl;
- MOSDMap *m = new MOSDMap(osdmap.get_fsid());
+ MOSDMap *m = new MOSDMap(osdmap.get_fsid(), features);
m->oldest_map = get_first_committed();
m->newest_map = osdmap.get_epoch();
m->maps[first] = bl;
}
while (first <= osdmap.get_epoch()) {
- epoch_t last = MIN(first + g_conf->osd_map_message_max - 1,
- osdmap.get_epoch());
- MOSDMap *m = build_incremental(first, last);
+ epoch_t last = std::min<epoch_t>(first + g_conf->osd_map_message_max - 1,
+ osdmap.get_epoch());
+ MOSDMap *m = build_incremental(first, last, features);
if (req) {
// send some maps. it may not be all of them, but it will get them
int OSDMonitor::get_version(version_t ver, bufferlist& bl)
{
- if (inc_osd_cache.lookup(ver, &bl)) {
- return 0;
- }
- int ret = PaxosService::get_version(ver, bl);
- if (!ret) {
- inc_osd_cache.add(ver, bl);
- }
+ return get_version(ver, mon->get_quorum_con_features(), bl);
+}
+
+void OSDMonitor::reencode_incremental_map(bufferlist& bl, uint64_t features)
+{
+ OSDMap::Incremental inc;
+ bufferlist::iterator q = bl.begin();
+ inc.decode(q);
+ // always encode with subset of osdmap's canonical features
+ uint64_t f = features & inc.encode_features;
+ dout(20) << __func__ << " " << inc.epoch << " with features " << f
+ << dendl;
+ bl.clear();
+ if (inc.fullmap.length()) {
+ // embedded full map?
+ OSDMap m;
+ m.decode(inc.fullmap);
+ inc.fullmap.clear();
+ m.encode(inc.fullmap, f | CEPH_FEATURE_RESERVED);
+ }
+ if (inc.crush.length()) {
+ // embedded crush map
+ CrushWrapper c;
+ auto p = inc.crush.begin();
+ c.decode(p);
+ inc.crush.clear();
+ c.encode(inc.crush, f);
+ }
+ inc.encode(bl, f | CEPH_FEATURE_RESERVED);
+}
+
+void OSDMonitor::reencode_full_map(bufferlist& bl, uint64_t features)
+{
+ OSDMap m;
+ bufferlist::iterator q = bl.begin();
+ m.decode(q);
+ // always encode with subset of osdmap's canonical features
+ uint64_t f = features & m.get_encoding_features();
+ dout(20) << __func__ << " " << m.get_epoch() << " with features " << f
+ << dendl;
+ bl.clear();
+ m.encode(bl, f | CEPH_FEATURE_RESERVED);
+}
+
+int OSDMonitor::get_version(version_t ver, uint64_t features, bufferlist& bl)
+{
+ uint64_t significant_features = OSDMap::get_significant_features(features);
+ if (inc_osd_cache.lookup({ver, significant_features}, &bl)) {
+ return 0;
+ }
+ int ret = PaxosService::get_version(ver, bl);
+ if (ret < 0) {
return ret;
+ }
+ // NOTE: this check is imprecise; the OSDMap encoding features may
+ // be a subset of the latest mon quorum features, but worst case we
+ // reencode once and then cache the (identical) result under both
+ // feature masks.
+ if (significant_features !=
+ OSDMap::get_significant_features(mon->get_quorum_con_features())) {
+ reencode_incremental_map(bl, features);
+ }
+ inc_osd_cache.add({ver, significant_features}, bl);
+ return 0;
}
int OSDMonitor::get_version_full(version_t ver, bufferlist& bl)
{
- if (full_osd_cache.lookup(ver, &bl)) {
- return 0;
- }
- int ret = PaxosService::get_version_full(ver, bl);
- if (!ret) {
- full_osd_cache.add(ver, bl);
- }
+ return get_version_full(ver, mon->get_quorum_con_features(), bl);
+}
+
+int OSDMonitor::get_version_full(version_t ver, uint64_t features,
+ bufferlist& bl)
+{
+ uint64_t significant_features = OSDMap::get_significant_features(features);
+ if (full_osd_cache.lookup({ver, significant_features}, &bl)) {
+ return 0;
+ }
+ int ret = PaxosService::get_version_full(ver, bl);
+ if (ret < 0) {
return ret;
+ }
+ // NOTE: this check is imprecise; the OSDMap encoding features may
+ // be a subset of the latest mon quorum features, but worst case we
+ // reencode once and then cache the (identical) result under both
+ // feature masks.
+ if (significant_features !=
+ OSDMap::get_significant_features(mon->get_quorum_con_features())) {
+ reencode_full_map(bl, features);
+ }
+ full_osd_cache.add({ver, significant_features}, bl);
+ return 0;
}
epoch_t OSDMonitor::blacklist(const entity_addr_t& a, utime_t until)
if (sub->next >= 1)
send_incremental(sub->next, sub->session, sub->incremental_onetime);
else
- sub->session->con->send_message(build_latest_full());
+ sub->session->con->send_message(build_latest_full(sub->session->con_features));
if (sub->onetime)
mon->session_map.remove_sub(sub);
else
}
}
-void OSDMonitor::scan_for_creating_pgs(
+void OSDMonitor::do_application_enable(int64_t pool_id,
+ const std::string &app_name)
+{
+ assert(paxos->is_plugged() && is_writeable());
+
+ dout(20) << __func__ << ": pool_id=" << pool_id << ", app_name=" << app_name
+ << dendl;
+
+ assert(osdmap.require_osd_release >= CEPH_RELEASE_LUMINOUS ||
+ pending_inc.new_require_osd_release >= CEPH_RELEASE_LUMINOUS);
+
+ auto pp = osdmap.get_pg_pool(pool_id);
+ assert(pp != nullptr);
+
+ pg_pool_t p = *pp;
+ if (pending_inc.new_pools.count(pool_id)) {
+ p = pending_inc.new_pools[pool_id];
+ }
+
+ p.application_metadata.insert({app_name, {}});
+ p.last_change = pending_inc.epoch;
+ pending_inc.new_pools[pool_id] = p;
+}
+
+unsigned OSDMonitor::scan_for_creating_pgs(
const mempool::osdmap::map<int64_t,pg_pool_t>& pools,
const mempool::osdmap::set<int64_t>& removed_pools,
utime_t modified,
creating_pgs_t* creating_pgs) const
{
+ unsigned queued = 0;
for (auto& p : pools) {
int64_t poolid = p.first;
const pg_pool_t& pool = p.second;
- int ruleno = osdmap.crush->find_rule(pool.get_crush_ruleset(),
+ int ruleno = osdmap.crush->find_rule(pool.get_crush_rule(),
pool.get_type(), pool.get_size());
if (ruleno < 0 || !osdmap.crush->rule_exists(ruleno))
continue;
<< " " << pool << dendl;
continue;
}
- dout(10) << __func__ << " scanning pool " << poolid
+ dout(10) << __func__ << " queueing pool create for " << poolid
<< " " << pool << dendl;
- if (creating_pgs->created_pools.count(poolid)) {
- // split pgs are skipped by OSD, so drop it early.
- continue;
- }
- // first pgs in this pool
- for (ps_t ps = 0; ps < pool.get_pg_num(); ps++) {
- const pg_t pgid{ps, static_cast<uint64_t>(poolid)};
- if (creating_pgs->pgs.count(pgid)) {
- dout(20) << __func__ << " already have " << pgid << dendl;
- continue;
- }
- creating_pgs->pgs.emplace(pgid, make_pair(created, modified));
- dout(10) << __func__ << " adding " << pgid
- << " at " << osdmap.get_epoch() << dendl;
+ if (creating_pgs->create_pool(poolid, pool.get_pg_num(),
+ created, modified)) {
+ queued++;
}
}
+ return queued;
}
void OSDMonitor::update_creating_pgs()
{
+ dout(10) << __func__ << " " << creating_pgs.pgs.size() << " pgs creating, "
+ << creating_pgs.queue.size() << " pools in queue" << dendl;
decltype(creating_pgs_by_osd_epoch) new_pgs_by_osd_epoch;
std::lock_guard<std::mutex> l(creating_pgs_lock);
- for (auto& pg : creating_pgs.pgs) {
+ for (const auto& pg : creating_pgs.pgs) {
int acting_primary = -1;
auto pgid = pg.first;
+ if (!osdmap.pg_exists(pgid)) {
+ dout(20) << __func__ << " ignoring " << pgid << " which should not exist"
+ << dendl;
+ continue;
+ }
auto mapped = pg.second.first;
+ dout(20) << __func__ << " looking up " << pgid << "@" << mapped << dendl;
mapping.get(pgid, nullptr, nullptr, nullptr, &acting_primary);
// check the previous creating_pgs, look for the target to whom the pg was
// previously mapped
mapped = mapping.get_epoch();
}
break;
- }
+ } else {
+ // newly creating
+ mapped = mapping.get_epoch();
+ }
}
}
dout(10) << __func__ << " will instruct osd." << acting_primary
- << " to create " << pgid << dendl;
+ << " to create " << pgid << "@" << mapped << dendl;
new_pgs_by_osd_epoch[acting_primary][mapped].insert(pgid);
}
creating_pgs_by_osd_epoch = std::move(new_pgs_by_osd_epoch);
creating_pgs_epoch = mapping.get_epoch();
}
-epoch_t OSDMonitor::send_pg_creates(int osd, Connection *con, epoch_t next)
+epoch_t OSDMonitor::send_pg_creates(int osd, Connection *con, epoch_t next) const
{
dout(30) << __func__ << " osd." << osd << " next=" << next
<< " " << creating_pgs_by_osd_epoch << dendl;
std::lock_guard<std::mutex> l(creating_pgs_lock);
+ if (creating_pgs_epoch <= creating_pgs.last_scan_epoch) {
+ dout(20) << __func__
+ << " not using stale creating_pgs@" << creating_pgs_epoch << dendl;
+ // the subscribers will be updated when the mapping is completed anyway
+ return next;
+ }
auto creating_pgs_by_epoch = creating_pgs_by_osd_epoch.find(osd);
if (creating_pgs_by_epoch == creating_pgs_by_osd_epoch.end())
return next;
m = new MOSDPGCreate(creating_pgs_epoch);
// Need the create time from the monitor using its clock to set
// last_scrub_stamp upon pg creation.
- const auto& creation = creating_pgs.pgs[pg];
- m->mkpg.emplace(pg, pg_create_t{creation.first, pg, 0});
- m->ctimes.emplace(pg, creation.second);
+ auto create = creating_pgs.pgs.find(pg);
+ assert(create != creating_pgs.pgs.end());
+ m->mkpg.emplace(pg, pg_create_t{create->second.first, pg, 0});
+ m->ctimes.emplace(pg, create->second.second);
dout(20) << __func__ << " will create " << pg
- << " at " << creation.first << dendl;
+ << " at " << create->second.first << dendl;
}
}
if (!m) {
bool do_propose = false;
utime_t now = ceph_clock_now();
- if (osdmap.test_flag(CEPH_OSDMAP_REQUIRE_LUMINOUS) &&
+ if (osdmap.require_osd_release >= CEPH_RELEASE_LUMINOUS &&
mon->monmap->get_required_features().contains_all(
ceph::features::mon::FEATURE_LUMINOUS)) {
if (handle_osd_timeouts(now, last_osd_report)) {
do_propose = true;
}
}
+ if (!osdmap.test_flag(CEPH_OSDMAP_PURGED_SNAPDIRS) &&
+ osdmap.require_osd_release >= CEPH_RELEASE_LUMINOUS &&
+ mon->mgrstatmon()->is_readable() &&
+ mon->mgrstatmon()->definitely_converted_snapsets()) {
+ dout(1) << __func__ << " all snapsets converted, setting purged_snapdirs"
+ << dendl;
+ add_flag(CEPH_OSDMAP_PURGED_SNAPDIRS);
+ do_propose = true;
+ }
// mark osds down?
if (check_failures(now))
}
}
- if (g_conf->mon_osd_down_out_interval > 0 &&
- down.sec() >= grace) {
+ bool down_out = !osdmap.is_destroyed(o) &&
+ g_conf->mon_osd_down_out_interval > 0 && down.sec() >= grace;
+ bool destroyed_out = osdmap.is_destroyed(o) &&
+ g_conf->mon_osd_destroyed_out_interval > 0 &&
+ // this is not precise enough as we did not make a note when this osd
+ // was marked as destroyed, but let's not bother with that
+ // complexity for now.
+ down.sec() >= g_conf->mon_osd_destroyed_out_interval;
+ if (down_out || destroyed_out) {
dout(10) << "tick marking osd." << o << " OUT after " << down
<< " sec (target " << grace << " = " << orig_grace << " + " << my_grace << ")" << dendl;
pending_inc.new_weight[o] = CEPH_OSD_OUT;
do_propose = true;
- mon->clog->info() << "osd." << o << " out (down for " << down << ")";
+ mon->clog->info() << "Marking osd." << o << " out (has been down for "
+ << int(down.sec()) << " seconds)";
} else
continue;
}
}
// if map full setting has changed, get that info out there!
- if (!osdmap.test_flag(CEPH_OSDMAP_REQUIRE_LUMINOUS) &&
- mon->pgmon()->is_readable()) {
+ if (osdmap.require_osd_release < CEPH_RELEASE_LUMINOUS &&
+ mon->pgservice->is_readable()) {
// for pre-luminous compat only!
- if (!mon->pgmon()->pg_map.full_osds.empty()) {
+ if (mon->pgservice->have_full_osds()) {
dout(5) << "There are full osds, setting full flag" << dendl;
add_flag(CEPH_OSDMAP_FULL);
} else if (osdmap.test_flag(CEPH_OSDMAP_FULL)){
remove_flag(CEPH_OSDMAP_FULL);
}
- if (!mon->pgmon()->pg_map.nearfull_osds.empty()) {
+ if (mon->pgservice->have_nearfull_osds()) {
dout(5) << "There are near full osds, setting nearfull flag" << dendl;
add_flag(CEPH_OSDMAP_NEARFULL);
} else if (osdmap.test_flag(CEPH_OSDMAP_NEARFULL)){
for (int i=0; i < max_osd; ++i) {
dout(30) << __func__ << ": checking up on osd " << i << dendl;
+ if (!osdmap.exists(i)) {
+ last_osd_report.erase(i); // if any
+ continue;
+ }
if (!osdmap.is_up(i))
continue;
const std::map<int,utime_t>::const_iterator t = last_osd_report.find(i);
} else if (can_mark_down(i)) {
utime_t diff = now - t->second;
if (diff > timeo) {
- mon->clog->info() << "osd." << i << " marked down after no pg stats for " << diff << "seconds";
- derr << "no osd or pg stats from osd." << i << " since " << t->second << ", " << diff
- << " seconds ago. marking down" << dendl;
+ mon->clog->info() << "osd." << i << " marked down after no beacon for "
+ << diff << " seconds";
+ derr << "no beacon from osd." << i << " since " << t->second
+ << ", " << diff << " seconds ago. marking down" << dendl;
pending_inc.new_state[i] = CEPH_OSD_UP;
new_down = true;
}
int num_in_osds = 0;
int num_down_in_osds = 0;
set<int> osds;
+ set<int> down_in_osds;
+ set<int> up_in_osds;
+ set<int> subtree_up;
+ unordered_map<int, set<int> > subtree_type_down;
+ unordered_map<int, int> num_osds_subtree;
+ int max_type = osdmap.crush->get_max_type_id();
+
for (int i = 0; i < osdmap.get_max_osd(); i++) {
if (!osdmap.exists(i)) {
if (osdmap.crush->item_exists(i)) {
osds.insert(i);
}
- continue;
- }
+ continue;
+ }
if (osdmap.is_out(i))
continue;
++num_in_osds;
+ if (down_in_osds.count(i) || up_in_osds.count(i))
+ continue;
if (!osdmap.is_up(i)) {
- ++num_down_in_osds;
- if (detail) {
- const osd_info_t& info = osdmap.get_info(i);
- ostringstream ss;
- ss << "osd." << i << " is down since epoch " << info.down_at
- << ", last address " << osdmap.get_addr(i);
- detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+ down_in_osds.insert(i);
+ int parent_id = 0;
+ int current = i;
+ for (int type = 0; type <= max_type; type++) {
+ if (!osdmap.crush->get_type_name(type))
+ continue;
+ int r = osdmap.crush->get_immediate_parent_id(current, &parent_id);
+ if (r == -ENOENT)
+ break;
+ // break early if this parent is already marked as up
+ if (subtree_up.count(parent_id))
+ break;
+ type = osdmap.crush->get_bucket_type(parent_id);
+ if (!osdmap.subtree_type_is_down(
+ g_ceph_context, parent_id, type,
+ &down_in_osds, &up_in_osds, &subtree_up, &subtree_type_down))
+ break;
+ current = parent_id;
+ }
+ }
+ }
+
+ // calculate the number of down osds in each down subtree and
+ // store it in num_osds_subtree
+ for (int type = 1; type <= max_type; type++) {
+ if (!osdmap.crush->get_type_name(type))
+ continue;
+ for (auto j = subtree_type_down[type].begin();
+ j != subtree_type_down[type].end();
+ ++j) {
+ if (type == 1) {
+ list<int> children;
+ int num = osdmap.crush->get_children(*j, &children);
+ num_osds_subtree[*j] = num;
+ } else {
+ list<int> children;
+ int num = 0;
+ int num_children = osdmap.crush->get_children(*j, &children);
+ if (num_children == 0)
+ continue;
+ for (auto l = children.begin(); l != children.end(); ++l) {
+ if (num_osds_subtree[*l] > 0) {
+ num = num + num_osds_subtree[*l];
+ }
+ }
+ num_osds_subtree[*j] = num;
}
}
}
+ num_down_in_osds = down_in_osds.size();
assert(num_down_in_osds <= num_in_osds);
if (num_down_in_osds > 0) {
+ // summary of down subtree types and osds
+ for (int type = max_type; type > 0; type--) {
+ if (!osdmap.crush->get_type_name(type))
+ continue;
+ if (subtree_type_down[type].size() > 0) {
+ ostringstream ss;
+ ss << subtree_type_down[type].size() << " "
+ << osdmap.crush->get_type_name(type);
+ if (subtree_type_down[type].size() > 1) {
+ ss << "s";
+ }
+ int sum_down_osds = 0;
+ for (auto j = subtree_type_down[type].begin();
+ j != subtree_type_down[type].end();
+ ++j) {
+ sum_down_osds = sum_down_osds + num_osds_subtree[*j];
+ }
+ ss << " (" << sum_down_osds << " osds) down";
+ summary.push_back(make_pair(HEALTH_WARN, ss.str()));
+ }
+ }
ostringstream ss;
- ss << num_down_in_osds << "/" << num_in_osds << " in osds are down";
+ ss << down_in_osds.size() << " osds down";
summary.push_back(make_pair(HEALTH_WARN, ss.str()));
+
+ if (detail) {
+ // details of down subtree types
+ for (int type = max_type; type > 0; type--) {
+ if (!osdmap.crush->get_type_name(type))
+ continue;
+ for (auto j = subtree_type_down[type].rbegin();
+ j != subtree_type_down[type].rend();
+ ++j) {
+ ostringstream ss;
+ ss << osdmap.crush->get_type_name(type);
+ ss << " ";
+ ss << osdmap.crush->get_item_name(*j);
+ // at the top level, do not print location
+ if (type != max_type) {
+ ss << " (";
+ ss << osdmap.crush->get_full_location_ordered_string(*j);
+ ss << ")";
+ }
+ int num = num_osds_subtree[*j];
+ ss << " (" << num << " osds)";
+ ss << " is down";
+ detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+ }
+ }
+ // details of down osds
+ for (auto it = down_in_osds.begin(); it != down_in_osds.end(); ++it) {
+ ostringstream ss;
+ ss << "osd." << *it << " (";
+ ss << osdmap.crush->get_full_location_ordered_string(*it);
+ ss << ") is down";
+ detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+ }
+ }
}
if (!osds.empty()) {
ostringstream ss;
- ss << "osds were removed from osdmap, but still kept in crushmap";
+ ss << osds.size() << " osds exist in the crush map but not in the osdmap";
summary.push_back(make_pair(HEALTH_WARN, ss.str()));
if (detail) {
- ss << " osds: [" << osds << "]";
+ ss << " (osds: " << osds << ")";
detail->push_back(make_pair(HEALTH_WARN, ss.str()));
}
}
- if (osdmap.test_flag(CEPH_OSDMAP_REQUIRE_LUMINOUS)) {
- // An osd could configure failsafe ratio, to something different
- // but for now assume it is the same here.
- float fsr = g_conf->osd_failsafe_full_ratio;
- if (fsr > 1.0) fsr /= 100;
- float fr = osdmap.get_full_ratio();
- float br = osdmap.get_backfillfull_ratio();
- float nr = osdmap.get_nearfull_ratio();
-
- bool out_of_order = false;
- // These checks correspond to how OSDService::check_full_status() in an OSD
- // handles the improper setting of these values.
- if (br < nr) {
- out_of_order = true;
- if (detail) {
- ostringstream ss;
- ss << "backfillfull_ratio (" << br << ") < nearfull_ratio (" << nr << "), increased";
- detail->push_back(make_pair(HEALTH_ERR, ss.str()));
- }
- br = nr;
- }
- if (fr < br) {
- out_of_order = true;
- if (detail) {
- ostringstream ss;
- ss << "full_ratio (" << fr << ") < backfillfull_ratio (" << br << "), increased";
- detail->push_back(make_pair(HEALTH_ERR, ss.str()));
- }
- fr = br;
- }
- if (fsr < fr) {
- out_of_order = true;
- if (detail) {
- ostringstream ss;
- ss << "osd_failsafe_full_ratio (" << fsr << ") < full_ratio (" << fr << "), increased";
- detail->push_back(make_pair(HEALTH_ERR, ss.str()));
- }
- }
- if (out_of_order) {
- ostringstream ss;
- ss << "Full ratio(s) out of order";
- summary.push_back(make_pair(HEALTH_ERR, ss.str()));
- }
-
- map<int, float> full, backfillfull, nearfull;
- osdmap.get_full_osd_util(mon->pgmon()->pg_map.osd_stat, &full, &backfillfull, &nearfull);
- if (full.size()) {
- ostringstream ss;
- ss << full.size() << " full osd(s)";
- summary.push_back(make_pair(HEALTH_ERR, ss.str()));
- }
- if (backfillfull.size()) {
- ostringstream ss;
- ss << backfillfull.size() << " backfillfull osd(s)";
- summary.push_back(make_pair(HEALTH_WARN, ss.str()));
- }
- if (nearfull.size()) {
- ostringstream ss;
- ss << nearfull.size() << " nearfull osd(s)";
- summary.push_back(make_pair(HEALTH_WARN, ss.str()));
- }
- if (detail) {
- for (auto& i: full) {
- ostringstream ss;
- ss << "osd." << i.first << " is full at " << roundf(i.second * 100) << "%";
- detail->push_back(make_pair(HEALTH_ERR, ss.str()));
- }
- for (auto& i: backfillfull) {
- ostringstream ss;
- ss << "osd." << i.first << " is backfill full at " << roundf(i.second * 100) << "%";
- detail->push_back(make_pair(HEALTH_WARN, ss.str()));
- }
- for (auto& i: nearfull) {
- ostringstream ss;
- ss << "osd." << i.first << " is near full at " << roundf(i.second * 100) << "%";
- detail->push_back(make_pair(HEALTH_WARN, ss.str()));
- }
- }
- }
// note: we leave it to ceph-mgr to generate details health warnings
// with actual osd utilizations
if (g_conf->mon_debug_no_require_luminous) {
// ignore these checks
} else if (HAVE_FEATURE(osdmap.get_up_osd_features(), SERVER_LUMINOUS) &&
- !osdmap.test_flag(CEPH_OSDMAP_REQUIRE_LUMINOUS)) {
- string msg = "all OSDs are running luminous or later but the"
- " 'require_luminous_osds' osdmap flag is not set";
+ osdmap.require_osd_release < CEPH_RELEASE_LUMINOUS) {
+ string msg = "all OSDs are running luminous or later but"
+ " require_osd_release < luminous";
summary.push_back(make_pair(HEALTH_WARN, msg));
if (detail) {
detail->push_back(make_pair(HEALTH_WARN, msg));
}
} else if (HAVE_FEATURE(osdmap.get_up_osd_features(), SERVER_KRAKEN) &&
- !osdmap.test_flag(CEPH_OSDMAP_REQUIRE_KRAKEN)) {
- string msg = "all OSDs are running kraken or later but the"
- " 'require_kraken_osds' osdmap flag is not set";
+ osdmap.require_osd_release < CEPH_RELEASE_KRAKEN) {
+ string msg = "all OSDs are running kraken or later but"
+ " require_osd_release < kraken";
summary.push_back(make_pair(HEALTH_WARN, msg));
if (detail) {
detail->push_back(make_pair(HEALTH_WARN, msg));
}
} else if (HAVE_FEATURE(osdmap.get_up_osd_features(), SERVER_JEWEL) &&
- !osdmap.test_flag(CEPH_OSDMAP_REQUIRE_JEWEL)) {
- string msg = "all OSDs are running jewel or later but the"
- " 'require_jewel_osds' osdmap flag is not set";
+ osdmap.require_osd_release < CEPH_RELEASE_JEWEL) {
+ string msg = "all OSDs are running jewel or later but"
+ " require_osd_release < jewel";
summary.push_back(make_pair(HEALTH_WARN, msg));
if (detail) {
detail->push_back(make_pair(HEALTH_WARN, msg));
}
}
- get_pools_health(summary, detail);
+ for (auto it : osdmap.get_pools()) {
+ const pg_pool_t &pool = it.second;
+ if (pool.has_flag(pg_pool_t::FLAG_FULL)) {
+ const string& pool_name = osdmap.get_pool_name(it.first);
+ stringstream ss;
+ ss << "pool '" << pool_name << "' is full";
+ summary.push_back(make_pair(HEALTH_WARN, ss.str()));
+ if (detail)
+ detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+ }
+ }
}
}
namespace {
enum osd_pool_get_choices {
SIZE, MIN_SIZE, CRASH_REPLAY_INTERVAL,
- PG_NUM, PGP_NUM, CRUSH_RULE, CRUSH_RULESET, HASHPSPOOL,
+ PG_NUM, PGP_NUM, CRUSH_RULE, HASHPSPOOL, EC_OVERWRITES,
NODELETE, NOPGCHANGE, NOSIZECHANGE,
WRITE_FADVISE_DONTNEED, NOSCRUB, NODEEP_SCRUB,
HIT_SET_TYPE, HIT_SET_PERIOD, HIT_SET_COUNT, HIT_SET_FPP,
boost::scoped_ptr<Formatter> f(Formatter::create(format));
if (prefix == "osd stat") {
- osdmap.print_summary(f.get(), ds);
+ osdmap.print_summary(f.get(), ds, "");
if (f)
f->flush(rdata);
else
}
else if (prefix == "osd perf" ||
prefix == "osd blocked-by") {
- const PGMap &pgm = mon->pgmon()->pg_map;
- r = process_pg_map_command(prefix, cmdmap, pgm, osdmap,
- f.get(), &ss, &rdata);
+ r = mon->pgservice->process_pg_command(prefix, cmdmap,
+ osdmap, f.get(), &ss, &rdata);
}
else if (prefix == "osd dump" ||
prefix == "osd tree" ||
prefix == "osd ls" ||
prefix == "osd getmap" ||
- prefix == "osd getcrushmap") {
+ prefix == "osd getcrushmap" ||
+ prefix == "osd ls-tree") {
string val;
epoch_t epoch = 0;
p->decode(osdmap_bl);
}
+ auto sg = make_scope_guard([&] {
+ if (p != &osdmap) {
+ delete p;
+ }
+ });
+
if (prefix == "osd dump") {
stringstream ds;
if (f) {
}
rdata.append(ds);
} else if (prefix == "osd tree") {
+ vector<string> states;
+ cmd_getval(g_ceph_context, cmdmap, "states", states);
+ unsigned filter = 0;
+ for (auto& s : states) {
+ if (s == "up") {
+ filter |= OSDMap::DUMP_UP;
+ } else if (s == "down") {
+ filter |= OSDMap::DUMP_DOWN;
+ } else if (s == "in") {
+ filter |= OSDMap::DUMP_IN;
+ } else if (s == "out") {
+ filter |= OSDMap::DUMP_OUT;
+ } else if (s == "destroyed") {
+ filter |= OSDMap::DUMP_DESTROYED;
+ } else {
+ ss << "unrecognized state '" << s << "'";
+ r = -EINVAL;
+ goto reply;
+ }
+ }
+ if ((filter & (OSDMap::DUMP_IN|OSDMap::DUMP_OUT)) ==
+ (OSDMap::DUMP_IN|OSDMap::DUMP_OUT)) {
+ ss << "cannot specify both 'in' and 'out'";
+ r = -EINVAL;
+ goto reply;
+ }
+ if (((filter & (OSDMap::DUMP_UP|OSDMap::DUMP_DOWN)) ==
+ (OSDMap::DUMP_UP|OSDMap::DUMP_DOWN)) ||
+ ((filter & (OSDMap::DUMP_UP|OSDMap::DUMP_DESTROYED)) ==
+ (OSDMap::DUMP_UP|OSDMap::DUMP_DESTROYED)) ||
+ ((filter & (OSDMap::DUMP_DOWN|OSDMap::DUMP_DESTROYED)) ==
+ (OSDMap::DUMP_DOWN|OSDMap::DUMP_DESTROYED))) {
+ ss << "can specify only one of 'up', 'down' and 'destroyed'";
+ r = -EINVAL;
+ goto reply;
+ }
if (f) {
f->open_object_section("tree");
- p->print_tree(f.get(), NULL);
+ p->print_tree(f.get(), NULL, filter);
f->close_section();
f->flush(ds);
} else {
- p->print_tree(NULL, &ds);
+ p->print_tree(NULL, &ds, filter);
}
rdata.append(ds);
} else if (prefix == "osd getmap") {
ss << "got osdmap epoch " << p->get_epoch();
} else if (prefix == "osd getcrushmap") {
p->crush->encode(rdata, mon->get_quorum_con_features());
- ss << "got crush map from osdmap epoch " << p->get_epoch();
+ ss << p->get_crush_version();
+ } else if (prefix == "osd ls-tree") {
+ string bucket_name;
+ cmd_getval(g_ceph_context, cmdmap, "name", bucket_name);
+ set<int> osds;
+ r = p->get_osds_by_bucket_name(bucket_name, &osds);
+ if (r == -ENOENT) {
+ ss << "\"" << bucket_name << "\" does not exist";
+ goto reply;
+ } else if (r < 0) {
+ ss << "can not parse bucket name:\"" << bucket_name << "\"";
+ goto reply;
+ }
+
+ if (f) {
+ f->open_array_section("osds");
+ for (auto &i : osds) {
+ if (osdmap.exists(i)) {
+ f->dump_int("osd", i);
+ }
+ }
+ f->close_section();
+ f->flush(ds);
+ } else {
+ bool first = true;
+ for (auto &i : osds) {
+ if (osdmap.exists(i)) {
+ if (!first)
+ ds << "\n";
+ first = false;
+ ds << i;
+ }
+ }
+ }
+
+ rdata.append(ds);
}
- if (p != &osdmap)
- delete p;
} else if (prefix == "osd df") {
string method;
cmd_getval(g_ceph_context, cmdmap, "output_method", method);
- print_utilization(ds, f ? f.get() : NULL, method == "tree");
+ print_osd_utilization(osdmap, mon->pgservice, ds,
+ f.get(), method == "tree");
rdata.append(ds);
} else if (prefix == "osd getmaxosd") {
if (f) {
f->close_section();
}
f->flush(rdata);
+ } else if (prefix == "osd versions") {
+ if (!f)
+ f.reset(Formatter::create("json-pretty"));
+ count_metadata("ceph_version", f.get());
+ f->flush(rdata);
+ r = 0;
+ } else if (prefix == "osd count-metadata") {
+ if (!f)
+ f.reset(Formatter::create("json-pretty"));
+ string field;
+ cmd_getval(g_ceph_context, cmdmap, "property", field);
+ count_metadata(field, f.get());
+ f->flush(rdata);
+ r = 0;
} else if (prefix == "osd map") {
string poolstr, objstr, namespacestr;
cmd_getval(g_ceph_context, cmdmap, "pool", poolstr);
}
goto reply;
- } else if ((prefix == "osd scrub" ||
- prefix == "osd deep-scrub" ||
- prefix == "osd repair")) {
+ } else if (prefix == "osd scrub" ||
+ prefix == "osd deep-scrub" ||
+ prefix == "osd repair") {
string whostr;
cmd_getval(g_ceph_context, cmdmap, "who", whostr);
vector<string> pvec;
get_str_vec(prefix, pvec);
- if (whostr == "*") {
+ if (whostr == "*" || whostr == "all" || whostr == "any") {
ss << "osds ";
int c = 0;
for (int i = 0; i < osdmap.get_max_osd(); i++)
{"min_size", MIN_SIZE},
{"crash_replay_interval", CRASH_REPLAY_INTERVAL},
{"pg_num", PG_NUM}, {"pgp_num", PGP_NUM},
- {"crush_rule", CRUSH_RULE},
- {"crush_ruleset", CRUSH_RULESET},
- {"hashpspool", HASHPSPOOL}, {"nodelete", NODELETE},
+ {"crush_rule", CRUSH_RULE}, {"hashpspool", HASHPSPOOL},
+ {"allow_ec_overwrites", EC_OVERWRITES}, {"nodelete", NODELETE},
{"nopgchange", NOPGCHANGE}, {"nosizechange", NOSIZECHANGE},
{"noscrub", NOSCRUB}, {"nodeep-scrub", NODEEP_SCRUB},
{"write_fadvise_dontneed", WRITE_FADVISE_DONTNEED},
CACHE_TARGET_DIRTY_RATIO, CACHE_TARGET_DIRTY_HIGH_RATIO,
CACHE_MIN_FLUSH_AGE, CACHE_MIN_EVICT_AGE,
MIN_READ_RECENCY_FOR_PROMOTE,
+ MIN_WRITE_RECENCY_FOR_PROMOTE,
HIT_SET_GRADE_DECAY_RATE, HIT_SET_SEARCH_LAST_N
};
const choices_set_t ONLY_ERASURE_CHOICES = {
- ERASURE_CODE_PROFILE
+ EC_OVERWRITES, ERASURE_CODE_PROFILE
};
choices_set_t selected_choices;
goto reply;
}
+ if (pool_opts_t::is_opt_name(var) &&
+ !p->opts.is_set(pool_opts_t::get_opt_desc(var).key)) {
+ ss << "option '" << var << "' is not set on pool '" << poolstr << "'";
+ r = -ENOENT;
+ goto reply;
+ }
+
selected_choices.insert(selected);
}
if (f) {
+ f->open_object_section("pool");
+ f->dump_string("pool", poolstr);
+ f->dump_int("pool_id", pool);
for(choices_set_t::const_iterator it = selected_choices.begin();
it != selected_choices.end(); ++it) {
choices_map_t::const_iterator i;
- f->open_object_section("pool");
- f->dump_string("pool", poolstr);
- f->dump_int("pool_id", pool);
+ for (i = ALL_CHOICES.begin(); i != ALL_CHOICES.end(); ++i) {
+ if (i->second == *it) {
+ break;
+ }
+ }
+ assert(i != ALL_CHOICES.end());
switch(*it) {
case PG_NUM:
f->dump_int("pg_num", p->get_pg_num());
p->get_crash_replay_interval());
break;
case CRUSH_RULE:
- if (osdmap.crush->rule_exists(p->get_crush_ruleset())) {
+ if (osdmap.crush->rule_exists(p->get_crush_rule())) {
f->dump_string("crush_rule", osdmap.crush->get_rule_name(
- p->get_crush_ruleset()));
+ p->get_crush_rule()));
} else {
- f->dump_string("crush_rule", stringify(p->get_crush_ruleset()));
+ f->dump_string("crush_rule", stringify(p->get_crush_rule()));
}
break;
- case CRUSH_RULESET:
- f->dump_int("crush_ruleset", p->get_crush_ruleset());
+ case EC_OVERWRITES:
+ f->dump_bool("allow_ec_overwrites",
+ p->has_flag(pg_pool_t::FLAG_EC_OVERWRITES));
break;
case HASHPSPOOL:
case NODELETE:
case WRITE_FADVISE_DONTNEED:
case NOSCRUB:
case NODEEP_SCRUB:
- for (i = ALL_CHOICES.begin(); i != ALL_CHOICES.end(); ++i) {
- if (i->second == *it)
- break;
- }
- assert(i != ALL_CHOICES.end());
- f->dump_string(i->first.c_str(),
- p->has_flag(pg_pool_t::get_flag_by_name(i->first)) ?
- "true" : "false");
+ f->dump_bool(i->first.c_str(),
+ p->has_flag(pg_pool_t::get_flag_by_name(i->first)));
break;
case HIT_SET_PERIOD:
f->dump_int("hit_set_period", p->hit_set_period);
case CSUM_TYPE:
case CSUM_MAX_BLOCK:
case CSUM_MIN_BLOCK:
- for (i = ALL_CHOICES.begin(); i != ALL_CHOICES.end(); ++i) {
- if (i->second == *it)
- break;
+ pool_opts_t::key_t key = pool_opts_t::get_opt_desc(i->first).key;
+ if (p->opts.is_set(key)) {
+ if(*it == CSUM_TYPE) {
+ int val;
+ p->opts.get(pool_opts_t::CSUM_TYPE, &val);
+ f->dump_string(i->first.c_str(), Checksummer::get_csum_type_string(val));
+ } else {
+ p->opts.dump(i->first, f.get());
+ }
}
- assert(i != ALL_CHOICES.end());
- if(*it == CSUM_TYPE) {
- int val;
- p->opts.get(pool_opts_t::CSUM_TYPE, &val);
- f->dump_string(i->first.c_str(), Checksummer::get_csum_type_string(val));
- }
- else {
- p->opts.dump(i->first, f.get());
- }
break;
}
- f->close_section();
- f->flush(rdata);
}
-
+ f->close_section();
+ f->flush(rdata);
} else /* !f */ {
for(choices_set_t::const_iterator it = selected_choices.begin();
it != selected_choices.end(); ++it) {
p->get_crash_replay_interval() << "\n";
break;
case CRUSH_RULE:
- if (osdmap.crush->rule_exists(p->get_crush_ruleset())) {
+ if (osdmap.crush->rule_exists(p->get_crush_rule())) {
ss << "crush_rule: " << osdmap.crush->get_rule_name(
- p->get_crush_ruleset()) << "\n";
+ p->get_crush_rule()) << "\n";
} else {
- ss << "crush_rule: " << p->get_crush_ruleset() << "\n";
+ ss << "crush_rule: " << p->get_crush_rule() << "\n";
}
break;
- case CRUSH_RULESET:
- ss << "crush_ruleset: " << p->get_crush_ruleset() << "\n";
- break;
case HIT_SET_PERIOD:
ss << "hit_set_period: " << p->hit_set_period << "\n";
break;
ss << "hit_set_search_last_n: " <<
p->hit_set_search_last_n << "\n";
break;
+ case EC_OVERWRITES:
+ ss << "allow_ec_overwrites: " <<
+ (p->has_flag(pg_pool_t::FLAG_EC_OVERWRITES) ? "true" : "false") <<
+ "\n";
+ break;
case HASHPSPOOL:
case NODELETE:
case NOPGCHANGE:
}
r = 0;
} else if (prefix == "osd pool stats") {
- const auto &pgm = mon->pgmon()->pg_map;
- r = process_pg_map_command(prefix, cmdmap, pgm, osdmap,
- f.get(), &ss, &rdata);
+ r = mon->pgservice->process_pg_command(prefix, cmdmap,
+ osdmap, f.get(), &ss, &rdata);
} else if (prefix == "osd pool get-quota") {
string pool_name;
cmd_getval(g_ceph_context, cmdmap, "pool", pool_name);
if (p->quota_max_objects == 0)
rs << "N/A";
else
- rs << si_t(p->quota_max_objects) << " objects";
+ rs << si_u_t(p->quota_max_objects) << " objects";
rs << "\n"
<< " max bytes : ";
if (p->quota_max_bytes == 0)
rs << "N/A";
else
- rs << si_t(p->quota_max_bytes) << "B";
+ rs << byte_u_t(p->quota_max_bytes);
rdata.append(rs.str());
}
rdata.append("\n");
r = 0;
} else if (prefix == "osd crush rule list" ||
prefix == "osd crush rule ls") {
- string format;
- cmd_getval(g_ceph_context, cmdmap, "format", format);
- boost::scoped_ptr<Formatter> f(Formatter::create(format, "json-pretty", "json-pretty"));
- f->open_array_section("rules");
- osdmap.crush->list_rules(f.get());
- f->close_section();
- ostringstream rs;
- f->flush(rs);
- rs << "\n";
- rdata.append(rs.str());
+ if (f) {
+ f->open_array_section("rules");
+ osdmap.crush->list_rules(f.get());
+ f->close_section();
+ f->flush(rdata);
+ } else {
+ ostringstream ss;
+ osdmap.crush->list_rules(&ss);
+ rdata.append(ss.str());
+ }
+ } else if (prefix == "osd crush rule ls-by-class") {
+ string class_name;
+ cmd_getval(g_ceph_context, cmdmap, "class", class_name);
+ if (class_name.empty()) {
+ ss << "no class specified";
+ r = -EINVAL;
+ goto reply;
+ }
+ set<int> rules;
+ r = osdmap.crush->get_rules_by_class(class_name, &rules);
+ if (r < 0) {
+ ss << "failed to get rules by class '" << class_name << "'";
+ goto reply;
+ }
+ if (f) {
+ f->open_array_section("rules");
+ for (auto &rule: rules) {
+ f->dump_string("name", osdmap.crush->get_rule_name(rule));
+ }
+ f->close_section();
+ f->flush(rdata);
+ } else {
+ ostringstream rs;
+ for (auto &rule: rules) {
+ rs << osdmap.crush->get_rule_name(rule) << "\n";
+ }
+ rdata.append(rs.str());
+ }
} else if (prefix == "osd crush rule dump") {
string name;
cmd_getval(g_ceph_context, cmdmap, "name", name);
} else {
int ruleno = osdmap.crush->get_rule_id(name);
if (ruleno < 0) {
- ss << "unknown crush ruleset '" << name << "'";
+ ss << "unknown crush rule '" << name << "'";
r = ruleno;
goto reply;
}
rs << "\n";
rdata.append(rs.str());
} else if (prefix == "osd crush tree") {
- boost::scoped_ptr<Formatter> f(Formatter::create(format, "json-pretty", "json-pretty"));
- f->open_array_section("crush_map_roots");
- osdmap.crush->dump_tree(f.get());
- f->close_section();
- f->flush(rdata);
+ string shadow;
+ cmd_getval(g_ceph_context, cmdmap, "shadow", shadow);
+ bool show_shadow = shadow == "--show-shadow";
+ boost::scoped_ptr<Formatter> f(Formatter::create(format));
+ if (f) {
+ osdmap.crush->dump_tree(nullptr,
+ f.get(),
+ osdmap.get_pool_names(),
+ show_shadow);
+ f->flush(rdata);
+ } else {
+ ostringstream ss;
+ osdmap.crush->dump_tree(&ss,
+ nullptr,
+ osdmap.get_pool_names(),
+ show_shadow);
+ rdata.append(ss.str());
+ }
+ } else if (prefix == "osd crush ls") {
+ string name;
+ if (!cmd_getval(g_ceph_context, cmdmap, "node", name)) {
+ ss << "no node specified";
+ r = -EINVAL;
+ goto reply;
+ }
+ if (!osdmap.crush->name_exists(name)) {
+ ss << "node '" << name << "' does not exist";
+ r = -ENOENT;
+ goto reply;
+ }
+ int id = osdmap.crush->get_item_id(name);
+ list<int> result;
+ if (id >= 0) {
+ result.push_back(id);
+ } else {
+ int num = osdmap.crush->get_bucket_size(id);
+ for (int i = 0; i < num; ++i) {
+ result.push_back(osdmap.crush->get_bucket_item(id, i));
+ }
+ }
+ if (f) {
+ f->open_array_section("items");
+ for (auto i : result) {
+ f->dump_string("item", osdmap.crush->get_item_name(i));
+ }
+ f->close_section();
+ f->flush(rdata);
+ } else {
+ ostringstream ss;
+ for (auto i : result) {
+ ss << osdmap.crush->get_item_name(i) << "\n";
+ }
+ rdata.append(ss.str());
+ }
+ r = 0;
} else if (prefix == "osd crush class ls") {
boost::scoped_ptr<Formatter> f(Formatter::create(format, "json-pretty", "json-pretty"));
f->open_array_section("crush_classes");
f->dump_string("class", i.second);
f->close_section();
f->flush(rdata);
+ } else if (prefix == "osd crush class ls-osd") {
+ string name;
+ cmd_getval(g_ceph_context, cmdmap, "class", name);
+ set<int> osds;
+ osdmap.crush->get_devices_by_class(name, &osds);
+ if (f) {
+ f->open_array_section("osds");
+ for (auto &osd: osds)
+ f->dump_int("osd", osd);
+ f->close_section();
+ f->flush(rdata);
+ } else {
+ bool first = true;
+ for (auto &osd : osds) {
+ if (!first)
+ ds << "\n";
+ first = false;
+ ds << osd;
+ }
+ rdata.append(ds);
+ }
} else if (prefix == "osd erasure-code-profile ls") {
const auto &profiles = osdmap.get_erasure_code_profiles();
if (f)
rs << "\n";
rdata.append(rs.str());
}
+ } else if (prefix == "osd crush weight-set ls") {
+ boost::scoped_ptr<Formatter> f(Formatter::create(format));
+ if (f) {
+ f->open_array_section("weight_sets");
+ if (osdmap.crush->have_choose_args(CrushWrapper::DEFAULT_CHOOSE_ARGS)) {
+ f->dump_string("pool", "(compat)");
+ }
+ for (auto& i : osdmap.crush->choose_args) {
+ if (i.first >= 0) {
+ f->dump_string("pool", osdmap.get_pool_name(i.first));
+ }
+ }
+ f->close_section();
+ f->flush(rdata);
+ } else {
+ ostringstream rs;
+ if (osdmap.crush->have_choose_args(CrushWrapper::DEFAULT_CHOOSE_ARGS)) {
+ rs << "(compat)\n";
+ }
+ for (auto& i : osdmap.crush->choose_args) {
+ if (i.first >= 0) {
+ rs << osdmap.get_pool_name(i.first) << "\n";
+ }
+ }
+ rdata.append(rs.str());
+ }
+ } else if (prefix == "osd crush weight-set dump") {
+ boost::scoped_ptr<Formatter> f(Formatter::create(format, "json-pretty",
+ "json-pretty"));
+ osdmap.crush->dump_choose_args(f.get());
+ f->flush(rdata);
} else if (prefix == "osd erasure-code-profile get") {
string name;
cmd_getval(g_ceph_context, cmdmap, "name", name);
rs << "\n";
rdata.append(rs.str());
}
+ } else if (prefix == "osd pool application get") {
+ boost::scoped_ptr<Formatter> f(Formatter::create(format, "json-pretty",
+ "json-pretty"));
+ string pool_name;
+ cmd_getval(g_ceph_context, cmdmap, "pool", pool_name);
+ string app;
+ cmd_getval(g_ceph_context, cmdmap, "app", app);
+ string key;
+ cmd_getval(g_ceph_context, cmdmap, "key", key);
+
+ if (pool_name.empty()) {
+ // all
+ f->open_object_section("pools");
+ for (const auto &pool : osdmap.pools) {
+ std::string name("<unknown>");
+ const auto &pni = osdmap.pool_name.find(pool.first);
+ if (pni != osdmap.pool_name.end())
+ name = pni->second;
+ f->open_object_section(name.c_str());
+ for (auto &app_pair : pool.second.application_metadata) {
+ f->open_object_section(app_pair.first.c_str());
+ for (auto &kv_pair : app_pair.second) {
+ f->dump_string(kv_pair.first.c_str(), kv_pair.second);
+ }
+ f->close_section();
+ }
+ f->close_section(); // name
+ }
+ f->close_section(); // pools
+ f->flush(rdata);
+ } else {
+ int64_t pool = osdmap.lookup_pg_pool_name(pool_name.c_str());
+ if (pool < 0) {
+ ss << "unrecognized pool '" << pool_name << "'";
+ r = -ENOENT;
+ goto reply;
+ }
+ auto p = osdmap.get_pg_pool(pool);
+ // filter by pool
+ if (app.empty()) {
+ f->open_object_section(pool_name.c_str());
+ for (auto &app_pair : p->application_metadata) {
+ f->open_object_section(app_pair.first.c_str());
+ for (auto &kv_pair : app_pair.second) {
+ f->dump_string(kv_pair.first.c_str(), kv_pair.second);
+ }
+ f->close_section(); // application
+ }
+ f->close_section(); // pool_name
+ f->flush(rdata);
+ goto reply;
+ }
+
+ auto app_it = p->application_metadata.find(app);
+ if (app_it == p->application_metadata.end()) {
+ ss << "pool '" << pool_name << "' has no application '" << app << "'";
+ r = -ENOENT;
+ goto reply;
+ }
+ // filter by pool + app
+ if (key.empty()) {
+ f->open_object_section(app_it->first.c_str());
+ for (auto &kv_pair : app_it->second) {
+ f->dump_string(kv_pair.first.c_str(), kv_pair.second);
+ }
+ f->close_section(); // application
+ f->flush(rdata);
+ goto reply;
+ }
+ // filter by pool + app + key
+ auto key_it = app_it->second.find(key);
+ if (key_it == app_it->second.end()) {
+ ss << "application '" << app << "' on pool '" << pool_name
+ << "' does not have key '" << key << "'";
+ r = -ENOENT;
+ goto reply;
+ }
+ ss << key_it->second << "\n";
+ rdata.append(ss.str());
+ ss.str("");
+ }
} else {
// try prepare update
return false;
return true;
}
-void OSDMonitor::update_pool_flags(int64_t pool_id, uint64_t flags)
+void OSDMonitor::set_pool_flags(int64_t pool_id, uint64_t flags)
+{
+ pg_pool_t *pool = pending_inc.get_new_pool(pool_id,
+ osdmap.get_pg_pool(pool_id));
+ assert(pool);
+ pool->set_flag(flags);
+}
+
+void OSDMonitor::clear_pool_flags(int64_t pool_id, uint64_t flags)
{
- const pg_pool_t *pool = osdmap.get_pg_pool(pool_id);
- pending_inc.get_new_pool(pool_id, pool)->flags = flags;
+ pg_pool_t *pool = pending_inc.get_new_pool(pool_id,
+ osdmap.get_pg_pool(pool_id));
+ assert(pool);
+ pool->unset_flag(flags);
}
bool OSDMonitor::update_pools_status()
{
- if (!mon->pgmon()->is_readable())
+ if (!mon->pgservice->is_readable())
return false;
bool ret = false;
auto& pools = osdmap.get_pools();
for (auto it = pools.begin(); it != pools.end(); ++it) {
- if (!mon->pgmon()->pg_map.pg_pool_sum.count(it->first))
+ const pool_stat_t *pstat = mon->pgservice->get_pool_stat(it->first);
+ if (!pstat)
continue;
- pool_stat_t& stats = mon->pgmon()->pg_map.pg_pool_sum[it->first];
- object_stat_sum_t& sum = stats.stats.sum;
+ const object_stat_sum_t& sum = pstat->stats.sum;
const pg_pool_t &pool = it->second;
const string& pool_name = osdmap.get_pool_name(it->first);
(pool.quota_max_bytes > 0 && (uint64_t)sum.num_bytes >= pool.quota_max_bytes) ||
(pool.quota_max_objects > 0 && (uint64_t)sum.num_objects >= pool.quota_max_objects);
- if (pool.has_flag(pg_pool_t::FLAG_FULL)) {
+ if (pool.has_flag(pg_pool_t::FLAG_FULL_NO_QUOTA)) {
if (pool_is_full)
continue;
mon->clog->info() << "pool '" << pool_name
- << "' no longer full; removing FULL flag";
-
- update_pool_flags(it->first, pool.get_flags() & ~pg_pool_t::FLAG_FULL);
+ << "' no longer out of quota; removing NO_QUOTA flag";
+ // below we cancel FLAG_FULL too, we'll set it again in
+ // OSDMonitor::encode_pending if it still fails the osd-full checking.
+ clear_pool_flags(it->first,
+ pg_pool_t::FLAG_FULL_NO_QUOTA | pg_pool_t::FLAG_FULL);
ret = true;
} else {
if (!pool_is_full)
(uint64_t)sum.num_bytes >= pool.quota_max_bytes) {
mon->clog->warn() << "pool '" << pool_name << "' is full"
<< " (reached quota's max_bytes: "
- << si_t(pool.quota_max_bytes) << ")";
+ << byte_u_t(pool.quota_max_bytes) << ")";
}
if (pool.quota_max_objects > 0 &&
(uint64_t)sum.num_objects >= pool.quota_max_objects) {
<< " (reached quota's max_objects: "
<< pool.quota_max_objects << ")";
}
- update_pool_flags(it->first, pool.get_flags() | pg_pool_t::FLAG_FULL);
+ // set both FLAG_FULL_NO_QUOTA and FLAG_FULL
+ // note that below we try to cancel FLAG_BACKFILLFULL/NEARFULL too
+ // since FLAG_FULL should always take precedence
+ set_pool_flags(it->first,
+ pg_pool_t::FLAG_FULL_NO_QUOTA | pg_pool_t::FLAG_FULL);
+ clear_pool_flags(it->first,
+ pg_pool_t::FLAG_NEARFULL |
+ pg_pool_t::FLAG_BACKFILLFULL);
ret = true;
}
}
return ret;
}
-void OSDMonitor::get_pools_health(
- list<pair<health_status_t,string> >& summary,
- list<pair<health_status_t,string> > *detail) const
-{
- auto& pools = osdmap.get_pools();
- for (auto it = pools.begin(); it != pools.end(); ++it) {
- if (!mon->pgmon()->pg_map.pg_pool_sum.count(it->first))
- continue;
- pool_stat_t& stats = mon->pgmon()->pg_map.pg_pool_sum[it->first];
- object_stat_sum_t& sum = stats.stats.sum;
- const pg_pool_t &pool = it->second;
- const string& pool_name = osdmap.get_pool_name(it->first);
-
- if (pool.has_flag(pg_pool_t::FLAG_FULL)) {
- // uncomment these asserts if/when we update the FULL flag on pg_stat update
- //assert((pool.quota_max_objects > 0) || (pool.quota_max_bytes > 0));
-
- stringstream ss;
- ss << "pool '" << pool_name << "' is full";
- summary.push_back(make_pair(HEALTH_WARN, ss.str()));
- if (detail)
- detail->push_back(make_pair(HEALTH_WARN, ss.str()));
- }
-
- float warn_threshold = (float)g_conf->mon_pool_quota_warn_threshold/100;
- float crit_threshold = (float)g_conf->mon_pool_quota_crit_threshold/100;
-
- if (pool.quota_max_objects > 0) {
- stringstream ss;
- health_status_t status = HEALTH_OK;
- if ((uint64_t)sum.num_objects >= pool.quota_max_objects) {
- // uncomment these asserts if/when we update the FULL flag on pg_stat update
- //assert(pool.has_flag(pg_pool_t::FLAG_FULL));
- } else if (crit_threshold > 0 &&
- sum.num_objects >= pool.quota_max_objects*crit_threshold) {
- ss << "pool '" << pool_name
- << "' has " << sum.num_objects << " objects"
- << " (max " << pool.quota_max_objects << ")";
- status = HEALTH_ERR;
- } else if (warn_threshold > 0 &&
- sum.num_objects >= pool.quota_max_objects*warn_threshold) {
- ss << "pool '" << pool_name
- << "' has " << sum.num_objects << " objects"
- << " (max " << pool.quota_max_objects << ")";
- status = HEALTH_WARN;
- }
- if (status != HEALTH_OK) {
- pair<health_status_t,string> s(status, ss.str());
- summary.push_back(s);
- if (detail)
- detail->push_back(s);
- }
- }
-
- if (pool.quota_max_bytes > 0) {
- health_status_t status = HEALTH_OK;
- stringstream ss;
- if ((uint64_t)sum.num_bytes >= pool.quota_max_bytes) {
- // uncomment these asserts if/when we update the FULL flag on pg_stat update
- //assert(pool.has_flag(pg_pool_t::FLAG_FULL));
- } else if (crit_threshold > 0 &&
- sum.num_bytes >= pool.quota_max_bytes*crit_threshold) {
- ss << "pool '" << pool_name
- << "' has " << si_t(sum.num_bytes) << " bytes"
- << " (max " << si_t(pool.quota_max_bytes) << ")";
- status = HEALTH_ERR;
- } else if (warn_threshold > 0 &&
- sum.num_bytes >= pool.quota_max_bytes*warn_threshold) {
- ss << "pool '" << pool_name
- << "' has " << si_t(sum.num_bytes) << " bytes"
- << " (max " << si_t(pool.quota_max_bytes) << ")";
- status = HEALTH_WARN;
- }
- if (status != HEALTH_OK) {
- pair<health_status_t,string> s(status, ss.str());
- summary.push_back(s);
- if (detail)
- detail->push_back(s);
- }
- }
- }
-}
-
-
int OSDMonitor::prepare_new_pool(MonOpRequestRef op)
{
op->mark_osdmon_event(__func__);
return -EPERM;
string erasure_code_profile;
stringstream ss;
- string ruleset_name;
+ string rule_name;
+ int ret = 0;
if (m->auid)
- return prepare_new_pool(m->name, m->auid, m->crush_rule, ruleset_name,
+ ret = prepare_new_pool(m->name, m->auid, m->crush_rule, rule_name,
0, 0,
erasure_code_profile,
pg_pool_t::TYPE_REPLICATED, 0, FAST_READ_OFF, &ss);
else
- return prepare_new_pool(m->name, session->auid, m->crush_rule, ruleset_name,
+ ret = prepare_new_pool(m->name, session->auid, m->crush_rule, rule_name,
0, 0,
erasure_code_profile,
pg_pool_t::TYPE_REPLICATED, 0, FAST_READ_OFF, &ss);
+
+ if (ret < 0) {
+ dout(10) << __func__ << " got " << ret << " " << ss.str() << dendl;
+ }
+ return ret;
}
int OSDMonitor::crush_rename_bucket(const string& srcname,
auto it = profile.find("stripe_unit");
if (it != profile.end()) {
string err_str;
- uint32_t stripe_unit = strict_si_cast<uint32_t>(it->second.c_str(), &err_str);
+ uint32_t stripe_unit = strict_iecstrtoll(it->second.c_str(), &err_str);
if (!err_str.empty()) {
*ss << "could not parse stripe_unit '" << it->second
<< "': " << err_str << std::endl;
return 0;
}
-int OSDMonitor::crush_ruleset_create_erasure(const string &name,
+int OSDMonitor::crush_rule_create_erasure(const string &name,
const string &profile,
- int *ruleset,
+ int *rule,
ostream *ss)
{
int ruleid = osdmap.crush->get_rule_id(name);
if (ruleid != -ENOENT) {
- *ruleset = osdmap.crush->get_rule_mask_ruleset(ruleid);
+ *rule = osdmap.crush->get_rule_mask_ruleset(ruleid);
return -EEXIST;
}
ruleid = newcrush.get_rule_id(name);
if (ruleid != -ENOENT) {
- *ruleset = newcrush.get_rule_mask_ruleset(ruleid);
+ *rule = newcrush.get_rule_mask_ruleset(ruleid);
return -EALREADY;
} else {
ErasureCodeInterfaceRef erasure_code;
return err;
}
- err = erasure_code->create_ruleset(name, newcrush, ss);
+ err = erasure_code->create_rule(name, newcrush, ss);
erasure_code.reset();
if (err < 0)
return err;
- *ruleset = err;
+ *rule = err;
pending_inc.crush.clear();
newcrush.encode(pending_inc.crush, mon->get_quorum_con_features());
return 0;
newmap.apply_incremental(new_pending);
// client compat
- if (newmap.require_min_compat_client.length()) {
+ if (newmap.require_min_compat_client > 0) {
auto mv = newmap.get_min_compat_client();
- if (mv.first > newmap.require_min_compat_client) {
- ss << "new crush map requires client version " << mv
+ if (mv > newmap.require_min_compat_client) {
+ ss << "new crush map requires client version " << ceph_release_name(mv)
<< " but require_min_compat_client is "
- << newmap.require_min_compat_client;
+ << ceph_release_name(newmap.require_min_compat_client);
return false;
}
}
user_map[*i] = string();
(*erasure_code_profile_map)[*i] = string();
} else {
- const string key = i->substr(0, equal);
+ string key = i->substr(0, equal);
equal++;
const string value = i->substr(equal);
+ if (osdmap.require_osd_release >= CEPH_RELEASE_LUMINOUS &&
+ key.find("ruleset-") == 0) {
+ if (g_conf->get_val<bool>("mon_fixup_legacy_erasure_code_profiles")) {
+ mon->clog->warn() << "erasure code profile property '" << key
+ << "' is no longer supported; try "
+ << "'crush-" << key.substr(8) << "' instead";
+ key = string("crush-") + key.substr(8);
+ } else {
+ *ss << "property '" << key << "' is no longer supported; try "
+ << "'crush-" << key.substr(8) << "' instead";
+ return -EINVAL;
+ }
+ }
user_map[key] = value;
(*erasure_code_profile_map)[key] = value;
}
auto it = profile.find("stripe_unit");
if (it != profile.end()) {
string err_str;
- stripe_unit = strict_si_cast<uint32_t>(it->second.c_str(), &err_str);
+ stripe_unit = strict_iecstrtoll(it->second.c_str(), &err_str);
assert(err_str.empty());
}
*stripe_width = data_chunks *
return err;
}
-int OSDMonitor::prepare_pool_crush_ruleset(const unsigned pool_type,
- const string &erasure_code_profile,
- const string &ruleset_name,
- int *crush_ruleset,
- ostream *ss)
+int OSDMonitor::prepare_pool_crush_rule(const unsigned pool_type,
+ const string &erasure_code_profile,
+ const string &rule_name,
+ int *crush_rule,
+ ostream *ss)
{
- if (*crush_ruleset < 0) {
+ if (*crush_rule < 0) {
switch (pool_type) {
case pg_pool_t::TYPE_REPLICATED:
{
- if (ruleset_name == "") {
- //Use default ruleset
- *crush_ruleset = osdmap.crush->get_osd_pool_default_crush_replicated_ruleset(g_ceph_context);
- if (*crush_ruleset < 0) {
- // Errors may happen e.g. if no valid ruleset is available
- *ss << "No suitable CRUSH ruleset exists, check "
+ if (rule_name == "") {
+ // Use default rule
+ *crush_rule = osdmap.crush->get_osd_pool_default_crush_replicated_ruleset(g_ceph_context);
+ if (*crush_rule < 0) {
+ // Errors may happen e.g. if no valid rule is available
+ *ss << "No suitable CRUSH rule exists, check "
<< "'osd pool default crush *' config options";
return -ENOENT;
}
} else {
- return get_crush_ruleset(ruleset_name, crush_ruleset, ss);
+ return get_crush_rule(rule_name, crush_rule, ss);
}
}
break;
case pg_pool_t::TYPE_ERASURE:
{
- int err = crush_ruleset_create_erasure(ruleset_name,
+ int err = crush_rule_create_erasure(rule_name,
erasure_code_profile,
- crush_ruleset, ss);
+ crush_rule, ss);
switch (err) {
case -EALREADY:
- dout(20) << "prepare_pool_crush_ruleset: ruleset "
- << ruleset_name << " try again" << dendl;
+ dout(20) << "prepare_pool_crush_rule: rule "
+ << rule_name << " try again" << dendl;
// fall through
case 0:
// need to wait for the crush rule to be proposed before proceeding
}
break;
default:
- *ss << "prepare_pool_crush_ruleset: " << pool_type
+ *ss << "prepare_pool_crush_rule: " << pool_type
<< " is not a known pool type";
return -EINVAL;
break;
}
} else {
- if (!osdmap.crush->ruleset_exists(*crush_ruleset)) {
- *ss << "CRUSH ruleset " << *crush_ruleset << " not found";
+ if (!osdmap.crush->ruleset_exists(*crush_rule)) {
+ *ss << "CRUSH rule " << *crush_rule << " not found";
return -ENOENT;
}
}
return 0;
}
-int OSDMonitor::get_crush_ruleset(const string &ruleset_name,
- int *crush_ruleset,
- ostream *ss)
+int OSDMonitor::get_crush_rule(const string &rule_name,
+ int *crush_rule,
+ ostream *ss)
{
int ret;
- ret = osdmap.crush->get_rule_id(ruleset_name);
+ ret = osdmap.crush->get_rule_id(rule_name);
if (ret != -ENOENT) {
// found it, use it
- *crush_ruleset = ret;
+ *crush_rule = ret;
} else {
CrushWrapper newcrush;
_get_pending_crush(newcrush);
- ret = newcrush.get_rule_id(ruleset_name);
+ ret = newcrush.get_rule_id(rule_name);
if (ret != -ENOENT) {
// found it, wait for it to be proposed
- dout(20) << __func__ << ": ruleset " << ruleset_name
+ dout(20) << __func__ << ": rule " << rule_name
<< " try again" << dendl;
return -EAGAIN;
} else {
- //Cannot find it , return error
- *ss << "specified ruleset " << ruleset_name << " doesn't exist";
+ // Cannot find it , return error
+ *ss << "specified rule " << rule_name << " doesn't exist";
return ret;
}
}
return 0;
}
+int OSDMonitor::check_pg_num(int64_t pool, int pg_num, int size, ostream *ss)
+{
+ auto max_pgs_per_osd = g_conf->get_val<uint64_t>("mon_max_pg_per_osd");
+ auto num_osds = std::max(osdmap.get_num_in_osds(), 3u); // assume min cluster size 3
+ auto max_pgs = max_pgs_per_osd * num_osds;
+ uint64_t projected = 0;
+ if (pool < 0) {
+ projected += pg_num * size;
+ }
+ for (const auto& i : osdmap.get_pools()) {
+ if (i.first == pool) {
+ projected += pg_num * size;
+ } else {
+ projected += i.second.get_pg_num() * i.second.get_size();
+ }
+ }
+ if (projected > max_pgs) {
+ if (pool >= 0) {
+ *ss << "pool id " << pool;
+ }
+ *ss << " pg_num " << pg_num << " size " << size
+ << " would mean " << projected
+ << " total pgs, which exceeds max " << max_pgs
+ << " (mon_max_pg_per_osd " << max_pgs_per_osd
+ << " * num_in_osds " << num_osds << ")";
+ return -ERANGE;
+ }
+ return 0;
+}
+
/**
* @param name The name of the new pool
* @param auid The auid of the pool owner. Can be -1
- * @param crush_ruleset The crush rule to use. If <0, will use the system default
- * @param crush_ruleset_name The crush rule to use, if crush_rulset <0
+ * @param crush_rule The crush rule to use. If <0, will use the system default
+ * @param crush_rule_name The crush rule to use, if crush_rulset <0
* @param pg_num The pg_num to use. If set to 0, will use the system default
* @param pgp_num The pgp_num to use. If set to 0, will use the system default
* @param erasure_code_profile The profile name in OSDMap to be used for erasure code
* @return 0 on success, negative errno on failure.
*/
int OSDMonitor::prepare_new_pool(string& name, uint64_t auid,
- int crush_ruleset,
- const string &crush_ruleset_name,
+ int crush_rule,
+ const string &crush_rule_name,
unsigned pg_num, unsigned pgp_num,
const string &erasure_code_profile,
const unsigned pool_type,
return -EINVAL;
}
int r;
- r = prepare_pool_crush_ruleset(pool_type, erasure_code_profile,
- crush_ruleset_name, &crush_ruleset, ss);
+ r = prepare_pool_crush_rule(pool_type, erasure_code_profile,
+ crush_rule_name, &crush_rule, ss);
if (r) {
- dout(10) << " prepare_pool_crush_ruleset returns " << r << dendl;
+ dout(10) << "prepare_pool_crush_rule returns " << r << dendl;
return r;
}
- CrushWrapper newcrush;
- _get_pending_crush(newcrush);
- ostringstream err;
- CrushTester tester(newcrush, err);
- // use the internal crush tester if crushtool config is empty
- if (g_conf->crushtool.empty()) {
- r = tester.test();
- } else {
- r = tester.test_with_crushtool(g_conf->crushtool.c_str(),
- osdmap.get_max_osd(),
- g_conf->mon_lease,
- crush_ruleset);
+ if (g_conf->mon_osd_crush_smoke_test) {
+ CrushWrapper newcrush;
+ _get_pending_crush(newcrush);
+ ostringstream err;
+ CrushTester tester(newcrush, err);
+ tester.set_min_x(0);
+ tester.set_max_x(50);
+ tester.set_rule(crush_rule);
+ auto start = ceph::coarse_mono_clock::now();
+ r = tester.test_with_fork(g_conf->mon_lease);
+ auto duration = ceph::coarse_mono_clock::now() - start;
+ if (r < 0) {
+ dout(10) << "tester.test_with_fork returns " << r
+ << ": " << err.str() << dendl;
+ *ss << "crush test failed with " << r << ": " << err.str();
+ return r;
+ }
+ dout(10) << __func__ << " crush smoke test duration: "
+ << duration << dendl;
}
+ unsigned size, min_size;
+ r = prepare_pool_size(pool_type, erasure_code_profile, &size, &min_size, ss);
if (r) {
- dout(10) << " tester.test_with_crushtool returns " << r
- << ": " << err.str() << dendl;
- *ss << "crushtool check failed with " << r << ": " << err.str();
+ dout(10) << "prepare_pool_size returns " << r << dendl;
return r;
}
- unsigned size, min_size;
- r = prepare_pool_size(pool_type, erasure_code_profile, &size, &min_size, ss);
+ r = check_pg_num(-1, pg_num, size, ss);
if (r) {
- dout(10) << " prepare_pool_size returns " << r << dendl;
+ dout(10) << "check_pg_num returns " << r << dendl;
return r;
}
- if (!osdmap.crush->check_crush_rule(crush_ruleset, pool_type, size, *ss)) {
+ if (!osdmap.crush->check_crush_rule(crush_rule, pool_type, size, *ss)) {
return -EINVAL;
}
uint32_t stripe_width = 0;
r = prepare_pool_stripe_width(pool_type, erasure_code_profile, &stripe_width, ss);
if (r) {
- dout(10) << " prepare_pool_stripe_width returns " << r << dendl;
+ dout(10) << "prepare_pool_stripe_width returns " << r << dendl;
return r;
}
pi->size = size;
pi->min_size = min_size;
- pi->crush_ruleset = crush_ruleset;
+ pi->crush_rule = crush_rule;
pi->expected_num_objects = expected_num_objects;
pi->object_hash = CEPH_STR_HASH_RJENKINS;
pi->set_pg_num(pg_num);
return true;
}
-int OSDMonitor::parse_osd_id(const char *s, stringstream *pss)
-{
- // osd.NNN?
- if (strncmp(s, "osd.", 4) == 0) {
- s += 4;
- }
-
- // NNN?
- ostringstream ss;
- long id = parse_pos_long(s, &ss);
- if (id < 0) {
- *pss << ss.str();
- return id;
- }
- if (id > 0xffff) {
- *pss << "osd id " << id << " is too large";
- return -ERANGE;
- }
- return id;
-}
-
-
int OSDMonitor::prepare_command_pool_set(map<string,cmd_vartype> &cmdmap,
stringstream& ss)
{
ss << "pool size must be between 1 and 10";
return -EINVAL;
}
+ int r = check_pg_num(pool, p.get_pg_num(), n, &ss);
+ if (r < 0) {
+ return r;
+ }
p.size = n;
if (n < p.min_size)
p.min_size = n;
if (err == 0) {
k = erasure_code->get_data_chunk_count();
} else {
- ss << __func__ << " get_erasure_code failed: " << tmp.rdbuf();
+ ss << __func__ << " get_erasure_code failed: " << tmp.str();
return err;
}
return -EEXIST;
return 0;
}
+ if (n > (unsigned)g_conf->mon_max_pool_pg_num) {
+ ss << "'pg_num' must be greater than 0 and less than or equal to "
+ << g_conf->mon_max_pool_pg_num
+ << " (you may adjust 'mon max pool pg num' for higher values)";
+ return -ERANGE;
+ }
+ int r = check_pg_num(pool, n, p.get_size(), &ss);
+ if (r) {
+ return r;
+ }
string force;
cmd_getval(g_ceph_context,cmdmap, "force", force);
if (p.cache_mode != pg_pool_t::CACHEMODE_NONE &&
if (!osdmap.crush->check_crush_rule(id, p.get_type(), p.get_size(), ss)) {
return -EINVAL;
}
- p.crush_ruleset = id;
- } else if (var == "crush_ruleset") {
- if (interr.length()) {
- ss << "error parsing integer value '" << val << "': " << interr;
- return -EINVAL;
- }
- if (!osdmap.crush->ruleset_exists(n)) {
- ss << "crush ruleset " << n << " does not exist";
- return -ENOENT;
- }
-
- if (!osdmap.crush->check_crush_rule(n, p.get_type(), p.get_size(), ss)) {
- return -EINVAL;
- }
- p.crush_ruleset = n;
+ p.crush_rule = id;
} else if (var == "nodelete" || var == "nopgchange" ||
var == "nosizechange" || var == "write_fadvise_dontneed" ||
var == "noscrub" || var == "nodeep-scrub") {
bloomp->set_fpp(f);
} else if (var == "use_gmt_hitset") {
if (val == "true" || (interr.empty() && n == 1)) {
- if (!(osdmap.get_up_osd_features() & CEPH_FEATURE_OSD_HITSET_GMT)) {
+ string force;
+ cmd_getval(g_ceph_context, cmdmap, "force", force);
+ if (!osdmap.get_num_up_osds() && force != "--yes-i-really-mean-it") {
+ ss << "Not advisable to continue since no OSDs are up. Pass "
+ << "--yes-i-really-mean-it if you really wish to continue.";
+ return -EPERM;
+ }
+ if (!(osdmap.get_up_osd_features() & CEPH_FEATURE_OSD_HITSET_GMT)
+ && force != "--yes-i-really-mean-it") {
ss << "not all OSDs support GMT hit set.";
return -EINVAL;
}
ss << "ec overwrites can only be enabled for an erasure coded pool";
return -EINVAL;
}
+ stringstream err;
+ if (!g_conf->mon_debug_no_require_bluestore_for_ec_overwrites &&
+ !is_pool_currently_all_bluestore(pool, p, &err)) {
+ ss << "pool must only be stored on bluestore for scrubbing to work: " << err.str();
+ return -EINVAL;
+ }
if (val == "true" || (interr.empty() && n == 1)) {
p.flags |= pg_pool_t::FLAG_EC_OVERWRITES;
} else if (val == "false" || (interr.empty() && n == 0)) {
ss << "expecting value 'true', 'false', '0', or '1'";
return -EINVAL;
}
- stringstream err;
- if (!is_pool_currently_all_bluestore(pool, p, &err)) {
- ss << "pool must only be stored on bluestore for scrubbing to work: " << err.str();
- return -EINVAL;
- }
} else if (var == "target_max_objects") {
if (interr.length()) {
ss << "error parsing int '" << val << "': " << interr;
return -EINVAL;
}
} else if (pool_opts_t::is_opt_name(var)) {
+ bool unset = val == "unset";
if (var == "compression_mode") {
- auto cmode = Compressor::get_comp_mode_type(val);
- if (!cmode) {
- ss << "unrecognized compression mode '" << val << "'";
- return EINVAL;
+ if (!unset) {
+ auto cmode = Compressor::get_comp_mode_type(val);
+ if (!cmode) {
+ ss << "unrecognized compression mode '" << val << "'";
+ return -EINVAL;
+ }
}
} else if (var == "compression_algorithm") {
- auto alg = Compressor::get_comp_alg_type(val);
- if (!alg) {
- ss << "unrecognized compression_algorithm '" << val << "'";
- return EINVAL;
+ if (!unset) {
+ auto alg = Compressor::get_comp_alg_type(val);
+ if (!alg) {
+ ss << "unrecognized compression_algorithm '" << val << "'";
+ return -EINVAL;
+ }
}
} else if (var == "compression_required_ratio") {
if (floaterr.length()) {
ss << "error parsing float value '" << val << "': " << floaterr;
return -EINVAL;
}
- if (f < 0 || f>1) {
+ if (f < 0 || f > 1) {
ss << "compression_required_ratio is out of range (0-1): '" << val << "'";
- return EINVAL;
+ return -EINVAL;
}
} else if (var == "csum_type") {
- auto t = val != "unset" ? Checksummer::get_csum_string_type(val) : 0;
+ auto t = unset ? 0 : Checksummer::get_csum_string_type(val);
if (t < 0 ) {
ss << "unrecognized csum_type '" << val << "'";
- return EINVAL;
+ return -EINVAL;
}
//preserve csum_type numeric value
n = t;
pool_opts_t::opt_desc_t desc = pool_opts_t::get_opt_desc(var);
switch (desc.type) {
case pool_opts_t::STR:
- if (val.empty()) {
+ if (unset) {
p.opts.unset(desc.key);
} else {
p.opts.set(desc.key, static_cast<std::string>(val));
ss << "unrecognized variable '" << var << "'";
return -EINVAL;
}
- ss << "set pool " << pool << " " << var << " to " << val;
+ if (val != "unset") {
+ ss << "set pool " << pool << " " << var << " to " << val;
+ } else {
+ ss << "unset pool " << pool << " " << var;
+ }
p.last_change = pending_inc.epoch;
pending_inc.new_pools[pool] = p;
return 0;
}
-bool OSDMonitor::prepare_command(MonOpRequestRef op)
+int OSDMonitor::prepare_command_pool_application(const string &prefix,
+ map<string,cmd_vartype> &cmdmap,
+ stringstream& ss)
{
- op->mark_osdmon_event(__func__);
- MMonCommand *m = static_cast<MMonCommand*>(op->get_req());
- stringstream ss;
- map<string, cmd_vartype> cmdmap;
- if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
- string rs = ss.str();
- mon->reply_command(op, -EINVAL, rs, get_last_committed());
- return true;
+ string pool_name;
+ cmd_getval(g_ceph_context, cmdmap, "pool", pool_name);
+ int64_t pool = osdmap.lookup_pg_pool_name(pool_name.c_str());
+ if (pool < 0) {
+ ss << "unrecognized pool '" << pool_name << "'";
+ return -ENOENT;
}
- MonSession *session = m->get_session();
- if (!session) {
- mon->reply_command(op, -EACCES, "access denied", get_last_committed());
- return true;
+ pg_pool_t p = *osdmap.get_pg_pool(pool);
+ if (pending_inc.new_pools.count(pool)) {
+ p = pending_inc.new_pools[pool];
}
- return prepare_command_impl(op, cmdmap);
-}
+ string app;
+ cmd_getval(g_ceph_context, cmdmap, "app", app);
+ bool app_exists = (p.application_metadata.count(app) > 0);
-static int parse_reweights(CephContext *cct,
- const map<string,cmd_vartype> &cmdmap,
- const OSDMap& osdmap,
- map<int32_t, uint32_t>* weights)
-{
- string weights_str;
- if (!cmd_getval(g_ceph_context, cmdmap, "weights", weights_str)) {
- return -EINVAL;
- }
- std::replace(begin(weights_str), end(weights_str), '\'', '"');
- json_spirit::mValue json_value;
- if (!json_spirit::read(weights_str, json_value)) {
- return -EINVAL;
- }
- if (json_value.type() != json_spirit::obj_type) {
- return -EINVAL;
- }
- const auto obj = json_value.get_obj();
- try {
- for (auto& osd_weight : obj) {
- auto osd_id = std::stoi(osd_weight.first);
- if (!osdmap.exists(osd_id)) {
- return -ENOENT;
- }
- if (osd_weight.second.type() != json_spirit::str_type) {
- return -EINVAL;
- }
- auto weight = std::stoul(osd_weight.second.get_str());
- weights->insert({osd_id, weight});
+ if (boost::algorithm::ends_with(prefix, "enable")) {
+ if (app.empty()) {
+ ss << "application name must be provided";
+ return -EINVAL;
}
- } catch (const std::logic_error& e) {
- return -EINVAL;
- }
- return 0;
-}
-bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
- map<string,cmd_vartype> &cmdmap)
-{
- op->mark_osdmon_event(__func__);
- MMonCommand *m = static_cast<MMonCommand*>(op->get_req());
- bool ret = false;
- stringstream ss;
- string rs;
- bufferlist rdata;
- int err = 0;
+ if (p.is_tier()) {
+ ss << "application must be enabled on base tier";
+ return -EINVAL;
+ }
- string format;
- cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain"));
- boost::scoped_ptr<Formatter> f(Formatter::create(format));
+ string force;
+ cmd_getval(g_ceph_context, cmdmap, "force", force);
- string prefix;
- cmd_getval(g_ceph_context, cmdmap, "prefix", prefix);
+ if (!app_exists && !p.application_metadata.empty() &&
+ force != "--yes-i-really-mean-it") {
+ ss << "Are you SURE? Pool '" << pool_name << "' already has an enabled "
+ << "application; pass --yes-i-really-mean-it to proceed anyway";
+ return -EPERM;
+ }
- int64_t osdid;
- string name;
- bool osdid_present = cmd_getval(g_ceph_context, cmdmap, "id", osdid);
- if (osdid_present) {
+ if (!app_exists && p.application_metadata.size() >= MAX_POOL_APPLICATIONS) {
+ ss << "too many enabled applications on pool '" << pool_name << "'; "
+ << "max " << MAX_POOL_APPLICATIONS;
+ return -EINVAL;
+ }
+
+ if (app.length() > MAX_POOL_APPLICATION_LENGTH) {
+ ss << "application name '" << app << "' too long; max length "
+ << MAX_POOL_APPLICATION_LENGTH;
+ return -EINVAL;
+ }
+
+ if (!app_exists) {
+ p.application_metadata[app] = {};
+ }
+ ss << "enabled application '" << app << "' on pool '" << pool_name << "'";
+
+ } else if (boost::algorithm::ends_with(prefix, "disable")) {
+ string force;
+ cmd_getval(g_ceph_context, cmdmap, "force", force);
+
+ if (force != "--yes-i-really-mean-it") {
+ ss << "Are you SURE? Disabling an application within a pool might result "
+ << "in loss of application functionality; pass "
+ << "--yes-i-really-mean-it to proceed anyway";
+ return -EPERM;
+ }
+
+ if (!app_exists) {
+ ss << "application '" << app << "' is not enabled on pool '" << pool_name
+ << "'";
+ return 0; // idempotent
+ }
+
+ p.application_metadata.erase(app);
+ ss << "disable application '" << app << "' on pool '" << pool_name << "'";
+
+ } else if (boost::algorithm::ends_with(prefix, "set")) {
+ if (p.is_tier()) {
+ ss << "application metadata must be set on base tier";
+ return -EINVAL;
+ }
+
+ if (!app_exists) {
+ ss << "application '" << app << "' is not enabled on pool '" << pool_name
+ << "'";
+ return -ENOENT;
+ }
+
+ string key;
+ cmd_getval(g_ceph_context, cmdmap, "key", key);
+
+ if (key.empty()) {
+ ss << "key must be provided";
+ return -EINVAL;
+ }
+
+ auto &app_keys = p.application_metadata[app];
+ if (app_keys.count(key) == 0 &&
+ app_keys.size() >= MAX_POOL_APPLICATION_KEYS) {
+ ss << "too many keys set for application '" << app << "' on pool '"
+ << pool_name << "'; max " << MAX_POOL_APPLICATION_KEYS;
+ return -EINVAL;
+ }
+
+ if (key.length() > MAX_POOL_APPLICATION_LENGTH) {
+ ss << "key '" << app << "' too long; max length "
+ << MAX_POOL_APPLICATION_LENGTH;
+ return -EINVAL;
+ }
+
+ string value;
+ cmd_getval(g_ceph_context, cmdmap, "value", value);
+ if (value.length() > MAX_POOL_APPLICATION_LENGTH) {
+ ss << "value '" << value << "' too long; max length "
+ << MAX_POOL_APPLICATION_LENGTH;
+ return -EINVAL;
+ }
+
+ p.application_metadata[app][key] = value;
+ ss << "set application '" << app << "' key '" << key << "' to '"
+ << value << "' on pool '" << pool_name << "'";
+ } else if (boost::algorithm::ends_with(prefix, "rm")) {
+ if (!app_exists) {
+ ss << "application '" << app << "' is not enabled on pool '" << pool_name
+ << "'";
+ return -ENOENT;
+ }
+
+ string key;
+ cmd_getval(g_ceph_context, cmdmap, "key", key);
+ auto it = p.application_metadata[app].find(key);
+ if (it == p.application_metadata[app].end()) {
+ ss << "application '" << app << "' on pool '" << pool_name
+ << "' does not have key '" << key << "'";
+ return 0; // idempotent
+ }
+
+ p.application_metadata[app].erase(it);
+ ss << "removed application '" << app << "' key '" << key << "' on pool '"
+ << pool_name << "'";
+ } else {
+ assert(false);
+ }
+
+ p.last_change = pending_inc.epoch;
+ pending_inc.new_pools[pool] = p;
+ return 0;
+}
+
+int OSDMonitor::_prepare_command_osd_crush_remove(
+ CrushWrapper &newcrush,
+ int32_t id,
+ int32_t ancestor,
+ bool has_ancestor,
+ bool unlink_only)
+{
+ int err = 0;
+
+ if (has_ancestor) {
+ err = newcrush.remove_item_under(g_ceph_context, id, ancestor,
+ unlink_only);
+ } else {
+ err = newcrush.remove_item(g_ceph_context, id, unlink_only);
+ }
+ return err;
+}
+
+void OSDMonitor::do_osd_crush_remove(CrushWrapper& newcrush)
+{
+ pending_inc.crush.clear();
+ newcrush.encode(pending_inc.crush, mon->get_quorum_con_features());
+}
+
+int OSDMonitor::prepare_command_osd_crush_remove(
+ CrushWrapper &newcrush,
+ int32_t id,
+ int32_t ancestor,
+ bool has_ancestor,
+ bool unlink_only)
+{
+ int err = _prepare_command_osd_crush_remove(
+ newcrush, id, ancestor,
+ has_ancestor, unlink_only);
+
+ if (err < 0)
+ return err;
+
+ assert(err == 0);
+ do_osd_crush_remove(newcrush);
+
+ return 0;
+}
+
+int OSDMonitor::prepare_command_osd_remove(int32_t id)
+{
+ if (osdmap.is_up(id)) {
+ return -EBUSY;
+ }
+
+ pending_inc.new_state[id] = osdmap.get_state(id);
+ pending_inc.new_uuid[id] = uuid_d();
+ pending_metadata_rm.insert(id);
+ pending_metadata.erase(id);
+
+ return 0;
+}
+
+int32_t OSDMonitor::_allocate_osd_id(int32_t* existing_id)
+{
+ assert(existing_id);
+ *existing_id = -1;
+
+ for (int32_t i = 0; i < osdmap.get_max_osd(); ++i) {
+ if (!osdmap.exists(i) &&
+ pending_inc.new_up_client.count(i) == 0 &&
+ (pending_inc.new_state.count(i) == 0 ||
+ (pending_inc.new_state[i] & CEPH_OSD_EXISTS) == 0)) {
+ *existing_id = i;
+ return -1;
+ }
+ }
+
+ if (pending_inc.new_max_osd < 0) {
+ return osdmap.get_max_osd();
+ }
+ return pending_inc.new_max_osd;
+}
+
+void OSDMonitor::do_osd_create(
+ const int32_t id,
+ const uuid_d& uuid,
+ const string& device_class,
+ int32_t* new_id)
+{
+ dout(10) << __func__ << " uuid " << uuid << dendl;
+ assert(new_id);
+
+ // We presume validation has been performed prior to calling this
+ // function. We assert with prejudice.
+
+ int32_t allocated_id = -1; // declare here so we can jump
+ int32_t existing_id = -1;
+ if (!uuid.is_zero()) {
+ existing_id = osdmap.identify_osd(uuid);
+ if (existing_id >= 0) {
+ assert(id < 0 || id == existing_id);
+ *new_id = existing_id;
+ goto out;
+ } else if (id >= 0) {
+ // uuid does not exist, and id has been provided, so just create
+ // the new osd.id
+ *new_id = id;
+ goto out;
+ }
+ }
+
+ // allocate a new id
+ allocated_id = _allocate_osd_id(&existing_id);
+ dout(10) << __func__ << " allocated id " << allocated_id
+ << " existing id " << existing_id << dendl;
+ if (existing_id >= 0) {
+ assert(existing_id < osdmap.get_max_osd());
+ assert(allocated_id < 0);
+ pending_inc.new_weight[existing_id] = CEPH_OSD_OUT;
+ *new_id = existing_id;
+ } else if (allocated_id >= 0) {
+ assert(existing_id < 0);
+ // raise max_osd
+ if (pending_inc.new_max_osd < 0) {
+ pending_inc.new_max_osd = osdmap.get_max_osd() + 1;
+ } else {
+ ++pending_inc.new_max_osd;
+ }
+ *new_id = pending_inc.new_max_osd - 1;
+ assert(*new_id == allocated_id);
+ } else {
+ assert(0 == "unexpected condition");
+ }
+
+out:
+ if (device_class.size()) {
+ CrushWrapper newcrush;
+ _get_pending_crush(newcrush);
+ if (newcrush.get_max_devices() < *new_id + 1) {
+ newcrush.set_max_devices(*new_id + 1);
+ }
+ string name = string("osd.") + stringify(*new_id);
+ if (!newcrush.item_exists(*new_id)) {
+ newcrush.set_item_name(*new_id, name);
+ }
+ ostringstream ss;
+ int r = newcrush.update_device_class(*new_id, device_class, name, &ss);
+ if (r < 0) {
+ derr << __func__ << " failed to set " << name << " device_class "
+ << device_class << ": " << cpp_strerror(r) << " - " << ss.str()
+ << dendl;
+ // non-fatal... this might be a replay and we want to be idempotent.
+ } else {
+ dout(20) << __func__ << " set " << name << " device_class " << device_class
+ << dendl;
+ pending_inc.crush.clear();
+ newcrush.encode(pending_inc.crush, mon->get_quorum_con_features());
+ }
+ } else {
+ dout(20) << __func__ << " no device_class" << dendl;
+ }
+
+ dout(10) << __func__ << " using id " << *new_id << dendl;
+ if (osdmap.get_max_osd() <= *new_id && pending_inc.new_max_osd <= *new_id) {
+ pending_inc.new_max_osd = *new_id + 1;
+ }
+
+ pending_inc.new_state[*new_id] |= CEPH_OSD_EXISTS | CEPH_OSD_NEW;
+ if (!uuid.is_zero())
+ pending_inc.new_uuid[*new_id] = uuid;
+}
+
+int OSDMonitor::validate_osd_create(
+ const int32_t id,
+ const uuid_d& uuid,
+ const bool check_osd_exists,
+ int32_t* existing_id,
+ stringstream& ss)
+{
+
+ dout(10) << __func__ << " id " << id << " uuid " << uuid
+ << " check_osd_exists " << check_osd_exists << dendl;
+
+ assert(existing_id);
+
+ if (id < 0 && uuid.is_zero()) {
+ // we have nothing to validate
+ *existing_id = -1;
+ return 0;
+ } else if (uuid.is_zero()) {
+ // we have an id but we will ignore it - because that's what
+ // `osd create` does.
+ return 0;
+ }
+
+ /*
+ * This function will be used to validate whether we are able to
+ * create a new osd when the `uuid` is specified.
+ *
+ * It will be used by both `osd create` and `osd new`, as the checks
+ * are basically the same when it pertains to osd id and uuid validation.
+ * However, `osd create` presumes an `uuid` is optional, for legacy
+ * reasons, while `osd new` requires the `uuid` to be provided. This
+ * means that `osd create` will not be idempotent if an `uuid` is not
+ * provided, but we will always guarantee the idempotency of `osd new`.
+ */
+
+ assert(!uuid.is_zero());
+ if (pending_inc.identify_osd(uuid) >= 0) {
+ // osd is about to exist
+ return -EAGAIN;
+ }
+
+ int32_t i = osdmap.identify_osd(uuid);
+ if (i >= 0) {
+ // osd already exists
+ if (id >= 0 && i != id) {
+ ss << "uuid " << uuid << " already in use for different id " << i;
+ return -EEXIST;
+ }
+ // return a positive errno to distinguish between a blocking error
+ // and an error we consider to not be a problem (i.e., this would be
+ // an idempotent operation).
+ *existing_id = i;
+ return EEXIST;
+ }
+ // i < 0
+ if (id >= 0) {
+ if (pending_inc.new_state.count(id)) {
+ // osd is about to exist
+ return -EAGAIN;
+ }
+ // we may not care if an osd exists if we are recreating a previously
+ // destroyed osd.
+ if (check_osd_exists && osdmap.exists(id)) {
+ ss << "id " << id << " already in use and does not match uuid "
+ << uuid;
+ return -EINVAL;
+ }
+ }
+ return 0;
+}
+
+int OSDMonitor::prepare_command_osd_create(
+ const int32_t id,
+ const uuid_d& uuid,
+ int32_t* existing_id,
+ stringstream& ss)
+{
+ dout(10) << __func__ << " id " << id << " uuid " << uuid << dendl;
+ assert(existing_id);
+ if (osdmap.is_destroyed(id)) {
+ ss << "ceph osd create has been deprecated. Please use ceph osd new "
+ "instead.";
+ return -EINVAL;
+ }
+
+ if (uuid.is_zero()) {
+ dout(10) << __func__ << " no uuid; assuming legacy `osd create`" << dendl;
+ }
+
+ return validate_osd_create(id, uuid, true, existing_id, ss);
+}
+
+int OSDMonitor::prepare_command_osd_new(
+ MonOpRequestRef op,
+ const map<string,cmd_vartype>& cmdmap,
+ const map<string,string>& params,
+ stringstream &ss,
+ Formatter *f)
+{
+ uuid_d uuid;
+ string uuidstr;
+ int64_t id = -1;
+
+ assert(paxos->is_plugged());
+
+ dout(10) << __func__ << " " << op << dendl;
+
+ /* validate command. abort now if something's wrong. */
+
+ /* `osd new` will expect a `uuid` to be supplied; `id` is optional.
+ *
+ * If `id` is not specified, we will identify any existing osd based
+ * on `uuid`. Operation will be idempotent iff secrets match.
+ *
+ * If `id` is specified, we will identify any existing osd based on
+ * `uuid` and match against `id`. If they match, operation will be
+ * idempotent iff secrets match.
+ *
+ * `-i secrets.json` will be optional. If supplied, will be used
+ * to check for idempotency when `id` and `uuid` match.
+ *
+ * If `id` is not specified, and `uuid` does not exist, an id will
+ * be found or allocated for the osd.
+ *
+ * If `id` is specified, and the osd has been previously marked
+ * as destroyed, then the `id` will be reused.
+ */
+ if (!cmd_getval(g_ceph_context, cmdmap, "uuid", uuidstr)) {
+ ss << "requires the OSD's UUID to be specified.";
+ return -EINVAL;
+ } else if (!uuid.parse(uuidstr.c_str())) {
+ ss << "invalid UUID value '" << uuidstr << "'.";
+ return -EINVAL;
+ }
+
+ if (cmd_getval(g_ceph_context, cmdmap, "id", id) &&
+ (id < 0)) {
+ ss << "invalid OSD id; must be greater or equal than zero.";
+ return -EINVAL;
+ }
+
+ // are we running an `osd create`-like command, or recreating
+ // a previously destroyed osd?
+
+ bool is_recreate_destroyed = (id >= 0 && osdmap.is_destroyed(id));
+
+ // we will care about `id` to assess whether osd is `destroyed`, or
+ // to create a new osd.
+ // we will need an `id` by the time we reach auth.
+
+ int32_t existing_id = -1;
+ int err = validate_osd_create(id, uuid, !is_recreate_destroyed,
+ &existing_id, ss);
+
+ bool may_be_idempotent = false;
+ if (err == EEXIST) {
+ // this is idempotent from the osdmon's point-of-view
+ may_be_idempotent = true;
+ assert(existing_id >= 0);
+ id = existing_id;
+ } else if (err < 0) {
+ return err;
+ }
+
+ if (!may_be_idempotent) {
+ // idempotency is out of the window. We are either creating a new
+ // osd or recreating a destroyed osd.
+ //
+ // We now need to figure out if we have an `id` (and if it's valid),
+ // of find an `id` if we don't have one.
+
+ // NOTE: we need to consider the case where the `id` is specified for
+ // `osd create`, and we must honor it. So this means checking if
+ // the `id` is destroyed, and if so assume the destroy; otherwise,
+ // check if it `exists` - in which case we complain about not being
+ // `destroyed`. In the end, if nothing fails, we must allow the
+ // creation, so that we are compatible with `create`.
+ if (id >= 0 && osdmap.exists(id) && !osdmap.is_destroyed(id)) {
+ dout(10) << __func__ << " osd." << id << " isn't destroyed" << dendl;
+ ss << "OSD " << id << " has not yet been destroyed";
+ return -EINVAL;
+ } else if (id < 0) {
+ // find an `id`
+ id = _allocate_osd_id(&existing_id);
+ if (id < 0) {
+ assert(existing_id >= 0);
+ id = existing_id;
+ }
+ dout(10) << __func__ << " found id " << id << " to use" << dendl;
+ } else if (id >= 0 && osdmap.is_destroyed(id)) {
+ dout(10) << __func__ << " recreating osd." << id << dendl;
+ } else {
+ dout(10) << __func__ << " creating new osd." << id << dendl;
+ }
+ } else {
+ assert(id >= 0);
+ assert(osdmap.exists(id));
+ }
+
+ // we are now able to either create a brand new osd or reuse an existing
+ // osd that has been previously destroyed.
+
+ dout(10) << __func__ << " id " << id << " uuid " << uuid << dendl;
+
+ if (may_be_idempotent && params.empty()) {
+ // nothing to do, really.
+ dout(10) << __func__ << " idempotent and no params -- no op." << dendl;
+ assert(id >= 0);
+ if (f) {
+ f->open_object_section("created_osd");
+ f->dump_int("osdid", id);
+ f->close_section();
+ } else {
+ ss << id;
+ }
+ return EEXIST;
+ }
+
+ string device_class;
+ auto p = params.find("crush_device_class");
+ if (p != params.end()) {
+ device_class = p->second;
+ dout(20) << __func__ << " device_class will be " << device_class << dendl;
+ }
+ string cephx_secret, lockbox_secret, dmcrypt_key;
+ bool has_lockbox = false;
+ bool has_secrets = params.count("cephx_secret")
+ || params.count("cephx_lockbox_secret")
+ || params.count("dmcrypt_key");
+
+ ConfigKeyService *svc = nullptr;
+ AuthMonitor::auth_entity_t cephx_entity, lockbox_entity;
+
+ if (has_secrets) {
+ if (params.count("cephx_secret") == 0) {
+ ss << "requires a cephx secret.";
+ return -EINVAL;
+ }
+ cephx_secret = params.at("cephx_secret");
+
+ bool has_lockbox_secret = (params.count("cephx_lockbox_secret") > 0);
+ bool has_dmcrypt_key = (params.count("dmcrypt_key") > 0);
+
+ dout(10) << __func__ << " has lockbox " << has_lockbox_secret
+ << " dmcrypt " << has_dmcrypt_key << dendl;
+
+ if (has_lockbox_secret && has_dmcrypt_key) {
+ has_lockbox = true;
+ lockbox_secret = params.at("cephx_lockbox_secret");
+ dmcrypt_key = params.at("dmcrypt_key");
+ } else if (!has_lockbox_secret != !has_dmcrypt_key) {
+ ss << "requires both a cephx lockbox secret and a dm-crypt key.";
+ return -EINVAL;
+ }
+
+ dout(10) << __func__ << " validate secrets using osd id " << id << dendl;
+
+ err = mon->authmon()->validate_osd_new(id, uuid,
+ cephx_secret,
+ lockbox_secret,
+ cephx_entity,
+ lockbox_entity,
+ ss);
+ if (err < 0) {
+ return err;
+ } else if (may_be_idempotent && err != EEXIST) {
+ // for this to be idempotent, `id` should already be >= 0; no need
+ // to use validate_id.
+ assert(id >= 0);
+ ss << "osd." << id << " exists but secrets do not match";
+ return -EEXIST;
+ }
+
+ if (has_lockbox) {
+ svc = (ConfigKeyService*)mon->config_key_service;
+ err = svc->validate_osd_new(uuid, dmcrypt_key, ss);
+ if (err < 0) {
+ return err;
+ } else if (may_be_idempotent && err != EEXIST) {
+ assert(id >= 0);
+ ss << "osd." << id << " exists but dm-crypt key does not match.";
+ return -EEXIST;
+ }
+ }
+ }
+ assert(!has_secrets || !cephx_secret.empty());
+ assert(!has_lockbox || !lockbox_secret.empty());
+
+ if (may_be_idempotent) {
+ // we have nothing to do for either the osdmon or the authmon,
+ // and we have no lockbox - so the config key service will not be
+ // touched. This is therefore an idempotent operation, and we can
+ // just return right away.
+ dout(10) << __func__ << " idempotent -- no op." << dendl;
+ assert(id >= 0);
+ if (f) {
+ f->open_object_section("created_osd");
+ f->dump_int("osdid", id);
+ f->close_section();
+ } else {
+ ss << id;
+ }
+ return EEXIST;
+ }
+ assert(!may_be_idempotent);
+
+ // perform updates.
+ if (has_secrets) {
+ assert(!cephx_secret.empty());
+ assert((lockbox_secret.empty() && dmcrypt_key.empty()) ||
+ (!lockbox_secret.empty() && !dmcrypt_key.empty()));
+
+ err = mon->authmon()->do_osd_new(cephx_entity,
+ lockbox_entity,
+ has_lockbox);
+ assert(0 == err);
+
+ if (has_lockbox) {
+ assert(nullptr != svc);
+ svc->do_osd_new(uuid, dmcrypt_key);
+ }
+ }
+
+ if (is_recreate_destroyed) {
+ assert(id >= 0);
+ assert(osdmap.is_destroyed(id));
+ pending_inc.new_weight[id] = CEPH_OSD_OUT;
+ pending_inc.new_state[id] |= CEPH_OSD_DESTROYED | CEPH_OSD_NEW;
+ if (osdmap.get_state(id) & CEPH_OSD_UP) {
+ // due to http://tracker.ceph.com/issues/20751 some clusters may
+ // have UP set for non-existent OSDs; make sure it is cleared
+ // for a newly created osd.
+ pending_inc.new_state[id] |= CEPH_OSD_UP;
+ }
+ pending_inc.new_uuid[id] = uuid;
+ } else {
+ assert(id >= 0);
+ int32_t new_id = -1;
+ do_osd_create(id, uuid, device_class, &new_id);
+ assert(new_id >= 0);
+ assert(id == new_id);
+ }
+
+ if (f) {
+ f->open_object_section("created_osd");
+ f->dump_int("osdid", id);
+ f->close_section();
+ } else {
+ ss << id;
+ }
+
+ return 0;
+}
+
+bool OSDMonitor::prepare_command(MonOpRequestRef op)
+{
+ op->mark_osdmon_event(__func__);
+ MMonCommand *m = static_cast<MMonCommand*>(op->get_req());
+ stringstream ss;
+ map<string, cmd_vartype> cmdmap;
+ if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+ string rs = ss.str();
+ mon->reply_command(op, -EINVAL, rs, get_last_committed());
+ return true;
+ }
+
+ MonSession *session = m->get_session();
+ if (!session) {
+ mon->reply_command(op, -EACCES, "access denied", get_last_committed());
+ return true;
+ }
+
+ return prepare_command_impl(op, cmdmap);
+}
+
+static int parse_reweights(CephContext *cct,
+ const map<string,cmd_vartype> &cmdmap,
+ const OSDMap& osdmap,
+ map<int32_t, uint32_t>* weights)
+{
+ string weights_str;
+ if (!cmd_getval(g_ceph_context, cmdmap, "weights", weights_str)) {
+ return -EINVAL;
+ }
+ std::replace(begin(weights_str), end(weights_str), '\'', '"');
+ json_spirit::mValue json_value;
+ if (!json_spirit::read(weights_str, json_value)) {
+ return -EINVAL;
+ }
+ if (json_value.type() != json_spirit::obj_type) {
+ return -EINVAL;
+ }
+ const auto obj = json_value.get_obj();
+ try {
+ for (auto& osd_weight : obj) {
+ auto osd_id = std::stoi(osd_weight.first);
+ if (!osdmap.exists(osd_id)) {
+ return -ENOENT;
+ }
+ if (osd_weight.second.type() != json_spirit::str_type) {
+ return -EINVAL;
+ }
+ auto weight = std::stoul(osd_weight.second.get_str());
+ weights->insert({osd_id, weight});
+ }
+ } catch (const std::logic_error& e) {
+ return -EINVAL;
+ }
+ return 0;
+}
+
+int OSDMonitor::prepare_command_osd_destroy(
+ int32_t id,
+ stringstream& ss)
+{
+ assert(paxos->is_plugged());
+
+ // we check if the osd exists for the benefit of `osd purge`, which may
+ // have previously removed the osd. If the osd does not exist, return
+ // -ENOENT to convey this, and let the caller deal with it.
+ //
+ // we presume that all auth secrets and config keys were removed prior
+ // to this command being called. if they exist by now, we also assume
+ // they must have been created by some other command and do not pertain
+ // to this non-existent osd.
+ if (!osdmap.exists(id)) {
+ dout(10) << __func__ << " osd." << id << " does not exist." << dendl;
+ return -ENOENT;
+ }
+
+ uuid_d uuid = osdmap.get_uuid(id);
+ dout(10) << __func__ << " destroying osd." << id
+ << " uuid " << uuid << dendl;
+
+ // if it has been destroyed, we assume our work here is done.
+ if (osdmap.is_destroyed(id)) {
+ ss << "destroyed osd." << id;
+ return 0;
+ }
+
+ EntityName cephx_entity, lockbox_entity;
+ bool idempotent_auth = false, idempotent_cks = false;
+
+ int err = mon->authmon()->validate_osd_destroy(id, uuid,
+ cephx_entity,
+ lockbox_entity,
+ ss);
+ if (err < 0) {
+ if (err == -ENOENT) {
+ idempotent_auth = true;
+ } else {
+ return err;
+ }
+ }
+
+ ConfigKeyService *svc = (ConfigKeyService*)mon->config_key_service;
+ err = svc->validate_osd_destroy(id, uuid);
+ if (err < 0) {
+ assert(err == -ENOENT);
+ err = 0;
+ idempotent_cks = true;
+ }
+
+ if (!idempotent_auth) {
+ err = mon->authmon()->do_osd_destroy(cephx_entity, lockbox_entity);
+ assert(0 == err);
+ }
+
+ if (!idempotent_cks) {
+ svc->do_osd_destroy(id, uuid);
+ }
+
+ pending_inc.new_state[id] = CEPH_OSD_DESTROYED;
+ pending_inc.new_uuid[id] = uuid_d();
+
+ // we can only propose_pending() once per service, otherwise we'll be
+ // defying PaxosService and all laws of nature. Therefore, as we may
+ // be used during 'osd purge', let's keep the caller responsible for
+ // proposing.
+ assert(err == 0);
+ return 0;
+}
+
+int OSDMonitor::prepare_command_osd_purge(
+ int32_t id,
+ stringstream& ss)
+{
+ assert(paxos->is_plugged());
+ dout(10) << __func__ << " purging osd." << id << dendl;
+
+ assert(!osdmap.is_up(id));
+
+ /*
+ * This may look a bit weird, but this is what's going to happen:
+ *
+ * 1. we make sure that removing from crush works
+ * 2. we call `prepare_command_osd_destroy()`. If it returns an
+ * error, then we abort the whole operation, as no updates
+ * have been made. However, we this function will have
+ * side-effects, thus we need to make sure that all operations
+ * performed henceforth will *always* succeed.
+ * 3. we call `prepare_command_osd_remove()`. Although this
+ * function can return an error, it currently only checks if the
+ * osd is up - and we have made sure that it is not so, so there
+ * is no conflict, and it is effectively an update.
+ * 4. finally, we call `do_osd_crush_remove()`, which will perform
+ * the crush update we delayed from before.
+ */
+
+ CrushWrapper newcrush;
+ _get_pending_crush(newcrush);
+
+ bool may_be_idempotent = false;
+
+ int err = _prepare_command_osd_crush_remove(newcrush, id, 0, false, false);
+ if (err == -ENOENT) {
+ err = 0;
+ may_be_idempotent = true;
+ } else if (err < 0) {
+ ss << "error removing osd." << id << " from crush";
+ return err;
+ }
+
+ // no point destroying the osd again if it has already been marked destroyed
+ if (!osdmap.is_destroyed(id)) {
+ err = prepare_command_osd_destroy(id, ss);
+ if (err < 0) {
+ if (err == -ENOENT) {
+ err = 0;
+ } else {
+ return err;
+ }
+ } else {
+ may_be_idempotent = false;
+ }
+ }
+ assert(0 == err);
+
+ if (may_be_idempotent && !osdmap.exists(id)) {
+ dout(10) << __func__ << " osd." << id << " does not exist and "
+ << "we are idempotent." << dendl;
+ return -ENOENT;
+ }
+
+ err = prepare_command_osd_remove(id);
+ // we should not be busy, as we should have made sure this id is not up.
+ assert(0 == err);
+
+ do_osd_crush_remove(newcrush);
+ return 0;
+}
+
+bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
+ map<string,cmd_vartype> &cmdmap)
+{
+ op->mark_osdmon_event(__func__);
+ MMonCommand *m = static_cast<MMonCommand*>(op->get_req());
+ bool ret = false;
+ stringstream ss;
+ string rs;
+ bufferlist rdata;
+ int err = 0;
+
+ string format;
+ cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain"));
+ boost::scoped_ptr<Formatter> f(Formatter::create(format));
+
+ string prefix;
+ cmd_getval(g_ceph_context, cmdmap, "prefix", prefix);
+
+ int64_t osdid;
+ string name;
+ bool osdid_present = false;
+ if (prefix != "osd pg-temp" &&
+ prefix != "osd pg-upmap" &&
+ prefix != "osd pg-upmap-items") { // avoid commands with non-int id arg
+ osdid_present = cmd_getval(g_ceph_context, cmdmap, "id", osdid);
+ }
+ if (osdid_present) {
ostringstream oss;
oss << "osd." << osdid;
name = oss.str();
if (prefix == "osd setcrushmap" ||
(prefix == "osd crush set" && !osdid_present)) {
+ if (pending_inc.crush.length()) {
+ dout(10) << __func__ << " waiting for pending crush update " << dendl;
+ wait_for_finished_proposal(op, new C_RetryMessage(this, op));
+ return true;
+ }
dout(10) << "prepare_command setting new crush map" << dendl;
bufferlist data(m->get_data());
CrushWrapper crush;
ss << "Failed to parse crushmap: " << e.what();
goto reply;
}
+
+ int64_t prior_version = 0;
+ if (cmd_getval(g_ceph_context, cmdmap, "prior_version", prior_version)) {
+ if (prior_version == osdmap.get_crush_version() - 1) {
+ // see if we are a resend of the last update. this is imperfect
+ // (multiple racing updaters may not both get reliable success)
+ // but we expect crush updaters (via this interface) to be rare-ish.
+ bufferlist current, proposed;
+ osdmap.crush->encode(current, mon->get_quorum_con_features());
+ crush.encode(proposed, mon->get_quorum_con_features());
+ if (current.contents_equal(proposed)) {
+ dout(10) << __func__
+ << " proposed matches current and version equals previous"
+ << dendl;
+ err = 0;
+ ss << osdmap.get_crush_version();
+ goto reply;
+ }
+ }
+ if (prior_version != osdmap.get_crush_version()) {
+ err = -EPERM;
+ ss << "prior_version " << prior_version << " != crush version "
+ << osdmap.get_crush_version();
+ goto reply;
+ }
+ }
- if (!validate_crush_against_features(&crush, ss)) {
+ if (crush.has_legacy_rule_ids()) {
err = -EINVAL;
+ ss << "crush maps with ruleset != ruleid are no longer allowed";
goto reply;
}
-
- const auto& osdmap_pools = osdmap.get_pools();
- for (auto pit = osdmap_pools.begin(); pit != osdmap_pools.end(); ++pit) {
- const int64_t pool_id = pit->first;
- const pg_pool_t &pool = pit->second;
- int ruleno = pool.get_crush_ruleset();
- if (!crush.rule_exists(ruleno)) {
- ss << " the crush rule no "<< ruleno << " for pool id " << pool_id << " is in use";
- err = -EINVAL;
- goto reply;
- }
+ if (!validate_crush_against_features(&crush, ss)) {
+ err = -EINVAL;
+ goto reply;
}
- // sanity check: test some inputs to make sure this map isn't totally broken
- dout(10) << " testing map" << dendl;
- stringstream ess;
- CrushTester tester(crush, ess);
- // XXX: Use mon_lease as a timeout value for crushtool.
- // If the crushtool consistently takes longer than 'mon_lease' seconds,
- // then we would consistently trigger an election before the command
- // finishes, having a flapping monitor unable to hold quorum.
- int r = tester.test_with_crushtool(g_conf->crushtool.c_str(),
- osdmap.get_max_osd(),
- g_conf->mon_lease);
- if (r < 0) {
- derr << "error on crush map: " << ess.str() << dendl;
- ss << "Failed crushmap test: " << ess.str();
- err = r;
+ err = osdmap.validate_crush_rules(&crush, &ss);
+ if (err < 0) {
goto reply;
}
- dout(10) << " result " << ess.str() << dendl;
+ if (g_conf->mon_osd_crush_smoke_test) {
+ // sanity check: test some inputs to make sure this map isn't
+ // totally broken
+ dout(10) << " testing map" << dendl;
+ stringstream ess;
+ CrushTester tester(crush, ess);
+ tester.set_min_x(0);
+ tester.set_max_x(50);
+ auto start = ceph::coarse_mono_clock::now();
+ int r = tester.test_with_fork(g_conf->mon_lease);
+ auto duration = ceph::coarse_mono_clock::now() - start;
+ if (r < 0) {
+ dout(10) << " tester.test_with_fork returns " << r
+ << ": " << ess.str() << dendl;
+ ss << "crush smoke test failed with " << r << ": " << ess.str();
+ err = r;
+ goto reply;
+ }
+ dout(10) << __func__ << " crush somke test duration: "
+ << duration << ", result: " << ess.str() << dendl;
+ }
pending_inc.crush = data;
- ss << "set crush map";
+ ss << osdmap.get_crush_version() + 1;
goto update;
+ } else if (prefix == "osd crush set-all-straw-buckets-to-straw2") {
+ CrushWrapper newcrush;
+ _get_pending_crush(newcrush);
+ for (int b = 0; b < newcrush.get_max_buckets(); ++b) {
+ int bid = -1 - b;
+ if (newcrush.bucket_exists(bid) &&
+ newcrush.get_bucket_alg(bid)) {
+ dout(20) << " bucket " << bid << " is straw, can convert" << dendl;
+ newcrush.bucket_set_alg(bid, CRUSH_BUCKET_STRAW2);
+ }
+ }
+ if (!validate_crush_against_features(&newcrush, ss)) {
+ err = -EINVAL;
+ goto reply;
+ }
+ pending_inc.crush.clear();
+ newcrush.encode(pending_inc.crush, mon->get_quorum_con_features());
+ wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+ get_last_committed() + 1));
+ return true;
} else if (prefix == "osd crush set-device-class") {
- if (!osdmap.exists(osdid)) {
- err = -ENOENT;
- ss << name << " does not exist. create it before updating the crush map";
+ if (osdmap.require_osd_release < CEPH_RELEASE_LUMINOUS) {
+ ss << "you must complete the upgrade and 'ceph osd require-osd-release "
+ << "luminous' before using crush device classes";
+ err = -EPERM;
goto reply;
}
goto reply;
}
+ bool stop = false;
+ vector<string> idvec;
+ cmd_getval(g_ceph_context, cmdmap, "ids", idvec);
CrushWrapper newcrush;
_get_pending_crush(newcrush);
+ set<int> updated;
+ for (unsigned j = 0; j < idvec.size() && !stop; j++) {
+ set<int> osds;
+ // wildcard?
+ if (j == 0 &&
+ (idvec[0] == "any" || idvec[0] == "all" || idvec[0] == "*")) {
+ osdmap.get_all_osds(osds);
+ stop = true;
+ } else {
+ // try traditional single osd way
+ long osd = parse_osd_id(idvec[j].c_str(), &ss);
+ if (osd < 0) {
+ // ss has reason for failure
+ ss << ", unable to parse osd id:\"" << idvec[j] << "\". ";
+ err = -EINVAL;
+ continue;
+ }
+ osds.insert(osd);
+ }
- string action;
- if (newcrush.item_exists(osdid)) {
- action = "updating";
- } else {
- action = "creating";
- newcrush.set_item_name(osdid, name);
+ for (auto &osd : osds) {
+ if (!osdmap.exists(osd)) {
+ ss << "osd." << osd << " does not exist. ";
+ continue;
+ }
+
+ ostringstream oss;
+ oss << "osd." << osd;
+ string name = oss.str();
+
+ if (newcrush.get_max_devices() < osd + 1) {
+ newcrush.set_max_devices(osd + 1);
+ }
+ string action;
+ if (newcrush.item_exists(osd)) {
+ action = "updating";
+ } else {
+ action = "creating";
+ newcrush.set_item_name(osd, name);
+ }
+
+ dout(5) << action << " crush item id " << osd << " name '" << name
+ << "' device_class '" << device_class << "'"
+ << dendl;
+ err = newcrush.update_device_class(osd, device_class, name, &ss);
+ if (err < 0) {
+ goto reply;
+ }
+ if (err == 0 && !_have_pending_crush()) {
+ if (!stop) {
+ // for single osd only, wildcard makes too much noise
+ ss << "set-device-class item id " << osd << " name '" << name
+ << "' device_class '" << device_class << "': no change";
+ }
+ } else {
+ updated.insert(osd);
+ }
+ }
+ }
+
+ if (!updated.empty()) {
+ pending_inc.crush.clear();
+ newcrush.encode(pending_inc.crush, mon->get_quorum_con_features());
+ ss << "set osd(s) " << updated << " to class '" << device_class << "'";
+ getline(ss, rs);
+ wait_for_finished_proposal(op,
+ new Monitor::C_Command(mon,op, 0, rs, get_last_committed() + 1));
+ return true;
}
- dout(5) << action << " crush item id " << osdid << " name '"
- << name << "' device_class " << device_class << dendl;
- err = newcrush.update_device_class(g_ceph_context, osdid, device_class, name);
+ } else if (prefix == "osd crush rm-device-class") {
+ bool stop = false;
+ vector<string> idvec;
+ cmd_getval(g_ceph_context, cmdmap, "ids", idvec);
+ CrushWrapper newcrush;
+ _get_pending_crush(newcrush);
+ set<int> updated;
+
+ for (unsigned j = 0; j < idvec.size() && !stop; j++) {
+ set<int> osds;
- if (err < 0)
+ // wildcard?
+ if (j == 0 &&
+ (idvec[0] == "any" || idvec[0] == "all" || idvec[0] == "*")) {
+ osdmap.get_all_osds(osds);
+ stop = true;
+ } else {
+ // try traditional single osd way
+ long osd = parse_osd_id(idvec[j].c_str(), &ss);
+ if (osd < 0) {
+ // ss has reason for failure
+ ss << ", unable to parse osd id:\"" << idvec[j] << "\". ";
+ err = -EINVAL;
+ goto reply;
+ }
+ osds.insert(osd);
+ }
+
+ for (auto &osd : osds) {
+ if (!osdmap.exists(osd)) {
+ ss << "osd." << osd << " does not exist. ";
+ continue;
+ }
+
+ auto class_name = newcrush.get_item_class(osd);
+ if (!class_name) {
+ ss << "osd." << osd << " belongs to no class, ";
+ continue;
+ }
+ // note that we do not verify if class_is_in_use here
+ // in case the device is misclassified and user wants
+ // to overridely reset...
+
+ err = newcrush.remove_device_class(g_ceph_context, osd, &ss);
+ if (err < 0) {
+ // ss has reason for failure
+ goto reply;
+ }
+ updated.insert(osd);
+ }
+ }
+
+ if (!updated.empty()) {
+ pending_inc.crush.clear();
+ newcrush.encode(pending_inc.crush, mon->get_quorum_con_features());
+ ss << "done removing class of osd(s): " << updated;
+ getline(ss, rs);
+ wait_for_finished_proposal(op,
+ new Monitor::C_Command(mon,op, 0, rs, get_last_committed() + 1));
+ return true;
+ }
+ } else if (prefix == "osd crush class rename") {
+ string srcname, dstname;
+ if (!cmd_getval(g_ceph_context, cmdmap, "srcname", srcname)) {
+ err = -EINVAL;
+ goto reply;
+ }
+ if (!cmd_getval(g_ceph_context, cmdmap, "dstname", dstname)) {
+ err = -EINVAL;
+ goto reply;
+ }
+
+ CrushWrapper newcrush;
+ _get_pending_crush(newcrush);
+ if (!newcrush.class_exists(srcname) && newcrush.class_exists(dstname)) {
+ // suppose this is a replay and return success
+ // so command is idempotent
+ ss << "already renamed to '" << dstname << "'";
+ err = 0;
goto reply;
+ }
- if (err == 0 && !_have_pending_crush()) {
- ss << "set-device-class item id " << osdid << " name '" << name << "' device_class "
- << device_class << " : no change";
+ err = newcrush.rename_class(srcname, dstname);
+ if (err < 0) {
+ ss << "fail to rename '" << srcname << "' to '" << dstname << "' : "
+ << cpp_strerror(err);
goto reply;
}
pending_inc.crush.clear();
newcrush.encode(pending_inc.crush, mon->get_quorum_con_features());
- ss << "set-device-class item id " << osdid << " name '" << name << "' device_class "
- << device_class;
- getline(ss, rs);
- wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
- get_last_committed() + 1));
- return true;
-
+ ss << "rename class '" << srcname << "' to '" << dstname << "'";
+ goto update;
} else if (prefix == "osd crush add-bucket") {
// os crush add-bucket <name> <type>
string name, typestr;
goto reply;
else
goto update;
- } else if (prefix == "osd crush class create") {
- string device_class;
- if (!cmd_getval(g_ceph_context, cmdmap, "class", device_class)) {
- err = -EINVAL; // no value!
+ } else if (prefix == "osd crush weight-set create" ||
+ prefix == "osd crush weight-set create-compat") {
+ CrushWrapper newcrush;
+ _get_pending_crush(newcrush);
+ int64_t pool;
+ int positions;
+ if (newcrush.has_non_straw2_buckets()) {
+ ss << "crush map contains one or more bucket(s) that are not straw2";
+ err = -EPERM;
goto reply;
}
-
- if (!_have_pending_crush() &&
- _get_stable_crush().class_exists(device_class)) {
- ss << "class '" << device_class << "' already exists";
- goto reply;
+ if (prefix == "osd crush weight-set create") {
+ if (osdmap.require_min_compat_client > 0 &&
+ osdmap.require_min_compat_client < CEPH_RELEASE_LUMINOUS) {
+ ss << "require_min_compat_client "
+ << ceph_release_name(osdmap.require_min_compat_client)
+ << " < luminous, which is required for per-pool weight-sets. "
+ << "Try 'ceph osd set-require-min-compat-client luminous' "
+ << "before using the new interface";
+ err = -EPERM;
+ goto reply;
+ }
+ string poolname, mode;
+ cmd_getval(g_ceph_context, cmdmap, "pool", poolname);
+ pool = osdmap.lookup_pg_pool_name(poolname.c_str());
+ if (pool < 0) {
+ ss << "pool '" << poolname << "' not found";
+ err = -ENOENT;
+ goto reply;
+ }
+ cmd_getval(g_ceph_context, cmdmap, "mode", mode);
+ if (mode != "flat" && mode != "positional") {
+ ss << "unrecognized weight-set mode '" << mode << "'";
+ err = -EINVAL;
+ goto reply;
+ }
+ positions = mode == "flat" ? 1 : osdmap.get_pg_pool(pool)->get_size();
+ } else {
+ pool = CrushWrapper::DEFAULT_CHOOSE_ARGS;
+ positions = 1;
}
+ newcrush.create_choose_args(pool, positions);
+ pending_inc.crush.clear();
+ newcrush.encode(pending_inc.crush, mon->get_quorum_con_features());
+ goto update;
+ } else if (prefix == "osd crush weight-set rm" ||
+ prefix == "osd crush weight-set rm-compat") {
CrushWrapper newcrush;
_get_pending_crush(newcrush);
-
- if (newcrush.class_exists(name)) {
- ss << "class '" << device_class << "' already exists";
- goto update;
+ int64_t pool;
+ if (prefix == "osd crush weight-set rm") {
+ string poolname;
+ cmd_getval(g_ceph_context, cmdmap, "pool", poolname);
+ pool = osdmap.lookup_pg_pool_name(poolname.c_str());
+ if (pool < 0) {
+ ss << "pool '" << poolname << "' not found";
+ err = -ENOENT;
+ goto reply;
+ }
+ } else {
+ pool = CrushWrapper::DEFAULT_CHOOSE_ARGS;
}
-
- int class_id = newcrush.get_or_create_class_id(device_class);
-
+ newcrush.rm_choose_args(pool);
pending_inc.crush.clear();
newcrush.encode(pending_inc.crush, mon->get_quorum_con_features());
- ss << "created class " << device_class << " with id " << class_id
- << " to crush map";
goto update;
-
- } else if (prefix == "osd crush class rm") {
- string device_class;
- if (!cmd_getval(g_ceph_context, cmdmap, "class", device_class)) {
- err = -EINVAL; // no value!
- goto reply;
- }
+ } else if (prefix == "osd crush weight-set reweight" ||
+ prefix == "osd crush weight-set reweight-compat") {
+ string poolname, item;
+ vector<double> weight;
+ cmd_getval(g_ceph_context, cmdmap, "pool", poolname);
+ cmd_getval(g_ceph_context, cmdmap, "item", item);
+ cmd_getval(g_ceph_context, cmdmap, "weight", weight);
CrushWrapper newcrush;
_get_pending_crush(newcrush);
-
- if (!newcrush.class_exists(device_class)) {
- err = -ENOENT;
- ss << "class '" << device_class << "' does not exist";
- goto reply;
+ int64_t pool;
+ if (prefix == "osd crush weight-set reweight") {
+ pool = osdmap.lookup_pg_pool_name(poolname.c_str());
+ if (pool < 0) {
+ ss << "pool '" << poolname << "' not found";
+ err = -ENOENT;
+ goto reply;
+ }
+ if (!newcrush.have_choose_args(pool)) {
+ ss << "no weight-set for pool '" << poolname << "'";
+ err = -ENOENT;
+ goto reply;
+ }
+ auto arg_map = newcrush.choose_args_get(pool);
+ int positions = newcrush.get_choose_args_positions(arg_map);
+ if (weight.size() != (size_t)positions) {
+ ss << "must specify exact " << positions << " weight values";
+ err = -EINVAL;
+ goto reply;
+ }
+ } else {
+ pool = CrushWrapper::DEFAULT_CHOOSE_ARGS;
+ if (!newcrush.have_choose_args(pool)) {
+ ss << "no backward-compatible weight-set";
+ err = -ENOENT;
+ goto reply;
+ }
}
-
- int class_id = newcrush.get_class_id(device_class);
-
- if (newcrush.class_is_in_use(class_id)) {
- err = -EBUSY;
- ss << "class '" << device_class << "' is in use";
+ if (!newcrush.name_exists(item)) {
+ ss << "item '" << item << "' does not exist";
+ err = -ENOENT;
goto reply;
}
-
- err = newcrush.remove_class_name(device_class);
+ err = newcrush.choose_args_adjust_item_weightf(
+ g_ceph_context,
+ newcrush.choose_args_get(pool),
+ newcrush.get_item_id(item),
+ weight,
+ &ss);
if (err < 0) {
- ss << "class '" << device_class << "' cannot be removed '"
- << cpp_strerror(err) << "'";
goto reply;
}
-
+ err = 0;
pending_inc.crush.clear();
newcrush.encode(pending_inc.crush, mon->get_quorum_con_features());
- ss << "removed class " << device_class << " with id " << class_id
- << " from crush map";
goto update;
-
} else if (osdid_present &&
(prefix == "osd crush set" || prefix == "osd crush add")) {
// <OsdName> is 'osd.<id>' or '<id>', passed as int64_t id
if (!osdmap.exists(osdid)) {
err = -ENOENT;
- ss << name << " does not exist. create it before updating the crush map";
+ ss << name << " does not exist. Create it before updating the crush map";
goto reply;
}
err = 0;
}
} while (false);
-
+ } else if (prefix == "osd crush swap-bucket") {
+ string source, dest, force;
+ cmd_getval(g_ceph_context, cmdmap, "source", source);
+ cmd_getval(g_ceph_context, cmdmap, "dest", dest);
+ cmd_getval(g_ceph_context, cmdmap, "force", force);
+ CrushWrapper newcrush;
+ _get_pending_crush(newcrush);
+ if (!newcrush.name_exists(source)) {
+ ss << "source item " << source << " does not exist";
+ err = -ENOENT;
+ goto reply;
+ }
+ if (!newcrush.name_exists(dest)) {
+ ss << "dest item " << dest << " does not exist";
+ err = -ENOENT;
+ goto reply;
+ }
+ int sid = newcrush.get_item_id(source);
+ int did = newcrush.get_item_id(dest);
+ int sparent;
+ if (newcrush.get_immediate_parent_id(sid, &sparent) == 0 &&
+ force != "--yes-i-really-mean-it") {
+ ss << "source item " << source << " is not an orphan bucket; pass --yes-i-really-mean-it to proceed anyway";
+ err = -EPERM;
+ goto reply;
+ }
+ if (newcrush.get_bucket_alg(sid) != newcrush.get_bucket_alg(did) &&
+ force != "--yes-i-really-mean-it") {
+ ss << "source bucket alg " << crush_alg_name(newcrush.get_bucket_alg(sid)) << " != "
+ << "dest bucket alg " << crush_alg_name(newcrush.get_bucket_alg(did))
+ << "; pass --yes-i-really-mean-it to proceed anyway";
+ err = -EPERM;
+ goto reply;
+ }
+ int r = newcrush.swap_bucket(g_ceph_context, sid, did);
+ if (r < 0) {
+ ss << "failed to swap bucket contents: " << cpp_strerror(r);
+ err = r;
+ goto reply;
+ }
+ ss << "swapped bucket of " << source << " to " << dest;
+ pending_inc.crush.clear();
+ newcrush.encode(pending_inc.crush, mon->get_quorum_con_features());
+ wait_for_finished_proposal(op,
+ new Monitor::C_Command(mon, op, err, ss.str(),
+ get_last_committed() + 1));
+ return true;
} else if (prefix == "osd crush link") {
// osd crush link <name> <loc1> [<loc2> ...]
string name;
return true;
}
int id = newcrush.get_item_id(name);
+ int ancestor = 0;
+
bool unlink_only = prefix == "osd crush unlink";
string ancestor_str;
if (cmd_getval(g_ceph_context, cmdmap, "ancestor", ancestor_str)) {
<< "' does not appear in the crush map";
break;
}
- int ancestor = newcrush.get_item_id(ancestor_str);
- err = newcrush.remove_item_under(g_ceph_context, id, ancestor,
- unlink_only);
- } else {
- err = newcrush.remove_item(g_ceph_context, id, unlink_only);
+ ancestor = newcrush.get_item_id(ancestor_str);
}
+
+ err = prepare_command_osd_crush_remove(
+ newcrush,
+ id, ancestor,
+ (ancestor < 0), unlink_only);
+
if (err == -ENOENT) {
ss << "item " << id << " does not appear in that position";
err = 0;
break;
}
if (err == 0) {
- pending_inc.crush.clear();
- newcrush.encode(pending_inc.crush, mon->get_quorum_con_features());
ss << "removed item id " << id << " name '" << name << "' from crush map";
getline(ss, rs);
wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
} while (false);
} else if (prefix == "osd crush reweight-all") {
- // osd crush reweight <name> <weight>
CrushWrapper newcrush;
_get_pending_crush(newcrush);
}
if (tunable == "straw_calc_version") {
- if (value < 0 || value > 1) {
+ if (value != 0 && value != 1) {
ss << "value must be 0 or 1; got " << value;
err = -EINVAL;
goto reply;
mode = "firstn";
if (osdmap.crush->rule_exists(name)) {
- // The name is uniquely associated to a ruleid and the ruleset it contains
- // From the user point of view, the ruleset is more meaningfull.
- ss << "ruleset " << name << " already exists";
+ // The name is uniquely associated to a ruleid and the rule it contains
+ // From the user point of view, the rule is more meaningfull.
+ ss << "rule " << name << " already exists";
err = 0;
goto reply;
}
_get_pending_crush(newcrush);
if (newcrush.rule_exists(name)) {
- // The name is uniquely associated to a ruleid and the ruleset it contains
- // From the user point of view, the ruleset is more meaningfull.
- ss << "ruleset " << name << " already exists";
+ // The name is uniquely associated to a ruleid and the rule it contains
+ // From the user point of view, the rule is more meaningfull.
+ ss << "rule " << name << " already exists";
err = 0;
} else {
- int ruleno = newcrush.add_simple_ruleset(name, root, type, mode,
+ int ruleno = newcrush.add_simple_rule(name, root, type, "", mode,
pg_pool_t::TYPE_REPLICATED, &ss);
if (ruleno < 0) {
err = ruleno;
get_last_committed() + 1));
return true;
+ } else if (prefix == "osd crush rule create-replicated") {
+ string name, root, type, device_class;
+ cmd_getval(g_ceph_context, cmdmap, "name", name);
+ cmd_getval(g_ceph_context, cmdmap, "root", root);
+ cmd_getval(g_ceph_context, cmdmap, "type", type);
+ cmd_getval(g_ceph_context, cmdmap, "class", device_class);
+
+ if (!device_class.empty()) {
+ if (osdmap.require_osd_release < CEPH_RELEASE_LUMINOUS) {
+ ss << "you must complete the upgrade and 'ceph osd require-osd-release "
+ << "luminous' before using crush device classes";
+ err = -EPERM;
+ goto reply;
+ }
+ }
+
+ if (osdmap.crush->rule_exists(name)) {
+ // The name is uniquely associated to a ruleid and the rule it contains
+ // From the user point of view, the rule is more meaningfull.
+ ss << "rule " << name << " already exists";
+ err = 0;
+ goto reply;
+ }
+
+ CrushWrapper newcrush;
+ _get_pending_crush(newcrush);
+
+ if (newcrush.rule_exists(name)) {
+ // The name is uniquely associated to a ruleid and the rule it contains
+ // From the user point of view, the rule is more meaningfull.
+ ss << "rule " << name << " already exists";
+ err = 0;
+ } else {
+ int ruleno = newcrush.add_simple_rule(
+ name, root, type, device_class,
+ "firstn", pg_pool_t::TYPE_REPLICATED, &ss);
+ if (ruleno < 0) {
+ err = ruleno;
+ goto reply;
+ }
+
+ pending_inc.crush.clear();
+ newcrush.encode(pending_inc.crush, mon->get_quorum_con_features());
+ }
+ getline(ss, rs);
+ wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+ get_last_committed() + 1));
+ return true;
+
} else if (prefix == "osd erasure-code-profile rm") {
string name;
cmd_getval(g_ceph_context, cmdmap, "name", name);
}
}
- int ruleset;
- err = crush_ruleset_create_erasure(name, profile, &ruleset, &ss);
+ int rule;
+ err = crush_rule_create_erasure(name, profile, &rule, &ss);
if (err < 0) {
switch(err) {
case -EEXIST: // return immediately
break;
}
} else {
- ss << "created ruleset " << name << " at " << ruleset;
+ ss << "created rule " << name << " at " << rule;
}
getline(ss, rs);
// FIXME: this is ok in some situations, but let's not bother with that
// complexity now.
int ruleset = newcrush.get_rule_mask_ruleset(ruleno);
- if (osdmap.crush_ruleset_in_use(ruleset)) {
+ if (osdmap.crush_rule_in_use(ruleset)) {
ss << "crush ruleset " << name << " " << ruleset << " is in use";
err = -EBUSY;
goto reply;
get_last_committed() + 1));
return true;
+ } else if (prefix == "osd crush rule rename") {
+ string srcname;
+ string dstname;
+ cmd_getval(g_ceph_context, cmdmap, "srcname", srcname);
+ cmd_getval(g_ceph_context, cmdmap, "dstname", dstname);
+ if (srcname.empty() || dstname.empty()) {
+ ss << "must specify both source rule name and destination rule name";
+ err = -EINVAL;
+ goto reply;
+ }
+ if (srcname == dstname) {
+ ss << "destination rule name is equal to source rule name";
+ err = 0;
+ goto reply;
+ }
+
+ CrushWrapper newcrush;
+ _get_pending_crush(newcrush);
+ if (!newcrush.rule_exists(srcname) && newcrush.rule_exists(dstname)) {
+ // srcname does not exist and dstname already exists
+ // suppose this is a replay and return success
+ // (so this command is idempotent)
+ ss << "already renamed to '" << dstname << "'";
+ err = 0;
+ goto reply;
+ }
+
+ err = newcrush.rename_rule(srcname, dstname, &ss);
+ if (err < 0) {
+ // ss has reason for failure
+ goto reply;
+ }
+ pending_inc.crush.clear();
+ newcrush.encode(pending_inc.crush, mon->get_quorum_con_features());
+ getline(ss, rs);
+ wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+ get_last_committed() + 1));
+ return true;
+
} else if (prefix == "osd setmaxosd") {
int64_t newmax;
if (!cmd_getval(g_ceph_context, cmdmap, "newmax", newmax)) {
} else if (prefix == "osd set-full-ratio" ||
prefix == "osd set-backfillfull-ratio" ||
prefix == "osd set-nearfull-ratio") {
- if (!osdmap.test_flag(CEPH_OSDMAP_REQUIRE_LUMINOUS)) {
- ss << "you must complete the upgrade and set require_luminous_osds before"
- << " using the new interface";
+ if (osdmap.require_osd_release < CEPH_RELEASE_LUMINOUS) {
+ ss << "you must complete the upgrade and 'ceph osd require-osd-release "
+ << "luminous' before using the new interface";
err = -EPERM;
goto reply;
}
double n;
if (!cmd_getval(g_ceph_context, cmdmap, "ratio", n)) {
ss << "unable to parse 'ratio' value '"
- << cmd_vartype_stringify(cmdmap["who"]) << "'";
+ << cmd_vartype_stringify(cmdmap["ratio"]) << "'";
err = -EINVAL;
goto reply;
}
get_last_committed() + 1));
return true;
} else if (prefix == "osd set-require-min-compat-client") {
- if (!osdmap.test_flag(CEPH_OSDMAP_REQUIRE_LUMINOUS)) {
- ss << "you must complete the upgrade and set require_luminous_osds before"
- << " using the new interface";
+ if (osdmap.require_osd_release < CEPH_RELEASE_LUMINOUS) {
+ ss << "you must complete the upgrade and 'ceph osd require-osd-release "
+ << "luminous' before using the new interface";
err = -EPERM;
goto reply;
}
string v;
cmd_getval(g_ceph_context, cmdmap, "version", v);
- if (v != "luminous" && v != "kraken" && v != "jewel" && v != "infernalis" &&
- v != "hammer" && v != "giant" && v != "firefly" && v != "emperor" &&
- v != "dumpling" && v != "cuttlefish" && v != "bobtail" && v != "argonaut") {
+ int vno = ceph_release_from_name(v.c_str());
+ if (vno <= 0) {
ss << "version " << v << " is not recognized";
err = -EINVAL;
goto reply;
OSDMap newmap;
newmap.deepish_copy_from(osdmap);
newmap.apply_incremental(pending_inc);
- newmap.require_min_compat_client = v;
- auto mv = newmap.get_min_compat_client();
- if (v < mv.first) {
- ss << "osdmap current utilizes features that require " << mv
- << "; cannot set require_min_compat_client below that to " << v;
+ newmap.require_min_compat_client = vno;
+ auto mvno = newmap.get_min_compat_client();
+ if (vno < mvno) {
+ ss << "osdmap current utilizes features that require "
+ << ceph_release_name(mvno)
+ << "; cannot set require_min_compat_client below that to "
+ << ceph_release_name(vno);
err = -EPERM;
goto reply;
}
- ss << "set require_min_compat_client to " << v;
- pending_inc.new_require_min_compat_client = v;
+ string sure;
+ cmd_getval(g_ceph_context, cmdmap, "sure", sure);
+ if (sure != "--yes-i-really-mean-it") {
+ FeatureMap m;
+ mon->get_combined_feature_map(&m);
+ uint64_t features = ceph_release_features(vno);
+ bool first = true;
+ bool ok = true;
+ for (int type : {
+ CEPH_ENTITY_TYPE_CLIENT,
+ CEPH_ENTITY_TYPE_MDS,
+ CEPH_ENTITY_TYPE_MGR }) {
+ auto p = m.m.find(type);
+ if (p == m.m.end()) {
+ continue;
+ }
+ for (auto& q : p->second) {
+ uint64_t missing = ~q.first & features;
+ if (missing) {
+ if (first) {
+ ss << "cannot set require_min_compat_client to " << v << ": ";
+ } else {
+ ss << "; ";
+ }
+ first = false;
+ ss << q.second << " connected " << ceph_entity_type_name(type)
+ << "(s) look like " << ceph_release_name(
+ ceph_release_from_features(q.first))
+ << " (missing 0x" << std::hex << missing << std::dec << ")";
+ ok = false;
+ }
+ }
+ }
+ if (!ok) {
+ ss << "; add --yes-i-really-mean-it to do it anyway";
+ err = -EPERM;
+ goto reply;
+ }
+ }
+ ss << "set require_min_compat_client to " << ceph_release_name(vno);
+ pending_inc.new_require_min_compat_client = vno;
getline(ss, rs);
wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
get_last_committed() + 1));
return prepare_unset_flag(op, CEPH_OSDMAP_PAUSERD | CEPH_OSDMAP_PAUSEWR);
} else if (prefix == "osd set") {
+ string sure;
+ cmd_getval(g_ceph_context, cmdmap, "sure", sure);
string key;
cmd_getval(g_ceph_context, cmdmap, "key", key);
if (key == "full")
else if (key == "notieragent")
return prepare_set_flag(op, CEPH_OSDMAP_NOTIERAGENT);
else if (key == "sortbitwise") {
- if (osdmap.get_up_osd_features() & CEPH_FEATURE_OSD_BITWISE_HOBJ_SORT) {
+ if (!osdmap.get_num_up_osds() && sure != "--yes-i-really-mean-it") {
+ ss << "Not advisable to continue since no OSDs are up. Pass "
+ << "--yes-i-really-mean-it if you really wish to continue.";
+ err = -EPERM;
+ goto reply;
+ }
+ if ((osdmap.get_up_osd_features() & CEPH_FEATURE_OSD_BITWISE_HOBJ_SORT)
+ || sure == "--yes-i-really-mean-it") {
return prepare_set_flag(op, CEPH_OSDMAP_SORTBITWISE);
} else {
ss << "not all up OSDs have OSD_BITWISE_HOBJ_SORT feature";
err = -EPERM;
+ goto reply;
+ }
+ } else if (key == "recovery_deletes") {
+ if (!osdmap.get_num_up_osds() && sure != "--yes-i-really-mean-it") {
+ ss << "Not advisable to continue since no OSDs are up. Pass "
+ << "--yes-i-really-mean-it if you really wish to continue.";
+ err = -EPERM;
+ goto reply;
+ }
+ if (HAVE_FEATURE(osdmap.get_up_osd_features(), OSD_RECOVERY_DELETES)
+ || sure == "--yes-i-really-mean-it") {
+ return prepare_set_flag(op, CEPH_OSDMAP_RECOVERY_DELETES);
+ } else {
+ ss << "not all up OSDs have OSD_RECOVERY_DELETES feature";
+ err = -EPERM;
+ goto reply;
}
} else if (key == "require_jewel_osds") {
+ if (!osdmap.get_num_up_osds() && sure != "--yes-i-really-mean-it") {
+ ss << "Not advisable to continue since no OSDs are up. Pass "
+ << "--yes-i-really-mean-it if you really wish to continue.";
+ err = -EPERM;
+ goto reply;
+ }
if (!osdmap.test_flag(CEPH_OSDMAP_SORTBITWISE)) {
ss << "the sortbitwise flag must be set before require_jewel_osds";
err = -EPERM;
- } else if (HAVE_FEATURE(osdmap.get_up_osd_features(), SERVER_JEWEL)) {
+ goto reply;
+ } else if (osdmap.require_osd_release >= CEPH_RELEASE_JEWEL) {
+ ss << "require_osd_release is already >= jewel";
+ err = 0;
+ goto reply;
+ } else if (HAVE_FEATURE(osdmap.get_up_osd_features(), SERVER_JEWEL)
+ || sure == "--yes-i-really-mean-it") {
return prepare_set_flag(op, CEPH_OSDMAP_REQUIRE_JEWEL);
} else {
ss << "not all up OSDs have CEPH_FEATURE_SERVER_JEWEL feature";
err = -EPERM;
}
} else if (key == "require_kraken_osds") {
+ if (!osdmap.get_num_up_osds() && sure != "--yes-i-really-mean-it") {
+ ss << "Not advisable to continue since no OSDs are up. Pass "
+ << "--yes-i-really-mean-it if you really wish to continue.";
+ err = -EPERM;
+ goto reply;
+ }
if (!osdmap.test_flag(CEPH_OSDMAP_SORTBITWISE)) {
ss << "the sortbitwise flag must be set before require_kraken_osds";
err = -EPERM;
- } else if (HAVE_FEATURE(osdmap.get_up_osd_features(), SERVER_KRAKEN)) {
+ goto reply;
+ } else if (osdmap.require_osd_release >= CEPH_RELEASE_KRAKEN) {
+ ss << "require_osd_release is already >= kraken";
+ err = 0;
+ goto reply;
+ } else if (HAVE_FEATURE(osdmap.get_up_osd_features(), SERVER_KRAKEN)
+ || sure == "--yes-i-really-mean-it") {
bool r = prepare_set_flag(op, CEPH_OSDMAP_REQUIRE_KRAKEN);
// ensure JEWEL is also set
pending_inc.new_flags |= CEPH_OSDMAP_REQUIRE_JEWEL;
ss << "not all up OSDs have CEPH_FEATURE_SERVER_KRAKEN feature";
err = -EPERM;
}
- } else if (key == "require_luminous_osds") {
- if (!osdmap.test_flag(CEPH_OSDMAP_SORTBITWISE)) {
- ss << "the sortbitwise flag must be set before require_luminous_osds";
- err = -EPERM;
- } else if (HAVE_FEATURE(osdmap.get_up_osd_features(), SERVER_LUMINOUS)) {
- bool r = prepare_set_flag(op, CEPH_OSDMAP_REQUIRE_LUMINOUS);
- // ensure JEWEL and KRAKEN are also set
- pending_inc.new_flags |= CEPH_OSDMAP_REQUIRE_JEWEL;
- pending_inc.new_flags |= CEPH_OSDMAP_REQUIRE_KRAKEN;
- return r;
- } else {
- ss << "not all up OSDs have CEPH_FEATURE_SERVER_LUMINOUS feature";
- err = -EPERM;
- }
} else {
ss << "unrecognized flag '" << key << "'";
err = -EINVAL;
return prepare_unset_flag(op, CEPH_OSDMAP_NODEEP_SCRUB);
else if (key == "notieragent")
return prepare_unset_flag(op, CEPH_OSDMAP_NOTIERAGENT);
- else if (key == "sortbitwise") {
- ss << "the sortbitwise flag is required and cannot be unset";
- err = -EPERM;
- } else {
+ else {
ss << "unrecognized flag '" << key << "'";
err = -EINVAL;
}
- } else if (prefix == "osd cluster_snap") {
- // ** DISABLE THIS FOR NOW **
- ss << "cluster snapshot currently disabled (broken implementation)";
- // ** DISABLE THIS FOR NOW **
+ } else if (prefix == "osd require-osd-release") {
+ string release;
+ cmd_getval(g_ceph_context, cmdmap, "release", release);
+ string sure;
+ cmd_getval(g_ceph_context, cmdmap, "sure", sure);
+ if (!osdmap.test_flag(CEPH_OSDMAP_SORTBITWISE)) {
+ ss << "the sortbitwise flag must be set first";
+ err = -EPERM;
+ goto reply;
+ }
+ int rel = ceph_release_from_name(release.c_str());
+ if (rel <= 0) {
+ ss << "unrecognized release " << release;
+ err = -EINVAL;
+ goto reply;
+ }
+ if (rel < CEPH_RELEASE_LUMINOUS) {
+ ss << "use this command only for luminous and later";
+ err = -EINVAL;
+ goto reply;
+ }
+ if (rel == osdmap.require_osd_release) {
+ // idempotent
+ err = 0;
+ goto reply;
+ }
+ if (rel == CEPH_RELEASE_LUMINOUS) {
+ if (!HAVE_FEATURE(osdmap.get_up_osd_features(), SERVER_LUMINOUS)) {
+ ss << "not all up OSDs have CEPH_FEATURE_SERVER_LUMINOUS feature";
+ err = -EPERM;
+ goto reply;
+ }
+ } else {
+ ss << "not supported for this release yet";
+ err = -EPERM;
+ goto reply;
+ }
+ if (rel < osdmap.require_osd_release) {
+ ss << "require_osd_release cannot be lowered once it has been set";
+ err = -EPERM;
+ goto reply;
+ }
+ pending_inc.new_require_osd_release = rel;
+ if (rel >= CEPH_RELEASE_LUMINOUS &&
+ !osdmap.test_flag(CEPH_OSDMAP_RECOVERY_DELETES)) {
+ return prepare_set_flag(op, CEPH_OSDMAP_RECOVERY_DELETES);
+ }
+ goto update;
+ } else if (prefix == "osd cluster_snap") {
+ // ** DISABLE THIS FOR NOW **
+ ss << "cluster snapshot currently disabled (broken implementation)";
+ // ** DISABLE THIS FOR NOW **
+
+ } else if (prefix == "osd down" ||
+ prefix == "osd out" ||
+ prefix == "osd in" ||
+ prefix == "osd rm") {
+
+ bool any = false;
+ bool stop = false;
+ bool verbose = true;
+
+ vector<string> idvec;
+ cmd_getval(g_ceph_context, cmdmap, "ids", idvec);
+ for (unsigned j = 0; j < idvec.size() && !stop; j++) {
+ set<int> osds;
+
+ // wildcard?
+ if (j == 0 &&
+ (idvec[0] == "any" || idvec[0] == "all" || idvec[0] == "*")) {
+ if (prefix == "osd in") {
+ // touch out osds only
+ osdmap.get_out_osds(osds);
+ } else {
+ osdmap.get_all_osds(osds);
+ }
+ stop = true;
+ verbose = false; // so the output is less noisy.
+ } else {
+ long osd = parse_osd_id(idvec[j].c_str(), &ss);
+ if (osd < 0) {
+ ss << "invalid osd id" << osd;
+ err = -EINVAL;
+ continue;
+ } else if (!osdmap.exists(osd)) {
+ ss << "osd." << osd << " does not exist. ";
+ continue;
+ }
+
+ osds.insert(osd);
+ }
+
+ for (auto &osd : osds) {
+ if (prefix == "osd down") {
+ if (osdmap.is_down(osd)) {
+ if (verbose)
+ ss << "osd." << osd << " is already down. ";
+ } else {
+ pending_inc.pending_osd_state_set(osd, CEPH_OSD_UP);
+ ss << "marked down osd." << osd << ". ";
+ any = true;
+ }
+ } else if (prefix == "osd out") {
+ if (osdmap.is_out(osd)) {
+ if (verbose)
+ ss << "osd." << osd << " is already out. ";
+ } else {
+ pending_inc.new_weight[osd] = CEPH_OSD_OUT;
+ if (osdmap.osd_weight[osd]) {
+ if (pending_inc.new_xinfo.count(osd) == 0) {
+ pending_inc.new_xinfo[osd] = osdmap.osd_xinfo[osd];
+ }
+ pending_inc.new_xinfo[osd].old_weight = osdmap.osd_weight[osd];
+ }
+ ss << "marked out osd." << osd << ". ";
+ std::ostringstream msg;
+ msg << "Client " << op->get_session()->entity_name
+ << " marked osd." << osd << " out";
+ if (osdmap.is_up(osd)) {
+ msg << ", while it was still marked up";
+ } else {
+ auto period = ceph_clock_now() - down_pending_out[osd];
+ msg << ", after it was down for " << int(period.sec())
+ << " seconds";
+ }
+
+ mon->clog->info() << msg.str();
+ any = true;
+ }
+ } else if (prefix == "osd in") {
+ if (osdmap.is_in(osd)) {
+ if (verbose)
+ ss << "osd." << osd << " is already in. ";
+ } else {
+ if (osdmap.osd_xinfo[osd].old_weight > 0) {
+ pending_inc.new_weight[osd] = osdmap.osd_xinfo[osd].old_weight;
+ if (pending_inc.new_xinfo.count(osd) == 0) {
+ pending_inc.new_xinfo[osd] = osdmap.osd_xinfo[osd];
+ }
+ pending_inc.new_xinfo[osd].old_weight = 0;
+ } else {
+ pending_inc.new_weight[osd] = CEPH_OSD_IN;
+ }
+ ss << "marked in osd." << osd << ". ";
+ any = true;
+ }
+ } else if (prefix == "osd rm") {
+ err = prepare_command_osd_remove(osd);
+
+ if (err == -EBUSY) {
+ if (any)
+ ss << ", ";
+ ss << "osd." << osd << " is still up; must be down before removal. ";
+ } else {
+ assert(err == 0);
+ if (any) {
+ ss << ", osd." << osd;
+ } else {
+ ss << "removed osd." << osd;
+ }
+ any = true;
+ }
+ }
+ }
+ }
+ if (any) {
+ getline(ss, rs);
+ wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, err, rs,
+ get_last_committed() + 1));
+ return true;
+ }
+ } else if (prefix == "osd add-noup" ||
+ prefix == "osd add-nodown" ||
+ prefix == "osd add-noin" ||
+ prefix == "osd add-noout") {
+
+ enum {
+ OP_NOUP,
+ OP_NODOWN,
+ OP_NOIN,
+ OP_NOOUT,
+ } option;
+
+ if (prefix == "osd add-noup") {
+ option = OP_NOUP;
+ } else if (prefix == "osd add-nodown") {
+ option = OP_NODOWN;
+ } else if (prefix == "osd add-noin") {
+ option = OP_NOIN;
+ } else {
+ option = OP_NOOUT;
+ }
+
+ bool any = false;
+ bool stop = false;
+
+ vector<string> idvec;
+ cmd_getval(g_ceph_context, cmdmap, "ids", idvec);
+ for (unsigned j = 0; j < idvec.size() && !stop; j++) {
+
+ set<int> osds;
+
+ // wildcard?
+ if (j == 0 &&
+ (idvec[0] == "any" || idvec[0] == "all" || idvec[0] == "*")) {
+ osdmap.get_all_osds(osds);
+ stop = true;
+ } else {
+ // try traditional single osd way
+
+ long osd = parse_osd_id(idvec[j].c_str(), &ss);
+ if (osd < 0) {
+ // ss has reason for failure
+ ss << ", unable to parse osd id:\"" << idvec[j] << "\". ";
+ err = -EINVAL;
+ continue;
+ }
+
+ osds.insert(osd);
+ }
+
+ for (auto &osd : osds) {
+
+ if (!osdmap.exists(osd)) {
+ ss << "osd." << osd << " does not exist. ";
+ continue;
+ }
+
+ switch (option) {
+ case OP_NOUP:
+ if (osdmap.is_up(osd)) {
+ ss << "osd." << osd << " is already up. ";
+ continue;
+ }
+
+ if (osdmap.is_noup(osd)) {
+ if (pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NOUP))
+ any = true;
+ } else {
+ pending_inc.pending_osd_state_set(osd, CEPH_OSD_NOUP);
+ any = true;
+ }
+
+ break;
+
+ case OP_NODOWN:
+ if (osdmap.is_down(osd)) {
+ ss << "osd." << osd << " is already down. ";
+ continue;
+ }
+
+ if (osdmap.is_nodown(osd)) {
+ if (pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NODOWN))
+ any = true;
+ } else {
+ pending_inc.pending_osd_state_set(osd, CEPH_OSD_NODOWN);
+ any = true;
+ }
+
+ break;
+
+ case OP_NOIN:
+ if (osdmap.is_in(osd)) {
+ ss << "osd." << osd << " is already in. ";
+ continue;
+ }
+
+ if (osdmap.is_noin(osd)) {
+ if (pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NOIN))
+ any = true;
+ } else {
+ pending_inc.pending_osd_state_set(osd, CEPH_OSD_NOIN);
+ any = true;
+ }
- } else if (prefix == "osd down" ||
- prefix == "osd out" ||
- prefix == "osd in" ||
- prefix == "osd rm") {
+ break;
+
+ case OP_NOOUT:
+ if (osdmap.is_out(osd)) {
+ ss << "osd." << osd << " is already out. ";
+ continue;
+ }
+
+ if (osdmap.is_noout(osd)) {
+ if (pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NOOUT))
+ any = true;
+ } else {
+ pending_inc.pending_osd_state_set(osd, CEPH_OSD_NOOUT);
+ any = true;
+ }
+
+ break;
+
+ default:
+ assert(0 == "invalid option");
+ }
+ }
+ }
+
+ if (any) {
+ getline(ss, rs);
+ wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, err, rs,
+ get_last_committed() + 1));
+ return true;
+ }
+ } else if (prefix == "osd rm-noup" ||
+ prefix == "osd rm-nodown" ||
+ prefix == "osd rm-noin" ||
+ prefix == "osd rm-noout") {
+
+ enum {
+ OP_NOUP,
+ OP_NODOWN,
+ OP_NOIN,
+ OP_NOOUT,
+ } option;
+
+ if (prefix == "osd rm-noup") {
+ option = OP_NOUP;
+ } else if (prefix == "osd rm-nodown") {
+ option = OP_NODOWN;
+ } else if (prefix == "osd rm-noin") {
+ option = OP_NOIN;
+ } else {
+ option = OP_NOOUT;
+ }
bool any = false;
+ bool stop = false;
vector<string> idvec;
cmd_getval(g_ceph_context, cmdmap, "ids", idvec);
- for (unsigned j = 0; j < idvec.size(); j++) {
- long osd = parse_osd_id(idvec[j].c_str(), &ss);
- if (osd < 0) {
- ss << "invalid osd id" << osd;
- err = -EINVAL;
- continue;
- } else if (!osdmap.exists(osd)) {
- ss << "osd." << osd << " does not exist. ";
- continue;
- }
- if (prefix == "osd down") {
- if (osdmap.is_down(osd)) {
- ss << "osd." << osd << " is already down. ";
- } else {
- pending_inc.new_state[osd] = CEPH_OSD_UP;
- ss << "marked down osd." << osd << ". ";
- any = true;
- }
- } else if (prefix == "osd out") {
- if (osdmap.is_out(osd)) {
- ss << "osd." << osd << " is already out. ";
- } else {
- pending_inc.new_weight[osd] = CEPH_OSD_OUT;
- if (osdmap.osd_weight[osd]) {
- if (pending_inc.new_xinfo.count(osd) == 0) {
- pending_inc.new_xinfo[osd] = osdmap.osd_xinfo[osd];
- }
- pending_inc.new_xinfo[osd].old_weight = osdmap.osd_weight[osd];
- }
- ss << "marked out osd." << osd << ". ";
- any = true;
- }
- } else if (prefix == "osd in") {
- if (osdmap.is_in(osd)) {
- ss << "osd." << osd << " is already in. ";
- } else {
- if (osdmap.osd_xinfo[osd].old_weight > 0) {
- pending_inc.new_weight[osd] = osdmap.osd_xinfo[osd].old_weight;
- if (pending_inc.new_xinfo.count(osd) == 0) {
- pending_inc.new_xinfo[osd] = osdmap.osd_xinfo[osd];
- }
- pending_inc.new_xinfo[osd].old_weight = 0;
- } else {
- pending_inc.new_weight[osd] = CEPH_OSD_IN;
- }
- ss << "marked in osd." << osd << ". ";
- any = true;
- }
- } else if (prefix == "osd rm") {
- if (osdmap.is_up(osd)) {
- if (any)
- ss << ", ";
- ss << "osd." << osd << " is still up; must be down before removal. ";
- err = -EBUSY;
- } else {
- pending_inc.new_state[osd] = osdmap.get_state(osd);
- pending_inc.new_uuid[osd] = uuid_d();
- pending_metadata_rm.insert(osd);
- if (any) {
- ss << ", osd." << osd;
- } else {
- ss << "removed osd." << osd;
+
+ for (unsigned j = 0; j < idvec.size() && !stop; j++) {
+
+ vector<int> osds;
+
+ // wildcard?
+ if (j == 0 &&
+ (idvec[0] == "any" || idvec[0] == "all" || idvec[0] == "*")) {
+
+ // touch previous noup/nodown/noin/noout osds only
+ switch (option) {
+ case OP_NOUP:
+ osdmap.get_noup_osds(&osds);
+ break;
+ case OP_NODOWN:
+ osdmap.get_nodown_osds(&osds);
+ break;
+ case OP_NOIN:
+ osdmap.get_noin_osds(&osds);
+ break;
+ case OP_NOOUT:
+ osdmap.get_noout_osds(&osds);
+ break;
+ default:
+ assert(0 == "invalid option");
+ }
+
+ // cancel any pending noup/nodown/noin/noout requests too
+ vector<int> pending_state_osds;
+ (void) pending_inc.get_pending_state_osds(&pending_state_osds);
+ for (auto &p : pending_state_osds) {
+
+ switch (option) {
+ case OP_NOUP:
+ if (!osdmap.is_noup(p) &&
+ pending_inc.pending_osd_state_clear(p, CEPH_OSD_NOUP)) {
+ any = true;
+ }
+ break;
+
+ case OP_NODOWN:
+ if (!osdmap.is_nodown(p) &&
+ pending_inc.pending_osd_state_clear(p, CEPH_OSD_NODOWN)) {
+ any = true;
+ }
+ break;
+
+ case OP_NOIN:
+ if (!osdmap.is_noin(p) &&
+ pending_inc.pending_osd_state_clear(p, CEPH_OSD_NOIN)) {
+ any = true;
+ }
+ break;
+
+ case OP_NOOUT:
+ if (!osdmap.is_noout(p) &&
+ pending_inc.pending_osd_state_clear(p, CEPH_OSD_NOOUT)) {
+ any = true;
+ }
+ break;
+
+ default:
+ assert(0 == "invalid option");
}
- any = true;
- }
+ }
+
+ stop = true;
+ } else {
+ // try traditional single osd way
+
+ long osd = parse_osd_id(idvec[j].c_str(), &ss);
+ if (osd < 0) {
+ // ss has reason for failure
+ ss << ", unable to parse osd id:\"" << idvec[j] << "\". ";
+ err = -EINVAL;
+ continue;
+ }
+
+ osds.push_back(osd);
+ }
+
+ for (auto &osd : osds) {
+
+ if (!osdmap.exists(osd)) {
+ ss << "osd." << osd << " does not exist. ";
+ continue;
+ }
+
+ switch (option) {
+ case OP_NOUP:
+ if (osdmap.is_noup(osd)) {
+ pending_inc.pending_osd_state_set(osd, CEPH_OSD_NOUP);
+ any = true;
+ } else if (pending_inc.pending_osd_state_clear(
+ osd, CEPH_OSD_NOUP)) {
+ any = true;
+ }
+ break;
+
+ case OP_NODOWN:
+ if (osdmap.is_nodown(osd)) {
+ pending_inc.pending_osd_state_set(osd, CEPH_OSD_NODOWN);
+ any = true;
+ } else if (pending_inc.pending_osd_state_clear(
+ osd, CEPH_OSD_NODOWN)) {
+ any = true;
+ }
+ break;
+
+ case OP_NOIN:
+ if (osdmap.is_noin(osd)) {
+ pending_inc.pending_osd_state_set(osd, CEPH_OSD_NOIN);
+ any = true;
+ } else if (pending_inc.pending_osd_state_clear(
+ osd, CEPH_OSD_NOIN)) {
+ any = true;
+ }
+ break;
+
+ case OP_NOOUT:
+ if (osdmap.is_noout(osd)) {
+ pending_inc.pending_osd_state_set(osd, CEPH_OSD_NOOUT);
+ any = true;
+ } else if (pending_inc.pending_osd_state_clear(
+ osd, CEPH_OSD_NOOUT)) {
+ any = true;
+ }
+ break;
+
+ default:
+ assert(0 == "invalid option");
+ }
}
}
+
if (any) {
getline(ss, rs);
wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, err, rs,
- get_last_committed() + 1));
+ get_last_committed() + 1));
return true;
}
} else if (prefix == "osd pg-temp") {
new_pg_temp.push_back(osd);
}
+ int pool_min_size = osdmap.get_pg_pool_min_size(pgid);
+ if ((int)new_pg_temp.size() < pool_min_size) {
+ ss << "num of osds (" << new_pg_temp.size() <<") < pool min size ("
+ << pool_min_size << ")";
+ err = -EINVAL;
+ goto reply;
+ }
+
+ int pool_size = osdmap.get_pg_pool_size(pgid);
+ if ((int)new_pg_temp.size() > pool_size) {
+ ss << "num of osds (" << new_pg_temp.size() <<") > pool size ("
+ << pool_size << ")";
+ err = -EINVAL;
+ goto reply;
+ }
+
pending_inc.new_pg_temp[pgid] = mempool::osdmap::vector<int>(
new_pg_temp.begin(), new_pg_temp.end());
ss << "set " << pgid << " pg_temp mapping to " << new_pg_temp;
goto reply;
}
- if (osdmap.require_min_compat_client.length() &&
- osdmap.require_min_compat_client < "firefly") {
- ss << "require_min_compat_client " << osdmap.require_min_compat_client
+ if (osdmap.require_min_compat_client > 0 &&
+ osdmap.require_min_compat_client < CEPH_RELEASE_FIREFLY) {
+ ss << "require_min_compat_client "
+ << ceph_release_name(osdmap.require_min_compat_client)
<< " < firefly, which is required for primary-temp";
err = -EPERM;
goto reply;
pending_inc.new_primary_temp[pgid] = osd;
ss << "set " << pgid << " primary_temp mapping to " << osd;
goto update;
- } else if (prefix == "osd pg-upmap") {
- if (!osdmap.test_flag(CEPH_OSDMAP_REQUIRE_LUMINOUS)) {
- ss << "you must set the require_luminous_osds flag to use this feature";
+ } else if (prefix == "osd pg-upmap" ||
+ prefix == "osd rm-pg-upmap" ||
+ prefix == "osd pg-upmap-items" ||
+ prefix == "osd rm-pg-upmap-items") {
+ if (osdmap.require_osd_release < CEPH_RELEASE_LUMINOUS) {
+ ss << "you must complete the upgrade and 'ceph osd require-osd-release "
+ << "luminous' before using the new interface";
err = -EPERM;
goto reply;
}
- if (osdmap.require_min_compat_client < "luminous") {
- ss << "min_compat_client " << osdmap.require_min_compat_client
- << " < luminous, which is required for pg-upmap";
+ if (osdmap.require_min_compat_client < CEPH_RELEASE_LUMINOUS) {
+ ss << "min_compat_client "
+ << ceph_release_name(osdmap.require_min_compat_client)
+ << " < luminous, which is required for pg-upmap. "
+ << "Try 'ceph osd set-require-min-compat-client luminous' "
+ << "before using the new interface";
err = -EPERM;
goto reply;
}
err = -ENOENT;
goto reply;
}
- if (pending_inc.new_pg_upmap.count(pgid) ||
- pending_inc.old_pg_upmap.count(pgid)) {
- dout(10) << __func__ << " waiting for pending update on " << pgid << dendl;
- wait_for_finished_proposal(op, new C_RetryMessage(this, op));
+ if (pending_inc.old_pools.count(pgid.pool())) {
+ ss << "pool of " << pgid << " is pending removal";
+ err = -ENOENT;
+ getline(ss, rs);
+ wait_for_finished_proposal(op,
+ new Monitor::C_Command(mon, op, err, rs, get_last_committed() + 1));
return true;
}
- vector<int64_t> id_vec;
- if (!cmd_getval(g_ceph_context, cmdmap, "id", id_vec)) {
- ss << "unable to parse 'id' value(s) '"
- << cmd_vartype_stringify(cmdmap["id"]) << "'";
- err = -EINVAL;
- goto reply;
- }
- vector<int32_t> new_pg_upmap;
- for (auto osd : id_vec) {
- if (osd != CRUSH_ITEM_NONE && !osdmap.exists(osd)) {
- ss << "osd." << osd << " does not exist";
- err = -ENOENT;
- goto reply;
+
+ enum {
+ OP_PG_UPMAP,
+ OP_RM_PG_UPMAP,
+ OP_PG_UPMAP_ITEMS,
+ OP_RM_PG_UPMAP_ITEMS,
+ } option;
+
+ if (prefix == "osd pg-upmap") {
+ option = OP_PG_UPMAP;
+ } else if (prefix == "osd rm-pg-upmap") {
+ option = OP_RM_PG_UPMAP;
+ } else if (prefix == "osd pg-upmap-items") {
+ option = OP_PG_UPMAP_ITEMS;
+ } else {
+ option = OP_RM_PG_UPMAP_ITEMS;
+ }
+
+ // check pending upmap changes
+ switch (option) {
+ case OP_PG_UPMAP: // fall through
+ case OP_RM_PG_UPMAP:
+ if (pending_inc.new_pg_upmap.count(pgid) ||
+ pending_inc.old_pg_upmap.count(pgid)) {
+ dout(10) << __func__ << " waiting for pending update on "
+ << pgid << dendl;
+ wait_for_finished_proposal(op, new C_RetryMessage(this, op));
+ return true;
}
- new_pg_upmap.push_back(osd);
- }
+ break;
- pending_inc.new_pg_upmap[pgid] = mempool::osdmap::vector<int32_t>(
- new_pg_upmap.begin(), new_pg_upmap.end());
- ss << "set " << pgid << " pg_upmap mapping to " << new_pg_upmap;
- goto update;
- } else if (prefix == "osd rm-pg-upmap") {
- if (!osdmap.test_flag(CEPH_OSDMAP_REQUIRE_LUMINOUS)) {
- ss << "you must set the require_luminous_osds flag to use this feature";
- err = -EPERM;
- goto reply;
- }
- if (osdmap.require_min_compat_client < "luminous") {
- ss << "require_min_compat_client " << osdmap.require_min_compat_client
- << " < luminous, which is required for pg-upmap";
- err = -EPERM;
- goto reply;
- }
- err = check_cluster_features(CEPH_FEATUREMASK_OSDMAP_PG_UPMAP, ss);
- if (err == -EAGAIN)
- goto wait;
- if (err < 0)
- goto reply;
- string pgidstr;
- if (!cmd_getval(g_ceph_context, cmdmap, "pgid", pgidstr)) {
- ss << "unable to parse 'pgid' value '"
- << cmd_vartype_stringify(cmdmap["pgid"]) << "'";
- err = -EINVAL;
- goto reply;
- }
- pg_t pgid;
- if (!pgid.parse(pgidstr.c_str())) {
- ss << "invalid pgid '" << pgidstr << "'";
- err = -EINVAL;
- goto reply;
- }
- if (!osdmap.pg_exists(pgid)) {
- ss << "pg " << pgid << " does not exist";
- err = -ENOENT;
- goto reply;
- }
- if (pending_inc.new_pg_upmap.count(pgid) ||
- pending_inc.old_pg_upmap.count(pgid)) {
- dout(10) << __func__ << " waiting for pending update on " << pgid << dendl;
- wait_for_finished_proposal(op, new C_RetryMessage(this, op));
- return true;
- }
+ case OP_PG_UPMAP_ITEMS: // fall through
+ case OP_RM_PG_UPMAP_ITEMS:
+ if (pending_inc.new_pg_upmap_items.count(pgid) ||
+ pending_inc.old_pg_upmap_items.count(pgid)) {
+ dout(10) << __func__ << " waiting for pending update on "
+ << pgid << dendl;
+ wait_for_finished_proposal(op, new C_RetryMessage(this, op));
+ return true;
+ }
+ break;
- pending_inc.old_pg_upmap.insert(pgid);
- ss << "clear " << pgid << " pg_upmap mapping";
- goto update;
- } else if (prefix == "osd pg-upmap-items") {
- if (!osdmap.test_flag(CEPH_OSDMAP_REQUIRE_LUMINOUS)) {
- ss << "you must set the require_luminous_osds flag to use this feature";
- err = -EPERM;
- goto reply;
- }
- if (osdmap.require_min_compat_client < "luminous") {
- ss << "require_min_compat_client " << osdmap.require_min_compat_client
- << " < luminous, which is required for pg-upmap";
- err = -EPERM;
- goto reply;
- }
- err = check_cluster_features(CEPH_FEATUREMASK_OSDMAP_PG_UPMAP, ss);
- if (err == -EAGAIN)
- goto wait;
- if (err < 0)
- goto reply;
- string pgidstr;
- if (!cmd_getval(g_ceph_context, cmdmap, "pgid", pgidstr)) {
- ss << "unable to parse 'pgid' value '"
- << cmd_vartype_stringify(cmdmap["pgid"]) << "'";
- err = -EINVAL;
- goto reply;
- }
- pg_t pgid;
- if (!pgid.parse(pgidstr.c_str())) {
- ss << "invalid pgid '" << pgidstr << "'";
- err = -EINVAL;
- goto reply;
- }
- if (!osdmap.pg_exists(pgid)) {
- ss << "pg " << pgid << " does not exist";
- err = -ENOENT;
- goto reply;
- }
- if (pending_inc.new_pg_upmap_items.count(pgid) ||
- pending_inc.old_pg_upmap_items.count(pgid)) {
- dout(10) << __func__ << " waiting for pending update on " << pgid << dendl;
- wait_for_finished_proposal(op, new C_RetryMessage(this, op));
- return true;
- }
- vector<int64_t> id_vec;
- if (!cmd_getval(g_ceph_context, cmdmap, "id", id_vec)) {
- ss << "unable to parse 'id' value(s) '"
- << cmd_vartype_stringify(cmdmap["id"]) << "'";
- err = -EINVAL;
- goto reply;
- }
- if (id_vec.size() % 2) {
- ss << "you must specify pairs of osd ids to be remapped";
- err = -EINVAL;
- goto reply;
+ default:
+ assert(0 == "invalid option");
}
- vector<pair<int32_t,int32_t>> new_pg_upmap_items;
- for (auto p = id_vec.begin(); p != id_vec.end(); ++p) {
- int from = *p++;
- int to = *p;
- if (!osdmap.exists(from)) {
- ss << "osd." << from << " does not exist";
- err = -ENOENT;
- goto reply;
+
+ switch (option) {
+ case OP_PG_UPMAP:
+ {
+ vector<int64_t> id_vec;
+ if (!cmd_getval(g_ceph_context, cmdmap, "id", id_vec)) {
+ ss << "unable to parse 'id' value(s) '"
+ << cmd_vartype_stringify(cmdmap["id"]) << "'";
+ err = -EINVAL;
+ goto reply;
+ }
+
+ int pool_min_size = osdmap.get_pg_pool_min_size(pgid);
+ if ((int)id_vec.size() < pool_min_size) {
+ ss << "num of osds (" << id_vec.size() <<") < pool min size ("
+ << pool_min_size << ")";
+ err = -EINVAL;
+ goto reply;
+ }
+
+ int pool_size = osdmap.get_pg_pool_size(pgid);
+ if ((int)id_vec.size() > pool_size) {
+ ss << "num of osds (" << id_vec.size() <<") > pool size ("
+ << pool_size << ")";
+ err = -EINVAL;
+ goto reply;
+ }
+
+ vector<int32_t> new_pg_upmap;
+ for (auto osd : id_vec) {
+ if (osd != CRUSH_ITEM_NONE && !osdmap.exists(osd)) {
+ ss << "osd." << osd << " does not exist";
+ err = -ENOENT;
+ goto reply;
+ }
+ auto it = std::find(new_pg_upmap.begin(), new_pg_upmap.end(), osd);
+ if (it != new_pg_upmap.end()) {
+ ss << "osd." << osd << " already exists, ";
+ continue;
+ }
+ new_pg_upmap.push_back(osd);
+ }
+
+ if (new_pg_upmap.empty()) {
+ ss << "no valid upmap items(pairs) is specified";
+ err = -EINVAL;
+ goto reply;
+ }
+
+ pending_inc.new_pg_upmap[pgid] = mempool::osdmap::vector<int32_t>(
+ new_pg_upmap.begin(), new_pg_upmap.end());
+ ss << "set " << pgid << " pg_upmap mapping to " << new_pg_upmap;
+ }
+ break;
+
+ case OP_RM_PG_UPMAP:
+ {
+ pending_inc.old_pg_upmap.insert(pgid);
+ ss << "clear " << pgid << " pg_upmap mapping";
+ }
+ break;
+
+ case OP_PG_UPMAP_ITEMS:
+ {
+ vector<int64_t> id_vec;
+ if (!cmd_getval(g_ceph_context, cmdmap, "id", id_vec)) {
+ ss << "unable to parse 'id' value(s) '"
+ << cmd_vartype_stringify(cmdmap["id"]) << "'";
+ err = -EINVAL;
+ goto reply;
+ }
+
+ if (id_vec.size() % 2) {
+ ss << "you must specify pairs of osd ids to be remapped";
+ err = -EINVAL;
+ goto reply;
+ }
+
+ int pool_size = osdmap.get_pg_pool_size(pgid);
+ if ((int)(id_vec.size() / 2) > pool_size) {
+ ss << "num of osd pairs (" << id_vec.size() / 2 <<") > pool size ("
+ << pool_size << ")";
+ err = -EINVAL;
+ goto reply;
+ }
+
+ vector<pair<int32_t,int32_t>> new_pg_upmap_items;
+ ostringstream items;
+ items << "[";
+ for (auto p = id_vec.begin(); p != id_vec.end(); ++p) {
+ int from = *p++;
+ int to = *p;
+ if (from == to) {
+ ss << "from osd." << from << " == to osd." << to << ", ";
+ continue;
+ }
+ if (!osdmap.exists(from)) {
+ ss << "osd." << from << " does not exist";
+ err = -ENOENT;
+ goto reply;
+ }
+ if (to != CRUSH_ITEM_NONE && !osdmap.exists(to)) {
+ ss << "osd." << to << " does not exist";
+ err = -ENOENT;
+ goto reply;
+ }
+ pair<int32_t,int32_t> entry = make_pair(from, to);
+ auto it = std::find(new_pg_upmap_items.begin(),
+ new_pg_upmap_items.end(), entry);
+ if (it != new_pg_upmap_items.end()) {
+ ss << "osd." << from << " -> osd." << to << " already exists, ";
+ continue;
+ }
+ new_pg_upmap_items.push_back(entry);
+ items << from << "->" << to << ",";
+ }
+ string out(items.str());
+ out.resize(out.size() - 1); // drop last ','
+ out += "]";
+
+ if (new_pg_upmap_items.empty()) {
+ ss << "no valid upmap items(pairs) is specified";
+ err = -EINVAL;
+ goto reply;
+ }
+
+ pending_inc.new_pg_upmap_items[pgid] =
+ mempool::osdmap::vector<pair<int32_t,int32_t>>(
+ new_pg_upmap_items.begin(), new_pg_upmap_items.end());
+ ss << "set " << pgid << " pg_upmap_items mapping to " << out;
}
- if (to != CRUSH_ITEM_NONE && !osdmap.exists(to)) {
- ss << "osd." << to << " does not exist";
- err = -ENOENT;
- goto reply;
+ break;
+
+ case OP_RM_PG_UPMAP_ITEMS:
+ {
+ pending_inc.old_pg_upmap_items.insert(pgid);
+ ss << "clear " << pgid << " pg_upmap_items mapping";
}
- new_pg_upmap_items.push_back(make_pair(from, to));
- }
+ break;
- pending_inc.new_pg_upmap_items[pgid] =
- mempool::osdmap::vector<pair<int32_t,int32_t>>(
- new_pg_upmap_items.begin(), new_pg_upmap_items.end());
- ss << "set " << pgid << " pg_upmap_items mapping to " << new_pg_upmap_items;
- goto update;
- } else if (prefix == "osd rm-pg-upmap-items") {
- if (!osdmap.test_flag(CEPH_OSDMAP_REQUIRE_LUMINOUS)) {
- ss << "you must set the require_luminous_osds flag to use this feature";
- err = -EPERM;
- goto reply;
- }
- if (osdmap.require_min_compat_client < "luminous") {
- ss << "require_min_compat_client " << osdmap.require_min_compat_client
- << " < luminous, which is required for pg-upmap";
- err = -EPERM;
- goto reply;
- }
- err = check_cluster_features(CEPH_FEATUREMASK_OSDMAP_PG_UPMAP, ss);
- if (err == -EAGAIN)
- goto wait;
- if (err < 0)
- goto reply;
- string pgidstr;
- if (!cmd_getval(g_ceph_context, cmdmap, "pgid", pgidstr)) {
- ss << "unable to parse 'pgid' value '"
- << cmd_vartype_stringify(cmdmap["pgid"]) << "'";
- err = -EINVAL;
- goto reply;
- }
- pg_t pgid;
- if (!pgid.parse(pgidstr.c_str())) {
- ss << "invalid pgid '" << pgidstr << "'";
- err = -EINVAL;
- goto reply;
- }
- if (!osdmap.pg_exists(pgid)) {
- ss << "pg " << pgid << " does not exist";
- err = -ENOENT;
- goto reply;
- }
- if (pending_inc.new_pg_upmap_items.count(pgid) ||
- pending_inc.old_pg_upmap_items.count(pgid)) {
- dout(10) << __func__ << " waiting for pending update on " << pgid << dendl;
- wait_for_finished_proposal(op, new C_RetryMessage(this, op));
- return true;
+ default:
+ assert(0 == "invalid option");
}
- pending_inc.old_pg_upmap_items.insert(pgid);
- ss << "clear " << pgid << " pg_upmap_items mapping";
goto update;
} else if (prefix == "osd primary-affinity") {
int64_t id;
err = -EINVAL;
goto reply;
}
- if (osdmap.require_min_compat_client.length() &&
- osdmap.require_min_compat_client < "firefly") {
- ss << "require_min_compat_client " << osdmap.require_min_compat_client
+ if (osdmap.require_min_compat_client > 0 &&
+ osdmap.require_min_compat_client < CEPH_RELEASE_FIREFLY) {
+ ss << "require_min_compat_client "
+ << ceph_release_name(osdmap.require_min_compat_client)
<< " < firefly, which is required for primary-affinity";
err = -EPERM;
goto reply;
wait_for_finished_proposal(
op,
new Monitor::C_Command(mon, op, 0, rs, rdata, get_last_committed() + 1));
- return true;
+ return true;
} else if (prefix == "osd lost") {
int64_t id;
if (!cmd_getval(g_ceph_context, cmdmap, "id", id)) {
return true;
}
+ } else if (prefix == "osd destroy" || prefix == "osd purge") {
+ /* Destroying an OSD means that we don't expect to further make use of
+ * the OSDs data (which may even become unreadable after this operation),
+ * and that we are okay with scrubbing all its cephx keys and config-key
+ * data (which may include lockbox keys, thus rendering the osd's data
+ * unreadable).
+ *
+ * The OSD will not be removed. Instead, we will mark it as destroyed,
+ * such that a subsequent call to `create` will not reuse the osd id.
+ * This will play into being able to recreate the OSD, at the same
+ * crush location, with minimal data movement.
+ */
+
+ // make sure authmon is writeable.
+ if (!mon->authmon()->is_writeable()) {
+ dout(10) << __func__ << " waiting for auth mon to be writeable for "
+ << "osd destroy" << dendl;
+ mon->authmon()->wait_for_writeable(op, new C_RetryMessage(this, op));
+ return false;
+ }
+
+ int64_t id;
+ if (!cmd_getval(g_ceph_context, cmdmap, "id", id)) {
+ ss << "unable to parse osd id value '"
+ << cmd_vartype_stringify(cmdmap["id"]) << "";
+ err = -EINVAL;
+ goto reply;
+ }
+
+ bool is_destroy = (prefix == "osd destroy");
+ if (!is_destroy) {
+ assert("osd purge" == prefix);
+ }
+
+ string sure;
+ if (!cmd_getval(g_ceph_context, cmdmap, "sure", sure) ||
+ sure != "--yes-i-really-mean-it") {
+ ss << "Are you SURE? This will mean real, permanent data loss, as well "
+ << "as cephx and lockbox keys. Pass --yes-i-really-mean-it if you "
+ << "really do.";
+ err = -EPERM;
+ goto reply;
+ } else if (!osdmap.exists(id)) {
+ ss << "osd." << id << " does not exist";
+ err = 0; // idempotent
+ goto reply;
+ } else if (osdmap.is_up(id)) {
+ ss << "osd." << id << " is not `down`.";
+ err = -EBUSY;
+ goto reply;
+ } else if (is_destroy && osdmap.is_destroyed(id)) {
+ ss << "destroyed osd." << id;
+ err = 0;
+ goto reply;
+ }
+
+ bool goto_reply = false;
+
+ paxos->plug();
+ if (is_destroy) {
+ err = prepare_command_osd_destroy(id, ss);
+ // we checked above that it should exist.
+ assert(err != -ENOENT);
+ } else {
+ err = prepare_command_osd_purge(id, ss);
+ if (err == -ENOENT) {
+ err = 0;
+ ss << "osd." << id << " does not exist.";
+ goto_reply = true;
+ }
+ }
+ paxos->unplug();
+
+ if (err < 0 || goto_reply) {
+ goto reply;
+ }
+
+ if (is_destroy) {
+ ss << "destroyed osd." << id;
+ } else {
+ ss << "purged osd." << id;
+ }
+
+ getline(ss, rs);
+ wait_for_finished_proposal(op,
+ new Monitor::C_Command(mon, op, 0, rs, get_last_committed() + 1));
+ force_immediate_propose();
+ return true;
+
+ } else if (prefix == "osd new") {
+
+ // make sure authmon is writeable.
+ if (!mon->authmon()->is_writeable()) {
+ dout(10) << __func__ << " waiting for auth mon to be writeable for "
+ << "osd new" << dendl;
+ mon->authmon()->wait_for_writeable(op, new C_RetryMessage(this, op));
+ return false;
+ }
+
+ map<string,string> param_map;
+
+ bufferlist bl = m->get_data();
+ string param_json = bl.to_str();
+ dout(20) << __func__ << " osd new json = " << param_json << dendl;
+
+ err = get_json_str_map(param_json, ss, ¶m_map);
+ if (err < 0)
+ goto reply;
+
+ dout(20) << __func__ << " osd new params " << param_map << dendl;
+
+ paxos->plug();
+ err = prepare_command_osd_new(op, cmdmap, param_map, ss, f.get());
+ paxos->unplug();
+
+ if (err < 0) {
+ goto reply;
+ }
+
+ if (f) {
+ f->flush(rdata);
+ } else {
+ rdata.append(ss);
+ }
+
+ if (err == EEXIST) {
+ // idempotent operation
+ err = 0;
+ goto reply;
+ }
+
+ wait_for_finished_proposal(op,
+ new Monitor::C_Command(mon, op, 0, rs, rdata,
+ get_last_committed() + 1));
+ force_immediate_propose();
+ return true;
+
} else if (prefix == "osd create") {
- int i = -1;
// optional id provided?
- int64_t id = -1;
- if (cmd_getval(g_ceph_context, cmdmap, "id", id)) {
- if (id < 0) {
- ss << "invalid osd id value '" << id << "'";
+ int64_t id = -1, cmd_id = -1;
+ if (cmd_getval(g_ceph_context, cmdmap, "id", cmd_id)) {
+ if (cmd_id < 0) {
+ ss << "invalid osd id value '" << cmd_id << "'";
err = -EINVAL;
goto reply;
}
- dout(10) << " osd create got id " << id << dendl;
+ dout(10) << " osd create got id " << cmd_id << dendl;
}
- // optional uuid provided?
uuid_d uuid;
string uuidstr;
if (cmd_getval(g_ceph_context, cmdmap, "uuid", uuidstr)) {
if (!uuid.parse(uuidstr.c_str())) {
- ss << "invalid uuid value '" << uuidstr << "'";
- err = -EINVAL;
- goto reply;
- }
- dout(10) << " osd create got uuid " << uuid << dendl;
- i = osdmap.identify_osd(uuid);
- if (i >= 0) {
- // osd already exists
- if (id >= 0 && i != id) {
- ss << "uuid " << uuidstr << " already in use for different id " << i;
- err = -EINVAL;
- goto reply;
- }
- err = 0;
- if (f) {
- f->open_object_section("created_osd");
- f->dump_int("osdid", i);
- f->close_section();
- f->flush(rdata);
- } else {
- ss << i;
- rdata.append(ss);
- }
- goto reply;
- }
- // i < 0
- if (id >= 0) {
- if (osdmap.exists(id)) {
- ss << "id " << id << " already in use and does not match uuid "
- << uuid;
- err = -EINVAL;
- goto reply;
- }
- if (pending_inc.new_state.count(id)) {
- // osd is about to exist
- wait_for_finished_proposal(op, new C_RetryMessage(this, op));
- return true;
- }
- i = id;
- }
- if (pending_inc.identify_osd(uuid) >= 0) {
- // osd is about to exist
- wait_for_finished_proposal(op, new C_RetryMessage(this, op));
- return true;
- }
- if (i >= 0) {
- // raise max_osd
- if (osdmap.get_max_osd() <= i && pending_inc.new_max_osd <= i)
- pending_inc.new_max_osd = i + 1;
- goto done;
+ ss << "invalid uuid value '" << uuidstr << "'";
+ err = -EINVAL;
+ goto reply;
}
+ // we only care about the id if we also have the uuid, to
+ // ensure the operation's idempotency.
+ id = cmd_id;
}
- // allocate a new id
- for (i=0; i < osdmap.get_max_osd(); i++) {
- if (!osdmap.exists(i) &&
- pending_inc.new_up_client.count(i) == 0 &&
- (pending_inc.new_state.count(i) == 0 ||
- (pending_inc.new_state[i] & CEPH_OSD_EXISTS) == 0)) {
- pending_inc.new_weight[i] = CEPH_OSD_OUT;
- goto done;
+ int32_t new_id = -1;
+ err = prepare_command_osd_create(id, uuid, &new_id, ss);
+ if (err < 0) {
+ if (err == -EAGAIN) {
+ wait_for_finished_proposal(op, new C_RetryMessage(this, op));
+ return true;
+ }
+ // a check has failed; reply to the user.
+ goto reply;
+
+ } else if (err == EEXIST) {
+ // this is an idempotent operation; we can go ahead and reply.
+ if (f) {
+ f->open_object_section("created_osd");
+ f->dump_int("osdid", new_id);
+ f->close_section();
+ f->flush(rdata);
+ } else {
+ ss << new_id;
+ rdata.append(ss);
}
+ err = 0;
+ goto reply;
}
- // raise max_osd
- if (pending_inc.new_max_osd < 0)
- pending_inc.new_max_osd = osdmap.get_max_osd() + 1;
- else
- pending_inc.new_max_osd++;
- i = pending_inc.new_max_osd - 1;
-
-done:
- dout(10) << " creating osd." << i << dendl;
- pending_inc.new_state[i] |= CEPH_OSD_EXISTS | CEPH_OSD_NEW;
- if (!uuid.is_zero())
- pending_inc.new_uuid[i] = uuid;
+ string empty_device_class;
+ do_osd_create(id, uuid, empty_device_class, &new_id);
+
if (f) {
f->open_object_section("created_osd");
- f->dump_int("osdid", i);
+ f->dump_int("osdid", new_id);
f->close_section();
f->flush(rdata);
} else {
- ss << i;
+ ss << new_id;
rdata.append(ss);
}
- wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs, rdata,
- get_last_committed() + 1));
+ wait_for_finished_proposal(op,
+ new Monitor::C_Command(mon, op, 0, rs, rdata,
+ get_last_committed() + 1));
return true;
} else if (prefix == "osd blacklist clear") {
utime_t expires = ceph_clock_now();
double d;
// default one hour
- cmd_getval(g_ceph_context, cmdmap, "expire", d, double(60*60));
+ cmd_getval(g_ceph_context, cmdmap, "expire", d,
+ g_conf->mon_osd_blacklist_default_expire);
expires += d;
pending_inc.new_blacklist[addr] = expires;
+
+ {
+ // cancel any pending un-blacklisting request too
+ auto it = std::find(pending_inc.old_blacklist.begin(),
+ pending_inc.old_blacklist.end(), addr);
+ if (it != pending_inc.old_blacklist.end()) {
+ pending_inc.old_blacklist.erase(it);
+ }
+ }
+
ss << "blacklisting " << addr << " until " << expires << " (" << d << " sec)";
getline(ss, rs);
wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
string pool_type_str;
cmd_getval(g_ceph_context, cmdmap, "pool_type", pool_type_str);
if (pool_type_str.empty())
- pool_type_str = pg_pool_t::get_default_type();
+ pool_type_str = g_conf->osd_pool_default_type;
string poolstr;
cmd_getval(g_ceph_context, cmdmap, "pool", poolstr);
goto reply;
}
- bool implicit_ruleset_creation = false;
- string ruleset_name;
- cmd_getval(g_ceph_context, cmdmap, "ruleset", ruleset_name);
+ bool implicit_rule_creation = false;
+ int64_t expected_num_objects = 0;
+ string rule_name;
+ cmd_getval(g_ceph_context, cmdmap, "rule", rule_name);
string erasure_code_profile;
cmd_getval(g_ceph_context, cmdmap, "erasure_code_profile", erasure_code_profile);
goto wait;
}
}
- if (ruleset_name == "") {
- implicit_ruleset_creation = true;
+ if (rule_name == "") {
+ implicit_rule_creation = true;
if (erasure_code_profile == "default") {
- ruleset_name = "erasure-code";
+ rule_name = "erasure-code";
} else {
- dout(1) << "implicitly use ruleset named after the pool: "
+ dout(1) << "implicitly use rule named after the pool: "
<< poolstr << dendl;
- ruleset_name = poolstr;
+ rule_name = poolstr;
}
}
+ cmd_getval(g_ceph_context, cmdmap, "expected_num_objects",
+ expected_num_objects, int64_t(0));
} else {
- //NOTE:for replicated pool,cmd_map will put ruleset_name to erasure_code_profile field
- ruleset_name = erasure_code_profile;
+ //NOTE:for replicated pool,cmd_map will put rule_name to erasure_code_profile field
+ // and put expected_num_objects to rule field
+ if (erasure_code_profile != "") { // cmd is from CLI
+ if (rule_name != "") {
+ string interr;
+ expected_num_objects = strict_strtoll(rule_name.c_str(), 10, &interr);
+ if (interr.length()) {
+ ss << "error parsing integer value '" << rule_name << "': " << interr;
+ err = -EINVAL;
+ goto reply;
+ }
+ }
+ rule_name = erasure_code_profile;
+ } else { // cmd is well-formed
+ cmd_getval(g_ceph_context, cmdmap, "expected_num_objects",
+ expected_num_objects, int64_t(0));
+ }
}
- if (!implicit_ruleset_creation && ruleset_name != "") {
- int ruleset;
- err = get_crush_ruleset(ruleset_name, &ruleset, &ss);
+ if (!implicit_rule_creation && rule_name != "") {
+ int rule;
+ err = get_crush_rule(rule_name, &rule, &ss);
if (err == -EAGAIN) {
wait_for_finished_proposal(op, new C_RetryMessage(this, op));
return true;
goto reply;
}
- int64_t expected_num_objects;
- cmd_getval(g_ceph_context, cmdmap, "expected_num_objects", expected_num_objects, int64_t(0));
if (expected_num_objects < 0) {
ss << "'expected_num_objects' must be non-negative";
err = -EINVAL;
err = prepare_new_pool(poolstr, 0, // auid=0 for admin created pool
-1, // default crush rule
- ruleset_name,
+ rule_name,
pg_num, pgp_num,
erasure_code_profile, pool_type,
(uint64_t)expected_num_objects,
// make sure new tier is empty
string force_nonempty;
cmd_getval(g_ceph_context, cmdmap, "force_nonempty", force_nonempty);
- const pool_stat_t& tier_stats =
- mon->pgmon()->pg_map.get_pg_pool_sum_stat(tierpool_id);
- if (tier_stats.stats.sum.num_objects != 0 &&
+ const pool_stat_t *pstats = mon->pgservice->get_pool_stat(tierpool_id);
+ if (pstats && pstats->stats.sum.num_objects != 0 &&
force_nonempty != "--force-nonempty") {
ss << "tier pool '" << tierpoolstr << "' is not empty; --force-nonempty to force";
err = -ENOTEMPTY;
mode != pg_pool_t::CACHEMODE_PROXY &&
mode != pg_pool_t::CACHEMODE_READPROXY))) {
- const pool_stat_t& tier_stats =
- mon->pgmon()->pg_map.get_pg_pool_sum_stat(pool_id);
+ const pool_stat_t* pstats =
+ mon->pgservice->get_pool_stat(pool_id);
- if (tier_stats.stats.sum.num_objects_dirty > 0) {
+ if (pstats && pstats->stats.sum.num_objects_dirty > 0) {
ss << "unable to set cache-mode '"
<< pg_pool_t::get_cache_mode_name(mode) << "' on pool '" << poolstr
<< "': dirty objects found";
goto reply;
}
// make sure new tier is empty
- const pool_stat_t& tier_stats =
- mon->pgmon()->pg_map.get_pg_pool_sum_stat(tierpool_id);
- if (tier_stats.stats.sum.num_objects != 0) {
+ const pool_stat_t *pstats =
+ mon->pgservice->get_pool_stat(tierpool_id);
+ if (pstats && pstats->stats.sum.num_objects != 0) {
ss << "tier pool '" << tierpoolstr << "' is not empty";
err = -ENOTEMPTY;
goto reply;
// val could contain unit designations, so we treat as a string
string val;
cmd_getval(g_ceph_context, cmdmap, "val", val);
- stringstream tss;
- int64_t value = unit_to_bytesize(val, &tss);
- if (value < 0) {
- ss << "error parsing value '" << value << "': " << tss.str();
- err = value;
+ string tss;
+ int64_t value;
+ if (field == "max_objects") {
+ value = strict_sistrtoll(val.c_str(), &tss);
+ } else if (field == "max_bytes") {
+ value = strict_iecstrtoll(val.c_str(), &tss);
+ } else {
+ assert(0 == "unrecognized option");
+ }
+ if (!tss.empty()) {
+ ss << "error parsing value '" << val << "': " << tss;
+ err = -EINVAL;
goto reply;
}
wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
get_last_committed() + 1));
return true;
+ } else if (prefix == "osd pool application enable" ||
+ prefix == "osd pool application disable" ||
+ prefix == "osd pool application set" ||
+ prefix == "osd pool application rm") {
+ err = prepare_command_pool_application(prefix, cmdmap, ss);
+ if (err == -EAGAIN)
+ goto wait;
+ if (err < 0)
+ goto reply;
+ getline(ss, rs);
+ wait_for_finished_proposal(
+ op, new Monitor::C_Command(mon, op, 0, rs, get_last_committed() + 1));
+ return true;
} else if (prefix == "osd reweight-by-pg" ||
prefix == "osd reweight-by-utilization" ||
prefix == "osd test-reweight-by-pg" ||
cmd_getval(g_ceph_context, cmdmap, "no_increasing", no_increasing);
string out_str;
mempool::osdmap::map<int32_t, uint32_t> new_weights;
- err = reweight::by_utilization(osdmap,
- mon->pgmon()->pg_map,
- oload,
- max_change,
- max_osds,
- by_pg,
- pools.empty() ? NULL : &pools,
- no_increasing == "--no-increasing",
- &new_weights,
- &ss, &out_str, f.get());
+ err = mon->pgservice->reweight_by_utilization(osdmap,
+ oload,
+ max_change,
+ max_osds,
+ by_pg,
+ pools.empty() ? NULL : &pools,
+ no_increasing == "--no-increasing",
+ &new_weights,
+ &ss, &out_str, f.get());
if (err >= 0) {
dout(10) << "reweight::by_utilization: finished with " << out_str << dendl;
}
new Monitor::C_Command(mon, op, 0, rs, rdata, get_last_committed() + 1));
return true;
}
+ } else if (prefix == "osd force-create-pg") {
+ pg_t pgid;
+ string pgidstr;
+ cmd_getval(g_ceph_context, cmdmap, "pgid", pgidstr);
+ if (!pgid.parse(pgidstr.c_str())) {
+ ss << "invalid pgid '" << pgidstr << "'";
+ err = -EINVAL;
+ goto reply;
+ }
+ if (!osdmap.pg_exists(pgid)) {
+ ss << "pg " << pgid << " should not exist";
+ err = -ENOENT;
+ goto reply;
+ }
+ bool creating_now;
+ {
+ std::lock_guard<std::mutex> l(creating_pgs_lock);
+ auto emplaced = creating_pgs.pgs.emplace(pgid,
+ make_pair(osdmap.get_epoch(),
+ ceph_clock_now()));
+ creating_now = emplaced.second;
+ }
+ if (creating_now) {
+ ss << "pg " << pgidstr << " now creating, ok";
+ err = 0;
+ goto update;
+ } else {
+ ss << "pg " << pgid << " already creating";
+ err = 0;
+ goto reply;
+ }
} else {
err = -EINVAL;
}
return true;
}
-bool OSDMonitor::preprocess_pool_op(MonOpRequestRef op)
+bool OSDMonitor::enforce_pool_op_caps(MonOpRequestRef op)
{
op->mark_osdmon_event(__func__);
+
MPoolOp *m = static_cast<MPoolOp*>(op->get_req());
-
+ MonSession *session = m->get_session();
+ if (!session) {
+ _pool_op_reply(op, -EPERM, osdmap.get_epoch());
+ return true;
+ }
+
+ switch (m->op) {
+ case POOL_OP_CREATE_UNMANAGED_SNAP:
+ case POOL_OP_DELETE_UNMANAGED_SNAP:
+ {
+ const std::string* pool_name = nullptr;
+ const pg_pool_t *pg_pool = osdmap.get_pg_pool(m->pool);
+ if (pg_pool != nullptr) {
+ pool_name = &osdmap.get_pool_name(m->pool);
+ }
+
+ if (!is_unmanaged_snap_op_permitted(cct, mon->key_server,
+ session->entity_name, session->caps,
+ pool_name)) {
+ dout(0) << "got unmanaged-snap pool op from entity with insufficient "
+ << "privileges. message: " << *m << std::endl
+ << "caps: " << session->caps << dendl;
+ _pool_op_reply(op, -EPERM, osdmap.get_epoch());
+ return true;
+ }
+ }
+ break;
+ default:
+ if (!session->is_capable("osd", MON_CAP_W)) {
+ dout(0) << "got pool op from entity with insufficient privileges. "
+ << "message: " << *m << std::endl
+ << "caps: " << session->caps << dendl;
+ _pool_op_reply(op, -EPERM, osdmap.get_epoch());
+ return true;
+ }
+ break;
+ }
+
+ return false;
+}
+
+bool OSDMonitor::preprocess_pool_op(MonOpRequestRef op)
+{
+ op->mark_osdmon_event(__func__);
+ MPoolOp *m = static_cast<MPoolOp*>(op->get_req());
+
+ if (enforce_pool_op_caps(op)) {
+ return true;
+ }
+
if (m->fsid != mon->monmap->fsid) {
dout(0) << __func__ << " drop message on fsid " << m->fsid
<< " != " << mon->monmap->fsid << " for " << *m << dendl;
{
op->mark_osdmon_event(__func__);
MPoolOp *m = static_cast<MPoolOp*>(op->get_req());
- MonSession *session = m->get_session();
- if (!session) {
- _pool_op_reply(op, -EPERM, osdmap.get_epoch());
- return true;
- }
- if (!session->is_capable("osd", MON_CAP_W)) {
- dout(5) << "attempt to create new pool without sufficient auid privileges!"
- << "message: " << *m << std::endl
- << "caps: " << session->caps << dendl;
- _pool_op_reply(op, -EPERM, osdmap.get_epoch());
- return true;
- }
-
int64_t pool = osdmap.lookup_pg_pool_name(m->name.c_str());
if (pool >= 0) {
_pool_op_reply(op, 0, osdmap.get_epoch());
case POOL_OP_DELETE_UNMANAGED_SNAP:
if (!pp.is_removed_snap(m->snapid)) {
+ if (m->snapid > pp.get_snap_seq()) {
+ _pool_op_reply(op, -ENOENT, osdmap.get_epoch());
+ return false;
+ }
pp.remove_unmanaged_snap(m->snapid);
changed = true;
}
const string& poolstr = osdmap.get_pool_name(pool_id);
// If the Pool is in use by CephFS, refuse to delete it
- FSMap const &pending_fsmap = mon->mdsmon()->get_pending();
+ FSMap const &pending_fsmap = mon->mdsmon()->get_pending_fsmap();
if (pending_fsmap.pool_in_use(pool_id)) {
*ss << "pool '" << poolstr << "' is in use by CephFS";
return -EBUSY;
const std::string &tier_pool_name = osdmap.get_pool_name(tier_pool_id);
const std::string &base_pool_name = osdmap.get_pool_name(base_pool_id);
- const FSMap &pending_fsmap = mon->mdsmon()->get_pending();
+ const FSMap &pending_fsmap = mon->mdsmon()->get_pending_fsmap();
if (pending_fsmap.pool_in_use(tier_pool_id)) {
*ss << "pool '" << tier_pool_name << "' is in use by CephFS";
*err = -EBUSY;
const std::string &base_pool_name = osdmap.get_pool_name(base_pool_id);
// Apply CephFS-specific checks
- const FSMap &pending_fsmap = mon->mdsmon()->get_pending();
+ const FSMap &pending_fsmap = mon->mdsmon()->get_pending_fsmap();
if (pending_fsmap.pool_in_use(base_pool_id)) {
- if (base_pool->type != pg_pool_t::TYPE_REPLICATED) {
- // If the underlying pool is erasure coded, we can't permit the
- // removal of the replicated tier that CephFS relies on to access it
- *ss << "pool '" << base_pool_name << "' is in use by CephFS via its tier";
+ if (base_pool->is_erasure() && !base_pool->allows_ecoverwrites()) {
+ // If the underlying pool is erasure coded and does not allow EC
+ // overwrites, we can't permit the removal of the replicated tier that
+ // CephFS relies on to access it
+ *ss << "pool '" << base_pool_name <<
+ "' does not allow EC overwrites and is in use by CephFS"
+ " via its tier";
*err = -EBUSY;
return false;
}
int OSDMonitor::_prepare_remove_pool(
int64_t pool, ostream *ss, bool no_fake)
{
- dout(10) << "_prepare_remove_pool " << pool << dendl;
+ dout(10) << __func__ << " " << pool << dendl;
const pg_pool_t *p = osdmap.get_pg_pool(pool);
int r = _check_remove_pool(pool, *p, ss);
if (r < 0)
}
if (pending_inc.old_pools.count(pool)) {
- dout(10) << "_prepare_remove_pool " << pool << " already pending removal"
+ dout(10) << __func__ << " " << pool << " already pending removal"
<< dendl;
return 0;
}
// remove
pending_inc.old_pools.insert(pool);
- // remove any pg_temp mappings for this pool too
+ // remove any pg_temp mappings for this pool
for (auto p = osdmap.pg_temp->begin();
p != osdmap.pg_temp->end();
++p) {
if (p->first.pool() == (uint64_t)pool) {
- dout(10) << "_prepare_remove_pool " << pool << " removing obsolete pg_temp "
+ dout(10) << __func__ << " " << pool << " removing obsolete pg_temp "
<< p->first << dendl;
pending_inc.new_pg_temp[p->first].clear();
}
}
+ // remove any primary_temp mappings for this pool
for (auto p = osdmap.primary_temp->begin();
p != osdmap.primary_temp->end();
++p) {
if (p->first.pool() == (uint64_t)pool) {
- dout(10) << "_prepare_remove_pool " << pool
+ dout(10) << __func__ << " " << pool
<< " removing obsolete primary_temp" << p->first << dendl;
pending_inc.new_primary_temp[p->first] = -1;
}
}
+ // remove any pg_upmap mappings for this pool
+ for (auto& p : osdmap.pg_upmap) {
+ if (p.first.pool() == (uint64_t)pool) {
+ dout(10) << __func__ << " " << pool
+ << " removing obsolete pg_upmap "
+ << p.first << dendl;
+ pending_inc.old_pg_upmap.insert(p.first);
+ }
+ }
+ // remove any pending pg_upmap mappings for this pool
+ {
+ auto it = pending_inc.new_pg_upmap.begin();
+ while (it != pending_inc.new_pg_upmap.end()) {
+ if (it->first.pool() == (uint64_t)pool) {
+ dout(10) << __func__ << " " << pool
+ << " removing pending pg_upmap "
+ << it->first << dendl;
+ it = pending_inc.new_pg_upmap.erase(it);
+ } else {
+ it++;
+ }
+ }
+ }
+ // remove any pg_upmap_items mappings for this pool
+ for (auto& p : osdmap.pg_upmap_items) {
+ if (p.first.pool() == (uint64_t)pool) {
+ dout(10) << __func__ << " " << pool
+ << " removing obsolete pg_upmap_items " << p.first
+ << dendl;
+ pending_inc.old_pg_upmap_items.insert(p.first);
+ }
+ }
+ // remove any pending pg_upmap mappings for this pool
+ {
+ auto it = pending_inc.new_pg_upmap_items.begin();
+ while (it != pending_inc.new_pg_upmap_items.end()) {
+ if (it->first.pool() == (uint64_t)pool) {
+ dout(10) << __func__ << " " << pool
+ << " removing pending pg_upmap_items "
+ << it->first << dendl;
+ it = pending_inc.new_pg_upmap_items.erase(it);
+ } else {
+ it++;
+ }
+ }
+ }
+
+ // remove any choose_args for this pool
+ CrushWrapper newcrush;
+ _get_pending_crush(newcrush);
+ if (newcrush.have_choose_args(pool)) {
+ dout(10) << __func__ << " removing choose_args for pool " << pool << dendl;
+ newcrush.rm_choose_args(pool);
+ pending_inc.crush.clear();
+ newcrush.encode(pending_inc.crush, mon->get_quorum_con_features());
+ }
return 0;
}
projects / ceph.git / blobdiff