]> git.proxmox.com Git - ceph.git/blobdiff - ceph/src/osd/OSDMap.cc
update sources to v12.1.0
[ceph.git] / ceph / src / osd / OSDMap.cc
index 5043ff417151a6407111cd06e1091af8cf1f888f..594ff82fae28dd10584bd798ad1d569d77cd6972 100644 (file)
@@ -27,6 +27,7 @@
 
 #include "crush/CrushTreeDumper.h"
 #include "common/Clock.h"
+#include "mon/PGStatService.h"
  
 #define dout_subsys ceph_subsys_osd
 
@@ -293,6 +294,40 @@ bool OSDMap::containing_subtree_is_down(CephContext *cct, int id, int subtree_ty
   }
 }
 
+bool OSDMap::subtree_type_is_down(CephContext *cct, int id, int subtree_type, set<int> *down_in_osds, set<int> *up_in_osds,
+                                           set<int> *subtree_up, unordered_map<int, set<int> > *subtree_type_down) const
+{
+  if (id >= 0) {
+    bool is_down_ret = is_down(id);
+    if (!is_out(id)) {
+      if (is_down_ret) {
+        down_in_osds->insert(id);
+      } else {
+        up_in_osds->insert(id);
+      }
+    }
+    return is_down_ret;
+  }
+
+  if (subtree_type_down &&
+      (*subtree_type_down)[subtree_type].count(id)) {
+    return true;
+  }
+
+  list<int> children;
+  crush->get_children(id, &children);
+  for (const auto &child : children) {
+    if (!subtree_type_is_down(cct, child, crush->get_bucket_type(child), down_in_osds, up_in_osds, subtree_up, subtree_type_down)) {
+      subtree_up->insert(id);
+      return false;
+    }
+  }
+  if (subtree_type_down) {
+    (*subtree_type_down)[subtree_type].insert(id);
+  }
+  return true;
+}
+
 void OSDMap::Incremental::encode_client_old(bufferlist& bl) const
 {
   __u16 v = 5;
@@ -332,7 +367,15 @@ void OSDMap::Incremental::encode_client_old(bufferlist& bl) const
     ::encode(n, bl);
   }
   ::encode(new_up_client, bl, 0);
-  ::encode(new_state, bl);
+  {
+    // legacy is map<int32_t,uint8_t>
+    uint32_t n = new_state.size();
+    ::encode(n, bl);
+    for (auto p : new_state) {
+      ::encode(p.first, bl);
+      ::encode((uint8_t)p.second, bl);
+    }
+  }
   ::encode(new_weight, bl);
   // for ::encode(new_pg_temp, bl);
   n = new_pg_temp.size();
@@ -368,7 +411,14 @@ void OSDMap::Incremental::encode_classic(bufferlist& bl, uint64_t features) cons
   ::encode(new_pool_names, bl);
   ::encode(old_pools, bl);
   ::encode(new_up_client, bl, features);
-  ::encode(new_state, bl);
+  {
+    uint32_t n = new_state.size();
+    ::encode(n, bl);
+    for (auto p : new_state) {
+      ::encode(p.first, bl);
+      ::encode((uint8_t)p.second, bl);
+    }
+  }
   ::encode(new_weight, bl);
   ::encode(new_pg_temp, bl);
 
@@ -410,7 +460,7 @@ void OSDMap::Incremental::encode(bufferlist& bl, uint64_t features) const
   ENCODE_START(8, 7, bl);
 
   {
-    uint8_t v = 4;
+    uint8_t v = 5;
     if (!HAVE_FEATURE(features, SERVER_LUMINOUS)) {
       v = 3;
     }
@@ -428,7 +478,16 @@ void OSDMap::Incremental::encode(bufferlist& bl, uint64_t features) const
     ::encode(new_pool_names, bl);
     ::encode(old_pools, bl);
     ::encode(new_up_client, bl, features);
-    ::encode(new_state, bl);
+    if (v >= 5) {
+      ::encode(new_state, bl);
+    } else {
+      uint32_t n = new_state.size();
+      ::encode(n, bl);
+      for (auto p : new_state) {
+       ::encode(p.first, bl);
+       ::encode((uint8_t)p.second, bl);
+      }
+    }
     ::encode(new_weight, bl);
     ::encode(new_pg_temp, bl);
     ::encode(new_primary_temp, bl);
@@ -445,7 +504,7 @@ void OSDMap::Incremental::encode(bufferlist& bl, uint64_t features) const
   }
 
   {
-    uint8_t target_v = 5;
+    uint8_t target_v = 6;
     if (!HAVE_FEATURE(features, SERVER_LUMINOUS)) {
       target_v = 2;
     }
@@ -466,7 +525,11 @@ void OSDMap::Incremental::encode(bufferlist& bl, uint64_t features) const
       ::encode(new_nearfull_ratio, bl);
       ::encode(new_full_ratio, bl);
       ::encode(new_backfillfull_ratio, bl);
+    }
+    // 5 was string-based new_require_min_compat_client
+    if (target_v >= 6) {
       ::encode(new_require_min_compat_client, bl);
+      ::encode(new_require_osd_release, bl);
     }
     ENCODE_FINISH(bl); // osd-only data
   }
@@ -543,7 +606,13 @@ void OSDMap::Incremental::decode_classic(bufferlist::iterator &p)
     ::decode(old_pools, p);
   }
   ::decode(new_up_client, p);
-  ::decode(new_state, p);
+  {
+    map<int32_t,uint8_t> ns;
+    ::decode(ns, p);
+    for (auto q : ns) {
+      new_state[q.first] = q.second;
+    }
+  }
   ::decode(new_weight, p);
 
   if (v < 6) {
@@ -612,7 +681,7 @@ void OSDMap::Incremental::decode(bufferlist::iterator& bl)
     return;
   }
   {
-    DECODE_START(4, bl); // client-usable data
+    DECODE_START(5, bl); // client-usable data
     ::decode(fsid, bl);
     ::decode(epoch, bl);
     ::decode(modified, bl);
@@ -626,7 +695,15 @@ void OSDMap::Incremental::decode(bufferlist::iterator& bl)
     ::decode(new_pool_names, bl);
     ::decode(old_pools, bl);
     ::decode(new_up_client, bl);
-    ::decode(new_state, bl);
+    if (struct_v >= 5) {
+      ::decode(new_state, bl);
+    } else {
+      map<int32_t,uint8_t> ns;
+      ::decode(ns, bl);
+      for (auto q : ns) {
+       new_state[q.first] = q.second;
+      }
+    }
     ::decode(new_weight, bl);
     ::decode(new_pg_temp, bl);
     ::decode(new_primary_temp, bl);
@@ -651,7 +728,7 @@ void OSDMap::Incremental::decode(bufferlist::iterator& bl)
   }
 
   {
-    DECODE_START(5, bl); // extended, osd-only data
+    DECODE_START(6, bl); // extended, osd-only data
     ::decode(new_hb_back_up, bl);
     ::decode(new_up_thru, bl);
     ::decode(new_last_clean_interval, bl);
@@ -679,8 +756,29 @@ void OSDMap::Incremental::decode(bufferlist::iterator& bl)
     } else {
       new_backfillfull_ratio = -1;
     }
-    if (struct_v >= 5)
+    if (struct_v == 5) {
+      string r;
+      ::decode(r, bl);
+      if (r.length()) {
+       new_require_min_compat_client = ceph_release_from_name(r.c_str());
+      }
+    }
+    if (struct_v >= 6) {
       ::decode(new_require_min_compat_client, bl);
+      ::decode(new_require_osd_release, bl);
+    } else {
+      if (new_flags >= 0 && (new_flags & CEPH_OSDMAP_REQUIRE_LUMINOUS)) {
+       // only for compat with post-kraken pre-luminous test clusters
+       new_require_osd_release = CEPH_RELEASE_LUMINOUS;
+       new_flags &= ~(CEPH_OSDMAP_LEGACY_REQUIRE_FLAGS);
+      } else if (new_flags >= 0 && (new_flags & CEPH_OSDMAP_REQUIRE_KRAKEN)) {
+       new_require_osd_release = CEPH_RELEASE_KRAKEN;
+      } else if (new_flags >= 0 && (new_flags & CEPH_OSDMAP_REQUIRE_JEWEL)) {
+       new_require_osd_release = CEPH_RELEASE_JEWEL;
+      } else {
+       new_require_osd_release = -1;
+      }
+    }
     DECODE_FINISH(bl); // osd-only data
   }
 
@@ -725,7 +823,8 @@ void OSDMap::Incremental::dump(Formatter *f) const
   f->dump_float("new_full_ratio", new_full_ratio);
   f->dump_float("new_nearfull_ratio", new_nearfull_ratio);
   f->dump_float("new_backfillfull_ratio", new_backfillfull_ratio);
-  f->dump_string("new_require_min_compat_client", new_require_min_compat_client);
+  f->dump_int("new_require_min_compat_client", new_require_min_compat_client);
+  f->dump_int("new_require_osd_release", new_require_osd_release);
 
   if (fullmap.length()) {
     f->open_object_section("full_map");
@@ -985,6 +1084,13 @@ void OSDMap::get_blacklist(list<pair<entity_addr_t,utime_t> > *bl) const
    std::copy(blacklist.begin(), blacklist.end(), std::back_inserter(*bl));
 }
 
+void OSDMap::get_blacklist(std::set<entity_addr_t> *bl) const
+{
+  for (const auto &i : blacklist) {
+    bl->insert(i.first);
+  }
+}
+
 void OSDMap::set_max_osd(int m)
 {
   int o = max_osd;
@@ -1044,19 +1150,22 @@ void OSDMap::count_full_nearfull_osds(int *full, int *backfill, int *nearfull) c
   }
 }
 
-static bool get_osd_utilization(const ceph::unordered_map<int32_t,osd_stat_t> &osd_stat,
-   int id, int64_t* kb, int64_t* kb_used, int64_t* kb_avail) {
-    auto p = osd_stat.find(id);
-    if (p == osd_stat.end())
-      return false;
-    *kb = p->second.kb;
-    *kb_used = p->second.kb_used;
-    *kb_avail = p->second.kb_avail;
-    return *kb > 0;
+static bool get_osd_utilization(
+  const mempool::pgmap::unordered_map<int32_t,osd_stat_t> &osd_stat,
+  int id, int64_t* kb, int64_t* kb_used, int64_t* kb_avail)
+{
+  auto p = osd_stat.find(id);
+  if (p == osd_stat.end())
+    return false;
+  *kb = p->second.kb;
+  *kb_used = p->second.kb_used;
+  *kb_avail = p->second.kb_avail;
+  return *kb > 0;
 }
 
-void OSDMap::get_full_osd_util(const ceph::unordered_map<int32_t,osd_stat_t> &osd_stat,
-     map<int, float> *full, map<int, float> *backfill, map<int, float> *nearfull) const
+void OSDMap::get_full_osd_util(
+  const mempool::pgmap::unordered_map<int32_t,osd_stat_t> &osd_stat,
+  map<int, float> *full, map<int, float> *backfill, map<int, float> *nearfull) const
 {
   full->clear();
   backfill->clear();
@@ -1078,6 +1187,24 @@ void OSDMap::get_full_osd_util(const ceph::unordered_map<int32_t,osd_stat_t> &os
   }
 }
 
+void OSDMap::get_full_osd_counts(set<int> *full, set<int> *backfill,
+                                set<int> *nearfull) const
+{
+  full->clear();
+  backfill->clear();
+  nearfull->clear();
+  for (int i = 0; i < max_osd; ++i) {
+    if (exists(i) && is_up(i) && is_in(i)) {
+      if (osd_state[i] & CEPH_OSD_FULL)
+       full->emplace(i);
+      else if (osd_state[i] & CEPH_OSD_BACKFILLFULL)
+       backfill->emplace(i);
+      else if (osd_state[i] & CEPH_OSD_NEARFULL)
+       nearfull->emplace(i);
+    }
+  }
+}
+
 void OSDMap::get_all_osds(set<int32_t>& ls) const
 {
   for (int i=0; i<max_osd; i++)
@@ -1093,6 +1220,14 @@ void OSDMap::get_up_osds(set<int32_t>& ls) const
   }
 }
 
+void OSDMap::get_out_osds(set<int32_t>& ls) const
+{
+  for (int i = 0; i < max_osd; i++) {
+    if (is_out(i))
+      ls.insert(i);
+  }
+}
+
 void OSDMap::calc_state_set(int state, set<string>& st)
 {
   unsigned t = state;
@@ -1166,8 +1301,8 @@ uint64_t OSDMap::get_features(int entity_type, uint64_t *pmask) const
     features |= CEPH_FEATURE_CRUSH_V4;
   if (crush->has_nondefault_tunables5())
     features |= CEPH_FEATURE_CRUSH_TUNABLES5;
-  if (crush->has_incompat_chooseargs())
-    features |= CEPH_FEATURE_CRUSH_CHOOSEARGS;
+  if (crush->has_incompat_choose_args())
+    features |= CEPH_FEATURE_CRUSH_CHOOSE_ARGS;
   mask |= CEPH_FEATURES_CRUSH;
 
   if (!pg_upmap.empty() || !pg_upmap_items.empty())
@@ -1186,7 +1321,7 @@ uint64_t OSDMap::get_features(int entity_type, uint64_t *pmask) const
        pool.second.is_tier()) {
       features |= CEPH_FEATURE_OSD_CACHEPOOL;
     }
-    int ruleid = crush->find_rule(pool.second.get_crush_ruleset(),
+    int ruleid = crush->find_rule(pool.second.get_crush_rule(),
                                  pool.second.get_type(),
                                  pool.second.get_size());
     if (ruleid >= 0) {
@@ -1226,14 +1361,14 @@ uint64_t OSDMap::get_features(int entity_type, uint64_t *pmask) const
 
   if (entity_type == CEPH_ENTITY_TYPE_OSD) {
     const uint64_t jewel_features = CEPH_FEATURE_SERVER_JEWEL;
-    if (test_flag(CEPH_OSDMAP_REQUIRE_JEWEL)) {
+    if (require_osd_release >= CEPH_RELEASE_JEWEL) {
       features |= jewel_features;
     }
     mask |= jewel_features;
 
     const uint64_t kraken_features = CEPH_FEATUREMASK_SERVER_KRAKEN
       | CEPH_FEATURE_MSG_ADDR2;
-    if (test_flag(CEPH_OSDMAP_REQUIRE_KRAKEN)) {
+    if (require_osd_release >= CEPH_RELEASE_KRAKEN) {
       features |= kraken_features;
     }
     mask |= kraken_features;
@@ -1244,34 +1379,34 @@ uint64_t OSDMap::get_features(int entity_type, uint64_t *pmask) const
   return features;
 }
 
-pair<string,string> OSDMap::get_min_compat_client() const
+uint8_t OSDMap::get_min_compat_client() const
 {
   uint64_t f = get_features(CEPH_ENTITY_TYPE_CLIENT, nullptr);
 
   if (HAVE_FEATURE(f, OSDMAP_PG_UPMAP) ||      // v12.0.0-1733-g27d6f43
-      HAVE_FEATURE(f, CRUSH_CHOOSEARGS)) {     // v12.0.1-2172-gef1ef28
-    return make_pair("luminous", "12.2.0");
+      HAVE_FEATURE(f, CRUSH_CHOOSE_ARGS)) {    // v12.0.1-2172-gef1ef28
+    return CEPH_RELEASE_LUMINOUS;  // v12.2.0
   }
   if (HAVE_FEATURE(f, CRUSH_TUNABLES5)) {      // v10.0.0-612-g043a737
-    return make_pair("jewel", "10.2.0");
+    return CEPH_RELEASE_JEWEL;     // v10.2.0
   }
   if (HAVE_FEATURE(f, CRUSH_V4)) {             // v0.91-678-g325fc56
-    return make_pair("hammer", "0.94");
+    return CEPH_RELEASE_HAMMER;    // v0.94.0
   }
   if (HAVE_FEATURE(f, OSD_PRIMARY_AFFINITY) || // v0.76-553-gf825624
       HAVE_FEATURE(f, CRUSH_TUNABLES3) ||      // v0.76-395-ge20a55d
       HAVE_FEATURE(f, OSD_ERASURE_CODES) ||    // v0.73-498-gbfc86a8
       HAVE_FEATURE(f, OSD_CACHEPOOL)) {        // v0.67-401-gb91c1c5
-    return make_pair("firefly", "0.80");
+    return CEPH_RELEASE_FIREFLY;   // v0.80.0
   }
   if (HAVE_FEATURE(f, CRUSH_TUNABLES2) ||      // v0.54-684-g0cc47ff
       HAVE_FEATURE(f, OSDHASHPSPOOL)) {        // v0.57-398-g8cc2b0f
-    return make_pair("dumpling", "0.67");
+    return CEPH_RELEASE_DUMPLING;  // v0.67.0
   }
   if (HAVE_FEATURE(f, CRUSH_TUNABLES)) {       // v0.48argonaut-206-g6f381af
-    return make_pair("argonaut", "0.48argonaut-207");
+    return CEPH_RELEASE_ARGONAUT;  // v0.48argonaut-206-g6f381af
   }
-  return make_pair("argonaut", "0.48");
+  return CEPH_RELEASE_ARGONAUT;    // v0.48argonaut-206-g6f381af
 }
 
 void OSDMap::_calc_up_osd_features()
@@ -1342,10 +1477,8 @@ void OSDMap::dedup(const OSDMap *o, OSDMap *n)
   }
 
   // does pg_temp match?
-  if (o->pg_temp->size() == n->pg_temp->size()) {
-    if (*o->pg_temp == *n->pg_temp)
-      n->pg_temp = o->pg_temp;
-  }
+  if (*o->pg_temp == *n->pg_temp)
+    n->pg_temp = o->pg_temp;
 
   // does primary_temp match?
   if (o->primary_temp->size() == n->primary_temp->size()) {
@@ -1452,8 +1585,22 @@ int OSDMap::apply_incremental(const Incremental &inc)
   }
 
   // nope, incremental.
-  if (inc.new_flags >= 0)
+  if (inc.new_flags >= 0) {
     flags = inc.new_flags;
+    // the below is just to cover a newly-upgraded luminous mon
+    // cluster that has to set require_jewel_osds or
+    // require_kraken_osds before the osds can be upgraded to
+    // luminous.
+    if (flags & CEPH_OSDMAP_REQUIRE_KRAKEN) {
+      if (require_osd_release < CEPH_RELEASE_KRAKEN) {
+       require_osd_release = CEPH_RELEASE_KRAKEN;
+      }
+    } else if (flags & CEPH_OSDMAP_REQUIRE_JEWEL) {
+      if (require_osd_release < CEPH_RELEASE_JEWEL) {
+       require_osd_release = CEPH_RELEASE_JEWEL;
+      }
+    }
+  }
 
   if (inc.new_max_osd >= 0)
     set_max_osd(inc.new_max_osd);
@@ -1577,7 +1724,11 @@ int OSDMap::apply_incremental(const Incremental &inc)
     if (pg.second.empty())
       pg_temp->erase(pg.first);
     else
-      (*pg_temp)[pg.first] = pg.second;
+      pg_temp->set(pg.first, pg.second);
+  }
+  if (!inc.new_pg_temp.empty()) {
+    // make sure pg_temp is efficiently stored
+    pg_temp->rebuild();
   }
 
   for (const auto &pg : inc.new_primary_temp) {
@@ -1626,9 +1777,15 @@ int OSDMap::apply_incremental(const Incremental &inc)
   if (inc.new_full_ratio >= 0) {
     full_ratio = inc.new_full_ratio;
   }
-  if (inc.new_require_min_compat_client.length()) {
+  if (inc.new_require_min_compat_client > 0) {
     require_min_compat_client = inc.new_require_min_compat_client;
   }
+  if (inc.new_require_osd_release >= 0) {
+    require_osd_release = inc.new_require_osd_release;
+    if (require_osd_release >= CEPH_RELEASE_LUMINOUS) {
+      flags &= ~(CEPH_OSDMAP_LEGACY_REQUIRE_FLAGS);
+    }
+  }
 
   // do new crush map last (after up/down stuff)
   if (inc.crush.length()) {
@@ -1636,6 +1793,13 @@ int OSDMap::apply_incremental(const Incremental &inc)
     auto blp = bl.begin();
     crush.reset(new CrushWrapper);
     crush->decode(blp);
+    if (require_osd_release >= CEPH_RELEASE_LUMINOUS) {
+      // only increment if this is a luminous-encoded osdmap, lest
+      // the mon's crush_version diverge from what the osds or others
+      // are decoding and applying on their end.  if we won't encode
+      // it in the canonical version, don't change it.
+      ++crush_version;
+    }
   }
 
   calc_num_osds();
@@ -1713,7 +1877,7 @@ void OSDMap::_remove_nonexistent_osds(const pg_pool_t& pool,
   }
 }
 
-int OSDMap::_pg_to_raw_osds(
+void OSDMap::_pg_to_raw_osds(
   const pg_pool_t& pool, pg_t pg,
   vector<int> *osds,
   ps_t *ppps) const
@@ -1723,7 +1887,7 @@ int OSDMap::_pg_to_raw_osds(
   unsigned size = pool.get_size();
 
   // what crush rule?
-  int ruleno = crush->find_rule(pool.get_crush_ruleset(), pool.get_type(), size);
+  int ruleno = crush->find_rule(pool.get_crush_rule(), pool.get_type(), size);
   if (ruleno >= 0)
     crush->do_rule(ruleno, pps, *osds, size, osd_weight, pg.pool());
 
@@ -1731,8 +1895,6 @@ int OSDMap::_pg_to_raw_osds(
 
   if (ppps)
     *ppps = pps;
-
-  return osds->size();
 }
 
 int OSDMap::_pick_primary(const vector<int>& osds) const
@@ -1905,17 +2067,16 @@ void OSDMap::_get_temp_osds(const pg_pool_t& pool, pg_t pg,
   }
 }
 
-int OSDMap::pg_to_raw_osds(pg_t pg, vector<int> *raw, int *primary) const
+void OSDMap::pg_to_raw_osds(pg_t pg, vector<int> *raw, int *primary) const
 {
   *primary = -1;
   raw->clear();
   const pg_pool_t *pool = get_pg_pool(pg.pool());
   if (!pool)
-    return 0;
-  int r = _pg_to_raw_osds(*pool, pg, raw, NULL);
+    return;
+  _pg_to_raw_osds(*pool, pg, raw, NULL);
   if (primary)
     *primary = _pick_primary(*raw);
-  return r;
 }
 
 void OSDMap::pg_to_raw_up(pg_t pg, vector<int> *up, int *primary) const
@@ -1936,7 +2097,7 @@ void OSDMap::pg_to_raw_up(pg_t pg, vector<int> *up, int *primary) const
   *primary = _pick_primary(raw);
   _apply_primary_affinity(pps, *pool, up, primary);
 }
-  
+
 void OSDMap::_pg_to_up_acting_osds(
   const pg_t& pg, vector<int> *up, int *up_primary,
   vector<int> *acting, int *acting_primary,
@@ -2057,7 +2218,13 @@ void OSDMap::encode_client_old(bufferlist& bl) const
   ::encode(flags, bl);
 
   ::encode(max_osd, bl);
-  ::encode(osd_state, bl);
+  {
+    uint32_t n = osd_state.size();
+    ::encode(n, bl);
+    for (auto s : osd_state) {
+      ::encode((uint8_t)s, bl);
+    }
+  }
   ::encode(osd_weight, bl);
   ::encode(osd_addrs->client_addr, bl, 0);
 
@@ -2099,7 +2266,13 @@ void OSDMap::encode_classic(bufferlist& bl, uint64_t features) const
   ::encode(flags, bl);
 
   ::encode(max_osd, bl);
-  ::encode(osd_state, bl);
+  {
+    uint32_t n = osd_state.size();
+    ::encode(n, bl);
+    for (auto s : osd_state) {
+      ::encode((uint8_t)s, bl);
+    }
+  }
   ::encode(osd_weight, bl);
   ::encode(osd_addrs->client_addr, bl, features);
 
@@ -2146,8 +2319,8 @@ void OSDMap::encode(bufferlist& bl, uint64_t features) const
   ENCODE_START(8, 7, bl);
 
   {
-    uint8_t v = 4;
-    if (!HAVE_FEATURE(features, OSDMAP_PG_UPMAP)) {
+    uint8_t v = 6;
+    if (!HAVE_FEATURE(features, SERVER_LUMINOUS)) {
       v = 3;
     }
     ENCODE_START(v, 1, bl); // client-usable data
@@ -2161,10 +2334,29 @@ void OSDMap::encode(bufferlist& bl, uint64_t features) const
     ::encode(pool_name, bl);
     ::encode(pool_max, bl);
 
-    ::encode(flags, bl);
+    if (v < 4) {
+      decltype(flags) f = flags;
+      if (require_osd_release >= CEPH_RELEASE_LUMINOUS)
+       f |= CEPH_OSDMAP_REQUIRE_LUMINOUS;
+      else if (require_osd_release == CEPH_RELEASE_KRAKEN)
+       f |= CEPH_OSDMAP_REQUIRE_KRAKEN;
+      else if (require_osd_release == CEPH_RELEASE_JEWEL)
+       f |= CEPH_OSDMAP_REQUIRE_JEWEL;
+      ::encode(f, bl);
+    } else {
+      ::encode(flags, bl);
+    }
 
     ::encode(max_osd, bl);
-    ::encode(osd_state, bl);
+    if (v >= 5) {
+      ::encode(osd_state, bl);
+    } else {
+      uint32_t n = osd_state.size();
+      ::encode(n, bl);
+      for (auto s : osd_state) {
+       ::encode((uint8_t)s, bl);
+      }
+    }
     ::encode(osd_weight, bl);
     ::encode(osd_addrs->client_addr, bl, features);
 
@@ -2190,11 +2382,14 @@ void OSDMap::encode(bufferlist& bl, uint64_t features) const
       assert(pg_upmap.empty());
       assert(pg_upmap_items.empty());
     }
+    if (v >= 6) {
+      ::encode(crush_version, bl);
+    }
     ENCODE_FINISH(bl); // client-usable data
   }
 
   {
-    uint8_t target_v = 4;
+    uint8_t target_v = 5;
     if (!HAVE_FEATURE(features, SERVER_LUMINOUS)) {
       target_v = 1;
     }
@@ -2219,7 +2414,11 @@ void OSDMap::encode(bufferlist& bl, uint64_t features) const
       ::encode(nearfull_ratio, bl);
       ::encode(full_ratio, bl);
       ::encode(backfillfull_ratio, bl);
+    }
+    // 4 was string-based new_require_min_compat_client
+    if (target_v >= 5) {
       ::encode(require_min_compat_client, bl);
+      ::encode(require_osd_release, bl);
     }
     ENCODE_FINISH(bl); // osd-only data
   }
@@ -2302,7 +2501,14 @@ void OSDMap::decode_classic(bufferlist::iterator& p)
   ::decode(flags, p);
 
   ::decode(max_osd, p);
-  ::decode(osd_state, p);
+  {
+    vector<uint8_t> os;
+    ::decode(os, p);
+    osd_state.resize(os.size());
+    for (unsigned i = 0; i < os.size(); ++i) {
+      osd_state[i] = os[i];
+    }
+  }
   ::decode(osd_weight, p);
   ::decode(osd_addrs->client_addr, p);
   if (v <= 5) {
@@ -2311,7 +2517,9 @@ void OSDMap::decode_classic(bufferlist::iterator& p)
     while (n--) {
       old_pg_t opg;
       ::decode_raw(opg, p);
-      ::decode((*pg_temp)[pg_t(opg)], p);
+      mempool::osdmap::vector<int32_t> v;
+      ::decode(v, p);
+      pg_temp->set(pg_t(opg), v);
     }
   } else {
     ::decode(*pg_temp, p);
@@ -2387,7 +2595,7 @@ void OSDMap::decode(bufferlist::iterator& bl)
    * Since we made it past that hurdle, we can use our normal paths.
    */
   {
-    DECODE_START(4, bl); // client-usable data
+    DECODE_START(6, bl); // client-usable data
     // base
     ::decode(fsid, bl);
     ::decode(epoch, bl);
@@ -2401,7 +2609,16 @@ void OSDMap::decode(bufferlist::iterator& bl)
     ::decode(flags, bl);
 
     ::decode(max_osd, bl);
-    ::decode(osd_state, bl);
+    if (struct_v >= 5) {
+      ::decode(osd_state, bl);
+    } else {
+      vector<uint8_t> os;
+      ::decode(os, bl);
+      osd_state.resize(os.size());
+      for (unsigned i = 0; i < os.size(); ++i) {
+       osd_state[i] = os[i];
+      }
+    }
     ::decode(osd_weight, bl);
     ::decode(osd_addrs->client_addr, bl);
 
@@ -2433,11 +2650,14 @@ void OSDMap::decode(bufferlist::iterator& bl)
       pg_upmap.clear();
       pg_upmap_items.clear();
     }
+    if (struct_v >= 6) {
+      ::decode(crush_version, bl);
+    }
     DECODE_FINISH(bl); // client-usable data
   }
 
   {
-    DECODE_START(4, bl); // extended, osd-only data
+    DECODE_START(5, bl); // extended, osd-only data
     ::decode(osd_addrs->hb_back_addr, bl);
     ::decode(osd_info, bl);
     ::decode(blacklist, bl);
@@ -2459,8 +2679,31 @@ void OSDMap::decode(bufferlist::iterator& bl)
     } else {
       backfillfull_ratio = 0;
     }
-    if (struct_v >= 4)
+    if (struct_v == 4) {
+      string r;
+      ::decode(r, bl);
+      if (r.length())
+       require_min_compat_client = ceph_release_from_name(r.c_str());
+    }
+    if (struct_v >= 5) {
       ::decode(require_min_compat_client, bl);
+      ::decode(require_osd_release, bl);
+      if (require_osd_release >= CEPH_RELEASE_LUMINOUS) {
+       flags &= ~(CEPH_OSDMAP_LEGACY_REQUIRE_FLAGS);
+      }
+    } else {
+      if (flags & CEPH_OSDMAP_REQUIRE_LUMINOUS) {
+       // only for compat with post-kraken pre-luminous test clusters
+       require_osd_release = CEPH_RELEASE_LUMINOUS;
+       flags &= ~(CEPH_OSDMAP_LEGACY_REQUIRE_FLAGS);
+      } else if (flags & CEPH_OSDMAP_REQUIRE_KRAKEN) {
+       require_osd_release = CEPH_RELEASE_KRAKEN;
+      } else if (flags & CEPH_OSDMAP_REQUIRE_JEWEL) {
+       require_osd_release = CEPH_RELEASE_JEWEL;
+      } else {
+       require_osd_release = 0;
+      }
+    }
     DECODE_FINISH(bl); // osd-only data
   }
 
@@ -2529,16 +2772,19 @@ void OSDMap::dump(Formatter *f) const
   f->dump_stream("created") << get_created();
   f->dump_stream("modified") << get_modified();
   f->dump_string("flags", get_flag_string());
+  f->dump_unsigned("crush_version", get_crush_version());
   f->dump_float("full_ratio", full_ratio);
   f->dump_float("backfillfull_ratio", backfillfull_ratio);
   f->dump_float("nearfull_ratio", nearfull_ratio);
   f->dump_string("cluster_snapshot", get_cluster_snapshot());
   f->dump_int("pool_max", get_pool_max());
   f->dump_int("max_osd", get_max_osd());
-  f->dump_string("require_min_compat_client", require_min_compat_client);
-  auto mv = get_min_compat_client();
-  f->dump_string("min_compat_client", mv.first);
-  f->dump_string("min_compat_client_version", mv.second);
+  f->dump_string("require_min_compat_client",
+                ceph_release_name(require_min_compat_client));
+  f->dump_string("min_compat_client",
+                ceph_release_name(get_min_compat_client()));
+  f->dump_string("require_osd_release",
+                ceph_release_name(require_osd_release));
 
   f->open_array_section("pools");
   for (const auto &pool : pools) {
@@ -2620,15 +2866,7 @@ void OSDMap::dump(Formatter *f) const
   }
   f->close_section();
   f->open_array_section("pg_temp");
-  for (const auto &pg : *pg_temp) {
-    f->open_object_section("osds");
-    f->dump_stream("pgid") << pg.first;
-    f->open_array_section("osds");
-    for (const auto osd : pg.second)
-      f->dump_int("osd", osd);
-    f->close_section();
-    f->close_section();
-  }
+  pg_temp->dump(f);
   f->close_section();
 
   f->open_array_section("primary_temp");
@@ -2749,14 +2987,16 @@ void OSDMap::print(ostream& out) const
       << "modified " << get_modified() << "\n";
 
   out << "flags " << get_flag_string() << "\n";
+  out << "crush_version " << get_crush_version() << "\n";
   out << "full_ratio " << full_ratio << "\n";
   out << "backfillfull_ratio " << backfillfull_ratio << "\n";
   out << "nearfull_ratio " << nearfull_ratio << "\n";
-  if (require_min_compat_client.length()) {
-    out << "require_min_compat_client " << require_min_compat_client << "\n";
+  if (require_min_compat_client > 0) {
+    out << "require_min_compat_client "
+       << ceph_release_name(require_min_compat_client) << "\n";
   }
-  auto mv = get_min_compat_client();
-  out << "min_compat_client " << mv.first << " " << mv.second << "\n";
+  out << "min_compat_client " << ceph_release_name(get_min_compat_client())
+      << "\n";
   if (get_cluster_snapshot().length())
     out << "cluster_snapshot " << get_cluster_snapshot() << "\n";
   out << "\n";
@@ -2808,8 +3048,24 @@ void OSDMap::print(ostream& out) const
 class OSDTreePlainDumper : public CrushTreeDumper::Dumper<TextTable> {
 public:
   typedef CrushTreeDumper::Dumper<TextTable> Parent;
-  OSDTreePlainDumper(const CrushWrapper *crush, const OSDMap *osdmap_)
-    : Parent(crush), osdmap(osdmap_) {}
+
+  OSDTreePlainDumper(const CrushWrapper *crush, const OSDMap *osdmap_,
+                    unsigned f)
+    : Parent(crush), osdmap(osdmap_), filter(f) { }
+
+  bool should_dump_leaf(int i) const override {
+    if (((filter & OSDMap::DUMP_UP) && !osdmap->is_up(i)) ||
+       ((filter & OSDMap::DUMP_DOWN) && !osdmap->is_down(i)) ||
+       ((filter & OSDMap::DUMP_IN) && !osdmap->is_in(i)) ||
+       ((filter & OSDMap::DUMP_OUT) && !osdmap->is_out(i))) {
+      return false;
+    }
+    return true;
+  }
+
+  bool should_dump_empty_bucket() const override {
+    return !filter;
+  }
 
   void dump(TextTable *tbl) {
     tbl->define_column("ID", TextTable::LEFT, TextTable::RIGHT);
@@ -2822,8 +3078,9 @@ public:
     Parent::dump(tbl);
 
     for (int i = 0; i < osdmap->get_max_osd(); i++) {
-      if (osdmap->exists(i) && !is_touched(i))
+      if (osdmap->exists(i) && !is_touched(i) && should_dump_leaf(i)) {
        dump_item(CrushTreeDumper::Item(i, 0, 0), tbl);
+      }
     }
   }
 
@@ -2859,14 +3116,30 @@ protected:
 
 private:
   const OSDMap *osdmap;
+  const unsigned filter;
 };
 
 class OSDTreeFormattingDumper : public CrushTreeDumper::FormattingDumper {
 public:
   typedef CrushTreeDumper::FormattingDumper Parent;
 
-  OSDTreeFormattingDumper(const CrushWrapper *crush, const OSDMap *osdmap_)
-    : Parent(crush), osdmap(osdmap_) {}
+  OSDTreeFormattingDumper(const CrushWrapper *crush, const OSDMap *osdmap_,
+                         unsigned f)
+    : Parent(crush), osdmap(osdmap_), filter(f) { }
+
+  bool should_dump_leaf(int i) const override {
+    if (((filter & OSDMap::DUMP_UP) && !osdmap->is_up(i)) ||
+       ((filter & OSDMap::DUMP_DOWN) && !osdmap->is_down(i)) ||
+       ((filter & OSDMap::DUMP_IN) && !osdmap->is_in(i)) ||
+       ((filter & OSDMap::DUMP_OUT) && !osdmap->is_out(i))) {
+      return false;
+    }
+    return true;
+  }
+
+  bool should_dump_empty_bucket() const override {
+    return !filter;
+  }
 
   void dump(Formatter *f) {
     f->open_array_section("nodes");
@@ -2874,7 +3147,7 @@ public:
     f->close_section();
     f->open_array_section("stray");
     for (int i = 0; i < osdmap->get_max_osd(); i++) {
-      if (osdmap->exists(i) && !is_touched(i))
+      if (osdmap->exists(i) && !is_touched(i) && should_dump_leaf(i))
        dump_item(CrushTreeDumper::Item(i, 0, 0), f);
     }
     f->close_section();
@@ -2894,16 +3167,17 @@ protected:
 
 private:
   const OSDMap *osdmap;
+  const unsigned filter;
 };
 
-void OSDMap::print_tree(Formatter *f, ostream *out) const
+void OSDMap::print_tree(Formatter *f, ostream *out, unsigned filter) const
 {
-  if (f)
-    OSDTreeFormattingDumper(crush.get(), this).dump(f);
-  else {
+  if (f) {
+    OSDTreeFormattingDumper(crush.get(), this, filter).dump(f);
+  else {
     assert(out);
     TextTable tbl;
-    OSDTreePlainDumper(crush.get(), this).dump(&tbl);
+    OSDTreePlainDumper(crush.get(), this, filter).dump(&tbl);
     *out << tbl;
   }
 }
@@ -2921,8 +3195,7 @@ void OSDMap::print_summary(Formatter *f, ostream& out) const
     f->dump_unsigned("num_remapped_pgs", get_num_pg_temp());
     f->close_section();
   } else {
-    out << "     osdmap e" << get_epoch() << ": "
-       << get_num_osds() << " osds: "
+    out << get_num_osds() << " osds: "
        << get_num_up_osds() << " up, "
        << get_num_in_osds() << " in";
     if (get_num_pg_temp())
@@ -2937,7 +3210,7 @@ void OSDMap::print_summary(Formatter *f, ostream& out) const
 void OSDMap::print_oneline_summary(ostream& out) const
 {
   out << "e" << get_epoch() << ": "
-      << get_num_osds() << " osds: "
+      << get_num_osds() << " total, "
       << get_num_up_osds() << " up, "
       << get_num_in_osds() << " in";
   if (test_flag(CEPH_OSDMAP_FULL))
@@ -2949,7 +3222,7 @@ void OSDMap::print_oneline_summary(ostream& out) const
 bool OSDMap::crush_ruleset_in_use(int ruleset) const
 {
   for (const auto &pool : pools) {
-    if (pool.second.crush_ruleset == ruleset)
+    if (pool.second.crush_rule == ruleset)
       return true;
   }
   return false;
@@ -3013,8 +3286,9 @@ int OSDMap::build_simple(CephContext *cct, epoch_t e, uuid_d &fsid,
 
   int poolbase = get_max_osd() ? get_max_osd() : 1;
 
-  int const default_replicated_ruleset = crush->get_osd_pool_default_crush_replicated_ruleset(cct);
-  assert(default_replicated_ruleset >= 0);
+  int const default_replicated_rule =
+    crush->get_osd_pool_default_crush_replicated_ruleset(cct);
+  assert(default_replicated_rule >= 0);
 
   for (auto &plname : pool_names) {
     int64_t pool = ++pool_max;
@@ -3030,7 +3304,7 @@ int OSDMap::build_simple(CephContext *cct, epoch_t e, uuid_d &fsid,
       pools[pool].set_flag(pg_pool_t::FLAG_NOSIZECHANGE);
     pools[pool].size = cct->_conf->osd_pool_default_size;
     pools[pool].min_size = cct->_conf->get_osd_pool_default_min_size();
-    pools[pool].crush_ruleset = default_replicated_ruleset;
+    pools[pool].crush_rule = default_replicated_rule;
     pools[pool].object_hash = CEPH_STR_HASH_RJENKINS;
     pools[pool].set_pg_num(poolbase << pg_bits);
     pools[pool].set_pgp_num(poolbase << pgp_bits);
@@ -3104,7 +3378,7 @@ int OSDMap::build_simple_crush_map(CephContext *cct, CrushWrapper& crush,
     crush.insert_item(cct, o, 1.0, name, loc);
   }
 
-  build_simple_crush_rulesets(cct, crush, "default", ss);
+  build_simple_crush_rules(cct, crush, "default", ss);
 
   crush.finalize();
 
@@ -3173,7 +3447,7 @@ int OSDMap::build_simple_crush_map_from_conf(CephContext *cct,
     crush.insert_item(cct, o, 1.0, section, loc);
   }
 
-  build_simple_crush_rulesets(cct, crush, "default", ss);
+  build_simple_crush_rules(cct, crush, "default", ss);
 
   crush.finalize();
 
@@ -3181,23 +3455,21 @@ int OSDMap::build_simple_crush_map_from_conf(CephContext *cct,
 }
 
 
-int OSDMap::build_simple_crush_rulesets(CephContext *cct,
-                                       CrushWrapper& crush,
-                                       const string& root,
-                                       ostream *ss)
+int OSDMap::build_simple_crush_rules(
+  CephContext *cct,
+  CrushWrapper& crush,
+  const string& root,
+  ostream *ss)
 {
-  int crush_ruleset =
-      crush._get_osd_pool_default_crush_replicated_ruleset(cct, true);
+  int crush_rule = crush.get_osd_pool_default_crush_replicated_ruleset(cct);
   string failure_domain =
     crush.get_type_name(cct->_conf->osd_crush_chooseleaf_type);
 
-  if (crush_ruleset == CEPH_DEFAULT_CRUSH_REPLICATED_RULESET)
-    crush_ruleset = -1; // create ruleset 0 by default
-
   int r;
-  r = crush.add_simple_ruleset_at("replicated_ruleset", root, failure_domain,
-                                  "firstn", pg_pool_t::TYPE_REPLICATED,
-                                  crush_ruleset, ss);
+  r = crush.add_simple_rule_at(
+    "replicated_rule", root, failure_domain,
+    "firstn", pg_pool_t::TYPE_REPLICATED,
+    crush_rule, ss);
   if (r < 0)
     return r;
   // do not add an erasure rule by default or else we will implicitly
@@ -3225,20 +3497,18 @@ int OSDMap::summarize_mapping_stats(
   vector<unsigned> new_by_osd(get_max_osd(), 0);
   for (int64_t pool_id : ls) {
     const pg_pool_t *pi = get_pg_pool(pool_id);
-    vector<int> up, up2, acting;
-    int up_primary, acting_primary;
+    vector<int> up, up2;
+    int up_primary;
     for (unsigned ps = 0; ps < pi->get_pg_num(); ++ps) {
       pg_t pgid(ps, pool_id, -1);
       total_pg += pi->get_size();
-      pg_to_up_acting_osds(pgid, &up, &up_primary,
-                          &acting, &acting_primary);
+      pg_to_up_acting_osds(pgid, &up, &up_primary, nullptr, nullptr);
       for (int osd : up) {
        if (osd >= 0 && osd < get_max_osd())
          ++base_by_osd[osd];
       }
       if (newmap) {
-       newmap->pg_to_up_acting_osds(pgid, &up2, &up_primary,
-                                    &acting, &acting_primary);
+       newmap->pg_to_up_acting_osds(pgid, &up2, &up_primary, nullptr, nullptr);
        for (int osd : up2) {
          if (osd >= 0 && osd < get_max_osd())
            ++new_by_osd[osd];
@@ -3420,7 +3690,7 @@ bool OSDMap::try_pg_upmap(
   const pg_pool_t *pool = get_pg_pool(pg.pool());
   if (!pool)
     return false;
-  int rule = crush->find_rule(pool->get_crush_ruleset(), pool->get_type(),
+  int rule = crush->find_rule(pool->get_crush_rule(), pool->get_type(),
                              pool->get_size());
   if (rule < 0)
     return false;
@@ -3456,17 +3726,29 @@ bool OSDMap::try_pg_upmap(
 
 int OSDMap::calc_pg_upmaps(
   CephContext *cct,
-  float max_deviation,
+  float max_deviation_ratio,
   int max,
-  const set<int64_t>& only_pools,
+  const set<int64_t>& only_pools_orig,
   OSDMap::Incremental *pending_inc)
 {
+  set<int64_t> only_pools;
+  if (only_pools_orig.empty()) {
+    for (auto& i : pools) {
+      only_pools.insert(i.first);
+    }
+  } else {
+    only_pools = only_pools_orig;
+  }
   OSDMap tmp;
   tmp.deepish_copy_from(*this);
+  float start_deviation = 0;
+  float end_deviation = 0;
   int num_changed = 0;
   while (true) {
     map<int,set<pg_t>> pgs_by_osd;
     int total_pgs = 0;
+    float osd_weight_total = 0;
+    map<int,float> osd_weight;
     for (auto& i : pools) {
       if (!only_pools.empty() && !only_pools.count(i.first))
        continue;
@@ -3480,23 +3762,35 @@ int OSDMap::calc_pg_upmaps(
        }
       }
       total_pgs += i.second.get_size() * i.second.get_pg_num();
+
+      map<int,float> pmap;
+      int ruleno = tmp.crush->find_rule(i.second.get_crush_rule(),
+                                       i.second.get_type(),
+                                       i.second.get_size());
+      tmp.crush->get_rule_weight_osd_map(ruleno, &pmap);
+      ldout(cct,30) << __func__ << " pool " << i.first << " ruleno " << ruleno << dendl;
+      for (auto p : pmap) {
+       osd_weight[p.first] += p.second;
+       osd_weight_total += p.second;
+      }
     }
-    float osd_weight_total = 0;
-    map<int,float> osd_weight;
-    for (auto& i : pgs_by_osd) {
-      float w = crush->get_item_weightf(i.first);
-      osd_weight[i.first] = w;
-      osd_weight_total += w;
-      ldout(cct, 20) << " osd." << i.first << " weight " << w
-                    << " pgs " << i.second.size() << dendl;
+    for (auto& i : osd_weight) {
+      int pgs = 0;
+      auto p = pgs_by_osd.find(i.first);
+      if (p != pgs_by_osd.end())
+       pgs = p->second.size();
+      else
+       pgs_by_osd.emplace(i.first, set<pg_t>());
+      ldout(cct, 20) << " osd." << i.first << " weight " << i.second
+                    << " pgs " << pgs << dendl;
     }
 
-    // NOTE: we assume we touch all osds with CRUSH!
     float pgs_per_weight = total_pgs / osd_weight_total;
     ldout(cct, 10) << " osd_weight_total " << osd_weight_total << dendl;
     ldout(cct, 10) << " pgs_per_weight " << pgs_per_weight << dendl;
 
     // osd deviation
+    float total_deviation = 0;
     map<int,float> osd_deviation;       // osd, deviation(pgs)
     multimap<float,int> deviation_osd;  // deviation(pgs), osd
     set<int> overfull;
@@ -3510,9 +3804,14 @@ int OSDMap::calc_pg_upmaps(
                     << dendl;
       osd_deviation[i.first] = deviation;
       deviation_osd.insert(make_pair(deviation, i.first));
-      if (deviation > 0)
+      if (deviation >= 1.0)
        overfull.insert(i.first);
+      total_deviation += abs(deviation);
+    }
+    if (num_changed == 0) {
+      start_deviation = total_deviation;
     }
+    end_deviation = total_deviation;
 
     // build underfull, sorted from least-full to most-average
     vector<int> underfull;
@@ -3523,7 +3822,8 @@ int OSDMap::calc_pg_upmaps(
        break;
       underfull.push_back(i->second);
     }
-    ldout(cct, 10) << " overfull " << overfull
+    ldout(cct, 10) << " total_deviation " << total_deviation
+                  << " overfull " << overfull
                   << " underfull " << underfull << dendl;
     if (overfull.empty() || underfull.empty())
       break;
@@ -3532,14 +3832,14 @@ int OSDMap::calc_pg_upmaps(
     bool restart = false;
     for (auto p = deviation_osd.rbegin(); p != deviation_osd.rend(); ++p) {
       int osd = p->second;
+      float deviation = p->first;
       float target = osd_weight[osd] * pgs_per_weight;
-      float deviation = deviation_osd.rbegin()->first;
-      if (deviation/target < max_deviation) {
+      if (deviation/target < max_deviation_ratio) {
        ldout(cct, 10) << " osd." << osd
                       << " target " << target
                       << " deviation " << deviation
-                      << " -> " << deviation/target
-                      << " < max " << max_deviation << dendl;
+                      << " -> ratio " << deviation/target
+                      << " < max ratio " << max_deviation_ratio << dendl;
        break;
       }
       int num_to_move = deviation;
@@ -3611,5 +3911,332 @@ int OSDMap::calc_pg_upmaps(
       break;
     }
   }
+  ldout(cct, 10) << " start deviation " << start_deviation << dendl;
+  ldout(cct, 10) << " end deviation " << end_deviation << dendl;
   return num_changed;
 }
+
+int OSDMap::get_osds_by_bucket_name(const string &name, set<int> *osds) const
+{
+  return crush->get_leaves(name, osds);
+}
+
+template <typename F>
+class OSDUtilizationDumper : public CrushTreeDumper::Dumper<F> {
+public:
+  typedef CrushTreeDumper::Dumper<F> Parent;
+
+  OSDUtilizationDumper(const CrushWrapper *crush, const OSDMap *osdmap_,
+                      const PGStatService *pgs_, bool tree_) :
+    Parent(crush),
+    osdmap(osdmap_),
+    pgs(pgs_),
+    tree(tree_),
+    average_util(average_utilization()),
+    min_var(-1),
+    max_var(-1),
+    stddev(0),
+    sum(0) {
+  }
+
+protected:
+  void dump_stray(F *f) {
+    for (int i = 0; i < osdmap->get_max_osd(); i++) {
+      if (osdmap->exists(i) && !this->is_touched(i))
+       dump_item(CrushTreeDumper::Item(i, 0, 0), f);
+    }
+  }
+
+  void dump_item(const CrushTreeDumper::Item &qi, F *f) override {
+    if (!tree && qi.is_bucket())
+      return;
+
+    float reweight = qi.is_bucket() ? -1 : osdmap->get_weightf(qi.id);
+    int64_t kb = 0, kb_used = 0, kb_avail = 0;
+    double util = 0;
+    if (get_bucket_utilization(qi.id, &kb, &kb_used, &kb_avail))
+      if (kb_used && kb)
+        util = 100.0 * (double)kb_used / (double)kb;
+
+    double var = 1.0;
+    if (average_util)
+      var = util / average_util;
+
+    size_t num_pgs = qi.is_bucket() ? 0 : pgs->get_num_pg_by_osd(qi.id);
+
+    dump_item(qi, reweight, kb, kb_used, kb_avail, util, var, num_pgs, f);
+
+    if (!qi.is_bucket() && reweight > 0) {
+      if (min_var < 0 || var < min_var)
+       min_var = var;
+      if (max_var < 0 || var > max_var)
+       max_var = var;
+
+      double dev = util - average_util;
+      dev *= dev;
+      stddev += reweight * dev;
+      sum += reweight;
+    }
+  }
+
+  virtual void dump_item(const CrushTreeDumper::Item &qi,
+                        float &reweight,
+                        int64_t kb,
+                        int64_t kb_used,
+                        int64_t kb_avail,
+                        double& util,
+                        double& var,
+                        const size_t num_pgs,
+                        F *f) = 0;
+
+  double dev() {
+    return sum > 0 ? sqrt(stddev / sum) : 0;
+  }
+
+  double average_utilization() {
+    int64_t kb = 0, kb_used = 0;
+    for (int i = 0; i < osdmap->get_max_osd(); i++) {
+      if (!osdmap->exists(i) || osdmap->get_weight(i) == 0)
+       continue;
+      int64_t kb_i, kb_used_i, kb_avail_i;
+      if (get_osd_utilization(i, &kb_i, &kb_used_i, &kb_avail_i)) {
+       kb += kb_i;
+       kb_used += kb_used_i;
+      }
+    }
+    return kb > 0 ? 100.0 * (double)kb_used / (double)kb : 0;
+  }
+
+  bool get_osd_utilization(int id, int64_t* kb, int64_t* kb_used,
+                          int64_t* kb_avail) const {
+    const osd_stat_t *p = pgs->get_osd_stat(id);
+    if (!p) return false;
+    *kb = p->kb;
+    *kb_used = p->kb_used;
+    *kb_avail = p->kb_avail;
+    return *kb > 0;
+  }
+
+  bool get_bucket_utilization(int id, int64_t* kb, int64_t* kb_used,
+                             int64_t* kb_avail) const {
+    if (id >= 0) {
+      if (osdmap->is_out(id)) {
+        *kb = 0;
+        *kb_used = 0;
+        *kb_avail = 0;
+        return true;
+      }
+      return get_osd_utilization(id, kb, kb_used, kb_avail);
+    }
+
+    *kb = 0;
+    *kb_used = 0;
+    *kb_avail = 0;
+
+    for (int k = osdmap->crush->get_bucket_size(id) - 1; k >= 0; k--) {
+      int item = osdmap->crush->get_bucket_item(id, k);
+      int64_t kb_i = 0, kb_used_i = 0, kb_avail_i = 0;
+      if (!get_bucket_utilization(item, &kb_i, &kb_used_i, &kb_avail_i))
+       return false;
+      *kb += kb_i;
+      *kb_used += kb_used_i;
+      *kb_avail += kb_avail_i;
+    }
+    return *kb > 0;
+  }
+
+protected:
+  const OSDMap *osdmap;
+  const PGStatService *pgs;
+  bool tree;
+  double average_util;
+  double min_var;
+  double max_var;
+  double stddev;
+  double sum;
+};
+
+
+class OSDUtilizationPlainDumper : public OSDUtilizationDumper<TextTable> {
+public:
+  typedef OSDUtilizationDumper<TextTable> Parent;
+
+  OSDUtilizationPlainDumper(const CrushWrapper *crush, const OSDMap *osdmap,
+                    const PGStatService *pgs, bool tree) :
+    Parent(crush, osdmap, pgs, tree) {}
+
+  void dump(TextTable *tbl) {
+    tbl->define_column("ID", TextTable::LEFT, TextTable::RIGHT);
+    tbl->define_column("WEIGHT", TextTable::LEFT, TextTable::RIGHT);
+    tbl->define_column("REWEIGHT", TextTable::LEFT, TextTable::RIGHT);
+    tbl->define_column("SIZE", TextTable::LEFT, TextTable::RIGHT);
+    tbl->define_column("USE", TextTable::LEFT, TextTable::RIGHT);
+    tbl->define_column("AVAIL", TextTable::LEFT, TextTable::RIGHT);
+    tbl->define_column("%USE", TextTable::LEFT, TextTable::RIGHT);
+    tbl->define_column("VAR", TextTable::LEFT, TextTable::RIGHT);
+    tbl->define_column("PGS", TextTable::LEFT, TextTable::RIGHT);
+    if (tree)
+      tbl->define_column("TYPE NAME", TextTable::LEFT, TextTable::LEFT);
+
+    Parent::dump(tbl);
+
+    dump_stray(tbl);
+
+    *tbl << "" << "" << "TOTAL"
+        << si_t(pgs->get_osd_sum().kb << 10)
+        << si_t(pgs->get_osd_sum().kb_used << 10)
+        << si_t(pgs->get_osd_sum().kb_avail << 10)
+        << lowprecision_t(average_util)
+        << ""
+        << TextTable::endrow;
+  }
+
+protected:
+  struct lowprecision_t {
+    float v;
+    explicit lowprecision_t(float _v) : v(_v) {}
+  };
+  friend std::ostream &operator<<(ostream& out, const lowprecision_t& v);
+
+  using OSDUtilizationDumper<TextTable>::dump_item;
+  void dump_item(const CrushTreeDumper::Item &qi,
+                        float &reweight,
+                        int64_t kb,
+                        int64_t kb_used,
+                        int64_t kb_avail,
+                        double& util,
+                        double& var,
+                        const size_t num_pgs,
+                        TextTable *tbl) override {
+    *tbl << qi.id
+        << weightf_t(qi.weight)
+        << weightf_t(reweight)
+        << si_t(kb << 10)
+        << si_t(kb_used << 10)
+        << si_t(kb_avail << 10)
+        << lowprecision_t(util)
+        << lowprecision_t(var);
+
+    if (qi.is_bucket()) {
+      *tbl << "-";
+    } else {
+      *tbl << num_pgs;
+    }
+
+    if (tree) {
+      ostringstream name;
+      for (int k = 0; k < qi.depth; k++)
+       name << "    ";
+      if (qi.is_bucket()) {
+       int type = crush->get_bucket_type(qi.id);
+       name << crush->get_type_name(type) << " "
+            << crush->get_item_name(qi.id);
+      } else {
+       name << "osd." << qi.id;
+      }
+      *tbl << name.str();
+    }
+
+    *tbl << TextTable::endrow;
+  }
+
+public:
+  string summary() {
+    ostringstream out;
+    out << "MIN/MAX VAR: " << lowprecision_t(min_var)
+       << "/" << lowprecision_t(max_var) << "  "
+       << "STDDEV: " << lowprecision_t(dev());
+    return out.str();
+  }
+};
+
+ostream& operator<<(ostream& out,
+                   const OSDUtilizationPlainDumper::lowprecision_t& v)
+{
+  if (v.v < -0.01) {
+    return out << "-";
+  } else if (v.v < 0.001) {
+    return out << "0";
+  } else {
+    std::streamsize p = out.precision();
+    return out << std::fixed << std::setprecision(2) << v.v << std::setprecision(p);
+  }
+}
+
+class OSDUtilizationFormatDumper : public OSDUtilizationDumper<Formatter> {
+public:
+  typedef OSDUtilizationDumper<Formatter> Parent;
+
+  OSDUtilizationFormatDumper(const CrushWrapper *crush, const OSDMap *osdmap,
+                            const PGStatService *pgs, bool tree) :
+    Parent(crush, osdmap, pgs, tree) {}
+
+  void dump(Formatter *f) {
+    f->open_array_section("nodes");
+    Parent::dump(f);
+    f->close_section();
+
+    f->open_array_section("stray");
+    dump_stray(f);
+    f->close_section();
+  }
+
+protected:
+  using OSDUtilizationDumper<Formatter>::dump_item;
+  void dump_item(const CrushTreeDumper::Item &qi,
+                        float &reweight,
+                        int64_t kb,
+                        int64_t kb_used,
+                        int64_t kb_avail,
+                        double& util,
+                        double& var,
+                        const size_t num_pgs,
+                        Formatter *f) override {
+    f->open_object_section("item");
+    CrushTreeDumper::dump_item_fields(crush, qi, f);
+    f->dump_float("reweight", reweight);
+    f->dump_int("kb", kb);
+    f->dump_int("kb_used", kb_used);
+    f->dump_int("kb_avail", kb_avail);
+    f->dump_float("utilization", util);
+    f->dump_float("var", var);
+    f->dump_unsigned("pgs", num_pgs);
+    CrushTreeDumper::dump_bucket_children(crush, qi, f);
+    f->close_section();
+  }
+
+public:
+  void summary(Formatter *f) {
+    f->open_object_section("summary");
+    f->dump_int("total_kb", pgs->get_osd_sum().kb);
+    f->dump_int("total_kb_used", pgs->get_osd_sum().kb_used);
+    f->dump_int("total_kb_avail", pgs->get_osd_sum().kb_avail);
+    f->dump_float("average_utilization", average_util);
+    f->dump_float("min_var", min_var);
+    f->dump_float("max_var", max_var);
+    f->dump_float("dev", dev());
+    f->close_section();
+  }
+};
+
+void print_osd_utilization(const OSDMap& osdmap,
+                          const PGStatService *pgstat,
+                          ostream& out,
+                          Formatter *f,
+                          bool tree)
+{
+  const CrushWrapper *crush = osdmap.crush.get();
+  if (f) {
+    f->open_object_section("df");
+    OSDUtilizationFormatDumper d(crush, &osdmap, pgstat, tree);
+    d.dump(f);
+    d.summary(f);
+    f->close_section();
+    f->flush(out);
+  } else {
+    OSDUtilizationPlainDumper d(crush, &osdmap, pgstat, tree);
+    TextTable tbl;
+    d.dump(&tbl);
+    out << tbl << d.summary() << "\n";
+  }
+}