ceph/src/osd/MissingLoc.h

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #pragma once
   5
   6 #include <map>
   7 #include <set>
   8
   9 #include "OSDMap.h"
  10 #include "common/HBHandle.h"
  11 #include "common/ceph_context.h"
  12 #include "common/dout.h"
  13 #include "osd_types.h"
  14
  15 class MissingLoc {
  16  public:
  17
  18   class MappingInfo {
  19   public:
  20     virtual const set<pg_shard_t> &get_upset() const = 0;
  21     virtual bool is_ec_pg() const = 0;
  22     virtual int get_pg_size() const = 0;
  23     virtual ~MappingInfo() {}
  24   };
  25
  26   // a loc_count indicates how many locations we know in each of
  27   // these distinct sets
  28   struct loc_count_t {
  29     int up = 0;        //< up
  30     int other = 0;    //< other
  31
  32     friend bool operator<(const loc_count_t& l,
  33                           const loc_count_t& r) {
  34       return (l.up < r.up ||
  35               (l.up == r.up &&
  36                (l.other < r.other)));
  37     }
  38     friend ostream& operator<<(ostream& out, const loc_count_t& l) {
  39       ceph_assert(l.up >= 0);
  40       ceph_assert(l.other >= 0);
  41       return out << "(" << l.up << "+" << l.other << ")";
  42     }
  43   };
  44
  45
  46   using missing_by_count_t = map < shard_id_t, map<loc_count_t,int> >;
  47  private:
  48   loc_count_t _get_count(const set<pg_shard_t> &shards) {
  49     loc_count_t r;
  50     for (auto s : shards) {
  51       if (mapping_info->get_upset().count(s)) {
  52         r.up++;
  53       } else {
  54         r.other++;
  55       }
  56     }
  57     return r;
  58   }
  59
  60   map<hobject_t, pg_missing_item> needs_recovery_map;
  61   map<hobject_t, set<pg_shard_t> > missing_loc;
  62   set<pg_shard_t> missing_loc_sources;
  63
  64   // for every entry in missing_loc, we count how many of each type of shard we have,
  65   // and maintain totals here.  The sum of the values for this map will always equal
  66   // missing_loc.size().
  67   missing_by_count_t missing_by_count;
  68
  69   void pgs_by_shard_id(
  70     const set<pg_shard_t>& s,
  71     map< shard_id_t, set<pg_shard_t> >& pgsbs) {
  72     if (mapping_info->is_ec_pg()) {
  73       int num_shards = mapping_info->get_pg_size();
  74       // For completely missing shards initialize with empty set<pg_shard_t>
  75       for (int i = 0 ; i < num_shards ; ++i) {
  76         shard_id_t shard(i);
  77         pgsbs[shard];
  78       }
  79       for (auto pgs: s)
  80         pgsbs[pgs.shard].insert(pgs);
  81     } else {
  82       pgsbs[shard_id_t::NO_SHARD] = s;
  83     }
  84   }
  85
  86   void _inc_count(const set<pg_shard_t>& s) {
  87     map< shard_id_t, set<pg_shard_t> > pgsbs;
  88     pgs_by_shard_id(s, pgsbs);
  89     for (auto shard: pgsbs)
  90       ++missing_by_count[shard.first][_get_count(shard.second)];
  91   }
  92   void _dec_count(const set<pg_shard_t>& s) {
  93     map< shard_id_t, set<pg_shard_t> > pgsbs;
  94     pgs_by_shard_id(s, pgsbs);
  95     for (auto shard: pgsbs) {
  96       auto p = missing_by_count[shard.first].find(_get_count(shard.second));
  97       ceph_assert(p != missing_by_count[shard.first].end());
  98       if (--p->second == 0) {
  99         missing_by_count[shard.first].erase(p);
 100       }
 101     }
 102   }
 103
 104   spg_t pgid;
 105   MappingInfo *mapping_info;
 106   DoutPrefixProvider *dpp;
 107   CephContext *cct;
 108   set<pg_shard_t> empty_set;
 109  public:
 110   boost::scoped_ptr<IsPGReadablePredicate> is_readable;
 111   boost::scoped_ptr<IsPGRecoverablePredicate> is_recoverable;
 112   explicit MissingLoc(
 113     spg_t pgid,
 114     MappingInfo *mapping_info,
 115     DoutPrefixProvider *dpp,
 116     CephContext *cct)
 117     : pgid(pgid), mapping_info(mapping_info), dpp(dpp), cct(cct) { }
 118   void set_backend_predicates(
 119     IsPGReadablePredicate *_is_readable,
 120     IsPGRecoverablePredicate *_is_recoverable) {
 121     is_readable.reset(_is_readable);
 122     is_recoverable.reset(_is_recoverable);
 123   }
 124   const IsPGRecoverablePredicate &get_recoverable_predicate() const {
 125     return *is_recoverable;
 126   }
 127   std::ostream& gen_prefix(std::ostream& out) const {
 128     return dpp->gen_prefix(out);
 129   }
 130   bool needs_recovery(
 131     const hobject_t &hoid,
 132     eversion_t *v = 0) const {
 133     map<hobject_t, pg_missing_item>::const_iterator i =
 134       needs_recovery_map.find(hoid);
 135     if (i == needs_recovery_map.end())
 136       return false;
 137     if (v)
 138       *v = i->second.need;
 139     return true;
 140   }
 141   bool is_deleted(const hobject_t &hoid) const {
 142     auto i = needs_recovery_map.find(hoid);
 143     if (i == needs_recovery_map.end())
 144       return false;
 145     return i->second.is_delete();
 146   }
 147   bool is_unfound(const hobject_t &hoid) const {
 148     auto it = needs_recovery_map.find(hoid);
 149     if (it == needs_recovery_map.end()) {
 150       return false;
 151     }
 152     if (it->second.is_delete()) {
 153       return false;
 154     }
 155     auto mit = missing_loc.find(hoid);
 156     return mit == missing_loc.end() || !(*is_recoverable)(mit->second);
 157   }
 158   bool readable_with_acting(
 159     const hobject_t &hoid,
 160     const set<pg_shard_t> &acting) const;
 161   uint64_t num_unfound() const {
 162     uint64_t ret = 0;
 163     for (map<hobject_t, pg_missing_item>::const_iterator i =
 164            needs_recovery_map.begin();
 165          i != needs_recovery_map.end();
 166          ++i) {
 167       if (i->second.is_delete())
 168         continue;
 169       auto mi = missing_loc.find(i->first);
 170       if (mi == missing_loc.end() || !(*is_recoverable)(mi->second))
 171         ++ret;
 172     }
 173     return ret;
 174   }
 175
 176   bool have_unfound() const {
 177     for (map<hobject_t, pg_missing_item>::const_iterator i =
 178            needs_recovery_map.begin();
 179          i != needs_recovery_map.end();
 180          ++i) {
 181       if (i->second.is_delete())
 182         continue;
 183       auto mi = missing_loc.find(i->first);
 184       if (mi == missing_loc.end() || !(*is_recoverable)(mi->second))
 185         return true;
 186     }
 187     return false;
 188   }
 189   void clear() {
 190     needs_recovery_map.clear();
 191     missing_loc.clear();
 192     missing_loc_sources.clear();
 193     missing_by_count.clear();
 194   }
 195
 196   void add_location(const hobject_t &hoid, pg_shard_t location) {
 197     auto p = missing_loc.find(hoid);
 198     if (p == missing_loc.end()) {
 199       p = missing_loc.emplace(hoid, set<pg_shard_t>()).first;
 200     } else {
 201       _dec_count(p->second);
 202     }
 203     p->second.insert(location);
 204     _inc_count(p->second);
 205   }
 206   void remove_location(const hobject_t &hoid, pg_shard_t location) {
 207     auto p = missing_loc.find(hoid);
 208     if (p != missing_loc.end()) {
 209       _dec_count(p->second);
 210       p->second.erase(location);
 211       if (p->second.empty()) {
 212         missing_loc.erase(p);
 213       } else {
 214         _inc_count(p->second);
 215       }
 216     }
 217   }
 218
 219   void clear_location(const hobject_t &hoid) {
 220     auto p = missing_loc.find(hoid);
 221     if (p != missing_loc.end()) {
 222       _dec_count(p->second);
 223       missing_loc.erase(p);
 224     }
 225   }
 226
 227   void add_active_missing(const pg_missing_t &missing) {
 228     for (map<hobject_t, pg_missing_item>::const_iterator i =
 229            missing.get_items().begin();
 230          i != missing.get_items().end();
 231          ++i) {
 232       map<hobject_t, pg_missing_item>::const_iterator j =
 233         needs_recovery_map.find(i->first);
 234       if (j == needs_recovery_map.end()) {
 235         needs_recovery_map.insert(*i);
 236       } else {
 237         if (i->second.need != j->second.need) {
 238           lgeneric_dout(cct, 0) << this << " " << pgid << " unexpected need for "
 239                                 << i->first << " have " << j->second
 240                                 << " tried to add " << i->second << dendl;
 241           ceph_assert(0 == "unexpected need for missing item");
 242         }
 243       }
 244     }
 245   }
 246
 247   void add_missing(const hobject_t &hoid, eversion_t need, eversion_t have, bool is_delete=false) {
 248     needs_recovery_map[hoid] = pg_missing_item(need, have, is_delete);
 249   }
 250   void revise_need(const hobject_t &hoid, eversion_t need) {
 251     auto it = needs_recovery_map.find(hoid);
 252     ceph_assert(it != needs_recovery_map.end());
 253     it->second.need = need;
 254   }
 255
 256   /// Adds info about a possible recovery source
 257   bool add_source_info(
 258     pg_shard_t source,           ///< [in] source
 259     const pg_info_t &oinfo,      ///< [in] info
 260     const pg_missing_t &omissing, ///< [in] (optional) missing
 261     HBHandle *handle             ///< [in] ThreadPool handle
 262     ); ///< @return whether a new object location was discovered
 263
 264   /// Adds recovery sources in batch
 265   void add_batch_sources_info(
 266     const set<pg_shard_t> &sources,  ///< [in] a set of resources which can be used for all objects
 267     HBHandle *handle  ///< [in] ThreadPool handle
 268     );
 269
 270   /// Uses osdmap to update structures for now down sources
 271   void check_recovery_sources(const OSDMapRef& osdmap);
 272
 273   /// Remove stray from recovery sources
 274   void remove_stray_recovery_sources(pg_shard_t stray);
 275
 276   /// Call when hoid is no longer missing in acting set
 277   void recovered(const hobject_t &hoid) {
 278     needs_recovery_map.erase(hoid);
 279     auto p = missing_loc.find(hoid);
 280     if (p != missing_loc.end()) {
 281       _dec_count(p->second);
 282       missing_loc.erase(p);
 283     }
 284   }
 285
 286   /// Call to update structures for hoid after a change
 287   void rebuild(
 288     const hobject_t &hoid,
 289     pg_shard_t self,
 290     const set<pg_shard_t> &to_recover,
 291     const pg_info_t &info,
 292     const pg_missing_t &missing,
 293     const map<pg_shard_t, pg_missing_t> &pmissing,
 294     const map<pg_shard_t, pg_info_t> &pinfo) {
 295     recovered(hoid);
 296     std::optional<pg_missing_item> item;
 297     auto miter = missing.get_items().find(hoid);
 298     if (miter != missing.get_items().end()) {
 299       item = miter->second;
 300     } else {
 301       for (auto &&i: to_recover) {
 302         if (i == self)
 303           continue;
 304         auto pmiter = pmissing.find(i);
 305         ceph_assert(pmiter != pmissing.end());
 306         miter = pmiter->second.get_items().find(hoid);
 307         if (miter != pmiter->second.get_items().end()) {
 308           item = miter->second;
 309           break;
 310         }
 311       }
 312     }
 313     if (!item)
 314       return; // recovered!
 315
 316     needs_recovery_map[hoid] = *item;
 317     if (item->is_delete())
 318       return;
 319     auto mliter =
 320       missing_loc.emplace(hoid, set<pg_shard_t>()).first;
 321     ceph_assert(info.last_backfill.is_max());
 322     ceph_assert(info.last_update >= item->need);
 323     if (!missing.is_missing(hoid))
 324       mliter->second.insert(self);
 325     for (auto &&i: pmissing) {
 326       if (i.first == self)
 327         continue;
 328       auto pinfoiter = pinfo.find(i.first);
 329       ceph_assert(pinfoiter != pinfo.end());
 330       if (item->need <= pinfoiter->second.last_update &&
 331           hoid <= pinfoiter->second.last_backfill &&
 332           !i.second.is_missing(hoid))
 333         mliter->second.insert(i.first);
 334     }
 335     _inc_count(mliter->second);
 336   }
 337
 338   const set<pg_shard_t> &get_locations(const hobject_t &hoid) const {
 339     auto it = missing_loc.find(hoid);
 340     return it == missing_loc.end() ? empty_set : it->second;
 341   }
 342   const map<hobject_t, set<pg_shard_t>> &get_missing_locs() const {
 343     return missing_loc;
 344   }
 345   const map<hobject_t, pg_missing_item> &get_needs_recovery() const {
 346     return needs_recovery_map;
 347   }
 348
 349   const missing_by_count_t &get_missing_by_count() const {
 350     return missing_by_count;
 351   }
 352 };