ceph/src/osd/MissingLoc.h

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #pragma once
   5
   6 #include <map>
   7 #include <set>
   8
   9 #include "OSDMap.h"
  10 #include "common/HBHandle.h"
  11 #include "common/ceph_context.h"
  12 #include "common/dout.h"
  13 #include "osd_types.h"
  14
  15 class MissingLoc {
  16  public:
  17
  18   class MappingInfo {
  19   public:
  20     virtual const std::set<pg_shard_t> &get_upset() const = 0;
  21     virtual bool is_ec_pg() const = 0;
  22     virtual int get_pg_size() const = 0;
  23     virtual ~MappingInfo() {}
  24   };
  25
  26   // a loc_count indicates how many locations we know in each of
  27   // these distinct sets
  28   struct loc_count_t {
  29     int up = 0;        //< up
  30     int other = 0;    //< other
  31
  32     friend bool operator<(const loc_count_t& l,
  33                           const loc_count_t& r) {
  34       return (l.up < r.up ||
  35               (l.up == r.up &&
  36                (l.other < r.other)));
  37     }
  38     friend std::ostream& operator<<(std::ostream& out, const loc_count_t& l) {
  39       ceph_assert(l.up >= 0);
  40       ceph_assert(l.other >= 0);
  41       return out << "(" << l.up << "+" << l.other << ")";
  42     }
  43   };
  44
  45
  46   using missing_by_count_t = std::map<shard_id_t, std::map<loc_count_t,int>>;
  47  private:
  48   loc_count_t _get_count(const std::set<pg_shard_t> &shards) {
  49     loc_count_t r;
  50     for (auto s : shards) {
  51       if (mapping_info->get_upset().count(s)) {
  52         r.up++;
  53       } else {
  54         r.other++;
  55       }
  56     }
  57     return r;
  58   }
  59
  60   std::map<hobject_t, pg_missing_item> needs_recovery_map;
  61   std::map<hobject_t, std::set<pg_shard_t> > missing_loc;
  62   std::set<pg_shard_t> missing_loc_sources;
  63
  64   // for every entry in missing_loc, we count how many of each type of shard we have,
  65   // and maintain totals here.  The sum of the values for this std::map will always equal
  66   // missing_loc.size().
  67   missing_by_count_t missing_by_count;
  68
  69   void pgs_by_shard_id(
  70     const std::set<pg_shard_t>& s,
  71     std::map<shard_id_t, std::set<pg_shard_t> >& pgsbs) {
  72     if (mapping_info->is_ec_pg()) {
  73       int num_shards = mapping_info->get_pg_size();
  74       // For completely missing shards initialize with empty std::set<pg_shard_t>
  75       for (int i = 0 ; i < num_shards ; ++i) {
  76         shard_id_t shard(i);
  77         pgsbs[shard];
  78       }
  79       for (auto pgs: s)
  80         pgsbs[pgs.shard].insert(pgs);
  81     } else {
  82       pgsbs[shard_id_t::NO_SHARD] = s;
  83     }
  84   }
  85
  86   void _inc_count(const std::set<pg_shard_t>& s) {
  87     std::map< shard_id_t, std::set<pg_shard_t> > pgsbs;
  88     pgs_by_shard_id(s, pgsbs);
  89     for (auto shard: pgsbs)
  90       ++missing_by_count[shard.first][_get_count(shard.second)];
  91   }
  92   void _dec_count(const std::set<pg_shard_t>& s) {
  93     std::map< shard_id_t, std::set<pg_shard_t> > pgsbs;
  94     pgs_by_shard_id(s, pgsbs);
  95     for (auto shard: pgsbs) {
  96       auto p = missing_by_count[shard.first].find(_get_count(shard.second));
  97       ceph_assert(p != missing_by_count[shard.first].end());
  98       if (--p->second == 0) {
  99         missing_by_count[shard.first].erase(p);
 100       }
 101     }
 102   }
 103
 104   spg_t pgid;
 105   MappingInfo *mapping_info;
 106   DoutPrefixProvider *dpp;
 107   CephContext *cct;
 108   std::set<pg_shard_t> empty_set;
 109  public:
 110   boost::scoped_ptr<IsPGReadablePredicate> is_readable;
 111   boost::scoped_ptr<IsPGRecoverablePredicate> is_recoverable;
 112   explicit MissingLoc(
 113     spg_t pgid,
 114     MappingInfo *mapping_info,
 115     DoutPrefixProvider *dpp,
 116     CephContext *cct)
 117     : pgid(pgid), mapping_info(mapping_info), dpp(dpp), cct(cct) { }
 118   void set_backend_predicates(
 119     IsPGReadablePredicate *_is_readable,
 120     IsPGRecoverablePredicate *_is_recoverable) {
 121     is_readable.reset(_is_readable);
 122     is_recoverable.reset(_is_recoverable);
 123   }
 124   const IsPGRecoverablePredicate &get_recoverable_predicate() const {
 125     return *is_recoverable;
 126   }
 127   std::ostream& gen_prefix(std::ostream& out) const {
 128     return dpp->gen_prefix(out);
 129   }
 130   bool needs_recovery(
 131     const hobject_t &hoid,
 132     eversion_t *v = 0) const {
 133     std::map<hobject_t, pg_missing_item>::const_iterator i =
 134       needs_recovery_map.find(hoid);
 135     if (i == needs_recovery_map.end())
 136       return false;
 137     if (v)
 138       *v = i->second.need;
 139     return true;
 140   }
 141   bool is_deleted(const hobject_t &hoid) const {
 142     auto i = needs_recovery_map.find(hoid);
 143     if (i == needs_recovery_map.end())
 144       return false;
 145     return i->second.is_delete();
 146   }
 147   bool is_unfound(const hobject_t &hoid) const {
 148     auto it = needs_recovery_map.find(hoid);
 149     if (it == needs_recovery_map.end()) {
 150       return false;
 151     }
 152     if (it->second.is_delete()) {
 153       return false;
 154     }
 155     auto mit = missing_loc.find(hoid);
 156     return mit == missing_loc.end() || !(*is_recoverable)(mit->second);
 157   }
 158   bool readable_with_acting(
 159     const hobject_t &hoid,
 160     const std::set<pg_shard_t> &acting,
 161     eversion_t* v = 0) const;
 162   uint64_t num_unfound() const {
 163     uint64_t ret = 0;
 164     for (std::map<hobject_t, pg_missing_item>::const_iterator i =
 165            needs_recovery_map.begin();
 166          i != needs_recovery_map.end();
 167          ++i) {
 168       if (i->second.is_delete())
 169         continue;
 170       auto mi = missing_loc.find(i->first);
 171       if (mi == missing_loc.end() || !(*is_recoverable)(mi->second))
 172         ++ret;
 173     }
 174     return ret;
 175   }
 176
 177   bool have_unfound() const {
 178     for (std::map<hobject_t, pg_missing_item>::const_iterator i =
 179            needs_recovery_map.begin();
 180          i != needs_recovery_map.end();
 181          ++i) {
 182       if (i->second.is_delete())
 183         continue;
 184       auto mi = missing_loc.find(i->first);
 185       if (mi == missing_loc.end() || !(*is_recoverable)(mi->second))
 186         return true;
 187     }
 188     return false;
 189   }
 190   void clear() {
 191     needs_recovery_map.clear();
 192     missing_loc.clear();
 193     missing_loc_sources.clear();
 194     missing_by_count.clear();
 195   }
 196
 197   void add_location(const hobject_t &hoid, pg_shard_t location) {
 198     auto p = missing_loc.find(hoid);
 199     if (p == missing_loc.end()) {
 200       p = missing_loc.emplace(hoid, std::set<pg_shard_t>()).first;
 201     } else {
 202       _dec_count(p->second);
 203     }
 204     p->second.insert(location);
 205     _inc_count(p->second);
 206   }
 207   void remove_location(const hobject_t &hoid, pg_shard_t location) {
 208     auto p = missing_loc.find(hoid);
 209     if (p != missing_loc.end()) {
 210       _dec_count(p->second);
 211       p->second.erase(location);
 212       if (p->second.empty()) {
 213         missing_loc.erase(p);
 214       } else {
 215         _inc_count(p->second);
 216       }
 217     }
 218   }
 219
 220   void clear_location(const hobject_t &hoid) {
 221     auto p = missing_loc.find(hoid);
 222     if (p != missing_loc.end()) {
 223       _dec_count(p->second);
 224       missing_loc.erase(p);
 225     }
 226   }
 227
 228   void add_active_missing(const pg_missing_t &missing) {
 229     for (std::map<hobject_t, pg_missing_item>::const_iterator i =
 230            missing.get_items().begin();
 231          i != missing.get_items().end();
 232          ++i) {
 233       std::map<hobject_t, pg_missing_item>::const_iterator j =
 234         needs_recovery_map.find(i->first);
 235       if (j == needs_recovery_map.end()) {
 236         needs_recovery_map.insert(*i);
 237       } else {
 238         if (i->second.need != j->second.need) {
 239           lgeneric_dout(cct, 0) << this << " " << pgid << " unexpected need for "
 240                                 << i->first << " have " << j->second
 241                                 << " tried to add " << i->second << dendl;
 242           ceph_assert(0 == "unexpected need for missing item");
 243         }
 244       }
 245     }
 246   }
 247
 248   void add_missing(const hobject_t &hoid, eversion_t need, eversion_t have, bool is_delete=false) {
 249     needs_recovery_map[hoid] = pg_missing_item(need, have, is_delete);
 250   }
 251   void revise_need(const hobject_t &hoid, eversion_t need) {
 252     auto it = needs_recovery_map.find(hoid);
 253     ceph_assert(it != needs_recovery_map.end());
 254     it->second.need = need;
 255   }
 256
 257   /// Adds info about a possible recovery source
 258   bool add_source_info(
 259     pg_shard_t source,           ///< [in] source
 260     const pg_info_t &oinfo,      ///< [in] info
 261     const pg_missing_t &omissing, ///< [in] (optional) missing
 262     HBHandle *handle             ///< [in] ThreadPool handle
 263     ); ///< @return whether a new object location was discovered
 264
 265   /// Adds recovery sources in batch
 266   void add_batch_sources_info(
 267     const std::set<pg_shard_t> &sources,  ///< [in] a std::set of resources which can be used for all objects
 268     HBHandle *handle  ///< [in] ThreadPool handle
 269     );
 270
 271   /// Uses osdmap to update structures for now down sources
 272   void check_recovery_sources(const OSDMapRef& osdmap);
 273
 274   /// Remove stray from recovery sources
 275   void remove_stray_recovery_sources(pg_shard_t stray);
 276
 277   /// Call when hoid is no longer missing in acting std::set
 278   void recovered(const hobject_t &hoid) {
 279     needs_recovery_map.erase(hoid);
 280     auto p = missing_loc.find(hoid);
 281     if (p != missing_loc.end()) {
 282       _dec_count(p->second);
 283       missing_loc.erase(p);
 284     }
 285   }
 286
 287   /// Call to update structures for hoid after a change
 288   void rebuild(
 289     const hobject_t &hoid,
 290     pg_shard_t self,
 291     const std::set<pg_shard_t> &to_recover,
 292     const pg_info_t &info,
 293     const pg_missing_t &missing,
 294     const std::map<pg_shard_t, pg_missing_t> &pmissing,
 295     const std::map<pg_shard_t, pg_info_t> &pinfo) {
 296     recovered(hoid);
 297     std::optional<pg_missing_item> item;
 298     auto miter = missing.get_items().find(hoid);
 299     if (miter != missing.get_items().end()) {
 300       item = miter->second;
 301     } else {
 302       for (auto &&i: to_recover) {
 303         if (i == self)
 304           continue;
 305         auto pmiter = pmissing.find(i);
 306         ceph_assert(pmiter != pmissing.end());
 307         miter = pmiter->second.get_items().find(hoid);
 308         if (miter != pmiter->second.get_items().end()) {
 309           item = miter->second;
 310           break;
 311         }
 312       }
 313     }
 314     if (!item)
 315       return; // recovered!
 316
 317     needs_recovery_map[hoid] = *item;
 318     if (item->is_delete())
 319       return;
 320     auto mliter =
 321       missing_loc.emplace(hoid, std::set<pg_shard_t>()).first;
 322     ceph_assert(info.last_backfill.is_max());
 323     ceph_assert(info.last_update >= item->need);
 324     if (!missing.is_missing(hoid))
 325       mliter->second.insert(self);
 326     for (auto &&i: pmissing) {
 327       if (i.first == self)
 328         continue;
 329       auto pinfoiter = pinfo.find(i.first);
 330       ceph_assert(pinfoiter != pinfo.end());
 331       if (item->need <= pinfoiter->second.last_update &&
 332           hoid <= pinfoiter->second.last_backfill &&
 333           !i.second.is_missing(hoid))
 334         mliter->second.insert(i.first);
 335     }
 336     _inc_count(mliter->second);
 337   }
 338
 339   const std::set<pg_shard_t> &get_locations(const hobject_t &hoid) const {
 340     auto it = missing_loc.find(hoid);
 341     return it == missing_loc.end() ? empty_set : it->second;
 342   }
 343   const std::map<hobject_t, std::set<pg_shard_t>> &get_missing_locs() const {
 344     return missing_loc;
 345   }
 346   const std::map<hobject_t, pg_missing_item> &get_needs_recovery() const {
 347     return needs_recovery_map;
 348   }
 349
 350   const missing_by_count_t &get_missing_by_count() const {
 351     return missing_by_count;
 352   }
 353 };