]>
Commit | Line | Data |
---|---|---|
9f95a23c TL |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | ||
4 | #pragma once | |
5 | ||
6 | #include <map> | |
7 | #include <set> | |
8 | ||
9 | #include "OSDMap.h" | |
10 | #include "common/HBHandle.h" | |
11 | #include "common/ceph_context.h" | |
12 | #include "common/dout.h" | |
13 | #include "osd_types.h" | |
14 | ||
15 | class MissingLoc { | |
16 | public: | |
17 | ||
18 | class MappingInfo { | |
19 | public: | |
20 | virtual const set<pg_shard_t> &get_upset() const = 0; | |
21 | virtual bool is_ec_pg() const = 0; | |
22 | virtual int get_pg_size() const = 0; | |
23 | virtual ~MappingInfo() {} | |
24 | }; | |
25 | ||
26 | // a loc_count indicates how many locations we know in each of | |
27 | // these distinct sets | |
28 | struct loc_count_t { | |
29 | int up = 0; //< up | |
30 | int other = 0; //< other | |
31 | ||
32 | friend bool operator<(const loc_count_t& l, | |
33 | const loc_count_t& r) { | |
34 | return (l.up < r.up || | |
35 | (l.up == r.up && | |
36 | (l.other < r.other))); | |
37 | } | |
38 | friend ostream& operator<<(ostream& out, const loc_count_t& l) { | |
39 | ceph_assert(l.up >= 0); | |
40 | ceph_assert(l.other >= 0); | |
41 | return out << "(" << l.up << "+" << l.other << ")"; | |
42 | } | |
43 | }; | |
44 | ||
45 | ||
46 | using missing_by_count_t = map < shard_id_t, map<loc_count_t,int> >; | |
47 | private: | |
48 | loc_count_t _get_count(const set<pg_shard_t> &shards) { | |
49 | loc_count_t r; | |
50 | for (auto s : shards) { | |
51 | if (mapping_info->get_upset().count(s)) { | |
52 | r.up++; | |
53 | } else { | |
54 | r.other++; | |
55 | } | |
56 | } | |
57 | return r; | |
58 | } | |
59 | ||
60 | map<hobject_t, pg_missing_item> needs_recovery_map; | |
61 | map<hobject_t, set<pg_shard_t> > missing_loc; | |
62 | set<pg_shard_t> missing_loc_sources; | |
63 | ||
64 | // for every entry in missing_loc, we count how many of each type of shard we have, | |
65 | // and maintain totals here. The sum of the values for this map will always equal | |
66 | // missing_loc.size(). | |
67 | missing_by_count_t missing_by_count; | |
68 | ||
69 | void pgs_by_shard_id( | |
70 | const set<pg_shard_t>& s, | |
71 | map< shard_id_t, set<pg_shard_t> >& pgsbs) { | |
72 | if (mapping_info->is_ec_pg()) { | |
73 | int num_shards = mapping_info->get_pg_size(); | |
74 | // For completely missing shards initialize with empty set<pg_shard_t> | |
75 | for (int i = 0 ; i < num_shards ; ++i) { | |
76 | shard_id_t shard(i); | |
77 | pgsbs[shard]; | |
78 | } | |
79 | for (auto pgs: s) | |
80 | pgsbs[pgs.shard].insert(pgs); | |
81 | } else { | |
82 | pgsbs[shard_id_t::NO_SHARD] = s; | |
83 | } | |
84 | } | |
85 | ||
86 | void _inc_count(const set<pg_shard_t>& s) { | |
87 | map< shard_id_t, set<pg_shard_t> > pgsbs; | |
88 | pgs_by_shard_id(s, pgsbs); | |
89 | for (auto shard: pgsbs) | |
90 | ++missing_by_count[shard.first][_get_count(shard.second)]; | |
91 | } | |
92 | void _dec_count(const set<pg_shard_t>& s) { | |
93 | map< shard_id_t, set<pg_shard_t> > pgsbs; | |
94 | pgs_by_shard_id(s, pgsbs); | |
95 | for (auto shard: pgsbs) { | |
96 | auto p = missing_by_count[shard.first].find(_get_count(shard.second)); | |
97 | ceph_assert(p != missing_by_count[shard.first].end()); | |
98 | if (--p->second == 0) { | |
99 | missing_by_count[shard.first].erase(p); | |
100 | } | |
101 | } | |
102 | } | |
103 | ||
104 | spg_t pgid; | |
105 | MappingInfo *mapping_info; | |
106 | DoutPrefixProvider *dpp; | |
107 | CephContext *cct; | |
108 | set<pg_shard_t> empty_set; | |
109 | public: | |
110 | boost::scoped_ptr<IsPGReadablePredicate> is_readable; | |
111 | boost::scoped_ptr<IsPGRecoverablePredicate> is_recoverable; | |
112 | explicit MissingLoc( | |
113 | spg_t pgid, | |
114 | MappingInfo *mapping_info, | |
115 | DoutPrefixProvider *dpp, | |
116 | CephContext *cct) | |
117 | : pgid(pgid), mapping_info(mapping_info), dpp(dpp), cct(cct) { } | |
118 | void set_backend_predicates( | |
119 | IsPGReadablePredicate *_is_readable, | |
120 | IsPGRecoverablePredicate *_is_recoverable) { | |
121 | is_readable.reset(_is_readable); | |
122 | is_recoverable.reset(_is_recoverable); | |
123 | } | |
124 | const IsPGRecoverablePredicate &get_recoverable_predicate() const { | |
125 | return *is_recoverable; | |
126 | } | |
127 | std::ostream& gen_prefix(std::ostream& out) const { | |
128 | return dpp->gen_prefix(out); | |
129 | } | |
130 | bool needs_recovery( | |
131 | const hobject_t &hoid, | |
132 | eversion_t *v = 0) const { | |
133 | map<hobject_t, pg_missing_item>::const_iterator i = | |
134 | needs_recovery_map.find(hoid); | |
135 | if (i == needs_recovery_map.end()) | |
136 | return false; | |
137 | if (v) | |
138 | *v = i->second.need; | |
139 | return true; | |
140 | } | |
141 | bool is_deleted(const hobject_t &hoid) const { | |
142 | auto i = needs_recovery_map.find(hoid); | |
143 | if (i == needs_recovery_map.end()) | |
144 | return false; | |
145 | return i->second.is_delete(); | |
146 | } | |
147 | bool is_unfound(const hobject_t &hoid) const { | |
148 | auto it = needs_recovery_map.find(hoid); | |
149 | if (it == needs_recovery_map.end()) { | |
150 | return false; | |
151 | } | |
152 | if (it->second.is_delete()) { | |
153 | return false; | |
154 | } | |
155 | auto mit = missing_loc.find(hoid); | |
156 | return mit == missing_loc.end() || !(*is_recoverable)(mit->second); | |
157 | } | |
158 | bool readable_with_acting( | |
159 | const hobject_t &hoid, | |
160 | const set<pg_shard_t> &acting) const; | |
161 | uint64_t num_unfound() const { | |
162 | uint64_t ret = 0; | |
163 | for (map<hobject_t, pg_missing_item>::const_iterator i = | |
164 | needs_recovery_map.begin(); | |
165 | i != needs_recovery_map.end(); | |
166 | ++i) { | |
167 | if (i->second.is_delete()) | |
168 | continue; | |
169 | auto mi = missing_loc.find(i->first); | |
170 | if (mi == missing_loc.end() || !(*is_recoverable)(mi->second)) | |
171 | ++ret; | |
172 | } | |
173 | return ret; | |
174 | } | |
175 | ||
176 | bool have_unfound() const { | |
177 | for (map<hobject_t, pg_missing_item>::const_iterator i = | |
178 | needs_recovery_map.begin(); | |
179 | i != needs_recovery_map.end(); | |
180 | ++i) { | |
181 | if (i->second.is_delete()) | |
182 | continue; | |
183 | auto mi = missing_loc.find(i->first); | |
184 | if (mi == missing_loc.end() || !(*is_recoverable)(mi->second)) | |
185 | return true; | |
186 | } | |
187 | return false; | |
188 | } | |
189 | void clear() { | |
190 | needs_recovery_map.clear(); | |
191 | missing_loc.clear(); | |
192 | missing_loc_sources.clear(); | |
193 | missing_by_count.clear(); | |
194 | } | |
195 | ||
196 | void add_location(const hobject_t &hoid, pg_shard_t location) { | |
197 | auto p = missing_loc.find(hoid); | |
198 | if (p == missing_loc.end()) { | |
199 | p = missing_loc.emplace(hoid, set<pg_shard_t>()).first; | |
200 | } else { | |
201 | _dec_count(p->second); | |
202 | } | |
203 | p->second.insert(location); | |
204 | _inc_count(p->second); | |
205 | } | |
206 | void remove_location(const hobject_t &hoid, pg_shard_t location) { | |
207 | auto p = missing_loc.find(hoid); | |
208 | if (p != missing_loc.end()) { | |
209 | _dec_count(p->second); | |
210 | p->second.erase(location); | |
211 | if (p->second.empty()) { | |
212 | missing_loc.erase(p); | |
213 | } else { | |
214 | _inc_count(p->second); | |
215 | } | |
216 | } | |
217 | } | |
218 | ||
219 | void clear_location(const hobject_t &hoid) { | |
220 | auto p = missing_loc.find(hoid); | |
221 | if (p != missing_loc.end()) { | |
222 | _dec_count(p->second); | |
223 | missing_loc.erase(p); | |
224 | } | |
225 | } | |
226 | ||
227 | void add_active_missing(const pg_missing_t &missing) { | |
228 | for (map<hobject_t, pg_missing_item>::const_iterator i = | |
229 | missing.get_items().begin(); | |
230 | i != missing.get_items().end(); | |
231 | ++i) { | |
232 | map<hobject_t, pg_missing_item>::const_iterator j = | |
233 | needs_recovery_map.find(i->first); | |
234 | if (j == needs_recovery_map.end()) { | |
235 | needs_recovery_map.insert(*i); | |
236 | } else { | |
237 | if (i->second.need != j->second.need) { | |
238 | lgeneric_dout(cct, 0) << this << " " << pgid << " unexpected need for " | |
239 | << i->first << " have " << j->second | |
240 | << " tried to add " << i->second << dendl; | |
241 | ceph_assert(0 == "unexpected need for missing item"); | |
242 | } | |
243 | } | |
244 | } | |
245 | } | |
246 | ||
247 | void add_missing(const hobject_t &hoid, eversion_t need, eversion_t have, bool is_delete=false) { | |
248 | needs_recovery_map[hoid] = pg_missing_item(need, have, is_delete); | |
249 | } | |
250 | void revise_need(const hobject_t &hoid, eversion_t need) { | |
251 | auto it = needs_recovery_map.find(hoid); | |
252 | ceph_assert(it != needs_recovery_map.end()); | |
253 | it->second.need = need; | |
254 | } | |
255 | ||
256 | /// Adds info about a possible recovery source | |
257 | bool add_source_info( | |
258 | pg_shard_t source, ///< [in] source | |
259 | const pg_info_t &oinfo, ///< [in] info | |
260 | const pg_missing_t &omissing, ///< [in] (optional) missing | |
261 | HBHandle *handle ///< [in] ThreadPool handle | |
262 | ); ///< @return whether a new object location was discovered | |
263 | ||
264 | /// Adds recovery sources in batch | |
265 | void add_batch_sources_info( | |
266 | const set<pg_shard_t> &sources, ///< [in] a set of resources which can be used for all objects | |
267 | HBHandle *handle ///< [in] ThreadPool handle | |
268 | ); | |
269 | ||
270 | /// Uses osdmap to update structures for now down sources | |
271 | void check_recovery_sources(const OSDMapRef& osdmap); | |
272 | ||
273 | /// Remove stray from recovery sources | |
274 | void remove_stray_recovery_sources(pg_shard_t stray); | |
275 | ||
276 | /// Call when hoid is no longer missing in acting set | |
277 | void recovered(const hobject_t &hoid) { | |
278 | needs_recovery_map.erase(hoid); | |
279 | auto p = missing_loc.find(hoid); | |
280 | if (p != missing_loc.end()) { | |
281 | _dec_count(p->second); | |
282 | missing_loc.erase(p); | |
283 | } | |
284 | } | |
285 | ||
286 | /// Call to update structures for hoid after a change | |
287 | void rebuild( | |
288 | const hobject_t &hoid, | |
289 | pg_shard_t self, | |
290 | const set<pg_shard_t> &to_recover, | |
291 | const pg_info_t &info, | |
292 | const pg_missing_t &missing, | |
293 | const map<pg_shard_t, pg_missing_t> &pmissing, | |
294 | const map<pg_shard_t, pg_info_t> &pinfo) { | |
295 | recovered(hoid); | |
296 | std::optional<pg_missing_item> item; | |
297 | auto miter = missing.get_items().find(hoid); | |
298 | if (miter != missing.get_items().end()) { | |
299 | item = miter->second; | |
300 | } else { | |
301 | for (auto &&i: to_recover) { | |
302 | if (i == self) | |
303 | continue; | |
304 | auto pmiter = pmissing.find(i); | |
305 | ceph_assert(pmiter != pmissing.end()); | |
306 | miter = pmiter->second.get_items().find(hoid); | |
307 | if (miter != pmiter->second.get_items().end()) { | |
308 | item = miter->second; | |
309 | break; | |
310 | } | |
311 | } | |
312 | } | |
313 | if (!item) | |
314 | return; // recovered! | |
315 | ||
316 | needs_recovery_map[hoid] = *item; | |
317 | if (item->is_delete()) | |
318 | return; | |
319 | auto mliter = | |
320 | missing_loc.emplace(hoid, set<pg_shard_t>()).first; | |
321 | ceph_assert(info.last_backfill.is_max()); | |
322 | ceph_assert(info.last_update >= item->need); | |
323 | if (!missing.is_missing(hoid)) | |
324 | mliter->second.insert(self); | |
325 | for (auto &&i: pmissing) { | |
326 | if (i.first == self) | |
327 | continue; | |
328 | auto pinfoiter = pinfo.find(i.first); | |
329 | ceph_assert(pinfoiter != pinfo.end()); | |
330 | if (item->need <= pinfoiter->second.last_update && | |
331 | hoid <= pinfoiter->second.last_backfill && | |
332 | !i.second.is_missing(hoid)) | |
333 | mliter->second.insert(i.first); | |
334 | } | |
335 | _inc_count(mliter->second); | |
336 | } | |
337 | ||
338 | const set<pg_shard_t> &get_locations(const hobject_t &hoid) const { | |
339 | auto it = missing_loc.find(hoid); | |
340 | return it == missing_loc.end() ? empty_set : it->second; | |
341 | } | |
342 | const map<hobject_t, set<pg_shard_t>> &get_missing_locs() const { | |
343 | return missing_loc; | |
344 | } | |
345 | const map<hobject_t, pg_missing_item> &get_needs_recovery() const { | |
346 | return needs_recovery_map; | |
347 | } | |
348 | ||
349 | const missing_by_count_t &get_missing_by_count() const { | |
350 | return missing_by_count; | |
351 | } | |
352 | }; |