]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
9f95a23c | 2 | // vim: ts=8 sw=2 smarttab ft=cpp |
11fdf7f2 | 3 | |
7c673cae FG |
4 | /* |
5 | * Ceph - scalable distributed file system | |
6 | * | |
7 | * Copyright (C) 2015 Red Hat | |
8 | * | |
9 | * This is free software; you can redistribute it and/or | |
10 | * modify it under the terms of the GNU Lesser General Public | |
11 | * License version 2.1, as published by the Free Software | |
12 | * Foundation. See file COPYING. | |
13 | * | |
14 | */ | |
15 | ||
f67539c2 | 16 | #pragma once |
7c673cae FG |
17 | |
18 | #include "common/config.h" | |
19 | #include "common/Formatter.h" | |
20 | #include "common/errno.h" | |
21 | ||
20effc67 | 22 | #include "rgw_sal_rados.h" |
7c673cae FG |
23 | |
24 | #define dout_subsys ceph_subsys_rgw | |
25 | ||
26 | #define RGW_ORPHAN_INDEX_OID "orphan.index" | |
27 | #define RGW_ORPHAN_INDEX_PREFIX "orphan.scan" | |
28 | ||
29 | ||
30 | enum RGWOrphanSearchStageId { | |
31 | ORPHAN_SEARCH_STAGE_UNKNOWN = 0, | |
32 | ORPHAN_SEARCH_STAGE_INIT = 1, | |
33 | ORPHAN_SEARCH_STAGE_LSPOOL = 2, | |
34 | ORPHAN_SEARCH_STAGE_LSBUCKETS = 3, | |
35 | ORPHAN_SEARCH_STAGE_ITERATE_BI = 4, | |
36 | ORPHAN_SEARCH_STAGE_COMPARE = 5, | |
37 | }; | |
38 | ||
39 | ||
40 | struct RGWOrphanSearchStage { | |
41 | RGWOrphanSearchStageId stage; | |
42 | int shard; | |
20effc67 | 43 | std::string marker; |
7c673cae FG |
44 | |
45 | RGWOrphanSearchStage() : stage(ORPHAN_SEARCH_STAGE_UNKNOWN), shard(0) {} | |
46 | explicit RGWOrphanSearchStage(RGWOrphanSearchStageId _stage) : stage(_stage), shard(0) {} | |
20effc67 | 47 | RGWOrphanSearchStage(RGWOrphanSearchStageId _stage, int _shard, const std::string& _marker) : stage(_stage), shard(_shard), marker(_marker) {} |
7c673cae FG |
48 | |
49 | void encode(bufferlist& bl) const { | |
50 | ENCODE_START(1, 1, bl); | |
11fdf7f2 TL |
51 | encode((int)stage, bl); |
52 | encode(shard, bl); | |
53 | encode(marker, bl); | |
7c673cae FG |
54 | ENCODE_FINISH(bl); |
55 | } | |
56 | ||
11fdf7f2 | 57 | void decode(bufferlist::const_iterator& bl) { |
7c673cae FG |
58 | DECODE_START(1, bl); |
59 | int s; | |
11fdf7f2 | 60 | decode(s, bl); |
7c673cae | 61 | stage = (RGWOrphanSearchStageId)s; |
11fdf7f2 TL |
62 | decode(shard, bl); |
63 | decode(marker, bl); | |
7c673cae FG |
64 | DECODE_FINISH(bl); |
65 | } | |
66 | ||
67 | void dump(Formatter *f) const; | |
68 | }; | |
69 | WRITE_CLASS_ENCODER(RGWOrphanSearchStage) | |
70 | ||
71 | struct RGWOrphanSearchInfo { | |
20effc67 | 72 | std::string job_name; |
7c673cae FG |
73 | rgw_pool pool; |
74 | uint16_t num_shards; | |
75 | utime_t start_time; | |
76 | ||
77 | void encode(bufferlist& bl) const { | |
78 | ENCODE_START(2, 1, bl); | |
11fdf7f2 TL |
79 | encode(job_name, bl); |
80 | encode(pool.to_str(), bl); | |
81 | encode(num_shards, bl); | |
82 | encode(start_time, bl); | |
7c673cae FG |
83 | ENCODE_FINISH(bl); |
84 | } | |
85 | ||
11fdf7f2 | 86 | void decode(bufferlist::const_iterator& bl) { |
7c673cae | 87 | DECODE_START(2, bl); |
11fdf7f2 | 88 | decode(job_name, bl); |
20effc67 | 89 | std::string s; |
11fdf7f2 | 90 | decode(s, bl); |
7c673cae | 91 | pool.from_str(s); |
11fdf7f2 TL |
92 | decode(num_shards, bl); |
93 | decode(start_time, bl); | |
7c673cae FG |
94 | DECODE_FINISH(bl); |
95 | } | |
96 | ||
97 | void dump(Formatter *f) const; | |
98 | }; | |
99 | WRITE_CLASS_ENCODER(RGWOrphanSearchInfo) | |
100 | ||
101 | struct RGWOrphanSearchState { | |
102 | RGWOrphanSearchInfo info; | |
103 | RGWOrphanSearchStage stage; | |
104 | ||
105 | RGWOrphanSearchState() : stage(ORPHAN_SEARCH_STAGE_UNKNOWN) {} | |
106 | ||
107 | void encode(bufferlist& bl) const { | |
108 | ENCODE_START(1, 1, bl); | |
11fdf7f2 TL |
109 | encode(info, bl); |
110 | encode(stage, bl); | |
7c673cae FG |
111 | ENCODE_FINISH(bl); |
112 | } | |
113 | ||
11fdf7f2 | 114 | void decode(bufferlist::const_iterator& bl) { |
7c673cae | 115 | DECODE_START(1, bl); |
11fdf7f2 TL |
116 | decode(info, bl); |
117 | decode(stage, bl); | |
7c673cae FG |
118 | DECODE_FINISH(bl); |
119 | } | |
120 | ||
121 | void dump(Formatter *f) const; | |
122 | }; | |
123 | WRITE_CLASS_ENCODER(RGWOrphanSearchState) | |
124 | ||
125 | class RGWOrphanStore { | |
20effc67 | 126 | rgw::sal::RadosStore* store; |
7c673cae FG |
127 | librados::IoCtx ioctx; |
128 | ||
20effc67 | 129 | std::string oid; |
7c673cae FG |
130 | |
131 | public: | |
20effc67 | 132 | explicit RGWOrphanStore(rgw::sal::RadosStore* _store) : store(_store), oid(RGW_ORPHAN_INDEX_OID) {} |
7c673cae FG |
133 | |
134 | librados::IoCtx& get_ioctx() { return ioctx; } | |
135 | ||
b3b6e05e | 136 | int init(const DoutPrefixProvider *dpp); |
7c673cae | 137 | |
20effc67 TL |
138 | int read_job(const std::string& job_name, RGWOrphanSearchState& state); |
139 | int write_job(const std::string& job_name, const RGWOrphanSearchState& state); | |
140 | int remove_job(const std::string& job_name); | |
141 | int list_jobs(std::map<std::string,RGWOrphanSearchState> &job_list); | |
7c673cae FG |
142 | |
143 | ||
20effc67 TL |
144 | int store_entries(const DoutPrefixProvider *dpp, const std::string& oid, const std::map<std::string, bufferlist>& entries); |
145 | int read_entries(const std::string& oid, const std::string& marker, std::map<std::string, bufferlist> *entries, bool *truncated); | |
7c673cae FG |
146 | }; |
147 | ||
148 | ||
149 | class RGWOrphanSearch { | |
20effc67 | 150 | rgw::sal::RadosStore* store; |
7c673cae FG |
151 | |
152 | RGWOrphanStore orphan_store; | |
153 | ||
154 | RGWOrphanSearchInfo search_info; | |
155 | RGWOrphanSearchStage search_stage; | |
156 | ||
20effc67 TL |
157 | std::map<int, std::string> all_objs_index; |
158 | std::map<int, std::string> buckets_instance_index; | |
159 | std::map<int, std::string> linked_objs_index; | |
7c673cae | 160 | |
20effc67 | 161 | std::string index_objs_prefix; |
7c673cae FG |
162 | |
163 | uint16_t max_concurrent_ios; | |
164 | uint64_t stale_secs; | |
11fdf7f2 TL |
165 | int64_t max_list_bucket_entries; |
166 | ||
167 | bool detailed_mode; | |
7c673cae FG |
168 | |
169 | struct log_iter_info { | |
20effc67 TL |
170 | std::string oid; |
171 | std::list<std::string>::iterator cur; | |
172 | std::list<std::string>::iterator end; | |
7c673cae FG |
173 | }; |
174 | ||
20effc67 | 175 | int log_oids(const DoutPrefixProvider *dpp, std::map<int, std::string>& log_shards, std::map<int, std::list<std::string> >& oids); |
7c673cae FG |
176 | |
177 | #define RGW_ORPHANSEARCH_HASH_PRIME 7877 | |
20effc67 | 178 | int orphan_shard(const std::string& str) { |
7c673cae FG |
179 | return ceph_str_hash_linux(str.c_str(), str.size()) % RGW_ORPHANSEARCH_HASH_PRIME % search_info.num_shards; |
180 | } | |
181 | ||
20effc67 TL |
182 | int handle_stat_result(const DoutPrefixProvider *dpp, std::map<int, std::list<std::string> >& oids, RGWRados::Object::Stat::Result& result); |
183 | int pop_and_handle_stat_op(const DoutPrefixProvider *dpp, std::map<int, std::list<std::string> >& oids, std::deque<RGWRados::Object::Stat>& ops); | |
7c673cae | 184 | |
20effc67 | 185 | int remove_index(std::map<int, std::string>& index); |
7c673cae | 186 | public: |
20effc67 | 187 | RGWOrphanSearch(rgw::sal::RadosStore* _store, int _max_ios, uint64_t _stale_secs) : store(_store), orphan_store(store), max_concurrent_ios(_max_ios), stale_secs(_stale_secs) {} |
7c673cae FG |
188 | |
189 | int save_state() { | |
190 | RGWOrphanSearchState state; | |
191 | state.info = search_info; | |
192 | state.stage = search_stage; | |
193 | return orphan_store.write_job(search_info.job_name, state); | |
194 | } | |
195 | ||
20effc67 | 196 | int init(const DoutPrefixProvider *dpp, const std::string& job_name, RGWOrphanSearchInfo *info, bool _detailed_mode=false); |
7c673cae | 197 | |
20effc67 | 198 | int create(const std::string& job_name, int num_shards); |
7c673cae | 199 | |
b3b6e05e TL |
200 | int build_all_oids_index(const DoutPrefixProvider *dpp); |
201 | int build_buckets_instance_index(const DoutPrefixProvider *dpp); | |
20effc67 | 202 | int build_linked_oids_for_bucket(const DoutPrefixProvider *dpp, const std::string& bucket_instance_id, std::map<int, std::list<std::string> >& oids); |
b3b6e05e TL |
203 | int build_linked_oids_index(const DoutPrefixProvider *dpp); |
204 | int compare_oid_indexes(const DoutPrefixProvider *dpp); | |
7c673cae | 205 | |
b3b6e05e | 206 | int run(const DoutPrefixProvider *dpp); |
7c673cae FG |
207 | int finish(); |
208 | }; | |
209 | ||
210 | ||
e306af50 TL |
211 | class RGWRadosList { |
212 | ||
213 | /* | |
214 | * process_t describes how to process a irectory, we will either | |
215 | * process the whole thing (entire_container == true) or a portion | |
216 | * of it (entire_container == false). When we only process a | |
217 | * portion, we will list the specific keys and/or specific lexical | |
218 | * prefixes. | |
219 | */ | |
220 | struct process_t { | |
221 | bool entire_container; | |
222 | std::set<rgw_obj_key> filter_keys; | |
223 | std::set<std::string> prefixes; | |
224 | ||
225 | process_t() : | |
226 | entire_container(false) | |
227 | {} | |
228 | }; | |
229 | ||
230 | std::map<std::string,process_t> bucket_process_map; | |
231 | std::set<std::string> visited_oids; | |
232 | ||
233 | void add_bucket_entire(const std::string& bucket_name) { | |
234 | auto p = bucket_process_map.emplace(std::make_pair(bucket_name, | |
235 | process_t())); | |
236 | p.first->second.entire_container = true; | |
237 | } | |
238 | ||
239 | void add_bucket_prefix(const std::string& bucket_name, | |
240 | const std::string& prefix) { | |
241 | auto p = bucket_process_map.emplace(std::make_pair(bucket_name, | |
242 | process_t())); | |
243 | p.first->second.prefixes.insert(prefix); | |
244 | } | |
245 | ||
246 | void add_bucket_filter(const std::string& bucket_name, | |
247 | const rgw_obj_key& obj_key) { | |
248 | auto p = bucket_process_map.emplace(std::make_pair(bucket_name, | |
249 | process_t())); | |
250 | p.first->second.filter_keys.insert(obj_key); | |
251 | } | |
252 | ||
20effc67 | 253 | rgw::sal::RadosStore* store; |
e306af50 TL |
254 | |
255 | uint16_t max_concurrent_ios; | |
256 | uint64_t stale_secs; | |
257 | std::string tenant_name; | |
258 | ||
f67539c2 TL |
259 | bool include_rgw_obj_name; |
260 | std::string field_separator; | |
261 | ||
20effc67 TL |
262 | int handle_stat_result(const DoutPrefixProvider *dpp, |
263 | RGWRados::Object::Stat::Result& result, | |
f67539c2 TL |
264 | std::string& bucket_name, |
265 | rgw_obj_key& obj_key, | |
20effc67 TL |
266 | std::set<std::string>& obj_oids); |
267 | int pop_and_handle_stat_op(const DoutPrefixProvider *dpp, | |
b3b6e05e | 268 | RGWObjectCtx& obj_ctx, |
e306af50 TL |
269 | std::deque<RGWRados::Object::Stat>& ops); |
270 | ||
271 | public: | |
272 | ||
20effc67 | 273 | RGWRadosList(rgw::sal::RadosStore* _store, |
e306af50 TL |
274 | int _max_ios, |
275 | uint64_t _stale_secs, | |
276 | const std::string& _tenant_name) : | |
277 | store(_store), | |
278 | max_concurrent_ios(_max_ios), | |
279 | stale_secs(_stale_secs), | |
f67539c2 TL |
280 | tenant_name(_tenant_name), |
281 | include_rgw_obj_name(false) | |
e306af50 TL |
282 | {} |
283 | ||
b3b6e05e TL |
284 | int process_bucket(const DoutPrefixProvider *dpp, |
285 | const std::string& bucket_instance_id, | |
e306af50 TL |
286 | const std::string& prefix, |
287 | const std::set<rgw_obj_key>& entries_filter); | |
288 | ||
20effc67 TL |
289 | int do_incomplete_multipart(const DoutPrefixProvider *dpp, |
290 | rgw::sal::Bucket* bucket); | |
e306af50 TL |
291 | |
292 | int build_linked_oids_index(); | |
293 | ||
b3b6e05e TL |
294 | int run(const DoutPrefixProvider *dpp, const std::string& bucket_id); |
295 | int run(const DoutPrefixProvider *dpp); | |
7c673cae | 296 | |
f67539c2 TL |
297 | // if there's a non-empty field separator, that means we'll display |
298 | // bucket and object names | |
299 | void set_field_separator(const std::string& fs) { | |
300 | field_separator = fs; | |
301 | include_rgw_obj_name = !field_separator.empty(); | |
302 | } | |
303 | }; // class RGWRadosList |