]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
9f95a23c | 2 | // vim: ts=8 sw=2 smarttab ft=cpp |
11fdf7f2 | 3 | |
7c673cae FG |
4 | /* |
5 | * Ceph - scalable distributed file system | |
6 | * | |
7 | * Copyright (C) 2015 Red Hat | |
8 | * | |
9 | * This is free software; you can redistribute it and/or | |
10 | * modify it under the terms of the GNU Lesser General Public | |
11 | * License version 2.1, as published by the Free Software | |
12 | * Foundation. See file COPYING. | |
13 | * | |
14 | */ | |
15 | ||
7f7e6c64 | 16 | #pragma once |
7c673cae FG |
17 | |
18 | #include "common/config.h" | |
19 | #include "common/Formatter.h" | |
20 | #include "common/errno.h" | |
21 | ||
9f95a23c | 22 | #include "rgw_sal.h" |
7c673cae FG |
23 | |
24 | #define dout_subsys ceph_subsys_rgw | |
25 | ||
26 | #define RGW_ORPHAN_INDEX_OID "orphan.index" | |
27 | #define RGW_ORPHAN_INDEX_PREFIX "orphan.scan" | |
28 | ||
29 | ||
30 | enum RGWOrphanSearchStageId { | |
31 | ORPHAN_SEARCH_STAGE_UNKNOWN = 0, | |
32 | ORPHAN_SEARCH_STAGE_INIT = 1, | |
33 | ORPHAN_SEARCH_STAGE_LSPOOL = 2, | |
34 | ORPHAN_SEARCH_STAGE_LSBUCKETS = 3, | |
35 | ORPHAN_SEARCH_STAGE_ITERATE_BI = 4, | |
36 | ORPHAN_SEARCH_STAGE_COMPARE = 5, | |
37 | }; | |
38 | ||
39 | ||
40 | struct RGWOrphanSearchStage { | |
41 | RGWOrphanSearchStageId stage; | |
42 | int shard; | |
43 | string marker; | |
44 | ||
45 | RGWOrphanSearchStage() : stage(ORPHAN_SEARCH_STAGE_UNKNOWN), shard(0) {} | |
46 | explicit RGWOrphanSearchStage(RGWOrphanSearchStageId _stage) : stage(_stage), shard(0) {} | |
47 | RGWOrphanSearchStage(RGWOrphanSearchStageId _stage, int _shard, const string& _marker) : stage(_stage), shard(_shard), marker(_marker) {} | |
48 | ||
49 | void encode(bufferlist& bl) const { | |
50 | ENCODE_START(1, 1, bl); | |
11fdf7f2 TL |
51 | encode((int)stage, bl); |
52 | encode(shard, bl); | |
53 | encode(marker, bl); | |
7c673cae FG |
54 | ENCODE_FINISH(bl); |
55 | } | |
56 | ||
11fdf7f2 | 57 | void decode(bufferlist::const_iterator& bl) { |
7c673cae FG |
58 | DECODE_START(1, bl); |
59 | int s; | |
11fdf7f2 | 60 | decode(s, bl); |
7c673cae | 61 | stage = (RGWOrphanSearchStageId)s; |
11fdf7f2 TL |
62 | decode(shard, bl); |
63 | decode(marker, bl); | |
7c673cae FG |
64 | DECODE_FINISH(bl); |
65 | } | |
66 | ||
67 | void dump(Formatter *f) const; | |
68 | }; | |
69 | WRITE_CLASS_ENCODER(RGWOrphanSearchStage) | |
70 | ||
71 | struct RGWOrphanSearchInfo { | |
72 | string job_name; | |
73 | rgw_pool pool; | |
74 | uint16_t num_shards; | |
75 | utime_t start_time; | |
76 | ||
77 | void encode(bufferlist& bl) const { | |
78 | ENCODE_START(2, 1, bl); | |
11fdf7f2 TL |
79 | encode(job_name, bl); |
80 | encode(pool.to_str(), bl); | |
81 | encode(num_shards, bl); | |
82 | encode(start_time, bl); | |
7c673cae FG |
83 | ENCODE_FINISH(bl); |
84 | } | |
85 | ||
11fdf7f2 | 86 | void decode(bufferlist::const_iterator& bl) { |
7c673cae | 87 | DECODE_START(2, bl); |
11fdf7f2 | 88 | decode(job_name, bl); |
7c673cae | 89 | string s; |
11fdf7f2 | 90 | decode(s, bl); |
7c673cae | 91 | pool.from_str(s); |
11fdf7f2 TL |
92 | decode(num_shards, bl); |
93 | decode(start_time, bl); | |
7c673cae FG |
94 | DECODE_FINISH(bl); |
95 | } | |
96 | ||
97 | void dump(Formatter *f) const; | |
98 | }; | |
99 | WRITE_CLASS_ENCODER(RGWOrphanSearchInfo) | |
100 | ||
101 | struct RGWOrphanSearchState { | |
102 | RGWOrphanSearchInfo info; | |
103 | RGWOrphanSearchStage stage; | |
104 | ||
105 | RGWOrphanSearchState() : stage(ORPHAN_SEARCH_STAGE_UNKNOWN) {} | |
106 | ||
107 | void encode(bufferlist& bl) const { | |
108 | ENCODE_START(1, 1, bl); | |
11fdf7f2 TL |
109 | encode(info, bl); |
110 | encode(stage, bl); | |
7c673cae FG |
111 | ENCODE_FINISH(bl); |
112 | } | |
113 | ||
11fdf7f2 | 114 | void decode(bufferlist::const_iterator& bl) { |
7c673cae | 115 | DECODE_START(1, bl); |
11fdf7f2 TL |
116 | decode(info, bl); |
117 | decode(stage, bl); | |
7c673cae FG |
118 | DECODE_FINISH(bl); |
119 | } | |
120 | ||
121 | void dump(Formatter *f) const; | |
122 | }; | |
123 | WRITE_CLASS_ENCODER(RGWOrphanSearchState) | |
124 | ||
125 | class RGWOrphanStore { | |
9f95a23c | 126 | rgw::sal::RGWRadosStore *store; |
7c673cae FG |
127 | librados::IoCtx ioctx; |
128 | ||
129 | string oid; | |
130 | ||
131 | public: | |
9f95a23c | 132 | explicit RGWOrphanStore(rgw::sal::RGWRadosStore *_store) : store(_store), oid(RGW_ORPHAN_INDEX_OID) {} |
7c673cae FG |
133 | |
134 | librados::IoCtx& get_ioctx() { return ioctx; } | |
135 | ||
136 | int init(); | |
137 | ||
138 | int read_job(const string& job_name, RGWOrphanSearchState& state); | |
139 | int write_job(const string& job_name, const RGWOrphanSearchState& state); | |
140 | int remove_job(const string& job_name); | |
141 | int list_jobs(map<string,RGWOrphanSearchState> &job_list); | |
142 | ||
143 | ||
144 | int store_entries(const string& oid, const map<string, bufferlist>& entries); | |
145 | int read_entries(const string& oid, const string& marker, map<string, bufferlist> *entries, bool *truncated); | |
146 | }; | |
147 | ||
148 | ||
149 | class RGWOrphanSearch { | |
9f95a23c | 150 | rgw::sal::RGWRadosStore *store; |
7c673cae FG |
151 | |
152 | RGWOrphanStore orphan_store; | |
153 | ||
154 | RGWOrphanSearchInfo search_info; | |
155 | RGWOrphanSearchStage search_stage; | |
156 | ||
157 | map<int, string> all_objs_index; | |
158 | map<int, string> buckets_instance_index; | |
159 | map<int, string> linked_objs_index; | |
160 | ||
161 | string index_objs_prefix; | |
162 | ||
163 | uint16_t max_concurrent_ios; | |
164 | uint64_t stale_secs; | |
11fdf7f2 TL |
165 | int64_t max_list_bucket_entries; |
166 | ||
167 | bool detailed_mode; | |
7c673cae FG |
168 | |
169 | struct log_iter_info { | |
170 | string oid; | |
171 | list<string>::iterator cur; | |
172 | list<string>::iterator end; | |
173 | }; | |
174 | ||
175 | int log_oids(map<int, string>& log_shards, map<int, list<string> >& oids); | |
176 | ||
177 | #define RGW_ORPHANSEARCH_HASH_PRIME 7877 | |
178 | int orphan_shard(const string& str) { | |
179 | return ceph_str_hash_linux(str.c_str(), str.size()) % RGW_ORPHANSEARCH_HASH_PRIME % search_info.num_shards; | |
180 | } | |
181 | ||
182 | int handle_stat_result(map<int, list<string> >& oids, RGWRados::Object::Stat::Result& result); | |
183 | int pop_and_handle_stat_op(map<int, list<string> >& oids, std::deque<RGWRados::Object::Stat>& ops); | |
184 | ||
185 | ||
186 | int remove_index(map<int, string>& index); | |
187 | public: | |
9f95a23c | 188 | RGWOrphanSearch(rgw::sal::RGWRadosStore *_store, int _max_ios, uint64_t _stale_secs) : store(_store), orphan_store(store), max_concurrent_ios(_max_ios), stale_secs(_stale_secs) {} |
7c673cae FG |
189 | |
190 | int save_state() { | |
191 | RGWOrphanSearchState state; | |
192 | state.info = search_info; | |
193 | state.stage = search_stage; | |
194 | return orphan_store.write_job(search_info.job_name, state); | |
195 | } | |
196 | ||
11fdf7f2 | 197 | int init(const string& job_name, RGWOrphanSearchInfo *info, bool _detailed_mode=false); |
7c673cae FG |
198 | |
199 | int create(const string& job_name, int num_shards); | |
200 | ||
201 | int build_all_oids_index(); | |
202 | int build_buckets_instance_index(); | |
203 | int build_linked_oids_for_bucket(const string& bucket_instance_id, map<int, list<string> >& oids); | |
204 | int build_linked_oids_index(); | |
205 | int compare_oid_indexes(); | |
206 | ||
207 | int run(); | |
208 | int finish(); | |
209 | }; | |
210 | ||
211 | ||
e306af50 TL |
212 | class RGWRadosList { |
213 | ||
214 | /* | |
215 | * process_t describes how to process a irectory, we will either | |
216 | * process the whole thing (entire_container == true) or a portion | |
217 | * of it (entire_container == false). When we only process a | |
218 | * portion, we will list the specific keys and/or specific lexical | |
219 | * prefixes. | |
220 | */ | |
221 | struct process_t { | |
222 | bool entire_container; | |
223 | std::set<rgw_obj_key> filter_keys; | |
224 | std::set<std::string> prefixes; | |
225 | ||
226 | process_t() : | |
227 | entire_container(false) | |
228 | {} | |
229 | }; | |
230 | ||
231 | std::map<std::string,process_t> bucket_process_map; | |
232 | std::set<std::string> visited_oids; | |
233 | ||
234 | void add_bucket_entire(const std::string& bucket_name) { | |
235 | auto p = bucket_process_map.emplace(std::make_pair(bucket_name, | |
236 | process_t())); | |
237 | p.first->second.entire_container = true; | |
238 | } | |
239 | ||
240 | void add_bucket_prefix(const std::string& bucket_name, | |
241 | const std::string& prefix) { | |
242 | auto p = bucket_process_map.emplace(std::make_pair(bucket_name, | |
243 | process_t())); | |
244 | p.first->second.prefixes.insert(prefix); | |
245 | } | |
246 | ||
247 | void add_bucket_filter(const std::string& bucket_name, | |
248 | const rgw_obj_key& obj_key) { | |
249 | auto p = bucket_process_map.emplace(std::make_pair(bucket_name, | |
250 | process_t())); | |
251 | p.first->second.filter_keys.insert(obj_key); | |
252 | } | |
253 | ||
254 | rgw::sal::RGWRadosStore* store; | |
255 | ||
256 | uint16_t max_concurrent_ios; | |
257 | uint64_t stale_secs; | |
258 | std::string tenant_name; | |
259 | ||
7f7e6c64 TL |
260 | bool include_rgw_obj_name; |
261 | std::string field_separator; | |
262 | ||
e306af50 | 263 | int handle_stat_result(RGWRados::Object::Stat::Result& result, |
7f7e6c64 TL |
264 | std::string& bucket_name, |
265 | rgw_obj_key& obj_key, | |
e306af50 TL |
266 | std::set<string>& obj_oids); |
267 | int pop_and_handle_stat_op(RGWObjectCtx& obj_ctx, | |
268 | std::deque<RGWRados::Object::Stat>& ops); | |
269 | ||
270 | public: | |
271 | ||
272 | RGWRadosList(rgw::sal::RGWRadosStore* _store, | |
273 | int _max_ios, | |
274 | uint64_t _stale_secs, | |
275 | const std::string& _tenant_name) : | |
276 | store(_store), | |
277 | max_concurrent_ios(_max_ios), | |
278 | stale_secs(_stale_secs), | |
7f7e6c64 TL |
279 | tenant_name(_tenant_name), |
280 | include_rgw_obj_name(false) | |
e306af50 TL |
281 | {} |
282 | ||
283 | int process_bucket(const std::string& bucket_instance_id, | |
284 | const std::string& prefix, | |
285 | const std::set<rgw_obj_key>& entries_filter); | |
286 | ||
287 | int do_incomplete_multipart(rgw::sal::RGWRadosStore* store, | |
288 | RGWBucketInfo& bucket_info); | |
289 | ||
290 | int build_linked_oids_index(); | |
291 | ||
292 | int run(const std::string& bucket_id); | |
293 | int run(); | |
7c673cae | 294 | |
7f7e6c64 TL |
295 | // if there's a non-empty field separator, that means we'll display |
296 | // bucket and object names | |
297 | void set_field_separator(const std::string& fs) { | |
298 | field_separator = fs; | |
299 | include_rgw_obj_name = !field_separator.empty(); | |
300 | } | |
301 | }; // class RGWRadosList |