]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | ||
4 | #ifndef CEPH_RGW_BUCKET_H | |
5 | #define CEPH_RGW_BUCKET_H | |
6 | ||
7 | #include <string> | |
8 | #include <memory> | |
9 | ||
10 | #include "include/types.h" | |
11 | #include "rgw_common.h" | |
12 | #include "rgw_tools.h" | |
13 | ||
14 | #include "rgw_rados.h" | |
15 | ||
16 | #include "rgw_string.h" | |
17 | ||
18 | #include "common/Formatter.h" | |
19 | #include "common/lru_map.h" | |
20 | #include "common/ceph_time.h" | |
21 | #include "rgw_formats.h" | |
22 | ||
7c673cae FG |
23 | // define as static when RGWBucket implementation compete |
24 | extern void rgw_get_buckets_obj(const rgw_user& user_id, string& buckets_obj_id); | |
25 | ||
26 | extern int rgw_bucket_store_info(RGWRados *store, const string& bucket_name, bufferlist& bl, bool exclusive, | |
27 | map<string, bufferlist> *pattrs, RGWObjVersionTracker *objv_tracker, | |
28 | real_time mtime); | |
29 | extern int rgw_bucket_instance_store_info(RGWRados *store, string& oid, bufferlist& bl, bool exclusive, | |
30 | map<string, bufferlist> *pattrs, RGWObjVersionTracker *objv_tracker, | |
31 | real_time mtime); | |
32 | ||
33 | extern int rgw_bucket_parse_bucket_instance(const string& bucket_instance, string *target_bucket_instance, int *shard_id); | |
34 | extern int rgw_bucket_parse_bucket_key(CephContext *cct, const string& key, | |
35 | rgw_bucket* bucket, int *shard_id); | |
36 | ||
37 | extern int rgw_bucket_instance_remove_entry(RGWRados *store, string& entry, RGWObjVersionTracker *objv_tracker); | |
38 | extern void rgw_bucket_instance_key_to_oid(string& key); | |
39 | extern void rgw_bucket_instance_oid_to_key(string& oid); | |
40 | ||
41 | extern int rgw_bucket_delete_bucket_obj(RGWRados *store, | |
42 | const string& tenant_name, | |
43 | const string& bucket_name, | |
44 | RGWObjVersionTracker& objv_tracker); | |
45 | ||
46 | extern int rgw_bucket_sync_user_stats(RGWRados *store, const rgw_user& user_id, const RGWBucketInfo& bucket_info); | |
47 | extern int rgw_bucket_sync_user_stats(RGWRados *store, const string& tenant_name, const string& bucket_name); | |
48 | ||
49 | extern std::string rgw_make_bucket_entry_name(const std::string& tenant_name, | |
50 | const std::string& bucket_name); | |
51 | static inline void rgw_make_bucket_entry_name(const string& tenant_name, | |
52 | const string& bucket_name, | |
53 | std::string& bucket_entry) { | |
54 | bucket_entry = rgw_make_bucket_entry_name(tenant_name, bucket_name); | |
55 | } | |
56 | ||
57 | extern void rgw_parse_url_bucket(const string& bucket, | |
58 | const string& auth_tenant, | |
59 | string &tenant_name, string &bucket_name); | |
60 | ||
61 | struct RGWBucketCompleteInfo { | |
62 | RGWBucketInfo info; | |
63 | map<string, bufferlist> attrs; | |
64 | ||
65 | void dump(Formatter *f) const; | |
66 | void decode_json(JSONObj *obj); | |
67 | }; | |
68 | ||
69 | class RGWBucketEntryMetadataObject : public RGWMetadataObject { | |
70 | RGWBucketEntryPoint ep; | |
71 | public: | |
72 | RGWBucketEntryMetadataObject(RGWBucketEntryPoint& _ep, obj_version& v, real_time m) : ep(_ep) { | |
73 | objv = v; | |
74 | mtime = m; | |
75 | } | |
76 | ||
77 | void dump(Formatter *f) const override { | |
78 | ep.dump(f); | |
79 | } | |
80 | }; | |
81 | ||
82 | class RGWBucketInstanceMetadataObject : public RGWMetadataObject { | |
83 | RGWBucketCompleteInfo info; | |
84 | public: | |
85 | RGWBucketInstanceMetadataObject() {} | |
86 | RGWBucketInstanceMetadataObject(RGWBucketCompleteInfo& i, obj_version& v, real_time m) : info(i) { | |
87 | objv = v; | |
88 | mtime = m; | |
89 | } | |
90 | ||
91 | void dump(Formatter *f) const override { | |
92 | info.dump(f); | |
93 | } | |
94 | ||
95 | void decode_json(JSONObj *obj) { | |
96 | info.decode_json(obj); | |
97 | } | |
98 | ||
99 | RGWBucketInfo& get_bucket_info() { return info.info; } | |
100 | }; | |
101 | ||
102 | /** | |
103 | * Store a list of the user's buckets, with associated functinos. | |
104 | */ | |
105 | class RGWUserBuckets | |
106 | { | |
107 | map<string, RGWBucketEnt> buckets; | |
108 | ||
109 | public: | |
110 | RGWUserBuckets() {} | |
111 | void encode(bufferlist& bl) const { | |
112 | ::encode(buckets, bl); | |
113 | } | |
114 | void decode(bufferlist::iterator& bl) { | |
115 | ::decode(buckets, bl); | |
116 | } | |
117 | /** | |
118 | * Check if the user owns a bucket by the given name. | |
119 | */ | |
120 | bool owns(string& name) { | |
121 | map<string, RGWBucketEnt>::iterator iter; | |
122 | iter = buckets.find(name); | |
123 | return (iter != buckets.end()); | |
124 | } | |
125 | ||
126 | /** | |
127 | * Add a (created) bucket to the user's bucket list. | |
128 | */ | |
129 | void add(const RGWBucketEnt& bucket) { | |
130 | buckets[bucket.bucket.name] = bucket; | |
131 | } | |
132 | ||
133 | /** | |
134 | * Remove a bucket from the user's list by name. | |
135 | */ | |
136 | void remove(string& name) { | |
137 | map<string, RGWBucketEnt>::iterator iter; | |
138 | iter = buckets.find(name); | |
139 | if (iter != buckets.end()) { | |
140 | buckets.erase(iter); | |
141 | } | |
142 | } | |
143 | ||
144 | /** | |
145 | * Get the user's buckets as a map. | |
146 | */ | |
147 | map<string, RGWBucketEnt>& get_buckets() { return buckets; } | |
148 | ||
149 | /** | |
150 | * Cleanup data structure | |
151 | */ | |
152 | void clear() { buckets.clear(); } | |
153 | ||
154 | size_t count() { return buckets.size(); } | |
155 | }; | |
156 | WRITE_CLASS_ENCODER(RGWUserBuckets) | |
157 | ||
158 | class RGWMetadataManager; | |
159 | ||
160 | extern void rgw_bucket_init(RGWMetadataManager *mm); | |
161 | /** | |
162 | * Get all the buckets owned by a user and fill up an RGWUserBuckets with them. | |
163 | * Returns: 0 on success, -ERR# on failure. | |
164 | */ | |
165 | extern int rgw_read_user_buckets(RGWRados *store, | |
166 | const rgw_user& user_id, | |
167 | RGWUserBuckets& buckets, | |
168 | const string& marker, | |
169 | const string& end_marker, | |
170 | uint64_t max, | |
171 | bool need_stats, | |
172 | bool* is_truncated, | |
173 | uint64_t default_amount = 1000); | |
174 | ||
175 | extern int rgw_link_bucket(RGWRados *store, const rgw_user& user_id, rgw_bucket& bucket, real_time creation_time, bool update_entrypoint = true); | |
176 | extern int rgw_unlink_bucket(RGWRados *store, const rgw_user& user_id, | |
177 | const string& tenant_name, const string& bucket_name, bool update_entrypoint = true); | |
178 | ||
179 | extern int rgw_remove_object(RGWRados *store, RGWBucketInfo& bucket_info, rgw_bucket& bucket, rgw_obj_key& key); | |
180 | extern int rgw_remove_bucket(RGWRados *store, rgw_bucket& bucket, bool delete_children); | |
181 | extern int rgw_remove_bucket_bypass_gc(RGWRados *store, rgw_bucket& bucket, int concurrent_max); | |
182 | ||
183 | extern int rgw_bucket_set_attrs(RGWRados *store, RGWBucketInfo& bucket_info, | |
184 | map<string, bufferlist>& attrs, | |
185 | RGWObjVersionTracker *objv_tracker); | |
186 | ||
187 | extern void check_bad_user_bucket_mapping(RGWRados *store, const rgw_user& user_id, bool fix); | |
188 | ||
189 | struct RGWBucketAdminOpState { | |
190 | rgw_user uid; | |
191 | std::string display_name; | |
192 | std::string bucket_name; | |
193 | std::string bucket_id; | |
194 | std::string object_name; | |
195 | ||
196 | bool list_buckets; | |
197 | bool stat_buckets; | |
198 | bool check_objects; | |
199 | bool fix_index; | |
200 | bool delete_child_objects; | |
201 | bool bucket_stored; | |
202 | int max_aio; | |
203 | ||
204 | rgw_bucket bucket; | |
205 | ||
206 | void set_fetch_stats(bool value) { stat_buckets = value; } | |
207 | void set_check_objects(bool value) { check_objects = value; } | |
208 | void set_fix_index(bool value) { fix_index = value; } | |
209 | void set_delete_children(bool value) { delete_child_objects = value; } | |
210 | ||
211 | void set_max_aio(int value) { max_aio = value; } | |
212 | ||
31f18b77 | 213 | void set_user_id(const rgw_user& user_id) { |
7c673cae FG |
214 | if (!user_id.empty()) |
215 | uid = user_id; | |
216 | } | |
31f18b77 | 217 | void set_bucket_name(const std::string& bucket_str) { |
7c673cae FG |
218 | bucket_name = bucket_str; |
219 | } | |
220 | void set_object(std::string& object_str) { | |
221 | object_name = object_str; | |
222 | } | |
223 | ||
224 | rgw_user& get_user_id() { return uid; } | |
225 | std::string& get_user_display_name() { return display_name; } | |
226 | std::string& get_bucket_name() { return bucket_name; } | |
227 | std::string& get_object_name() { return object_name; } | |
228 | ||
229 | rgw_bucket& get_bucket() { return bucket; } | |
230 | void set_bucket(rgw_bucket& _bucket) { | |
231 | bucket = _bucket; | |
232 | bucket_stored = true; | |
233 | } | |
234 | ||
235 | void set_bucket_id(const string& bi) { | |
236 | bucket_id = bi; | |
237 | } | |
238 | const string& get_bucket_id() { return bucket_id; } | |
239 | ||
240 | bool will_fetch_stats() { return stat_buckets; } | |
241 | bool will_fix_index() { return fix_index; } | |
242 | bool will_delete_children() { return delete_child_objects; } | |
243 | bool will_check_objects() { return check_objects; } | |
244 | bool is_user_op() { return !uid.empty(); } | |
245 | bool is_system_op() { return uid.empty(); } | |
246 | bool has_bucket_stored() { return bucket_stored; } | |
247 | int get_max_aio() { return max_aio; } | |
248 | ||
249 | RGWBucketAdminOpState() : list_buckets(false), stat_buckets(false), check_objects(false), | |
250 | fix_index(false), delete_child_objects(false), | |
251 | bucket_stored(false) {} | |
252 | }; | |
253 | ||
254 | /* | |
255 | * A simple wrapper class for administrative bucket operations | |
256 | */ | |
257 | ||
258 | class RGWBucket | |
259 | { | |
260 | RGWUserBuckets buckets; | |
261 | RGWRados *store; | |
262 | RGWAccessHandle handle; | |
263 | ||
264 | RGWUserInfo user_info; | |
265 | std::string tenant; | |
266 | std::string bucket_name; | |
267 | ||
268 | bool failure; | |
269 | ||
270 | RGWBucketInfo bucket_info; | |
271 | ||
272 | public: | |
273 | RGWBucket() : store(NULL), handle(NULL), failure(false) {} | |
274 | int init(RGWRados *storage, RGWBucketAdminOpState& op_state); | |
275 | ||
276 | int check_bad_index_multipart(RGWBucketAdminOpState& op_state, | |
277 | list<rgw_obj_index_key>& objs_to_unlink, std::string *err_msg = NULL); | |
278 | ||
279 | int check_object_index(RGWBucketAdminOpState& op_state, | |
280 | RGWFormatterFlusher& flusher, | |
281 | std::string *err_msg = NULL); | |
282 | ||
283 | int check_index(RGWBucketAdminOpState& op_state, | |
284 | map<RGWObjCategory, RGWStorageStats>& existing_stats, | |
285 | map<RGWObjCategory, RGWStorageStats>& calculated_stats, | |
286 | std::string *err_msg = NULL); | |
287 | ||
288 | int remove(RGWBucketAdminOpState& op_state, bool bypass_gc = false, bool keep_index_consistent = true, std::string *err_msg = NULL); | |
289 | int link(RGWBucketAdminOpState& op_state, std::string *err_msg = NULL); | |
290 | int unlink(RGWBucketAdminOpState& op_state, std::string *err_msg = NULL); | |
291 | ||
292 | int remove_object(RGWBucketAdminOpState& op_state, std::string *err_msg = NULL); | |
293 | int policy_bl_to_stream(bufferlist& bl, ostream& o); | |
294 | int get_policy(RGWBucketAdminOpState& op_state, RGWAccessControlPolicy& policy); | |
295 | ||
296 | void clear_failure() { failure = false; } | |
297 | }; | |
298 | ||
299 | class RGWBucketAdminOp | |
300 | { | |
301 | public: | |
302 | static int get_policy(RGWRados *store, RGWBucketAdminOpState& op_state, | |
303 | RGWFormatterFlusher& flusher); | |
304 | static int get_policy(RGWRados *store, RGWBucketAdminOpState& op_state, | |
305 | RGWAccessControlPolicy& policy); | |
306 | static int dump_s3_policy(RGWRados *store, RGWBucketAdminOpState& op_state, | |
307 | ostream& os); | |
308 | ||
309 | static int unlink(RGWRados *store, RGWBucketAdminOpState& op_state); | |
310 | static int link(RGWRados *store, RGWBucketAdminOpState& op_state, string *err_msg = NULL); | |
311 | ||
312 | static int check_index(RGWRados *store, RGWBucketAdminOpState& op_state, | |
313 | RGWFormatterFlusher& flusher); | |
314 | ||
315 | static int remove_bucket(RGWRados *store, RGWBucketAdminOpState& op_state, bool bypass_gc = false, bool keep_index_consistent = true); | |
316 | static int remove_object(RGWRados *store, RGWBucketAdminOpState& op_state); | |
317 | static int info(RGWRados *store, RGWBucketAdminOpState& op_state, RGWFormatterFlusher& flusher); | |
318 | static int limit_check(RGWRados *store, RGWBucketAdminOpState& op_state, | |
319 | const std::list<std::string>& user_ids, | |
320 | RGWFormatterFlusher& flusher, | |
321 | bool warnings_only = false); | |
322 | }; | |
323 | ||
324 | ||
325 | enum DataLogEntityType { | |
326 | ENTITY_TYPE_UNKNOWN = 0, | |
327 | ENTITY_TYPE_BUCKET = 1, | |
328 | }; | |
329 | ||
330 | struct rgw_data_change { | |
331 | DataLogEntityType entity_type; | |
332 | string key; | |
333 | real_time timestamp; | |
334 | ||
335 | void encode(bufferlist& bl) const { | |
336 | ENCODE_START(1, 1, bl); | |
337 | uint8_t t = (uint8_t)entity_type; | |
338 | ::encode(t, bl); | |
339 | ::encode(key, bl); | |
340 | ::encode(timestamp, bl); | |
341 | ENCODE_FINISH(bl); | |
342 | } | |
343 | ||
344 | void decode(bufferlist::iterator& bl) { | |
345 | DECODE_START(1, bl); | |
346 | uint8_t t; | |
347 | ::decode(t, bl); | |
348 | entity_type = (DataLogEntityType)t; | |
349 | ::decode(key, bl); | |
350 | ::decode(timestamp, bl); | |
351 | DECODE_FINISH(bl); | |
352 | } | |
353 | ||
354 | void dump(Formatter *f) const; | |
355 | void decode_json(JSONObj *obj); | |
356 | }; | |
357 | WRITE_CLASS_ENCODER(rgw_data_change) | |
358 | ||
359 | struct rgw_data_change_log_entry { | |
360 | string log_id; | |
361 | real_time log_timestamp; | |
362 | rgw_data_change entry; | |
363 | ||
364 | void encode(bufferlist& bl) const { | |
365 | ENCODE_START(1, 1, bl); | |
366 | ::encode(log_id, bl); | |
367 | ::encode(log_timestamp, bl); | |
368 | ::encode(entry, bl); | |
369 | ENCODE_FINISH(bl); | |
370 | } | |
371 | ||
372 | void decode(bufferlist::iterator& bl) { | |
373 | DECODE_START(1, bl); | |
374 | ::decode(log_id, bl); | |
375 | ::decode(log_timestamp, bl); | |
376 | ::decode(entry, bl); | |
377 | DECODE_FINISH(bl); | |
378 | } | |
379 | ||
380 | void dump(Formatter *f) const; | |
381 | void decode_json(JSONObj *obj); | |
382 | }; | |
383 | WRITE_CLASS_ENCODER(rgw_data_change_log_entry) | |
384 | ||
385 | struct RGWDataChangesLogInfo { | |
386 | string marker; | |
387 | real_time last_update; | |
388 | ||
389 | void dump(Formatter *f) const; | |
390 | void decode_json(JSONObj *obj); | |
391 | }; | |
392 | ||
393 | class RGWDataChangesLog { | |
394 | CephContext *cct; | |
395 | RGWRados *store; | |
396 | ||
397 | int num_shards; | |
398 | string *oids; | |
399 | ||
400 | Mutex lock; | |
401 | RWLock modified_lock; | |
402 | map<int, set<string> > modified_shards; | |
403 | ||
404 | std::atomic<bool> down_flag = { false }; | |
405 | ||
406 | struct ChangeStatus { | |
407 | real_time cur_expiration; | |
408 | real_time cur_sent; | |
409 | bool pending; | |
410 | RefCountedCond *cond; | |
411 | Mutex *lock; | |
412 | ||
413 | ChangeStatus() : pending(false), cond(NULL) { | |
414 | lock = new Mutex("RGWDataChangesLog::ChangeStatus"); | |
415 | } | |
416 | ||
417 | ~ChangeStatus() { | |
418 | delete lock; | |
419 | } | |
420 | }; | |
421 | ||
422 | typedef ceph::shared_ptr<ChangeStatus> ChangeStatusPtr; | |
423 | ||
424 | lru_map<rgw_bucket_shard, ChangeStatusPtr> changes; | |
425 | ||
426 | map<rgw_bucket_shard, bool> cur_cycle; | |
427 | ||
428 | void _get_change(const rgw_bucket_shard& bs, ChangeStatusPtr& status); | |
429 | void register_renew(rgw_bucket_shard& bs); | |
430 | void update_renewed(rgw_bucket_shard& bs, real_time& expiration); | |
431 | ||
432 | class ChangesRenewThread : public Thread { | |
433 | CephContext *cct; | |
434 | RGWDataChangesLog *log; | |
435 | Mutex lock; | |
436 | Cond cond; | |
437 | ||
438 | public: | |
439 | ChangesRenewThread(CephContext *_cct, RGWDataChangesLog *_log) : cct(_cct), log(_log), lock("ChangesRenewThread::lock") {} | |
440 | void *entry() override; | |
441 | void stop(); | |
442 | }; | |
443 | ||
444 | ChangesRenewThread *renew_thread; | |
445 | ||
446 | public: | |
447 | ||
448 | RGWDataChangesLog(CephContext *_cct, RGWRados *_store) : cct(_cct), store(_store), | |
449 | lock("RGWDataChangesLog::lock"), modified_lock("RGWDataChangesLog::modified_lock"), | |
450 | changes(cct->_conf->rgw_data_log_changes_size) { | |
451 | num_shards = cct->_conf->rgw_data_log_num_shards; | |
452 | ||
453 | oids = new string[num_shards]; | |
454 | ||
455 | string prefix = cct->_conf->rgw_data_log_obj_prefix; | |
456 | ||
457 | if (prefix.empty()) { | |
458 | prefix = "data_log"; | |
459 | } | |
460 | ||
461 | for (int i = 0; i < num_shards; i++) { | |
462 | char buf[16]; | |
463 | snprintf(buf, sizeof(buf), "%s.%d", prefix.c_str(), i); | |
464 | oids[i] = buf; | |
465 | } | |
466 | ||
467 | renew_thread = new ChangesRenewThread(cct, this); | |
468 | renew_thread->create("rgw_dt_lg_renew"); | |
469 | } | |
470 | ||
471 | ~RGWDataChangesLog(); | |
472 | ||
473 | int choose_oid(const rgw_bucket_shard& bs); | |
474 | const std::string& get_oid(int shard_id) const { return oids[shard_id]; } | |
475 | int add_entry(rgw_bucket& bucket, int shard_id); | |
476 | int get_log_shard_id(rgw_bucket& bucket, int shard_id); | |
477 | int renew_entries(); | |
478 | int list_entries(int shard, const real_time& start_time, const real_time& end_time, int max_entries, | |
479 | list<rgw_data_change_log_entry>& entries, | |
480 | const string& marker, | |
481 | string *out_marker, | |
482 | bool *truncated); | |
483 | int trim_entries(int shard_id, const real_time& start_time, const real_time& end_time, | |
484 | const string& start_marker, const string& end_marker); | |
485 | int trim_entries(const real_time& start_time, const real_time& end_time, | |
486 | const string& start_marker, const string& end_marker); | |
487 | int get_info(int shard_id, RGWDataChangesLogInfo *info); | |
488 | int lock_exclusive(int shard_id, timespan duration, string& zone_id, string& owner_id) { | |
489 | return store->lock_exclusive(store->get_zone_params().log_pool, oids[shard_id], duration, zone_id, owner_id); | |
490 | } | |
491 | int unlock(int shard_id, string& zone_id, string& owner_id) { | |
492 | return store->unlock(store->get_zone_params().log_pool, oids[shard_id], zone_id, owner_id); | |
493 | } | |
494 | struct LogMarker { | |
495 | int shard; | |
496 | string marker; | |
497 | ||
498 | LogMarker() : shard(0) {} | |
499 | }; | |
500 | int list_entries(const real_time& start_time, const real_time& end_time, int max_entries, | |
501 | list<rgw_data_change_log_entry>& entries, LogMarker& marker, bool *ptruncated); | |
502 | ||
503 | void mark_modified(int shard_id, const rgw_bucket_shard& bs); | |
504 | void read_clear_modified(map<int, set<string> > &modified); | |
505 | ||
506 | bool going_down(); | |
507 | }; | |
508 | ||
509 | ||
510 | #endif |