]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
9f95a23c | 2 | // vim: ts=8 sw=2 smarttab ft=cpp |
7c673cae FG |
3 | |
4 | #ifndef CEPH_RGWRADOS_H | |
5 | #define CEPH_RGWRADOS_H | |
6 | ||
7 | #include <functional> | |
9f95a23c | 8 | #include <boost/container/flat_map.hpp> |
7c673cae FG |
9 | |
10 | #include "include/rados/librados.hpp" | |
11 | #include "include/Context.h" | |
12 | #include "common/RefCountedObj.h" | |
13 | #include "common/RWLock.h" | |
14 | #include "common/ceph_time.h" | |
7c673cae FG |
15 | #include "rgw_common.h" |
16 | #include "cls/rgw/cls_rgw_types.h" | |
17 | #include "cls/version/cls_version_types.h" | |
18 | #include "cls/log/cls_log_types.h" | |
7c673cae | 19 | #include "cls/timeindex/cls_timeindex_types.h" |
11fdf7f2 | 20 | #include "cls/otp/cls_otp_types.h" |
7c673cae FG |
21 | #include "rgw_log.h" |
22 | #include "rgw_metadata.h" | |
23 | #include "rgw_meta_sync_status.h" | |
24 | #include "rgw_period_puller.h" | |
9f95a23c | 25 | #include "rgw_obj_manifest.h" |
7c673cae | 26 | #include "rgw_sync_module.h" |
9f95a23c | 27 | #include "rgw_trim_bilog.h" |
11fdf7f2 | 28 | #include "rgw_service.h" |
f67539c2 | 29 | #include "rgw_sal.h" |
11fdf7f2 TL |
30 | |
31 | #include "services/svc_rados.h" | |
9f95a23c | 32 | #include "services/svc_bi_rados.h" |
7c673cae FG |
33 | |
34 | class RGWWatcher; | |
35 | class SafeTimer; | |
36 | class ACLOwner; | |
37 | class RGWGC; | |
38 | class RGWMetaNotifier; | |
39 | class RGWDataNotifier; | |
40 | class RGWLC; | |
41 | class RGWObjectExpirer; | |
42 | class RGWMetaSyncProcessorThread; | |
43 | class RGWDataSyncProcessorThread; | |
44 | class RGWSyncLogTrimThread; | |
11fdf7f2 | 45 | class RGWSyncTraceManager; |
7c673cae FG |
46 | struct RGWZoneGroup; |
47 | struct RGWZoneParams; | |
31f18b77 FG |
48 | class RGWReshard; |
49 | class RGWReshardWait; | |
7c673cae | 50 | |
11fdf7f2 TL |
51 | class RGWSysObjectCtx; |
52 | ||
7c673cae FG |
53 | /* flags for put_obj_meta() */ |
54 | #define PUT_OBJ_CREATE 0x01 | |
55 | #define PUT_OBJ_EXCL 0x02 | |
56 | #define PUT_OBJ_CREATE_EXCL (PUT_OBJ_CREATE | PUT_OBJ_EXCL) | |
57 | ||
58 | #define RGW_OBJ_NS_MULTIPART "multipart" | |
59 | #define RGW_OBJ_NS_SHADOW "shadow" | |
60 | ||
7c673cae FG |
61 | static inline void prepend_bucket_marker(const rgw_bucket& bucket, const string& orig_oid, string& oid) |
62 | { | |
63 | if (bucket.marker.empty() || orig_oid.empty()) { | |
64 | oid = orig_oid; | |
65 | } else { | |
66 | oid = bucket.marker; | |
67 | oid.append("_"); | |
68 | oid.append(orig_oid); | |
69 | } | |
70 | } | |
71 | ||
72 | static inline void get_obj_bucket_and_oid_loc(const rgw_obj& obj, string& oid, string& locator) | |
73 | { | |
74 | const rgw_bucket& bucket = obj.bucket; | |
75 | prepend_bucket_marker(bucket, obj.get_oid(), oid); | |
76 | const string& loc = obj.key.get_loc(); | |
77 | if (!loc.empty()) { | |
78 | prepend_bucket_marker(bucket, loc, locator); | |
79 | } else { | |
80 | locator.clear(); | |
81 | } | |
82 | } | |
83 | ||
b3b6e05e | 84 | int rgw_policy_from_attrset(const DoutPrefixProvider *dpp, CephContext *cct, map<string, bufferlist>& attrset, RGWAccessControlPolicy *policy); |
7c673cae | 85 | |
7c673cae FG |
86 | struct RGWOLHInfo { |
87 | rgw_obj target; | |
88 | bool removed; | |
89 | ||
90 | RGWOLHInfo() : removed(false) {} | |
91 | ||
92 | void encode(bufferlist& bl) const { | |
93 | ENCODE_START(1, 1, bl); | |
11fdf7f2 TL |
94 | encode(target, bl); |
95 | encode(removed, bl); | |
7c673cae FG |
96 | ENCODE_FINISH(bl); |
97 | } | |
98 | ||
11fdf7f2 | 99 | void decode(bufferlist::const_iterator& bl) { |
7c673cae | 100 | DECODE_START(1, bl); |
11fdf7f2 TL |
101 | decode(target, bl); |
102 | decode(removed, bl); | |
7c673cae FG |
103 | DECODE_FINISH(bl); |
104 | } | |
105 | static void generate_test_instances(list<RGWOLHInfo*>& o); | |
106 | void dump(Formatter *f) const; | |
107 | }; | |
108 | WRITE_CLASS_ENCODER(RGWOLHInfo) | |
109 | ||
110 | struct RGWOLHPendingInfo { | |
111 | ceph::real_time time; | |
112 | ||
113 | RGWOLHPendingInfo() {} | |
114 | ||
115 | void encode(bufferlist& bl) const { | |
116 | ENCODE_START(1, 1, bl); | |
11fdf7f2 | 117 | encode(time, bl); |
7c673cae FG |
118 | ENCODE_FINISH(bl); |
119 | } | |
120 | ||
11fdf7f2 | 121 | void decode(bufferlist::const_iterator& bl) { |
7c673cae | 122 | DECODE_START(1, bl); |
11fdf7f2 | 123 | decode(time, bl); |
7c673cae FG |
124 | DECODE_FINISH(bl); |
125 | } | |
126 | ||
127 | void dump(Formatter *f) const; | |
128 | }; | |
129 | WRITE_CLASS_ENCODER(RGWOLHPendingInfo) | |
130 | ||
131 | struct RGWUsageBatch { | |
132 | map<ceph::real_time, rgw_usage_log_entry> m; | |
133 | ||
134 | void insert(ceph::real_time& t, rgw_usage_log_entry& entry, bool *account) { | |
135 | bool exists = m.find(t) != m.end(); | |
136 | *account = !exists; | |
137 | m[t].aggregate(entry); | |
138 | } | |
139 | }; | |
140 | ||
7c673cae | 141 | class RGWGetDataCB { |
7c673cae FG |
142 | public: |
143 | virtual int handle_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) = 0; | |
11fdf7f2 | 144 | RGWGetDataCB() {} |
7c673cae | 145 | virtual ~RGWGetDataCB() {} |
7c673cae FG |
146 | }; |
147 | ||
148 | struct RGWCloneRangeInfo { | |
149 | rgw_obj src; | |
150 | off_t src_ofs; | |
151 | off_t dst_ofs; | |
152 | uint64_t len; | |
153 | }; | |
154 | ||
7c673cae FG |
155 | struct RGWObjState { |
156 | rgw_obj obj; | |
9f95a23c TL |
157 | bool is_atomic{false}; |
158 | bool has_attrs{false}; | |
159 | bool exists{false}; | |
160 | uint64_t size{0}; //< size of raw object | |
7c673cae FG |
161 | uint64_t accounted_size{0}; //< size before compression, encryption |
162 | ceph::real_time mtime; | |
9f95a23c | 163 | uint64_t epoch{0}; |
7c673cae | 164 | bufferlist obj_tag; |
181888fb | 165 | bufferlist tail_tag; |
7c673cae | 166 | string write_tag; |
9f95a23c TL |
167 | bool fake_tag{false}; |
168 | std::optional<RGWObjManifest> manifest; | |
7c673cae | 169 | string shadow_obj; |
9f95a23c | 170 | bool has_data{false}; |
7c673cae | 171 | bufferlist data; |
9f95a23c TL |
172 | bool prefetch_data{false}; |
173 | bool keep_tail{false}; | |
174 | bool is_olh{false}; | |
7c673cae | 175 | bufferlist olh_tag; |
9f95a23c TL |
176 | uint64_t pg_ver{false}; |
177 | uint32_t zone_short_id{0}; | |
7c673cae FG |
178 | |
179 | /* important! don't forget to update copy constructor */ | |
180 | ||
181 | RGWObjVersionTracker objv_tracker; | |
182 | ||
183 | map<string, bufferlist> attrset; | |
9f95a23c TL |
184 | |
185 | RGWObjState(); | |
186 | RGWObjState(const RGWObjState& rhs); | |
187 | ~RGWObjState(); | |
7c673cae FG |
188 | |
189 | bool get_attr(string name, bufferlist& dest) { | |
190 | map<string, bufferlist>::iterator iter = attrset.find(name); | |
191 | if (iter != attrset.end()) { | |
192 | dest = iter->second; | |
193 | return true; | |
194 | } | |
195 | return false; | |
196 | } | |
197 | }; | |
198 | ||
9f95a23c TL |
199 | class RGWFetchObjFilter { |
200 | public: | |
201 | virtual ~RGWFetchObjFilter() {} | |
202 | ||
203 | virtual int filter(CephContext *cct, | |
204 | const rgw_obj_key& source_key, | |
205 | const RGWBucketInfo& dest_bucket_info, | |
206 | std::optional<rgw_placement_rule> dest_placement_rule, | |
207 | const map<string, bufferlist>& obj_attrs, | |
208 | std::optional<rgw_user> *poverride_owner, | |
209 | const rgw_placement_rule **prule) = 0; | |
210 | }; | |
211 | ||
212 | class RGWFetchObjFilter_Default : public RGWFetchObjFilter { | |
213 | protected: | |
214 | rgw_placement_rule dest_rule; | |
215 | public: | |
216 | RGWFetchObjFilter_Default() {} | |
217 | ||
218 | int filter(CephContext *cct, | |
219 | const rgw_obj_key& source_key, | |
220 | const RGWBucketInfo& dest_bucket_info, | |
221 | std::optional<rgw_placement_rule> dest_placement_rule, | |
222 | const map<string, bufferlist>& obj_attrs, | |
223 | std::optional<rgw_user> *poverride_owner, | |
224 | const rgw_placement_rule **prule) override; | |
225 | }; | |
226 | ||
227 | class RGWObjectCtx { | |
228 | rgw::sal::RGWRadosStore *store; | |
229 | ceph::shared_mutex lock = ceph::make_shared_mutex("RGWObjectCtx"); | |
230 | void *s{nullptr}; | |
231 | ||
232 | std::map<rgw_obj, RGWObjState> objs_state; | |
233 | public: | |
234 | explicit RGWObjectCtx(rgw::sal::RGWRadosStore *_store) : store(_store) {} | |
235 | explicit RGWObjectCtx(rgw::sal::RGWRadosStore *_store, void *_s) : store(_store), s(_s) {} | |
236 | ||
237 | void *get_private() { | |
238 | return s; | |
239 | } | |
240 | ||
241 | rgw::sal::RGWRadosStore *get_store() { | |
242 | return store; | |
243 | } | |
244 | ||
245 | RGWObjState *get_state(const rgw_obj& obj); | |
246 | ||
247 | void set_atomic(rgw_obj& obj); | |
248 | void set_prefetch_data(const rgw_obj& obj); | |
249 | void invalidate(const rgw_obj& obj); | |
250 | }; | |
251 | ||
252 | ||
7c673cae FG |
253 | struct RGWRawObjState { |
254 | rgw_raw_obj obj; | |
255 | bool has_attrs{false}; | |
256 | bool exists{false}; | |
257 | uint64_t size{0}; | |
258 | ceph::real_time mtime; | |
11fdf7f2 | 259 | uint64_t epoch{0}; |
7c673cae FG |
260 | bufferlist obj_tag; |
261 | bool has_data{false}; | |
262 | bufferlist data; | |
263 | bool prefetch_data{false}; | |
264 | uint64_t pg_ver{0}; | |
265 | ||
266 | /* important! don't forget to update copy constructor */ | |
267 | ||
268 | RGWObjVersionTracker objv_tracker; | |
269 | ||
270 | map<string, bufferlist> attrset; | |
271 | RGWRawObjState() {} | |
272 | RGWRawObjState(const RGWRawObjState& rhs) : obj (rhs.obj) { | |
273 | has_attrs = rhs.has_attrs; | |
274 | exists = rhs.exists; | |
275 | size = rhs.size; | |
276 | mtime = rhs.mtime; | |
277 | epoch = rhs.epoch; | |
278 | if (rhs.obj_tag.length()) { | |
279 | obj_tag = rhs.obj_tag; | |
280 | } | |
281 | has_data = rhs.has_data; | |
282 | if (rhs.data.length()) { | |
283 | data = rhs.data; | |
284 | } | |
285 | prefetch_data = rhs.prefetch_data; | |
286 | pg_ver = rhs.pg_ver; | |
287 | objv_tracker = rhs.objv_tracker; | |
288 | } | |
289 | }; | |
290 | ||
291 | struct RGWPoolIterCtx { | |
292 | librados::IoCtx io_ctx; | |
293 | librados::NObjectIterator iter; | |
294 | }; | |
295 | ||
296 | struct RGWListRawObjsCtx { | |
297 | bool initialized; | |
298 | RGWPoolIterCtx iter_ctx; | |
299 | ||
300 | RGWListRawObjsCtx() : initialized(false) {} | |
301 | }; | |
302 | ||
7c673cae FG |
303 | struct objexp_hint_entry { |
304 | string tenant; | |
305 | string bucket_name; | |
306 | string bucket_id; | |
307 | rgw_obj_key obj_key; | |
308 | ceph::real_time exp_time; | |
309 | ||
310 | void encode(bufferlist& bl) const { | |
311 | ENCODE_START(2, 1, bl); | |
11fdf7f2 TL |
312 | encode(bucket_name, bl); |
313 | encode(bucket_id, bl); | |
314 | encode(obj_key, bl); | |
315 | encode(exp_time, bl); | |
316 | encode(tenant, bl); | |
7c673cae FG |
317 | ENCODE_FINISH(bl); |
318 | } | |
319 | ||
11fdf7f2 | 320 | void decode(bufferlist::const_iterator& bl) { |
7c673cae FG |
321 | // XXX Do we want DECODE_START_LEGACY_COMPAT_LEN(2, 1, 1, bl); ? |
322 | DECODE_START(2, bl); | |
11fdf7f2 TL |
323 | decode(bucket_name, bl); |
324 | decode(bucket_id, bl); | |
325 | decode(obj_key, bl); | |
326 | decode(exp_time, bl); | |
7c673cae | 327 | if (struct_v >= 2) { |
11fdf7f2 | 328 | decode(tenant, bl); |
7c673cae FG |
329 | } else { |
330 | tenant.clear(); | |
331 | } | |
332 | DECODE_FINISH(bl); | |
333 | } | |
9f95a23c TL |
334 | |
335 | void dump(Formatter *f) const; | |
336 | static void generate_test_instances(list<objexp_hint_entry*>& o); | |
7c673cae FG |
337 | }; |
338 | WRITE_CLASS_ENCODER(objexp_hint_entry) | |
339 | ||
7c673cae FG |
340 | class RGWDataChangesLog; |
341 | class RGWMetaSyncStatusManager; | |
342 | class RGWDataSyncStatusManager; | |
7c673cae | 343 | class RGWCoroutinesManagerRegistry; |
7c673cae FG |
344 | |
345 | class RGWGetBucketStats_CB : public RefCountedObject { | |
346 | protected: | |
347 | rgw_bucket bucket; | |
348 | map<RGWObjCategory, RGWStorageStats> *stats; | |
349 | public: | |
224ce89b | 350 | explicit RGWGetBucketStats_CB(const rgw_bucket& _bucket) : bucket(_bucket), stats(NULL) {} |
7c673cae FG |
351 | ~RGWGetBucketStats_CB() override {} |
352 | virtual void handle_response(int r) = 0; | |
353 | virtual void set_response(map<RGWObjCategory, RGWStorageStats> *_stats) { | |
354 | stats = _stats; | |
355 | } | |
356 | }; | |
357 | ||
358 | class RGWGetUserStats_CB : public RefCountedObject { | |
359 | protected: | |
360 | rgw_user user; | |
361 | RGWStorageStats stats; | |
362 | public: | |
363 | explicit RGWGetUserStats_CB(const rgw_user& _user) : user(_user) {} | |
364 | ~RGWGetUserStats_CB() override {} | |
365 | virtual void handle_response(int r) = 0; | |
366 | virtual void set_response(RGWStorageStats& _stats) { | |
367 | stats = _stats; | |
368 | } | |
369 | }; | |
370 | ||
371 | class RGWGetDirHeader_CB; | |
372 | class RGWGetUserHeader_CB; | |
f67539c2 TL |
373 | namespace rgw { namespace sal { |
374 | class RGWRadosStore; | |
375 | class MPRadosSerializer; | |
376 | class LCRadosSerializer; | |
377 | } } | |
7c673cae | 378 | |
7c673cae FG |
379 | class RGWAsyncRadosProcessor; |
380 | ||
381 | template <class T> | |
382 | class RGWChainedCacheImpl; | |
383 | ||
384 | struct bucket_info_entry { | |
385 | RGWBucketInfo info; | |
386 | real_time mtime; | |
387 | map<string, bufferlist> attrs; | |
388 | }; | |
389 | ||
9f95a23c | 390 | struct tombstone_entry; |
7c673cae | 391 | |
9f95a23c TL |
392 | template <class K, class V> |
393 | class lru_map; | |
394 | using tombstone_cache_t = lru_map<rgw_obj, tombstone_entry>; | |
7c673cae | 395 | |
31f18b77 FG |
396 | class RGWIndexCompletionManager; |
397 | ||
9f95a23c | 398 | class RGWRados |
7c673cae FG |
399 | { |
400 | friend class RGWGC; | |
401 | friend class RGWMetaNotifier; | |
402 | friend class RGWDataNotifier; | |
7c673cae FG |
403 | friend class RGWObjectExpirer; |
404 | friend class RGWMetaSyncProcessorThread; | |
405 | friend class RGWDataSyncProcessorThread; | |
31f18b77 FG |
406 | friend class RGWReshard; |
407 | friend class RGWBucketReshard; | |
f64942e4 | 408 | friend class RGWBucketReshardLock; |
31f18b77 | 409 | friend class BucketIndexLockGuard; |
f67539c2 TL |
410 | friend class rgw::sal::MPRadosSerializer; |
411 | friend class rgw::sal::LCRadosSerializer; | |
412 | friend class rgw::sal::RGWRadosStore; | |
7c673cae FG |
413 | |
414 | /** Open the pool used as root for this gateway */ | |
b3b6e05e TL |
415 | int open_root_pool_ctx(const DoutPrefixProvider *dpp); |
416 | int open_gc_pool_ctx(const DoutPrefixProvider *dpp); | |
417 | int open_lc_pool_ctx(const DoutPrefixProvider *dpp); | |
418 | int open_objexp_pool_ctx(const DoutPrefixProvider *dpp); | |
419 | int open_reshard_pool_ctx(const DoutPrefixProvider *dpp); | |
420 | int open_notif_pool_ctx(const DoutPrefixProvider *dpp); | |
421 | ||
422 | int open_pool_ctx(const DoutPrefixProvider *dpp, const rgw_pool& pool, librados::IoCtx& io_ctx, | |
494da23a | 423 | bool mostly_omap); |
7c673cae FG |
424 | |
425 | std::atomic<int64_t> max_req_id = { 0 }; | |
9f95a23c | 426 | ceph::mutex lock = ceph::make_mutex("rados_timer_lock"); |
7c673cae FG |
427 | SafeTimer *timer; |
428 | ||
9f95a23c | 429 | rgw::sal::RGWRadosStore *store; |
7c673cae FG |
430 | RGWGC *gc; |
431 | RGWLC *lc; | |
432 | RGWObjectExpirer *obj_expirer; | |
433 | bool use_gc_thread; | |
434 | bool use_lc_thread; | |
435 | bool quota_threads; | |
436 | bool run_sync_thread; | |
31f18b77 | 437 | bool run_reshard_thread; |
7c673cae | 438 | |
7c673cae FG |
439 | RGWMetaNotifier *meta_notifier; |
440 | RGWDataNotifier *data_notifier; | |
441 | RGWMetaSyncProcessorThread *meta_sync_processor_thread; | |
11fdf7f2 | 442 | RGWSyncTraceManager *sync_tracer = nullptr; |
9f95a23c | 443 | map<rgw_zone_id, RGWDataSyncProcessorThread *> data_sync_processor_threads; |
7c673cae | 444 | |
b32b8144 | 445 | boost::optional<rgw::BucketTrimManager> bucket_trim; |
7c673cae FG |
446 | RGWSyncLogTrimThread *sync_log_trimmer{nullptr}; |
447 | ||
9f95a23c TL |
448 | ceph::mutex meta_sync_thread_lock = ceph::make_mutex("meta_sync_thread_lock"); |
449 | ceph::mutex data_sync_thread_lock = ceph::make_mutex("data_sync_thread_lock"); | |
7c673cae | 450 | |
7c673cae | 451 | librados::IoCtx root_pool_ctx; // .rgw |
11fdf7f2 TL |
452 | |
453 | double inject_notify_timeout_probability = 0; | |
454 | unsigned max_notify_retries = 0; | |
7c673cae FG |
455 | |
456 | friend class RGWWatcher; | |
457 | ||
9f95a23c | 458 | ceph::mutex bucket_id_lock = ceph::make_mutex("rados_bucket_id"); |
7c673cae FG |
459 | |
460 | // This field represents the number of bucket index object shards | |
461 | uint32_t bucket_index_max_shards; | |
462 | ||
b3b6e05e TL |
463 | int get_obj_head_ref(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_rados_ref *ref); |
464 | int get_system_obj_ref(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, rgw_rados_ref *ref); | |
7c673cae FG |
465 | uint64_t max_bucket_id; |
466 | ||
b3b6e05e | 467 | int get_olh_target_state(const DoutPrefixProvider *dpp, RGWObjectCtx& rctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, |
9f95a23c | 468 | RGWObjState *olh_state, RGWObjState **target_state, optional_yield y); |
b3b6e05e | 469 | int get_obj_state_impl(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWObjState **state, |
9f95a23c | 470 | bool follow_olh, optional_yield y, bool assume_noent = false); |
b3b6e05e | 471 | int append_atomic_test(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, |
9f95a23c | 472 | librados::ObjectOperation& op, RGWObjState **state, optional_yield y); |
b3b6e05e | 473 | int append_atomic_test(const DoutPrefixProvider *dpp, const RGWObjState* astate, librados::ObjectOperation& op); |
7c673cae FG |
474 | |
475 | int update_placement_map(); | |
476 | int store_bucket_info(RGWBucketInfo& info, map<string, bufferlist> *pattrs, RGWObjVersionTracker *objv_tracker, bool exclusive); | |
477 | ||
478 | void remove_rgw_head_obj(librados::ObjectWriteOperation& op); | |
479 | void cls_obj_check_prefix_exist(librados::ObjectOperation& op, const string& prefix, bool fail_if_exist); | |
480 | void cls_obj_check_mtime(librados::ObjectOperation& op, const real_time& mtime, bool high_precision_time, RGWCheckMTimeType type); | |
481 | protected: | |
482 | CephContext *cct; | |
483 | ||
494da23a | 484 | librados::Rados rados; |
7c673cae FG |
485 | |
486 | using RGWChainedCacheImpl_bucket_info_entry = RGWChainedCacheImpl<bucket_info_entry>; | |
487 | RGWChainedCacheImpl_bucket_info_entry *binfo_cache; | |
488 | ||
7c673cae FG |
489 | tombstone_cache_t *obj_tombstone_cache; |
490 | ||
491 | librados::IoCtx gc_pool_ctx; // .rgw.gc | |
492 | librados::IoCtx lc_pool_ctx; // .rgw.lc | |
493 | librados::IoCtx objexp_pool_ctx; | |
31f18b77 | 494 | librados::IoCtx reshard_pool_ctx; |
f67539c2 | 495 | librados::IoCtx notif_pool_ctx; // .rgw.notif |
7c673cae | 496 | |
11fdf7f2 | 497 | bool pools_initialized; |
7c673cae | 498 | |
11fdf7f2 | 499 | RGWQuotaHandler *quota_handler; |
7c673cae | 500 | |
11fdf7f2 | 501 | RGWCoroutinesManagerRegistry *cr_registry; |
7c673cae | 502 | |
11fdf7f2 TL |
503 | RGWSyncModuleInstanceRef sync_module; |
504 | bool writeable_zone{false}; | |
7c673cae | 505 | |
11fdf7f2 | 506 | RGWIndexCompletionManager *index_completion_manager{nullptr}; |
7c673cae | 507 | |
11fdf7f2 | 508 | bool use_cache{false}; |
f67539c2 | 509 | |
b3b6e05e | 510 | int get_obj_head_ioctx(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, librados::IoCtx *ioctx); |
11fdf7f2 | 511 | public: |
9f95a23c | 512 | RGWRados(): timer(NULL), |
11fdf7f2 | 513 | gc(NULL), lc(NULL), obj_expirer(NULL), use_gc_thread(false), use_lc_thread(false), quota_threads(false), |
9f95a23c | 514 | run_sync_thread(false), run_reshard_thread(false), meta_notifier(NULL), |
11fdf7f2 | 515 | data_notifier(NULL), meta_sync_processor_thread(NULL), |
11fdf7f2 TL |
516 | bucket_index_max_shards(0), |
517 | max_bucket_id(0), cct(NULL), | |
11fdf7f2 TL |
518 | binfo_cache(NULL), obj_tombstone_cache(nullptr), |
519 | pools_initialized(false), | |
520 | quota_handler(NULL), | |
521 | cr_registry(NULL), | |
9f95a23c TL |
522 | pctl(&ctl), |
523 | reshard(NULL) {} | |
7c673cae | 524 | |
11fdf7f2 TL |
525 | RGWRados& set_use_cache(bool status) { |
526 | use_cache = status; | |
527 | return *this; | |
7c673cae FG |
528 | } |
529 | ||
11fdf7f2 TL |
530 | RGWLC *get_lc() { |
531 | return lc; | |
7c673cae FG |
532 | } |
533 | ||
11fdf7f2 TL |
534 | RGWRados& set_run_gc_thread(bool _use_gc_thread) { |
535 | use_gc_thread = _use_gc_thread; | |
536 | return *this; | |
7c673cae FG |
537 | } |
538 | ||
11fdf7f2 TL |
539 | RGWRados& set_run_lc_thread(bool _use_lc_thread) { |
540 | use_lc_thread = _use_lc_thread; | |
541 | return *this; | |
7c673cae FG |
542 | } |
543 | ||
11fdf7f2 TL |
544 | RGWRados& set_run_quota_threads(bool _run_quota_threads) { |
545 | quota_threads = _run_quota_threads; | |
546 | return *this; | |
7c673cae FG |
547 | } |
548 | ||
11fdf7f2 TL |
549 | RGWRados& set_run_sync_thread(bool _run_sync_thread) { |
550 | run_sync_thread = _run_sync_thread; | |
551 | return *this; | |
7c673cae FG |
552 | } |
553 | ||
11fdf7f2 TL |
554 | RGWRados& set_run_reshard_thread(bool _run_reshard_thread) { |
555 | run_reshard_thread = _run_reshard_thread; | |
556 | return *this; | |
7c673cae FG |
557 | } |
558 | ||
11fdf7f2 TL |
559 | uint64_t get_new_req_id() { |
560 | return ++max_req_id; | |
7c673cae FG |
561 | } |
562 | ||
11fdf7f2 TL |
563 | librados::IoCtx* get_lc_pool_ctx() { |
564 | return &lc_pool_ctx; | |
7c673cae | 565 | } |
f67539c2 TL |
566 | |
567 | librados::IoCtx& get_notif_pool_ctx() { | |
568 | return notif_pool_ctx; | |
569 | } | |
570 | ||
11fdf7f2 TL |
571 | void set_context(CephContext *_cct) { |
572 | cct = _cct; | |
7c673cae | 573 | } |
9f95a23c TL |
574 | void set_store(rgw::sal::RGWRadosStore *_store) { |
575 | store = _store; | |
576 | } | |
31f18b77 | 577 | |
11fdf7f2 | 578 | RGWServices svc; |
9f95a23c TL |
579 | RGWCtl ctl; |
580 | ||
581 | RGWCtl *pctl{nullptr}; | |
11fdf7f2 TL |
582 | |
583 | /** | |
584 | * AmazonS3 errors contain a HostId string, but is an opaque base64 blob; we | |
585 | * try to be more transparent. This has a wrapper so we can update it when zonegroup/zone are changed. | |
586 | */ | |
587 | string host_id; | |
31f18b77 | 588 | |
31f18b77 FG |
589 | RGWReshard *reshard; |
590 | std::shared_ptr<RGWReshardWait> reshard_wait; | |
591 | ||
7c673cae FG |
592 | virtual ~RGWRados() = default; |
593 | ||
594 | tombstone_cache_t *get_tombstone_cache() { | |
595 | return obj_tombstone_cache; | |
596 | } | |
7c673cae FG |
597 | const RGWSyncModuleInstanceRef& get_sync_module() { |
598 | return sync_module; | |
599 | } | |
11fdf7f2 TL |
600 | RGWSyncTraceManager *get_sync_tracer() { |
601 | return sync_tracer; | |
602 | } | |
7c673cae | 603 | |
b3b6e05e | 604 | int get_required_alignment(const DoutPrefixProvider *dpp, const rgw_pool& pool, uint64_t *alignment); |
11fdf7f2 | 605 | void get_max_aligned_size(uint64_t size, uint64_t alignment, uint64_t *max_size); |
b3b6e05e TL |
606 | int get_max_chunk_size(const rgw_pool& pool, uint64_t *max_chunk_size, const DoutPrefixProvider *dpp, uint64_t *palignment = nullptr); |
607 | int get_max_chunk_size(const rgw_placement_rule& placement_rule, const rgw_obj& obj, uint64_t *max_chunk_size, const DoutPrefixProvider *dpp, uint64_t *palignment = nullptr); | |
7c673cae FG |
608 | |
609 | uint32_t get_max_bucket_shards() { | |
9f95a23c | 610 | return RGWSI_BucketIndex_RADOS::shards_max(); |
7c673cae FG |
611 | } |
612 | ||
181888fb | 613 | |
b3b6e05e | 614 | int get_raw_obj_ref(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, rgw_rados_ref *ref); |
7c673cae | 615 | |
b3b6e05e TL |
616 | int list_raw_objects_init(const DoutPrefixProvider *dpp, const rgw_pool& pool, const string& marker, RGWListRawObjsCtx *ctx); |
617 | int list_raw_objects_next(const DoutPrefixProvider *dpp, const string& prefix_filter, int max, | |
181888fb FG |
618 | RGWListRawObjsCtx& ctx, list<string>& oids, |
619 | bool *is_truncated); | |
b3b6e05e | 620 | int list_raw_objects(const DoutPrefixProvider *dpp, const rgw_pool& pool, const string& prefix_filter, int max, |
7c673cae FG |
621 | RGWListRawObjsCtx& ctx, list<string>& oids, |
622 | bool *is_truncated); | |
181888fb | 623 | string list_raw_objs_get_cursor(RGWListRawObjsCtx& ctx); |
7c673cae | 624 | |
7c673cae FG |
625 | CephContext *ctx() { return cct; } |
626 | /** do all necessary setup of the storage device */ | |
b3b6e05e | 627 | int initialize(CephContext *_cct, const DoutPrefixProvider *dpp) { |
7c673cae | 628 | set_context(_cct); |
b3b6e05e | 629 | return initialize(dpp); |
7c673cae FG |
630 | } |
631 | /** Initialize the RADOS instance and prepare to do other ops */ | |
b3b6e05e TL |
632 | int init_svc(bool raw, const DoutPrefixProvider *dpp); |
633 | int init_ctl(const DoutPrefixProvider *dpp); | |
11fdf7f2 | 634 | int init_rados(); |
b3b6e05e TL |
635 | int init_complete(const DoutPrefixProvider *dpp); |
636 | int initialize(const DoutPrefixProvider *dpp); | |
7c673cae FG |
637 | void finalize(); |
638 | ||
224ce89b | 639 | int register_to_service_map(const string& daemon_type, const map<string, string>& meta); |
11fdf7f2 | 640 | int update_service_map(std::map<std::string, std::string>&& status); |
7c673cae FG |
641 | |
642 | /// list logs | |
b3b6e05e | 643 | int log_list_init(const DoutPrefixProvider *dpp, const string& prefix, RGWAccessHandle *handle); |
7c673cae FG |
644 | int log_list_next(RGWAccessHandle handle, string *name); |
645 | ||
646 | /// remove log | |
b3b6e05e | 647 | int log_remove(const DoutPrefixProvider *dpp, const string& name); |
7c673cae FG |
648 | |
649 | /// show log | |
b3b6e05e | 650 | int log_show_init(const DoutPrefixProvider *dpp, const string& name, RGWAccessHandle *handle); |
7c673cae FG |
651 | int log_show_next(RGWAccessHandle handle, rgw_log_entry *entry); |
652 | ||
653 | // log bandwidth info | |
b3b6e05e TL |
654 | int log_usage(const DoutPrefixProvider *dpp, map<rgw_user_bucket, RGWUsageBatch>& usage_info); |
655 | int read_usage(const DoutPrefixProvider *dpp, const rgw_user& user, const string& bucket_name, uint64_t start_epoch, uint64_t end_epoch, | |
11fdf7f2 TL |
656 | uint32_t max_entries, bool *is_truncated, RGWUsageIter& read_iter, map<rgw_user_bucket, |
657 | rgw_usage_log_entry>& usage); | |
b3b6e05e TL |
658 | int trim_usage(const DoutPrefixProvider *dpp, const rgw_user& user, const string& bucket_name, uint64_t start_epoch, uint64_t end_epoch); |
659 | int clear_usage(const DoutPrefixProvider *dpp); | |
7c673cae | 660 | |
b3b6e05e | 661 | int create_pool(const DoutPrefixProvider *dpp, const rgw_pool& pool); |
7c673cae | 662 | |
7c673cae FG |
663 | void create_bucket_id(string *bucket_id); |
664 | ||
11fdf7f2 TL |
665 | bool get_obj_data_pool(const rgw_placement_rule& placement_rule, const rgw_obj& obj, rgw_pool *pool); |
666 | bool obj_to_raw(const rgw_placement_rule& placement_rule, const rgw_obj& obj, rgw_raw_obj *raw_obj); | |
7c673cae | 667 | |
11fdf7f2 | 668 | int create_bucket(const RGWUserInfo& owner, rgw_bucket& bucket, |
f67539c2 TL |
669 | const string& zonegroup_id, |
670 | const rgw_placement_rule& placement_rule, | |
671 | const string& swift_ver_location, | |
672 | const RGWQuotaInfo * pquota_info, | |
673 | map<std::string,bufferlist>& attrs, | |
674 | RGWBucketInfo& bucket_info, | |
675 | obj_version *pobjv, | |
676 | obj_version *pep_objv, | |
677 | ceph::real_time creation_time, | |
678 | rgw_bucket *master_bucket, | |
679 | uint32_t *master_num_shards, | |
680 | optional_yield y, | |
b3b6e05e | 681 | const DoutPrefixProvider *dpp, |
f67539c2 | 682 | bool exclusive = true); |
7c673cae FG |
683 | |
684 | RGWCoroutinesManagerRegistry *get_cr_registry() { return cr_registry; } | |
685 | ||
7c673cae FG |
686 | struct BucketShard { |
687 | RGWRados *store; | |
688 | rgw_bucket bucket; | |
689 | int shard_id; | |
9f95a23c | 690 | RGWSI_RADOS::Obj bucket_obj; |
7c673cae FG |
691 | |
692 | explicit BucketShard(RGWRados *_store) : store(_store), shard_id(-1) {} | |
b3b6e05e TL |
693 | int init(const rgw_bucket& _bucket, const rgw_obj& obj, RGWBucketInfo* out, const DoutPrefixProvider *dpp); |
694 | int init(const rgw_bucket& _bucket, int sid, const rgw::bucket_index_layout_generation& idx_layout, RGWBucketInfo* out, const DoutPrefixProvider *dpp); | |
695 | int init(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj); | |
696 | int init(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout, int sid); | |
7c673cae FG |
697 | }; |
698 | ||
699 | class Object { | |
700 | RGWRados *store; | |
701 | RGWBucketInfo bucket_info; | |
702 | RGWObjectCtx& ctx; | |
703 | rgw_obj obj; | |
704 | ||
705 | BucketShard bs; | |
706 | ||
707 | RGWObjState *state; | |
708 | ||
709 | bool versioning_disabled; | |
710 | ||
711 | bool bs_initialized; | |
712 | ||
713 | protected: | |
b3b6e05e | 714 | int get_state(const DoutPrefixProvider *dpp, RGWObjState **pstate, bool follow_olh, optional_yield y, bool assume_noent = false); |
7c673cae FG |
715 | void invalidate_state(); |
716 | ||
b3b6e05e | 717 | int prepare_atomic_modification(const DoutPrefixProvider *dpp, librados::ObjectWriteOperation& op, bool reset_obj, const string *ptag, |
9f95a23c | 718 | const char *ifmatch, const char *ifnomatch, bool removal_op, bool modify_tail, optional_yield y); |
b3b6e05e | 719 | int complete_atomic_modification(const DoutPrefixProvider *dpp); |
7c673cae FG |
720 | |
721 | public: | |
722 | Object(RGWRados *_store, const RGWBucketInfo& _bucket_info, RGWObjectCtx& _ctx, const rgw_obj& _obj) : store(_store), bucket_info(_bucket_info), | |
723 | ctx(_ctx), obj(_obj), bs(store), | |
724 | state(NULL), versioning_disabled(false), | |
725 | bs_initialized(false) {} | |
726 | ||
727 | RGWRados *get_store() { return store; } | |
728 | rgw_obj& get_obj() { return obj; } | |
729 | RGWObjectCtx& get_ctx() { return ctx; } | |
730 | RGWBucketInfo& get_bucket_info() { return bucket_info; } | |
b3b6e05e | 731 | int get_manifest(const DoutPrefixProvider *dpp, RGWObjManifest **pmanifest, optional_yield y); |
7c673cae | 732 | |
b3b6e05e | 733 | int get_bucket_shard(BucketShard **pbs, const DoutPrefixProvider *dpp) { |
7c673cae | 734 | if (!bs_initialized) { |
f64942e4 | 735 | int r = |
b3b6e05e | 736 | bs.init(bucket_info.bucket, obj, nullptr /* no RGWBucketInfo */, dpp); |
7c673cae FG |
737 | if (r < 0) { |
738 | return r; | |
739 | } | |
740 | bs_initialized = true; | |
741 | } | |
742 | *pbs = &bs; | |
743 | return 0; | |
744 | } | |
745 | ||
746 | void set_versioning_disabled(bool status) { | |
747 | versioning_disabled = status; | |
748 | } | |
749 | ||
750 | bool versioning_enabled() { | |
751 | return (!versioning_disabled && bucket_info.versioning_enabled()); | |
752 | } | |
753 | ||
754 | struct Read { | |
755 | RGWRados::Object *source; | |
756 | ||
757 | struct GetObjState { | |
11fdf7f2 TL |
758 | map<rgw_pool, librados::IoCtx> io_ctxs; |
759 | rgw_pool cur_pool; | |
760 | librados::IoCtx *cur_ioctx{nullptr}; | |
7c673cae FG |
761 | rgw_obj obj; |
762 | rgw_raw_obj head_obj; | |
763 | } state; | |
764 | ||
765 | struct ConditionParams { | |
766 | const ceph::real_time *mod_ptr; | |
767 | const ceph::real_time *unmod_ptr; | |
768 | bool high_precision_time; | |
769 | uint32_t mod_zone_id; | |
770 | uint64_t mod_pg_ver; | |
771 | const char *if_match; | |
772 | const char *if_nomatch; | |
773 | ||
774 | ConditionParams() : | |
775 | mod_ptr(NULL), unmod_ptr(NULL), high_precision_time(false), mod_zone_id(0), mod_pg_ver(0), | |
776 | if_match(NULL), if_nomatch(NULL) {} | |
777 | } conds; | |
778 | ||
779 | struct Params { | |
780 | ceph::real_time *lastmod; | |
781 | uint64_t *obj_size; | |
782 | map<string, bufferlist> *attrs; | |
eafe8130 | 783 | rgw_obj *target_obj; |
7c673cae | 784 | |
eafe8130 TL |
785 | Params() : lastmod(nullptr), obj_size(nullptr), attrs(nullptr), |
786 | target_obj(nullptr) {} | |
7c673cae FG |
787 | } params; |
788 | ||
789 | explicit Read(RGWRados::Object *_source) : source(_source) {} | |
790 | ||
b3b6e05e | 791 | int prepare(optional_yield y, const DoutPrefixProvider *dpp); |
7c673cae | 792 | static int range_to_ofs(uint64_t obj_size, int64_t &ofs, int64_t &end); |
b3b6e05e TL |
793 | int read(int64_t ofs, int64_t end, bufferlist& bl, optional_yield y, const DoutPrefixProvider *dpp); |
794 | int iterate(const DoutPrefixProvider *dpp, int64_t ofs, int64_t end, RGWGetDataCB *cb, optional_yield y); | |
795 | int get_attr(const DoutPrefixProvider *dpp, const char *name, bufferlist& dest, optional_yield y); | |
7c673cae FG |
796 | }; |
797 | ||
798 | struct Write { | |
799 | RGWRados::Object *target; | |
800 | ||
801 | struct MetaParams { | |
802 | ceph::real_time *mtime; | |
803 | map<std::string, bufferlist>* rmattrs; | |
804 | const bufferlist *data; | |
805 | RGWObjManifest *manifest; | |
806 | const string *ptag; | |
807 | list<rgw_obj_index_key> *remove_objs; | |
808 | ceph::real_time set_mtime; | |
809 | rgw_user owner; | |
810 | RGWObjCategory category; | |
811 | int flags; | |
812 | const char *if_match; | |
813 | const char *if_nomatch; | |
11fdf7f2 | 814 | std::optional<uint64_t> olh_epoch; |
7c673cae FG |
815 | ceph::real_time delete_at; |
816 | bool canceled; | |
817 | const string *user_data; | |
31f18b77 | 818 | rgw_zone_set *zones_trace; |
181888fb | 819 | bool modify_tail; |
3efd9988 | 820 | bool completeMultipart; |
11fdf7f2 | 821 | bool appendable; |
7c673cae FG |
822 | |
823 | MetaParams() : mtime(NULL), rmattrs(NULL), data(NULL), manifest(NULL), ptag(NULL), | |
11fdf7f2 | 824 | remove_objs(NULL), category(RGWObjCategory::Main), flags(0), |
91327a77 | 825 | if_match(NULL), if_nomatch(NULL), canceled(false), user_data(nullptr), zones_trace(nullptr), |
11fdf7f2 | 826 | modify_tail(false), completeMultipart(false), appendable(false) {} |
7c673cae FG |
827 | } meta; |
828 | ||
829 | explicit Write(RGWRados::Object *_target) : target(_target) {} | |
830 | ||
b3b6e05e TL |
831 | int _do_write_meta(const DoutPrefixProvider *dpp, |
832 | uint64_t size, uint64_t accounted_size, | |
7c673cae | 833 | map<std::string, bufferlist>& attrs, |
181888fb | 834 | bool modify_tail, bool assume_noent, |
9f95a23c | 835 | void *index_op, optional_yield y); |
b3b6e05e | 836 | int write_meta(const DoutPrefixProvider *dpp, uint64_t size, uint64_t accounted_size, |
9f95a23c | 837 | map<std::string, bufferlist>& attrs, optional_yield y); |
7c673cae | 838 | int write_data(const char *data, uint64_t ofs, uint64_t len, bool exclusive); |
11fdf7f2 TL |
839 | const req_state* get_req_state() { |
840 | return (req_state *)target->get_ctx().get_private(); | |
841 | } | |
7c673cae FG |
842 | }; |
843 | ||
844 | struct Delete { | |
845 | RGWRados::Object *target; | |
846 | ||
847 | struct DeleteParams { | |
848 | rgw_user bucket_owner; | |
b3b6e05e TL |
849 | int versioning_status; // versioning flags in enum RGWBucketFlags |
850 | ACLOwner obj_owner; // needed for creation of deletion marker | |
7c673cae FG |
851 | uint64_t olh_epoch; |
852 | string marker_version_id; | |
853 | uint32_t bilog_flags; | |
854 | list<rgw_obj_index_key> *remove_objs; | |
855 | ceph::real_time expiration_time; | |
856 | ceph::real_time unmod_since; | |
857 | ceph::real_time mtime; /* for setting delete marker mtime */ | |
858 | bool high_precision_time; | |
31f18b77 | 859 | rgw_zone_set *zones_trace; |
9f95a23c TL |
860 | bool abortmp; |
861 | uint64_t parts_accounted_size; | |
7c673cae | 862 | |
9f95a23c | 863 | DeleteParams() : versioning_status(0), olh_epoch(0), bilog_flags(0), remove_objs(NULL), high_precision_time(false), zones_trace(nullptr), abortmp(false), parts_accounted_size(0) {} |
7c673cae FG |
864 | } params; |
865 | ||
866 | struct DeleteResult { | |
867 | bool delete_marker; | |
868 | string version_id; | |
869 | ||
870 | DeleteResult() : delete_marker(false) {} | |
871 | } result; | |
872 | ||
873 | explicit Delete(RGWRados::Object *_target) : target(_target) {} | |
874 | ||
b3b6e05e | 875 | int delete_obj(optional_yield y, const DoutPrefixProvider *dpp); |
7c673cae FG |
876 | }; |
877 | ||
878 | struct Stat { | |
879 | RGWRados::Object *source; | |
880 | ||
881 | struct Result { | |
882 | rgw_obj obj; | |
9f95a23c TL |
883 | std::optional<RGWObjManifest> manifest; |
884 | uint64_t size{0}; | |
885 | struct timespec mtime {}; | |
7c673cae | 886 | map<string, bufferlist> attrs; |
7c673cae FG |
887 | } result; |
888 | ||
889 | struct State { | |
890 | librados::IoCtx io_ctx; | |
891 | librados::AioCompletion *completion; | |
892 | int ret; | |
893 | ||
894 | State() : completion(NULL), ret(0) {} | |
895 | } state; | |
896 | ||
897 | ||
898 | explicit Stat(RGWRados::Object *_source) : source(_source) {} | |
899 | ||
b3b6e05e | 900 | int stat_async(const DoutPrefixProvider *dpp); |
7c673cae FG |
901 | int wait(); |
902 | int stat(); | |
903 | private: | |
904 | int finish(); | |
905 | }; | |
906 | }; | |
907 | ||
908 | class Bucket { | |
909 | RGWRados *store; | |
910 | RGWBucketInfo bucket_info; | |
911 | rgw_bucket& bucket; | |
912 | int shard_id; | |
913 | ||
914 | public: | |
915 | Bucket(RGWRados *_store, const RGWBucketInfo& _bucket_info) : store(_store), bucket_info(_bucket_info), bucket(bucket_info.bucket), | |
916 | shard_id(RGW_NO_SHARD) {} | |
917 | RGWRados *get_store() { return store; } | |
918 | rgw_bucket& get_bucket() { return bucket; } | |
919 | RGWBucketInfo& get_bucket_info() { return bucket_info; } | |
920 | ||
b3b6e05e | 921 | int update_bucket_id(const string& new_bucket_id, const DoutPrefixProvider *dpp); |
31f18b77 | 922 | |
7c673cae FG |
923 | int get_shard_id() { return shard_id; } |
924 | void set_shard_id(int id) { | |
925 | shard_id = id; | |
926 | } | |
927 | ||
928 | class UpdateIndex { | |
929 | RGWRados::Bucket *target; | |
930 | string optag; | |
931 | rgw_obj obj; | |
932 | uint16_t bilog_flags{0}; | |
933 | BucketShard bs; | |
934 | bool bs_initialized{false}; | |
935 | bool blind; | |
936 | bool prepared{false}; | |
31f18b77 FG |
937 | rgw_zone_set *zones_trace{nullptr}; |
938 | ||
b3b6e05e | 939 | int init_bs(const DoutPrefixProvider *dpp) { |
f64942e4 | 940 | int r = |
b3b6e05e | 941 | bs.init(target->get_bucket(), obj, nullptr /* no RGWBucketInfo */, dpp); |
31f18b77 FG |
942 | if (r < 0) { |
943 | return r; | |
944 | } | |
945 | bs_initialized = true; | |
946 | return 0; | |
947 | } | |
948 | ||
949 | void invalidate_bs() { | |
950 | bs_initialized = false; | |
951 | } | |
952 | ||
b3b6e05e | 953 | int guard_reshard(const DoutPrefixProvider *dpp, BucketShard **pbs, std::function<int(BucketShard *)> call); |
7c673cae FG |
954 | public: |
955 | ||
956 | UpdateIndex(RGWRados::Bucket *_target, const rgw_obj& _obj) : target(_target), obj(_obj), | |
957 | bs(target->get_store()) { | |
f67539c2 | 958 | blind = (target->get_bucket_info().layout.current_index.layout.type == rgw::BucketIndexType::Indexless); |
7c673cae FG |
959 | } |
960 | ||
b3b6e05e | 961 | int get_bucket_shard(BucketShard **pbs, const DoutPrefixProvider *dpp) { |
7c673cae | 962 | if (!bs_initialized) { |
b3b6e05e | 963 | int r = init_bs(dpp); |
7c673cae FG |
964 | if (r < 0) { |
965 | return r; | |
966 | } | |
7c673cae FG |
967 | } |
968 | *pbs = &bs; | |
969 | return 0; | |
970 | } | |
971 | ||
972 | void set_bilog_flags(uint16_t flags) { | |
973 | bilog_flags = flags; | |
974 | } | |
31f18b77 FG |
975 | |
976 | void set_zones_trace(rgw_zone_set *_zones_trace) { | |
977 | zones_trace = _zones_trace; | |
978 | } | |
7c673cae | 979 | |
b3b6e05e TL |
980 | int prepare(const DoutPrefixProvider *dpp, RGWModifyOp, const string *write_tag, optional_yield y); |
981 | int complete(const DoutPrefixProvider *dpp, int64_t poolid, uint64_t epoch, uint64_t size, | |
7c673cae FG |
982 | uint64_t accounted_size, ceph::real_time& ut, |
983 | const string& etag, const string& content_type, | |
11fdf7f2 | 984 | const string& storage_class, |
7c673cae | 985 | bufferlist *acl_bl, RGWObjCategory category, |
11fdf7f2 | 986 | list<rgw_obj_index_key> *remove_objs, const string *user_data = nullptr, bool appendable = false); |
b3b6e05e TL |
987 | int complete_del(const DoutPrefixProvider *dpp, |
988 | int64_t poolid, uint64_t epoch, | |
7c673cae FG |
989 | ceph::real_time& removed_mtime, /* mtime of removed object */ |
990 | list<rgw_obj_index_key> *remove_objs); | |
b3b6e05e | 991 | int cancel(const DoutPrefixProvider *dpp); |
7c673cae FG |
992 | |
993 | const string *get_optag() { return &optag; } | |
994 | ||
995 | bool is_prepared() { return prepared; } | |
1adf2230 AA |
996 | }; // class UpdateIndex |
997 | ||
998 | class List { | |
999 | protected: | |
eafe8130 TL |
1000 | // absolute maximum number of objects that |
1001 | // list_objects_(un)ordered can return | |
1002 | static constexpr int64_t bucket_list_objects_absolute_max = 25000; | |
7c673cae | 1003 | |
7c673cae FG |
1004 | RGWRados::Bucket *target; |
1005 | rgw_obj_key next_marker; | |
1006 | ||
b3b6e05e TL |
1007 | int list_objects_ordered(const DoutPrefixProvider *dpp, |
1008 | int64_t max, | |
1adf2230 AA |
1009 | vector<rgw_bucket_dir_entry> *result, |
1010 | map<string, bool> *common_prefixes, | |
9f95a23c TL |
1011 | bool *is_truncated, |
1012 | optional_yield y); | |
b3b6e05e TL |
1013 | int list_objects_unordered(const DoutPrefixProvider *dpp, |
1014 | int64_t max, | |
1adf2230 AA |
1015 | vector<rgw_bucket_dir_entry> *result, |
1016 | map<string, bool> *common_prefixes, | |
9f95a23c TL |
1017 | bool *is_truncated, |
1018 | optional_yield y); | |
1adf2230 AA |
1019 | |
1020 | public: | |
1021 | ||
7c673cae FG |
1022 | struct Params { |
1023 | string prefix; | |
1024 | string delim; | |
1025 | rgw_obj_key marker; | |
1026 | rgw_obj_key end_marker; | |
1027 | string ns; | |
1028 | bool enforce_ns; | |
1029 | RGWAccessListFilter *filter; | |
1030 | bool list_versions; | |
1adf2230 AA |
1031 | bool allow_unordered; |
1032 | ||
1033 | Params() : | |
1034 | enforce_ns(true), | |
1035 | filter(NULL), | |
1036 | list_versions(false), | |
1037 | allow_unordered(false) | |
1038 | {} | |
7c673cae FG |
1039 | } params; |
1040 | ||
7c673cae FG |
1041 | explicit List(RGWRados::Bucket *_target) : target(_target) {} |
1042 | ||
b3b6e05e | 1043 | int list_objects(const DoutPrefixProvider *dpp, int64_t max, |
1adf2230 AA |
1044 | vector<rgw_bucket_dir_entry> *result, |
1045 | map<string, bool> *common_prefixes, | |
9f95a23c TL |
1046 | bool *is_truncated, |
1047 | optional_yield y) { | |
1adf2230 | 1048 | if (params.allow_unordered) { |
b3b6e05e | 1049 | return list_objects_unordered(dpp, max, result, common_prefixes, |
9f95a23c | 1050 | is_truncated, y); |
1adf2230 | 1051 | } else { |
b3b6e05e | 1052 | return list_objects_ordered(dpp, max, result, common_prefixes, |
9f95a23c | 1053 | is_truncated, y); |
1adf2230 AA |
1054 | } |
1055 | } | |
7c673cae FG |
1056 | rgw_obj_key& get_next_marker() { |
1057 | return next_marker; | |
1058 | } | |
1adf2230 AA |
1059 | }; // class List |
1060 | }; // class Bucket | |
7c673cae | 1061 | |
b3b6e05e TL |
1062 | int on_last_entry_in_listing(const DoutPrefixProvider *dpp, |
1063 | RGWBucketInfo& bucket_info, | |
7c673cae FG |
1064 | const std::string& obj_prefix, |
1065 | const std::string& obj_delim, | |
1066 | std::function<int(const rgw_bucket_dir_entry&)> handler); | |
1067 | ||
f67539c2 | 1068 | bool swift_versioning_enabled(rgw::sal::RGWBucket* bucket) const; |
7c673cae FG |
1069 | |
1070 | int swift_versioning_copy(RGWObjectCtx& obj_ctx, /* in/out */ | |
1071 | const rgw_user& user, /* in */ | |
f67539c2 TL |
1072 | rgw::sal::RGWBucket* bucket, /* in */ |
1073 | rgw::sal::RGWObject* obj, /* in */ | |
9f95a23c TL |
1074 | const DoutPrefixProvider *dpp, /* in/out */ |
1075 | optional_yield y); /* in */ | |
1076 | int swift_versioning_restore(RGWObjectCtx& obj_ctx, /* in/out */ | |
7c673cae | 1077 | const rgw_user& user, /* in */ |
f67539c2 TL |
1078 | rgw::sal::RGWBucket* bucket, /* in */ |
1079 | rgw::sal::RGWObject* obj, /* in */ | |
9f95a23c TL |
1080 | bool& restored, /* out */ |
1081 | const DoutPrefixProvider *dpp); /* in/out */ | |
b3b6e05e TL |
1082 | int copy_obj_to_remote_dest(const DoutPrefixProvider *dpp, |
1083 | RGWObjState *astate, | |
7c673cae FG |
1084 | map<string, bufferlist>& src_attrs, |
1085 | RGWRados::Object::Read& read_op, | |
1086 | const rgw_user& user_id, | |
f67539c2 | 1087 | rgw::sal::RGWObject* dest_obj, |
7c673cae FG |
1088 | ceph::real_time *mtime); |
1089 | ||
1090 | enum AttrsMod { | |
1091 | ATTRSMOD_NONE = 0, | |
1092 | ATTRSMOD_REPLACE = 1, | |
1093 | ATTRSMOD_MERGE = 2 | |
1094 | }; | |
1095 | ||
f67539c2 | 1096 | int rewrite_obj(RGWBucketInfo& dest_bucket_info, rgw::sal::RGWObject* obj, const DoutPrefixProvider *dpp, optional_yield y); |
7c673cae | 1097 | |
b3b6e05e TL |
1098 | int stat_remote_obj(const DoutPrefixProvider *dpp, |
1099 | RGWObjectCtx& obj_ctx, | |
7c673cae | 1100 | const rgw_user& user_id, |
7c673cae | 1101 | req_info *info, |
9f95a23c | 1102 | const rgw_zone_id& source_zone, |
f67539c2 | 1103 | rgw::sal::RGWObject* src_obj, |
9f95a23c | 1104 | const RGWBucketInfo *src_bucket_info, |
7c673cae FG |
1105 | real_time *src_mtime, |
1106 | uint64_t *psize, | |
1107 | const real_time *mod_ptr, | |
1108 | const real_time *unmod_ptr, | |
1109 | bool high_precision_time, | |
1110 | const char *if_match, | |
1111 | const char *if_nomatch, | |
1112 | map<string, bufferlist> *pattrs, | |
11fdf7f2 | 1113 | map<string, string> *pheaders, |
7c673cae FG |
1114 | string *version_id, |
1115 | string *ptag, | |
1116 | string *petag); | |
1117 | ||
1118 | int fetch_remote_obj(RGWObjectCtx& obj_ctx, | |
1119 | const rgw_user& user_id, | |
7c673cae | 1120 | req_info *info, |
9f95a23c | 1121 | const rgw_zone_id& source_zone, |
f67539c2 TL |
1122 | rgw::sal::RGWObject* dest_obj, |
1123 | rgw::sal::RGWObject* src_obj, | |
1124 | rgw::sal::RGWBucket* dest_bucket, | |
1125 | rgw::sal::RGWBucket* src_bucket, | |
11fdf7f2 | 1126 | std::optional<rgw_placement_rule> dest_placement, |
7c673cae FG |
1127 | ceph::real_time *src_mtime, |
1128 | ceph::real_time *mtime, | |
1129 | const ceph::real_time *mod_ptr, | |
1130 | const ceph::real_time *unmod_ptr, | |
1131 | bool high_precision_time, | |
1132 | const char *if_match, | |
1133 | const char *if_nomatch, | |
1134 | AttrsMod attrs_mod, | |
1135 | bool copy_if_newer, | |
f67539c2 | 1136 | rgw::sal::RGWAttrs& attrs, |
7c673cae | 1137 | RGWObjCategory category, |
11fdf7f2 | 1138 | std::optional<uint64_t> olh_epoch, |
7c673cae | 1139 | ceph::real_time delete_at, |
7c673cae | 1140 | string *ptag, |
11fdf7f2 | 1141 | string *petag, |
7c673cae | 1142 | void (*progress_cb)(off_t, void *), |
31f18b77 | 1143 | void *progress_data, |
9f95a23c TL |
1144 | const DoutPrefixProvider *dpp, |
1145 | RGWFetchObjFilter *filter, | |
81eedcae TL |
1146 | rgw_zone_set *zones_trace= nullptr, |
1147 | std::optional<uint64_t>* bytes_transferred = 0); | |
7c673cae FG |
1148 | /** |
1149 | * Copy an object. | |
1150 | * dest_obj: the object to copy into | |
1151 | * src_obj: the object to copy from | |
1152 | * attrs: usage depends on attrs_mod parameter | |
1153 | * attrs_mod: the modification mode of the attrs, may have the following values: | |
1154 | * ATTRSMOD_NONE - the attributes of the source object will be | |
1155 | * copied without modifications, attrs parameter is ignored; | |
1156 | * ATTRSMOD_REPLACE - new object will have the attributes provided by attrs | |
1157 | * parameter, source object attributes are not copied; | |
1158 | * ATTRSMOD_MERGE - any conflicting meta keys on the source object's attributes | |
1159 | * are overwritten by values contained in attrs parameter. | |
7c673cae FG |
1160 | * Returns: 0 on success, -ERR# otherwise. |
1161 | */ | |
1162 | int copy_obj(RGWObjectCtx& obj_ctx, | |
1163 | const rgw_user& user_id, | |
7c673cae | 1164 | req_info *info, |
9f95a23c | 1165 | const rgw_zone_id& source_zone, |
f67539c2 TL |
1166 | rgw::sal::RGWObject* dest_obj, |
1167 | rgw::sal::RGWObject* src_obj, | |
1168 | rgw::sal::RGWBucket* dest_bucket, | |
1169 | rgw::sal::RGWBucket* src_bucket, | |
11fdf7f2 | 1170 | const rgw_placement_rule& dest_placement, |
7c673cae FG |
1171 | ceph::real_time *src_mtime, |
1172 | ceph::real_time *mtime, | |
1173 | const ceph::real_time *mod_ptr, | |
1174 | const ceph::real_time *unmod_ptr, | |
1175 | bool high_precision_time, | |
1176 | const char *if_match, | |
1177 | const char *if_nomatch, | |
1178 | AttrsMod attrs_mod, | |
1179 | bool copy_if_newer, | |
1180 | map<std::string, bufferlist>& attrs, | |
1181 | RGWObjCategory category, | |
1182 | uint64_t olh_epoch, | |
1183 | ceph::real_time delete_at, | |
1184 | string *version_id, | |
1185 | string *ptag, | |
11fdf7f2 | 1186 | string *petag, |
7c673cae | 1187 | void (*progress_cb)(off_t, void *), |
9f95a23c TL |
1188 | void *progress_data, |
1189 | const DoutPrefixProvider *dpp, | |
1190 | optional_yield y); | |
7c673cae FG |
1191 | |
1192 | int copy_obj_data(RGWObjectCtx& obj_ctx, | |
f67539c2 | 1193 | rgw::sal::RGWBucket* bucket, |
11fdf7f2 | 1194 | const rgw_placement_rule& dest_placement, |
7c673cae | 1195 | RGWRados::Object::Read& read_op, off_t end, |
f67539c2 | 1196 | rgw::sal::RGWObject* dest_obj, |
7c673cae FG |
1197 | ceph::real_time *mtime, |
1198 | ceph::real_time set_mtime, | |
1199 | map<string, bufferlist>& attrs, | |
7c673cae FG |
1200 | uint64_t olh_epoch, |
1201 | ceph::real_time delete_at, | |
9f95a23c TL |
1202 | string *petag, |
1203 | const DoutPrefixProvider *dpp, | |
1204 | optional_yield y); | |
7c673cae | 1205 | |
11fdf7f2 | 1206 | int transition_obj(RGWObjectCtx& obj_ctx, |
f67539c2 TL |
1207 | rgw::sal::RGWBucket* bucket, |
1208 | rgw::sal::RGWObject& obj, | |
11fdf7f2 TL |
1209 | const rgw_placement_rule& placement_rule, |
1210 | const real_time& mtime, | |
9f95a23c TL |
1211 | uint64_t olh_epoch, |
1212 | const DoutPrefixProvider *dpp, | |
1213 | optional_yield y); | |
11fdf7f2 | 1214 | |
b3b6e05e | 1215 | int check_bucket_empty(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, optional_yield y); |
7c673cae FG |
1216 | |
1217 | /** | |
1218 | * Delete a bucket. | |
1219 | * bucket: the name of the bucket to delete | |
1220 | * Returns 0 on success, -ERR# otherwise. | |
1221 | */ | |
b3b6e05e | 1222 | int delete_bucket(RGWBucketInfo& bucket_info, RGWObjVersionTracker& objv_tracker, optional_yield y, const DoutPrefixProvider *dpp, bool check_empty = true); |
7c673cae | 1223 | |
7c673cae | 1224 | void wakeup_meta_sync_shards(set<int>& shard_ids); |
9f95a23c | 1225 | void wakeup_data_sync_shards(const rgw_zone_id& source_zone, map<int, set<string> >& shard_ids); |
7c673cae FG |
1226 | |
1227 | RGWMetaSyncStatusManager* get_meta_sync_manager(); | |
9f95a23c | 1228 | RGWDataSyncStatusManager* get_data_sync_manager(const rgw_zone_id& source_zone); |
7c673cae | 1229 | |
b3b6e05e TL |
1230 | int set_bucket_owner(rgw_bucket& bucket, ACLOwner& owner, const DoutPrefixProvider *dpp); |
1231 | int set_buckets_enabled(std::vector<rgw_bucket>& buckets, bool enabled, const DoutPrefixProvider *dpp); | |
1232 | int bucket_suspended(const DoutPrefixProvider *dpp, rgw_bucket& bucket, bool *suspended); | |
7c673cae FG |
1233 | |
1234 | /** Delete an object.*/ | |
b3b6e05e TL |
1235 | int delete_obj(const DoutPrefixProvider *dpp, |
1236 | RGWObjectCtx& obj_ctx, | |
f67539c2 TL |
1237 | const RGWBucketInfo& bucket_owner, |
1238 | const rgw_obj& src_obj, | |
b3b6e05e | 1239 | int versioning_status, // versioning flags in enum RGWBucketFlags |
f67539c2 TL |
1240 | uint16_t bilog_flags = 0, |
1241 | const ceph::real_time& expiration_time = ceph::real_time(), | |
1242 | rgw_zone_set *zones_trace = nullptr); | |
7c673cae | 1243 | |
b3b6e05e | 1244 | int delete_raw_obj(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj); |
7c673cae | 1245 | |
7c673cae | 1246 | /** Remove an object from the bucket index */ |
b3b6e05e | 1247 | int delete_obj_index(const rgw_obj& obj, ceph::real_time mtime, const DoutPrefixProvider *dpp); |
7c673cae | 1248 | |
7c673cae FG |
1249 | /** |
1250 | * Set an attr on an object. | |
1251 | * bucket: name of the bucket holding the object | |
1252 | * obj: name of the object to set the attr on | |
1253 | * name: the attr to set | |
1254 | * bl: the contents of the attr | |
1255 | * Returns: 0 on success, -ERR# otherwise. | |
1256 | */ | |
b3b6e05e | 1257 | int set_attr(const DoutPrefixProvider *dpp, void *ctx, const RGWBucketInfo& bucket_info, rgw_obj& obj, const char *name, bufferlist& bl); |
7c673cae | 1258 | |
b3b6e05e | 1259 | int set_attrs(const DoutPrefixProvider *dpp, void *ctx, const RGWBucketInfo& bucket_info, rgw_obj& obj, |
7c673cae | 1260 | map<string, bufferlist>& attrs, |
9f95a23c TL |
1261 | map<string, bufferlist>* rmattrs, |
1262 | optional_yield y); | |
7c673cae | 1263 | |
b3b6e05e | 1264 | int get_obj_state(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWObjState **state, |
9f95a23c | 1265 | bool follow_olh, optional_yield y, bool assume_noent = false); |
b3b6e05e TL |
1266 | int get_obj_state(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWObjState **state, optional_yield y) { |
1267 | return get_obj_state(dpp, rctx, bucket_info, obj, state, true, y); | |
7c673cae FG |
1268 | } |
1269 | ||
b3b6e05e | 1270 | using iterate_obj_cb = int (*)(const DoutPrefixProvider*, const rgw_raw_obj&, off_t, off_t, |
11fdf7f2 TL |
1271 | off_t, bool, RGWObjState*, void*); |
1272 | ||
b3b6e05e | 1273 | int iterate_obj(const DoutPrefixProvider *dpp, RGWObjectCtx& ctx, const RGWBucketInfo& bucket_info, |
11fdf7f2 | 1274 | const rgw_obj& obj, off_t ofs, off_t end, |
9f95a23c TL |
1275 | uint64_t max_chunk_size, iterate_obj_cb cb, void *arg, |
1276 | optional_yield y); | |
7c673cae | 1277 | |
b3b6e05e TL |
1278 | int get_obj_iterate_cb(const DoutPrefixProvider *dpp, |
1279 | const rgw_raw_obj& read_obj, off_t obj_ofs, | |
11fdf7f2 TL |
1280 | off_t read_ofs, off_t len, bool is_head_obj, |
1281 | RGWObjState *astate, void *arg); | |
7c673cae FG |
1282 | |
1283 | void get_obj_aio_completion_cb(librados::completion_t cb, void *arg); | |
1284 | ||
1285 | /** | |
1286 | * a simple object read without keeping state | |
1287 | */ | |
1288 | ||
b3b6e05e TL |
1289 | int raw_obj_stat(const DoutPrefixProvider *dpp, |
1290 | rgw_raw_obj& obj, uint64_t *psize, ceph::real_time *pmtime, uint64_t *epoch, | |
11fdf7f2 | 1291 | map<string, bufferlist> *attrs, bufferlist *first_chunk, |
9f95a23c | 1292 | RGWObjVersionTracker *objv_tracker, optional_yield y); |
7c673cae | 1293 | |
b3b6e05e TL |
1294 | int obj_operate(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, librados::ObjectWriteOperation *op); |
1295 | int obj_operate(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, librados::ObjectReadOperation *op); | |
7c673cae | 1296 | |
b3b6e05e TL |
1297 | int guard_reshard(const DoutPrefixProvider *dpp, |
1298 | BucketShard *bs, | |
f64942e4 AA |
1299 | const rgw_obj& obj_instance, |
1300 | const RGWBucketInfo& bucket_info, | |
1301 | std::function<int(BucketShard *)> call); | |
1302 | int block_while_resharding(RGWRados::BucketShard *bs, | |
1303 | string *new_bucket_id, | |
11fdf7f2 | 1304 | const RGWBucketInfo& bucket_info, |
b3b6e05e TL |
1305 | optional_yield y, |
1306 | const DoutPrefixProvider *dpp); | |
1307 | ||
1308 | void bucket_index_guard_olh_op(const DoutPrefixProvider *dpp, RGWObjState& olh_state, librados::ObjectOperation& op); | |
1309 | int olh_init_modification(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, RGWObjState& state, const rgw_obj& olh_obj, string *op_tag); | |
1310 | int olh_init_modification_impl(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, RGWObjState& state, const rgw_obj& olh_obj, string *op_tag); | |
1311 | int bucket_index_link_olh(const DoutPrefixProvider *dpp, | |
1312 | const RGWBucketInfo& bucket_info, RGWObjState& olh_state, | |
7c673cae FG |
1313 | const rgw_obj& obj_instance, bool delete_marker, |
1314 | const string& op_tag, struct rgw_bucket_dir_entry_meta *meta, | |
1315 | uint64_t olh_epoch, | |
91327a77 AA |
1316 | ceph::real_time unmod_since, bool high_precision_time, |
1317 | rgw_zone_set *zones_trace = nullptr, | |
1318 | bool log_data_change = false); | |
b3b6e05e TL |
1319 | int bucket_index_unlink_instance(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj_instance, const string& op_tag, const string& olh_tag, uint64_t olh_epoch, rgw_zone_set *zones_trace = nullptr); |
1320 | int bucket_index_read_olh_log(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, RGWObjState& state, const rgw_obj& obj_instance, uint64_t ver_marker, | |
7c673cae | 1321 | map<uint64_t, vector<rgw_bucket_olh_log_entry> > *log, bool *is_truncated); |
b3b6e05e TL |
1322 | int bucket_index_trim_olh_log(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, RGWObjState& obj_state, const rgw_obj& obj_instance, uint64_t ver); |
1323 | int bucket_index_clear_olh(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, RGWObjState& state, const rgw_obj& obj_instance); | |
1324 | int apply_olh_log(const DoutPrefixProvider *dpp, RGWObjectCtx& ctx, RGWObjState& obj_state, const RGWBucketInfo& bucket_info, const rgw_obj& obj, | |
7c673cae | 1325 | bufferlist& obj_tag, map<uint64_t, vector<rgw_bucket_olh_log_entry> >& log, |
31f18b77 | 1326 | uint64_t *plast_ver, rgw_zone_set *zones_trace = nullptr); |
b3b6e05e TL |
1327 | int update_olh(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, RGWObjState *state, const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_zone_set *zones_trace = nullptr); |
1328 | int set_olh(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, const RGWBucketInfo& bucket_info, const rgw_obj& target_obj, bool delete_marker, rgw_bucket_dir_entry_meta *meta, | |
91327a77 | 1329 | uint64_t olh_epoch, ceph::real_time unmod_since, bool high_precision_time, |
9f95a23c | 1330 | optional_yield y, rgw_zone_set *zones_trace = nullptr, bool log_data_change = false); |
b3b6e05e | 1331 | int repair_olh(const DoutPrefixProvider *dpp, RGWObjState* state, const RGWBucketInfo& bucket_info, |
a8e16298 | 1332 | const rgw_obj& obj); |
b3b6e05e | 1333 | int unlink_obj_instance(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, RGWBucketInfo& bucket_info, const rgw_obj& target_obj, |
9f95a23c | 1334 | uint64_t olh_epoch, optional_yield y, rgw_zone_set *zones_trace = nullptr); |
7c673cae FG |
1335 | |
1336 | void check_pending_olh_entries(map<string, bufferlist>& pending_entries, map<string, bufferlist> *rm_pending_entries); | |
b3b6e05e TL |
1337 | int remove_olh_pending_entries(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, RGWObjState& state, const rgw_obj& olh_obj, map<string, bufferlist>& pending_attrs); |
1338 | int follow_olh(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, RGWObjectCtx& ctx, RGWObjState *state, const rgw_obj& olh_obj, rgw_obj *target); | |
1339 | int get_olh(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWOLHInfo *olh); | |
7c673cae | 1340 | |
11fdf7f2 | 1341 | void gen_rand_obj_instance_name(rgw_obj_key *target_key); |
7c673cae FG |
1342 | void gen_rand_obj_instance_name(rgw_obj *target); |
1343 | ||
b3b6e05e TL |
1344 | int update_containers_stats(map<string, RGWBucketEnt>& m, const DoutPrefixProvider *dpp); |
1345 | int append_async(const DoutPrefixProvider *dpp, rgw_raw_obj& obj, size_t size, bufferlist& bl); | |
7c673cae | 1346 | |
11fdf7f2 | 1347 | public: |
7c673cae FG |
1348 | void set_atomic(void *ctx, rgw_obj& obj) { |
1349 | RGWObjectCtx *rctx = static_cast<RGWObjectCtx *>(ctx); | |
11fdf7f2 | 1350 | rctx->set_atomic(obj); |
7c673cae | 1351 | } |
11fdf7f2 | 1352 | void set_prefetch_data(void *ctx, const rgw_obj& obj) { |
7c673cae | 1353 | RGWObjectCtx *rctx = static_cast<RGWObjectCtx *>(ctx); |
11fdf7f2 | 1354 | rctx->set_prefetch_data(obj); |
7c673cae | 1355 | } |
7c673cae | 1356 | int decode_policy(bufferlist& bl, ACLOwner *owner); |
b3b6e05e | 1357 | int get_bucket_stats(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, int shard_id, string *bucket_ver, string *master_ver, |
c07f9fc5 | 1358 | map<RGWObjCategory, RGWStorageStats>& stats, string *max_marker, bool* syncstopped = NULL); |
b3b6e05e | 1359 | int get_bucket_stats_async(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, int shard_id, RGWGetBucketStats_CB *cb); |
3a9019d9 | 1360 | |
b3b6e05e | 1361 | int put_bucket_instance_info(RGWBucketInfo& info, bool exclusive, ceph::real_time mtime, map<string, bufferlist> *pattrs, const DoutPrefixProvider *dpp); |
9f95a23c | 1362 | /* xxx dang obj_ctx -> svc */ |
b3b6e05e TL |
1363 | int get_bucket_instance_info(RGWSysObjectCtx& obj_ctx, const string& meta_key, RGWBucketInfo& info, ceph::real_time *pmtime, map<string, bufferlist> *pattrs, optional_yield y, const DoutPrefixProvider *dpp); |
1364 | int get_bucket_instance_info(RGWSysObjectCtx& obj_ctx, const rgw_bucket& bucket, RGWBucketInfo& info, ceph::real_time *pmtime, map<string, bufferlist> *pattrs, optional_yield y, const DoutPrefixProvider *dpp); | |
3a9019d9 | 1365 | |
9f95a23c | 1366 | static void make_bucket_entry_name(const string& tenant_name, const string& bucket_name, string& bucket_entry); |
b32b8144 | 1367 | |
9f95a23c | 1368 | int get_bucket_info(RGWServices *svc, |
b32b8144 FG |
1369 | const string& tenant_name, const string& bucket_name, |
1370 | RGWBucketInfo& info, | |
b3b6e05e TL |
1371 | ceph::real_time *pmtime, optional_yield y, |
1372 | const DoutPrefixProvider *dpp, map<string, bufferlist> *pattrs = NULL); | |
b32b8144 | 1373 | |
81eedcae TL |
1374 | // Returns 0 on successful refresh. Returns error code if there was |
1375 | // an error or the version stored on the OSD is the same as that | |
b32b8144 FG |
1376 | // presented in the BucketInfo structure. |
1377 | // | |
1378 | int try_refresh_bucket_info(RGWBucketInfo& info, | |
1379 | ceph::real_time *pmtime, | |
b3b6e05e | 1380 | const DoutPrefixProvider *dpp, |
b32b8144 FG |
1381 | map<string, bufferlist> *pattrs = nullptr); |
1382 | ||
7c673cae | 1383 | int put_linked_bucket_info(RGWBucketInfo& info, bool exclusive, ceph::real_time mtime, obj_version *pep_objv, |
b3b6e05e TL |
1384 | map<string, bufferlist> *pattrs, bool create_entry_point, |
1385 | const DoutPrefixProvider *dpp); | |
7c673cae | 1386 | |
b3b6e05e | 1387 | int cls_obj_prepare_op(const DoutPrefixProvider *dpp, BucketShard& bs, RGWModifyOp op, string& tag, rgw_obj& obj, uint16_t bilog_flags, optional_yield y, rgw_zone_set *zones_trace = nullptr); |
31f18b77 FG |
1388 | int cls_obj_complete_op(BucketShard& bs, const rgw_obj& obj, RGWModifyOp op, string& tag, int64_t pool, uint64_t epoch, |
1389 | rgw_bucket_dir_entry& ent, RGWObjCategory category, list<rgw_obj_index_key> *remove_objs, uint16_t bilog_flags, rgw_zone_set *zones_trace = nullptr); | |
1390 | int cls_obj_complete_add(BucketShard& bs, const rgw_obj& obj, string& tag, int64_t pool, uint64_t epoch, rgw_bucket_dir_entry& ent, | |
1391 | RGWObjCategory category, list<rgw_obj_index_key> *remove_objs, uint16_t bilog_flags, rgw_zone_set *zones_trace = nullptr); | |
7c673cae | 1392 | int cls_obj_complete_del(BucketShard& bs, string& tag, int64_t pool, uint64_t epoch, rgw_obj& obj, |
31f18b77 FG |
1393 | ceph::real_time& removed_mtime, list<rgw_obj_index_key> *remove_objs, uint16_t bilog_flags, rgw_zone_set *zones_trace = nullptr); |
1394 | int cls_obj_complete_cancel(BucketShard& bs, string& tag, rgw_obj& obj, uint16_t bilog_flags, rgw_zone_set *zones_trace = nullptr); | |
b3b6e05e | 1395 | int cls_obj_set_bucket_tag_timeout(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, uint64_t timeout); |
9f95a23c TL |
1396 | |
1397 | using ent_map_t = | |
1398 | boost::container::flat_map<std::string, rgw_bucket_dir_entry>; | |
1399 | ||
1400 | using check_filter_t = bool (*)(const std::string&); | |
1401 | ||
b3b6e05e TL |
1402 | int cls_bucket_list_ordered(const DoutPrefixProvider *dpp, |
1403 | RGWBucketInfo& bucket_info, | |
9f95a23c TL |
1404 | const int shard_id, |
1405 | const rgw_obj_index_key& start_after, | |
11fdf7f2 | 1406 | const string& prefix, |
9f95a23c TL |
1407 | const string& delimiter, |
1408 | const uint32_t num_entries, | |
1409 | const bool list_versions, | |
1410 | const uint16_t exp_factor, // 0 means ignore | |
1411 | ent_map_t& m, | |
1412 | bool* is_truncated, | |
1413 | bool* cls_filtered, | |
1adf2230 | 1414 | rgw_obj_index_key *last_entry, |
9f95a23c TL |
1415 | optional_yield y, |
1416 | check_filter_t force_check_filter = nullptr); | |
b3b6e05e TL |
1417 | int cls_bucket_list_unordered(const DoutPrefixProvider *dpp, |
1418 | RGWBucketInfo& bucket_info, | |
9f95a23c TL |
1419 | int shard_id, |
1420 | const rgw_obj_index_key& start_after, | |
11fdf7f2 | 1421 | const string& prefix, |
9f95a23c TL |
1422 | uint32_t num_entries, |
1423 | bool list_versions, | |
1adf2230 | 1424 | vector<rgw_bucket_dir_entry>& ent_list, |
9f95a23c TL |
1425 | bool *is_truncated, |
1426 | rgw_obj_index_key *last_entry, | |
1427 | optional_yield y, | |
1428 | check_filter_t = nullptr); | |
b3b6e05e TL |
1429 | int cls_bucket_head(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, int shard_id, vector<rgw_bucket_dir_header>& headers, map<int, string> *bucket_instance_ids = NULL); |
1430 | int cls_bucket_head_async(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, int shard_id, RGWGetDirHeader_CB *ctx, int *num_aio); | |
7c673cae | 1431 | |
b3b6e05e TL |
1432 | int bi_get_instance(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_bucket_dir_entry *dirent); |
1433 | int bi_get_olh(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_bucket_olh_entry *olh); | |
1434 | int bi_get(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, BIIndexType index_type, rgw_cls_bi_entry *entry); | |
7c673cae FG |
1435 | void bi_put(librados::ObjectWriteOperation& op, BucketShard& bs, rgw_cls_bi_entry& entry); |
1436 | int bi_put(BucketShard& bs, rgw_cls_bi_entry& entry); | |
b3b6e05e TL |
1437 | int bi_put(const DoutPrefixProvider *dpp, rgw_bucket& bucket, rgw_obj& obj, rgw_cls_bi_entry& entry); |
1438 | int bi_list(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, int shard_id, const string& filter_obj, const string& marker, uint32_t max, list<rgw_cls_bi_entry> *entries, bool *is_truncated); | |
7c673cae | 1439 | int bi_list(BucketShard& bs, const string& filter_obj, const string& marker, uint32_t max, list<rgw_cls_bi_entry> *entries, bool *is_truncated); |
b3b6e05e | 1440 | int bi_list(const DoutPrefixProvider *dpp, rgw_bucket& bucket, const string& obj_name, const string& marker, uint32_t max, |
7c673cae FG |
1441 | list<rgw_cls_bi_entry> *entries, bool *is_truncated); |
1442 | int bi_remove(BucketShard& bs); | |
1443 | ||
b3b6e05e TL |
1444 | int cls_obj_usage_log_add(const DoutPrefixProvider *dpp, const string& oid, rgw_usage_log_info& info); |
1445 | int cls_obj_usage_log_read(const DoutPrefixProvider *dpp, const string& oid, const string& user, const string& bucket, uint64_t start_epoch, | |
11fdf7f2 TL |
1446 | uint64_t end_epoch, uint32_t max_entries, string& read_iter, map<rgw_user_bucket, |
1447 | rgw_usage_log_entry>& usage, bool *is_truncated); | |
b3b6e05e | 1448 | int cls_obj_usage_log_trim(const DoutPrefixProvider *dpp, const string& oid, const string& user, const string& bucket, uint64_t start_epoch, |
11fdf7f2 | 1449 | uint64_t end_epoch); |
b3b6e05e | 1450 | int cls_obj_usage_log_clear(const DoutPrefixProvider *dpp, string& oid); |
7c673cae | 1451 | |
f67539c2 | 1452 | int get_target_shard_id(const rgw::bucket_index_normal_layout& layout, const string& obj_key, int *shard_id); |
9f95a23c TL |
1453 | |
1454 | int lock_exclusive(const rgw_pool& pool, const string& oid, ceph::timespan& duration, rgw_zone_id& zone_id, string& owner_id); | |
1455 | int unlock(const rgw_pool& pool, const string& oid, rgw_zone_id& zone_id, string& owner_id); | |
7c673cae | 1456 | |
b3b6e05e | 1457 | void update_gc_chain(const DoutPrefixProvider *dpp, rgw_obj& head_obj, RGWObjManifest& manifest, cls_rgw_obj_chain *chain); |
9f95a23c | 1458 | int send_chain_to_gc(cls_rgw_obj_chain& chain, const string& tag); |
b3b6e05e TL |
1459 | void delete_objs_inline(const DoutPrefixProvider *dpp, cls_rgw_obj_chain& chain, const string& tag); |
1460 | int gc_operate(const DoutPrefixProvider *dpp, string& oid, librados::ObjectWriteOperation *op); | |
9f95a23c TL |
1461 | int gc_aio_operate(const std::string& oid, librados::AioCompletion *c, |
1462 | librados::ObjectWriteOperation *op); | |
b3b6e05e | 1463 | int gc_operate(const DoutPrefixProvider *dpp, string& oid, librados::ObjectReadOperation *op, bufferlist *pbl); |
7c673cae | 1464 | |
9f95a23c | 1465 | int list_gc_objs(int *index, string& marker, uint32_t max, bool expired_only, std::list<cls_rgw_gc_obj_info>& result, bool *truncated, bool& processing_queue); |
11fdf7f2 | 1466 | int process_gc(bool expired_only); |
b3b6e05e TL |
1467 | bool process_expire_objects(const DoutPrefixProvider *dpp); |
1468 | int defer_gc(const DoutPrefixProvider *dpp, void *ctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, optional_yield y); | |
7c673cae FG |
1469 | |
1470 | int process_lc(); | |
f6b5b4d7 | 1471 | int list_lc_progress(string& marker, uint32_t max_entries, |
f67539c2 | 1472 | vector<rgw::sal::Lifecycle::LCEntry>& progress_map, int& index); |
f6b5b4d7 | 1473 | |
b3b6e05e | 1474 | int bucket_check_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, |
7c673cae FG |
1475 | map<RGWObjCategory, RGWStorageStats> *existing_stats, |
1476 | map<RGWObjCategory, RGWStorageStats> *calculated_stats); | |
b3b6e05e TL |
1477 | int bucket_rebuild_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info); |
1478 | int bucket_set_reshard(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const cls_rgw_bucket_instance_entry& entry); | |
1479 | int remove_objs_from_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, list<rgw_obj_index_key>& oid_list); | |
1480 | int move_rados_obj(const DoutPrefixProvider *dpp, | |
1481 | librados::IoCtx& src_ioctx, | |
7c673cae FG |
1482 | const string& src_oid, const string& src_locator, |
1483 | librados::IoCtx& dst_ioctx, | |
1484 | const string& dst_oid, const string& dst_locator); | |
b3b6e05e TL |
1485 | int fix_head_obj_locator(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, bool copy_obj, bool remove_bad, rgw_obj_key& key); |
1486 | int fix_tail_obj_locator(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, rgw_obj_key& key, bool fix, bool *need_fix, optional_yield y); | |
7c673cae FG |
1487 | |
1488 | int check_quota(const rgw_user& bucket_owner, rgw_bucket& bucket, | |
f67539c2 TL |
1489 | RGWQuotaInfo& user_quota, RGWQuotaInfo& bucket_quota, uint64_t obj_size, |
1490 | optional_yield y, bool check_size_only = false); | |
7c673cae | 1491 | |
224ce89b | 1492 | int check_bucket_shards(const RGWBucketInfo& bucket_info, const rgw_bucket& bucket, |
b3b6e05e | 1493 | uint64_t num_objs, const DoutPrefixProvider *dpp); |
31f18b77 | 1494 | |
b3b6e05e | 1495 | int add_bucket_to_reshard(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, uint32_t new_num_shards); |
31f18b77 | 1496 | |
7c673cae | 1497 | uint64_t instance_id(); |
3efd9988 | 1498 | |
7c673cae FG |
1499 | librados::Rados* get_rados_handle(); |
1500 | ||
b3b6e05e TL |
1501 | int delete_raw_obj_aio(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, list<librados::AioCompletion *>& handles); |
1502 | int delete_obj_aio(const DoutPrefixProvider *dpp, const rgw_obj& obj, RGWBucketInfo& info, RGWObjState *astate, | |
9f95a23c TL |
1503 | list<librados::AioCompletion *>& handles, bool keep_index_consistent, |
1504 | optional_yield y); | |
11fdf7f2 | 1505 | |
11fdf7f2 | 1506 | private: |
7c673cae FG |
1507 | /** |
1508 | * Check the actual on-disk state of the object specified | |
1509 | * by list_state, and fill in the time and size of object. | |
1510 | * Then append any changes to suggested_updates for | |
1511 | * the rgw class' dir_suggest_changes function. | |
1512 | * | |
1513 | * Note that this can maul list_state; don't use it afterwards. Also | |
1514 | * it expects object to already be filled in from list_state; it only | |
1515 | * sets the size and mtime. | |
1516 | * | |
1517 | * Returns 0 on success, -ENOENT if the object doesn't exist on disk, | |
1518 | * and -errno on other failures. (-ENOENT is not a failure, and it | |
1519 | * will encode that info as a suggested update.) | |
1520 | */ | |
b3b6e05e TL |
1521 | int check_disk_state(const DoutPrefixProvider *dpp, |
1522 | librados::IoCtx io_ctx, | |
7c673cae FG |
1523 | const RGWBucketInfo& bucket_info, |
1524 | rgw_bucket_dir_entry& list_state, | |
1525 | rgw_bucket_dir_entry& object, | |
9f95a23c TL |
1526 | bufferlist& suggested_updates, |
1527 | optional_yield y); | |
7c673cae FG |
1528 | |
1529 | /** | |
1530 | * Init pool iteration | |
31f18b77 | 1531 | * pool: pool to use for the ctx initialization |
7c673cae FG |
1532 | * ctx: context object to use for the iteration |
1533 | * Returns: 0 on success, -ERR# otherwise. | |
1534 | */ | |
b3b6e05e | 1535 | int pool_iterate_begin(const DoutPrefixProvider *dpp, const rgw_pool& pool, RGWPoolIterCtx& ctx); |
31f18b77 | 1536 | |
181888fb FG |
1537 | /** |
1538 | * Init pool iteration | |
1539 | * pool: pool to use | |
1540 | * cursor: position to start iteration | |
1541 | * ctx: context object to use for the iteration | |
1542 | * Returns: 0 on success, -ERR# otherwise. | |
1543 | */ | |
b3b6e05e | 1544 | int pool_iterate_begin(const DoutPrefixProvider *dpp, const rgw_pool& pool, const string& cursor, RGWPoolIterCtx& ctx); |
181888fb FG |
1545 | |
1546 | /** | |
1547 | * Get pool iteration position | |
1548 | * ctx: context object to use for the iteration | |
1549 | * Returns: string representation of position | |
1550 | */ | |
1551 | string pool_iterate_get_cursor(RGWPoolIterCtx& ctx); | |
1552 | ||
7c673cae FG |
1553 | /** |
1554 | * Iterate over pool return object names, use optional filter | |
1555 | * ctx: iteration context, initialized with pool_iterate_begin() | |
1556 | * num: max number of objects to return | |
1557 | * objs: a vector that the results will append into | |
1558 | * is_truncated: if not NULL, will hold true iff iteration is complete | |
1559 | * filter: if not NULL, will be used to filter returned objects | |
1560 | * Returns: 0 on success, -ERR# otherwise. | |
1561 | */ | |
1562 | int pool_iterate(RGWPoolIterCtx& ctx, uint32_t num, vector<rgw_bucket_dir_entry>& objs, | |
1563 | bool *is_truncated, RGWAccessListFilter *filter); | |
1564 | ||
1565 | uint64_t next_bucket_id(); | |
11fdf7f2 | 1566 | |
9f95a23c TL |
1567 | /** |
1568 | * This is broken out to facilitate unit testing. | |
1569 | */ | |
1570 | static uint32_t calc_ordered_bucket_list_per_shard(uint32_t num_entries, | |
1571 | uint32_t num_shards); | |
11fdf7f2 TL |
1572 | }; |
1573 | ||
7c673cae | 1574 | #endif |