]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
9f95a23c | 2 | // vim: ts=8 sw=2 smarttab ft=cpp |
7c673cae FG |
3 | |
4 | #ifndef CEPH_RGWRADOS_H | |
5 | #define CEPH_RGWRADOS_H | |
6 | ||
7 | #include <functional> | |
9f95a23c | 8 | #include <boost/container/flat_map.hpp> |
7c673cae FG |
9 | |
10 | #include "include/rados/librados.hpp" | |
11 | #include "include/Context.h" | |
12 | #include "common/RefCountedObj.h" | |
13 | #include "common/RWLock.h" | |
14 | #include "common/ceph_time.h" | |
7c673cae FG |
15 | #include "rgw_common.h" |
16 | #include "cls/rgw/cls_rgw_types.h" | |
17 | #include "cls/version/cls_version_types.h" | |
18 | #include "cls/log/cls_log_types.h" | |
7c673cae | 19 | #include "cls/timeindex/cls_timeindex_types.h" |
11fdf7f2 | 20 | #include "cls/otp/cls_otp_types.h" |
7c673cae FG |
21 | #include "rgw_log.h" |
22 | #include "rgw_metadata.h" | |
23 | #include "rgw_meta_sync_status.h" | |
24 | #include "rgw_period_puller.h" | |
9f95a23c | 25 | #include "rgw_obj_manifest.h" |
7c673cae | 26 | #include "rgw_sync_module.h" |
9f95a23c | 27 | #include "rgw_trim_bilog.h" |
11fdf7f2 | 28 | #include "rgw_service.h" |
f67539c2 | 29 | #include "rgw_sal.h" |
11fdf7f2 TL |
30 | |
31 | #include "services/svc_rados.h" | |
9f95a23c | 32 | #include "services/svc_bi_rados.h" |
7c673cae FG |
33 | |
34 | class RGWWatcher; | |
35 | class SafeTimer; | |
36 | class ACLOwner; | |
37 | class RGWGC; | |
38 | class RGWMetaNotifier; | |
39 | class RGWDataNotifier; | |
40 | class RGWLC; | |
41 | class RGWObjectExpirer; | |
42 | class RGWMetaSyncProcessorThread; | |
43 | class RGWDataSyncProcessorThread; | |
44 | class RGWSyncLogTrimThread; | |
11fdf7f2 | 45 | class RGWSyncTraceManager; |
7c673cae FG |
46 | struct RGWZoneGroup; |
47 | struct RGWZoneParams; | |
31f18b77 FG |
48 | class RGWReshard; |
49 | class RGWReshardWait; | |
7c673cae | 50 | |
11fdf7f2 TL |
51 | class RGWSysObjectCtx; |
52 | ||
7c673cae FG |
53 | /* flags for put_obj_meta() */ |
54 | #define PUT_OBJ_CREATE 0x01 | |
55 | #define PUT_OBJ_EXCL 0x02 | |
56 | #define PUT_OBJ_CREATE_EXCL (PUT_OBJ_CREATE | PUT_OBJ_EXCL) | |
57 | ||
58 | #define RGW_OBJ_NS_MULTIPART "multipart" | |
59 | #define RGW_OBJ_NS_SHADOW "shadow" | |
60 | ||
7c673cae FG |
61 | static inline void prepend_bucket_marker(const rgw_bucket& bucket, const string& orig_oid, string& oid) |
62 | { | |
63 | if (bucket.marker.empty() || orig_oid.empty()) { | |
64 | oid = orig_oid; | |
65 | } else { | |
66 | oid = bucket.marker; | |
67 | oid.append("_"); | |
68 | oid.append(orig_oid); | |
69 | } | |
70 | } | |
71 | ||
72 | static inline void get_obj_bucket_and_oid_loc(const rgw_obj& obj, string& oid, string& locator) | |
73 | { | |
74 | const rgw_bucket& bucket = obj.bucket; | |
75 | prepend_bucket_marker(bucket, obj.get_oid(), oid); | |
76 | const string& loc = obj.key.get_loc(); | |
77 | if (!loc.empty()) { | |
78 | prepend_bucket_marker(bucket, loc, locator); | |
79 | } else { | |
80 | locator.clear(); | |
81 | } | |
82 | } | |
83 | ||
b3b6e05e | 84 | int rgw_policy_from_attrset(const DoutPrefixProvider *dpp, CephContext *cct, map<string, bufferlist>& attrset, RGWAccessControlPolicy *policy); |
7c673cae | 85 | |
7c673cae FG |
86 | struct RGWOLHInfo { |
87 | rgw_obj target; | |
88 | bool removed; | |
89 | ||
90 | RGWOLHInfo() : removed(false) {} | |
91 | ||
92 | void encode(bufferlist& bl) const { | |
93 | ENCODE_START(1, 1, bl); | |
11fdf7f2 TL |
94 | encode(target, bl); |
95 | encode(removed, bl); | |
7c673cae FG |
96 | ENCODE_FINISH(bl); |
97 | } | |
98 | ||
11fdf7f2 | 99 | void decode(bufferlist::const_iterator& bl) { |
7c673cae | 100 | DECODE_START(1, bl); |
11fdf7f2 TL |
101 | decode(target, bl); |
102 | decode(removed, bl); | |
7c673cae FG |
103 | DECODE_FINISH(bl); |
104 | } | |
105 | static void generate_test_instances(list<RGWOLHInfo*>& o); | |
106 | void dump(Formatter *f) const; | |
107 | }; | |
108 | WRITE_CLASS_ENCODER(RGWOLHInfo) | |
109 | ||
110 | struct RGWOLHPendingInfo { | |
111 | ceph::real_time time; | |
112 | ||
113 | RGWOLHPendingInfo() {} | |
114 | ||
115 | void encode(bufferlist& bl) const { | |
116 | ENCODE_START(1, 1, bl); | |
11fdf7f2 | 117 | encode(time, bl); |
7c673cae FG |
118 | ENCODE_FINISH(bl); |
119 | } | |
120 | ||
11fdf7f2 | 121 | void decode(bufferlist::const_iterator& bl) { |
7c673cae | 122 | DECODE_START(1, bl); |
11fdf7f2 | 123 | decode(time, bl); |
7c673cae FG |
124 | DECODE_FINISH(bl); |
125 | } | |
126 | ||
127 | void dump(Formatter *f) const; | |
128 | }; | |
129 | WRITE_CLASS_ENCODER(RGWOLHPendingInfo) | |
130 | ||
131 | struct RGWUsageBatch { | |
132 | map<ceph::real_time, rgw_usage_log_entry> m; | |
133 | ||
134 | void insert(ceph::real_time& t, rgw_usage_log_entry& entry, bool *account) { | |
135 | bool exists = m.find(t) != m.end(); | |
136 | *account = !exists; | |
137 | m[t].aggregate(entry); | |
138 | } | |
139 | }; | |
140 | ||
7c673cae | 141 | class RGWGetDataCB { |
7c673cae FG |
142 | public: |
143 | virtual int handle_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) = 0; | |
11fdf7f2 | 144 | RGWGetDataCB() {} |
7c673cae | 145 | virtual ~RGWGetDataCB() {} |
7c673cae FG |
146 | }; |
147 | ||
148 | struct RGWCloneRangeInfo { | |
149 | rgw_obj src; | |
150 | off_t src_ofs; | |
151 | off_t dst_ofs; | |
152 | uint64_t len; | |
153 | }; | |
154 | ||
7c673cae FG |
155 | struct RGWObjState { |
156 | rgw_obj obj; | |
9f95a23c TL |
157 | bool is_atomic{false}; |
158 | bool has_attrs{false}; | |
159 | bool exists{false}; | |
160 | uint64_t size{0}; //< size of raw object | |
7c673cae FG |
161 | uint64_t accounted_size{0}; //< size before compression, encryption |
162 | ceph::real_time mtime; | |
9f95a23c | 163 | uint64_t epoch{0}; |
7c673cae | 164 | bufferlist obj_tag; |
181888fb | 165 | bufferlist tail_tag; |
7c673cae | 166 | string write_tag; |
9f95a23c TL |
167 | bool fake_tag{false}; |
168 | std::optional<RGWObjManifest> manifest; | |
7c673cae | 169 | string shadow_obj; |
9f95a23c | 170 | bool has_data{false}; |
7c673cae | 171 | bufferlist data; |
9f95a23c TL |
172 | bool prefetch_data{false}; |
173 | bool keep_tail{false}; | |
174 | bool is_olh{false}; | |
7c673cae | 175 | bufferlist olh_tag; |
9f95a23c TL |
176 | uint64_t pg_ver{false}; |
177 | uint32_t zone_short_id{0}; | |
7c673cae FG |
178 | |
179 | /* important! don't forget to update copy constructor */ | |
180 | ||
181 | RGWObjVersionTracker objv_tracker; | |
182 | ||
183 | map<string, bufferlist> attrset; | |
9f95a23c TL |
184 | |
185 | RGWObjState(); | |
186 | RGWObjState(const RGWObjState& rhs); | |
187 | ~RGWObjState(); | |
7c673cae FG |
188 | |
189 | bool get_attr(string name, bufferlist& dest) { | |
190 | map<string, bufferlist>::iterator iter = attrset.find(name); | |
191 | if (iter != attrset.end()) { | |
192 | dest = iter->second; | |
193 | return true; | |
194 | } | |
195 | return false; | |
196 | } | |
197 | }; | |
198 | ||
9f95a23c TL |
199 | class RGWFetchObjFilter { |
200 | public: | |
201 | virtual ~RGWFetchObjFilter() {} | |
202 | ||
203 | virtual int filter(CephContext *cct, | |
204 | const rgw_obj_key& source_key, | |
205 | const RGWBucketInfo& dest_bucket_info, | |
206 | std::optional<rgw_placement_rule> dest_placement_rule, | |
207 | const map<string, bufferlist>& obj_attrs, | |
208 | std::optional<rgw_user> *poverride_owner, | |
209 | const rgw_placement_rule **prule) = 0; | |
210 | }; | |
211 | ||
212 | class RGWFetchObjFilter_Default : public RGWFetchObjFilter { | |
213 | protected: | |
214 | rgw_placement_rule dest_rule; | |
215 | public: | |
216 | RGWFetchObjFilter_Default() {} | |
217 | ||
218 | int filter(CephContext *cct, | |
219 | const rgw_obj_key& source_key, | |
220 | const RGWBucketInfo& dest_bucket_info, | |
221 | std::optional<rgw_placement_rule> dest_placement_rule, | |
222 | const map<string, bufferlist>& obj_attrs, | |
223 | std::optional<rgw_user> *poverride_owner, | |
224 | const rgw_placement_rule **prule) override; | |
225 | }; | |
226 | ||
227 | class RGWObjectCtx { | |
228 | rgw::sal::RGWRadosStore *store; | |
229 | ceph::shared_mutex lock = ceph::make_shared_mutex("RGWObjectCtx"); | |
230 | void *s{nullptr}; | |
231 | ||
232 | std::map<rgw_obj, RGWObjState> objs_state; | |
233 | public: | |
234 | explicit RGWObjectCtx(rgw::sal::RGWRadosStore *_store) : store(_store) {} | |
235 | explicit RGWObjectCtx(rgw::sal::RGWRadosStore *_store, void *_s) : store(_store), s(_s) {} | |
236 | ||
237 | void *get_private() { | |
238 | return s; | |
239 | } | |
240 | ||
241 | rgw::sal::RGWRadosStore *get_store() { | |
242 | return store; | |
243 | } | |
244 | ||
245 | RGWObjState *get_state(const rgw_obj& obj); | |
246 | ||
247 | void set_atomic(rgw_obj& obj); | |
248 | void set_prefetch_data(const rgw_obj& obj); | |
249 | void invalidate(const rgw_obj& obj); | |
250 | }; | |
251 | ||
252 | ||
7c673cae FG |
253 | struct RGWRawObjState { |
254 | rgw_raw_obj obj; | |
255 | bool has_attrs{false}; | |
256 | bool exists{false}; | |
257 | uint64_t size{0}; | |
258 | ceph::real_time mtime; | |
11fdf7f2 | 259 | uint64_t epoch{0}; |
7c673cae FG |
260 | bufferlist obj_tag; |
261 | bool has_data{false}; | |
262 | bufferlist data; | |
263 | bool prefetch_data{false}; | |
264 | uint64_t pg_ver{0}; | |
265 | ||
266 | /* important! don't forget to update copy constructor */ | |
267 | ||
268 | RGWObjVersionTracker objv_tracker; | |
269 | ||
270 | map<string, bufferlist> attrset; | |
271 | RGWRawObjState() {} | |
272 | RGWRawObjState(const RGWRawObjState& rhs) : obj (rhs.obj) { | |
273 | has_attrs = rhs.has_attrs; | |
274 | exists = rhs.exists; | |
275 | size = rhs.size; | |
276 | mtime = rhs.mtime; | |
277 | epoch = rhs.epoch; | |
278 | if (rhs.obj_tag.length()) { | |
279 | obj_tag = rhs.obj_tag; | |
280 | } | |
281 | has_data = rhs.has_data; | |
282 | if (rhs.data.length()) { | |
283 | data = rhs.data; | |
284 | } | |
285 | prefetch_data = rhs.prefetch_data; | |
286 | pg_ver = rhs.pg_ver; | |
287 | objv_tracker = rhs.objv_tracker; | |
288 | } | |
289 | }; | |
290 | ||
291 | struct RGWPoolIterCtx { | |
292 | librados::IoCtx io_ctx; | |
293 | librados::NObjectIterator iter; | |
294 | }; | |
295 | ||
296 | struct RGWListRawObjsCtx { | |
297 | bool initialized; | |
298 | RGWPoolIterCtx iter_ctx; | |
299 | ||
300 | RGWListRawObjsCtx() : initialized(false) {} | |
301 | }; | |
302 | ||
7c673cae FG |
303 | struct objexp_hint_entry { |
304 | string tenant; | |
305 | string bucket_name; | |
306 | string bucket_id; | |
307 | rgw_obj_key obj_key; | |
308 | ceph::real_time exp_time; | |
309 | ||
310 | void encode(bufferlist& bl) const { | |
311 | ENCODE_START(2, 1, bl); | |
11fdf7f2 TL |
312 | encode(bucket_name, bl); |
313 | encode(bucket_id, bl); | |
314 | encode(obj_key, bl); | |
315 | encode(exp_time, bl); | |
316 | encode(tenant, bl); | |
7c673cae FG |
317 | ENCODE_FINISH(bl); |
318 | } | |
319 | ||
11fdf7f2 | 320 | void decode(bufferlist::const_iterator& bl) { |
7c673cae FG |
321 | // XXX Do we want DECODE_START_LEGACY_COMPAT_LEN(2, 1, 1, bl); ? |
322 | DECODE_START(2, bl); | |
11fdf7f2 TL |
323 | decode(bucket_name, bl); |
324 | decode(bucket_id, bl); | |
325 | decode(obj_key, bl); | |
326 | decode(exp_time, bl); | |
7c673cae | 327 | if (struct_v >= 2) { |
11fdf7f2 | 328 | decode(tenant, bl); |
7c673cae FG |
329 | } else { |
330 | tenant.clear(); | |
331 | } | |
332 | DECODE_FINISH(bl); | |
333 | } | |
9f95a23c TL |
334 | |
335 | void dump(Formatter *f) const; | |
336 | static void generate_test_instances(list<objexp_hint_entry*>& o); | |
7c673cae FG |
337 | }; |
338 | WRITE_CLASS_ENCODER(objexp_hint_entry) | |
339 | ||
7c673cae FG |
340 | class RGWDataChangesLog; |
341 | class RGWMetaSyncStatusManager; | |
342 | class RGWDataSyncStatusManager; | |
7c673cae | 343 | class RGWCoroutinesManagerRegistry; |
7c673cae FG |
344 | |
345 | class RGWGetBucketStats_CB : public RefCountedObject { | |
346 | protected: | |
347 | rgw_bucket bucket; | |
348 | map<RGWObjCategory, RGWStorageStats> *stats; | |
349 | public: | |
224ce89b | 350 | explicit RGWGetBucketStats_CB(const rgw_bucket& _bucket) : bucket(_bucket), stats(NULL) {} |
7c673cae FG |
351 | ~RGWGetBucketStats_CB() override {} |
352 | virtual void handle_response(int r) = 0; | |
353 | virtual void set_response(map<RGWObjCategory, RGWStorageStats> *_stats) { | |
354 | stats = _stats; | |
355 | } | |
356 | }; | |
357 | ||
358 | class RGWGetUserStats_CB : public RefCountedObject { | |
359 | protected: | |
360 | rgw_user user; | |
361 | RGWStorageStats stats; | |
362 | public: | |
363 | explicit RGWGetUserStats_CB(const rgw_user& _user) : user(_user) {} | |
364 | ~RGWGetUserStats_CB() override {} | |
365 | virtual void handle_response(int r) = 0; | |
366 | virtual void set_response(RGWStorageStats& _stats) { | |
367 | stats = _stats; | |
368 | } | |
369 | }; | |
370 | ||
371 | class RGWGetDirHeader_CB; | |
372 | class RGWGetUserHeader_CB; | |
f67539c2 TL |
373 | namespace rgw { namespace sal { |
374 | class RGWRadosStore; | |
375 | class MPRadosSerializer; | |
376 | class LCRadosSerializer; | |
377 | } } | |
7c673cae | 378 | |
7c673cae FG |
379 | class RGWAsyncRadosProcessor; |
380 | ||
381 | template <class T> | |
382 | class RGWChainedCacheImpl; | |
383 | ||
384 | struct bucket_info_entry { | |
385 | RGWBucketInfo info; | |
386 | real_time mtime; | |
387 | map<string, bufferlist> attrs; | |
388 | }; | |
389 | ||
9f95a23c | 390 | struct tombstone_entry; |
7c673cae | 391 | |
9f95a23c TL |
392 | template <class K, class V> |
393 | class lru_map; | |
394 | using tombstone_cache_t = lru_map<rgw_obj, tombstone_entry>; | |
7c673cae | 395 | |
31f18b77 FG |
396 | class RGWIndexCompletionManager; |
397 | ||
9f95a23c | 398 | class RGWRados |
7c673cae FG |
399 | { |
400 | friend class RGWGC; | |
401 | friend class RGWMetaNotifier; | |
402 | friend class RGWDataNotifier; | |
7c673cae FG |
403 | friend class RGWObjectExpirer; |
404 | friend class RGWMetaSyncProcessorThread; | |
405 | friend class RGWDataSyncProcessorThread; | |
31f18b77 FG |
406 | friend class RGWReshard; |
407 | friend class RGWBucketReshard; | |
f64942e4 | 408 | friend class RGWBucketReshardLock; |
31f18b77 | 409 | friend class BucketIndexLockGuard; |
f67539c2 TL |
410 | friend class rgw::sal::MPRadosSerializer; |
411 | friend class rgw::sal::LCRadosSerializer; | |
412 | friend class rgw::sal::RGWRadosStore; | |
7c673cae FG |
413 | |
414 | /** Open the pool used as root for this gateway */ | |
b3b6e05e TL |
415 | int open_root_pool_ctx(const DoutPrefixProvider *dpp); |
416 | int open_gc_pool_ctx(const DoutPrefixProvider *dpp); | |
417 | int open_lc_pool_ctx(const DoutPrefixProvider *dpp); | |
418 | int open_objexp_pool_ctx(const DoutPrefixProvider *dpp); | |
419 | int open_reshard_pool_ctx(const DoutPrefixProvider *dpp); | |
420 | int open_notif_pool_ctx(const DoutPrefixProvider *dpp); | |
421 | ||
422 | int open_pool_ctx(const DoutPrefixProvider *dpp, const rgw_pool& pool, librados::IoCtx& io_ctx, | |
494da23a | 423 | bool mostly_omap); |
7c673cae FG |
424 | |
425 | std::atomic<int64_t> max_req_id = { 0 }; | |
9f95a23c | 426 | ceph::mutex lock = ceph::make_mutex("rados_timer_lock"); |
7c673cae FG |
427 | SafeTimer *timer; |
428 | ||
9f95a23c | 429 | rgw::sal::RGWRadosStore *store; |
522d829b | 430 | RGWGC *gc = nullptr; |
7c673cae FG |
431 | RGWLC *lc; |
432 | RGWObjectExpirer *obj_expirer; | |
433 | bool use_gc_thread; | |
434 | bool use_lc_thread; | |
435 | bool quota_threads; | |
436 | bool run_sync_thread; | |
31f18b77 | 437 | bool run_reshard_thread; |
7c673cae | 438 | |
7c673cae FG |
439 | RGWMetaNotifier *meta_notifier; |
440 | RGWDataNotifier *data_notifier; | |
441 | RGWMetaSyncProcessorThread *meta_sync_processor_thread; | |
11fdf7f2 | 442 | RGWSyncTraceManager *sync_tracer = nullptr; |
9f95a23c | 443 | map<rgw_zone_id, RGWDataSyncProcessorThread *> data_sync_processor_threads; |
7c673cae | 444 | |
b32b8144 | 445 | boost::optional<rgw::BucketTrimManager> bucket_trim; |
7c673cae FG |
446 | RGWSyncLogTrimThread *sync_log_trimmer{nullptr}; |
447 | ||
9f95a23c TL |
448 | ceph::mutex meta_sync_thread_lock = ceph::make_mutex("meta_sync_thread_lock"); |
449 | ceph::mutex data_sync_thread_lock = ceph::make_mutex("data_sync_thread_lock"); | |
7c673cae | 450 | |
7c673cae | 451 | librados::IoCtx root_pool_ctx; // .rgw |
11fdf7f2 TL |
452 | |
453 | double inject_notify_timeout_probability = 0; | |
454 | unsigned max_notify_retries = 0; | |
7c673cae FG |
455 | |
456 | friend class RGWWatcher; | |
457 | ||
9f95a23c | 458 | ceph::mutex bucket_id_lock = ceph::make_mutex("rados_bucket_id"); |
7c673cae FG |
459 | |
460 | // This field represents the number of bucket index object shards | |
461 | uint32_t bucket_index_max_shards; | |
462 | ||
b3b6e05e TL |
463 | int get_obj_head_ref(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_rados_ref *ref); |
464 | int get_system_obj_ref(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, rgw_rados_ref *ref); | |
7c673cae FG |
465 | uint64_t max_bucket_id; |
466 | ||
b3b6e05e | 467 | int get_olh_target_state(const DoutPrefixProvider *dpp, RGWObjectCtx& rctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, |
9f95a23c | 468 | RGWObjState *olh_state, RGWObjState **target_state, optional_yield y); |
b3b6e05e | 469 | int get_obj_state_impl(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWObjState **state, |
9f95a23c | 470 | bool follow_olh, optional_yield y, bool assume_noent = false); |
b3b6e05e | 471 | int append_atomic_test(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, |
9f95a23c | 472 | librados::ObjectOperation& op, RGWObjState **state, optional_yield y); |
b3b6e05e | 473 | int append_atomic_test(const DoutPrefixProvider *dpp, const RGWObjState* astate, librados::ObjectOperation& op); |
7c673cae FG |
474 | |
475 | int update_placement_map(); | |
476 | int store_bucket_info(RGWBucketInfo& info, map<string, bufferlist> *pattrs, RGWObjVersionTracker *objv_tracker, bool exclusive); | |
477 | ||
478 | void remove_rgw_head_obj(librados::ObjectWriteOperation& op); | |
479 | void cls_obj_check_prefix_exist(librados::ObjectOperation& op, const string& prefix, bool fail_if_exist); | |
480 | void cls_obj_check_mtime(librados::ObjectOperation& op, const real_time& mtime, bool high_precision_time, RGWCheckMTimeType type); | |
481 | protected: | |
482 | CephContext *cct; | |
483 | ||
494da23a | 484 | librados::Rados rados; |
7c673cae FG |
485 | |
486 | using RGWChainedCacheImpl_bucket_info_entry = RGWChainedCacheImpl<bucket_info_entry>; | |
487 | RGWChainedCacheImpl_bucket_info_entry *binfo_cache; | |
488 | ||
7c673cae FG |
489 | tombstone_cache_t *obj_tombstone_cache; |
490 | ||
491 | librados::IoCtx gc_pool_ctx; // .rgw.gc | |
492 | librados::IoCtx lc_pool_ctx; // .rgw.lc | |
493 | librados::IoCtx objexp_pool_ctx; | |
31f18b77 | 494 | librados::IoCtx reshard_pool_ctx; |
f67539c2 | 495 | librados::IoCtx notif_pool_ctx; // .rgw.notif |
7c673cae | 496 | |
11fdf7f2 | 497 | bool pools_initialized; |
7c673cae | 498 | |
11fdf7f2 | 499 | RGWQuotaHandler *quota_handler; |
7c673cae | 500 | |
11fdf7f2 | 501 | RGWCoroutinesManagerRegistry *cr_registry; |
7c673cae | 502 | |
11fdf7f2 TL |
503 | RGWSyncModuleInstanceRef sync_module; |
504 | bool writeable_zone{false}; | |
7c673cae | 505 | |
11fdf7f2 | 506 | RGWIndexCompletionManager *index_completion_manager{nullptr}; |
7c673cae | 507 | |
11fdf7f2 | 508 | bool use_cache{false}; |
522d829b | 509 | bool use_gc{true}; |
f67539c2 | 510 | |
b3b6e05e | 511 | int get_obj_head_ioctx(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, librados::IoCtx *ioctx); |
11fdf7f2 | 512 | public: |
9f95a23c | 513 | RGWRados(): timer(NULL), |
11fdf7f2 | 514 | gc(NULL), lc(NULL), obj_expirer(NULL), use_gc_thread(false), use_lc_thread(false), quota_threads(false), |
9f95a23c | 515 | run_sync_thread(false), run_reshard_thread(false), meta_notifier(NULL), |
11fdf7f2 | 516 | data_notifier(NULL), meta_sync_processor_thread(NULL), |
11fdf7f2 TL |
517 | bucket_index_max_shards(0), |
518 | max_bucket_id(0), cct(NULL), | |
11fdf7f2 TL |
519 | binfo_cache(NULL), obj_tombstone_cache(nullptr), |
520 | pools_initialized(false), | |
521 | quota_handler(NULL), | |
522 | cr_registry(NULL), | |
9f95a23c TL |
523 | pctl(&ctl), |
524 | reshard(NULL) {} | |
7c673cae | 525 | |
11fdf7f2 TL |
526 | RGWRados& set_use_cache(bool status) { |
527 | use_cache = status; | |
528 | return *this; | |
7c673cae FG |
529 | } |
530 | ||
522d829b TL |
531 | RGWRados& set_use_gc(bool status) { |
532 | use_gc = status; | |
533 | return *this; | |
534 | } | |
535 | ||
11fdf7f2 TL |
536 | RGWLC *get_lc() { |
537 | return lc; | |
7c673cae FG |
538 | } |
539 | ||
11fdf7f2 TL |
540 | RGWRados& set_run_gc_thread(bool _use_gc_thread) { |
541 | use_gc_thread = _use_gc_thread; | |
542 | return *this; | |
7c673cae FG |
543 | } |
544 | ||
11fdf7f2 TL |
545 | RGWRados& set_run_lc_thread(bool _use_lc_thread) { |
546 | use_lc_thread = _use_lc_thread; | |
547 | return *this; | |
7c673cae FG |
548 | } |
549 | ||
11fdf7f2 TL |
550 | RGWRados& set_run_quota_threads(bool _run_quota_threads) { |
551 | quota_threads = _run_quota_threads; | |
552 | return *this; | |
7c673cae FG |
553 | } |
554 | ||
11fdf7f2 TL |
555 | RGWRados& set_run_sync_thread(bool _run_sync_thread) { |
556 | run_sync_thread = _run_sync_thread; | |
557 | return *this; | |
7c673cae FG |
558 | } |
559 | ||
11fdf7f2 TL |
560 | RGWRados& set_run_reshard_thread(bool _run_reshard_thread) { |
561 | run_reshard_thread = _run_reshard_thread; | |
562 | return *this; | |
7c673cae FG |
563 | } |
564 | ||
11fdf7f2 TL |
565 | uint64_t get_new_req_id() { |
566 | return ++max_req_id; | |
7c673cae FG |
567 | } |
568 | ||
11fdf7f2 TL |
569 | librados::IoCtx* get_lc_pool_ctx() { |
570 | return &lc_pool_ctx; | |
7c673cae | 571 | } |
f67539c2 TL |
572 | |
573 | librados::IoCtx& get_notif_pool_ctx() { | |
574 | return notif_pool_ctx; | |
575 | } | |
576 | ||
11fdf7f2 TL |
577 | void set_context(CephContext *_cct) { |
578 | cct = _cct; | |
7c673cae | 579 | } |
9f95a23c TL |
580 | void set_store(rgw::sal::RGWRadosStore *_store) { |
581 | store = _store; | |
582 | } | |
31f18b77 | 583 | |
11fdf7f2 | 584 | RGWServices svc; |
9f95a23c TL |
585 | RGWCtl ctl; |
586 | ||
587 | RGWCtl *pctl{nullptr}; | |
11fdf7f2 TL |
588 | |
589 | /** | |
590 | * AmazonS3 errors contain a HostId string, but is an opaque base64 blob; we | |
591 | * try to be more transparent. This has a wrapper so we can update it when zonegroup/zone are changed. | |
592 | */ | |
593 | string host_id; | |
31f18b77 | 594 | |
31f18b77 FG |
595 | RGWReshard *reshard; |
596 | std::shared_ptr<RGWReshardWait> reshard_wait; | |
597 | ||
7c673cae FG |
598 | virtual ~RGWRados() = default; |
599 | ||
600 | tombstone_cache_t *get_tombstone_cache() { | |
601 | return obj_tombstone_cache; | |
602 | } | |
7c673cae FG |
603 | const RGWSyncModuleInstanceRef& get_sync_module() { |
604 | return sync_module; | |
605 | } | |
11fdf7f2 TL |
606 | RGWSyncTraceManager *get_sync_tracer() { |
607 | return sync_tracer; | |
608 | } | |
7c673cae | 609 | |
b3b6e05e | 610 | int get_required_alignment(const DoutPrefixProvider *dpp, const rgw_pool& pool, uint64_t *alignment); |
11fdf7f2 | 611 | void get_max_aligned_size(uint64_t size, uint64_t alignment, uint64_t *max_size); |
b3b6e05e TL |
612 | int get_max_chunk_size(const rgw_pool& pool, uint64_t *max_chunk_size, const DoutPrefixProvider *dpp, uint64_t *palignment = nullptr); |
613 | int get_max_chunk_size(const rgw_placement_rule& placement_rule, const rgw_obj& obj, uint64_t *max_chunk_size, const DoutPrefixProvider *dpp, uint64_t *palignment = nullptr); | |
7c673cae FG |
614 | |
615 | uint32_t get_max_bucket_shards() { | |
9f95a23c | 616 | return RGWSI_BucketIndex_RADOS::shards_max(); |
7c673cae FG |
617 | } |
618 | ||
181888fb | 619 | |
b3b6e05e | 620 | int get_raw_obj_ref(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, rgw_rados_ref *ref); |
7c673cae | 621 | |
b3b6e05e TL |
622 | int list_raw_objects_init(const DoutPrefixProvider *dpp, const rgw_pool& pool, const string& marker, RGWListRawObjsCtx *ctx); |
623 | int list_raw_objects_next(const DoutPrefixProvider *dpp, const string& prefix_filter, int max, | |
181888fb FG |
624 | RGWListRawObjsCtx& ctx, list<string>& oids, |
625 | bool *is_truncated); | |
b3b6e05e | 626 | int list_raw_objects(const DoutPrefixProvider *dpp, const rgw_pool& pool, const string& prefix_filter, int max, |
7c673cae FG |
627 | RGWListRawObjsCtx& ctx, list<string>& oids, |
628 | bool *is_truncated); | |
181888fb | 629 | string list_raw_objs_get_cursor(RGWListRawObjsCtx& ctx); |
7c673cae | 630 | |
7c673cae FG |
631 | CephContext *ctx() { return cct; } |
632 | /** do all necessary setup of the storage device */ | |
b3b6e05e | 633 | int initialize(CephContext *_cct, const DoutPrefixProvider *dpp) { |
7c673cae | 634 | set_context(_cct); |
b3b6e05e | 635 | return initialize(dpp); |
7c673cae FG |
636 | } |
637 | /** Initialize the RADOS instance and prepare to do other ops */ | |
b3b6e05e TL |
638 | int init_svc(bool raw, const DoutPrefixProvider *dpp); |
639 | int init_ctl(const DoutPrefixProvider *dpp); | |
11fdf7f2 | 640 | int init_rados(); |
b3b6e05e TL |
641 | int init_complete(const DoutPrefixProvider *dpp); |
642 | int initialize(const DoutPrefixProvider *dpp); | |
7c673cae FG |
643 | void finalize(); |
644 | ||
224ce89b | 645 | int register_to_service_map(const string& daemon_type, const map<string, string>& meta); |
11fdf7f2 | 646 | int update_service_map(std::map<std::string, std::string>&& status); |
7c673cae FG |
647 | |
648 | /// list logs | |
b3b6e05e | 649 | int log_list_init(const DoutPrefixProvider *dpp, const string& prefix, RGWAccessHandle *handle); |
7c673cae FG |
650 | int log_list_next(RGWAccessHandle handle, string *name); |
651 | ||
652 | /// remove log | |
b3b6e05e | 653 | int log_remove(const DoutPrefixProvider *dpp, const string& name); |
7c673cae FG |
654 | |
655 | /// show log | |
b3b6e05e | 656 | int log_show_init(const DoutPrefixProvider *dpp, const string& name, RGWAccessHandle *handle); |
7c673cae FG |
657 | int log_show_next(RGWAccessHandle handle, rgw_log_entry *entry); |
658 | ||
659 | // log bandwidth info | |
b3b6e05e TL |
660 | int log_usage(const DoutPrefixProvider *dpp, map<rgw_user_bucket, RGWUsageBatch>& usage_info); |
661 | int read_usage(const DoutPrefixProvider *dpp, const rgw_user& user, const string& bucket_name, uint64_t start_epoch, uint64_t end_epoch, | |
11fdf7f2 TL |
662 | uint32_t max_entries, bool *is_truncated, RGWUsageIter& read_iter, map<rgw_user_bucket, |
663 | rgw_usage_log_entry>& usage); | |
b3b6e05e TL |
664 | int trim_usage(const DoutPrefixProvider *dpp, const rgw_user& user, const string& bucket_name, uint64_t start_epoch, uint64_t end_epoch); |
665 | int clear_usage(const DoutPrefixProvider *dpp); | |
7c673cae | 666 | |
b3b6e05e | 667 | int create_pool(const DoutPrefixProvider *dpp, const rgw_pool& pool); |
7c673cae | 668 | |
7c673cae FG |
669 | void create_bucket_id(string *bucket_id); |
670 | ||
11fdf7f2 TL |
671 | bool get_obj_data_pool(const rgw_placement_rule& placement_rule, const rgw_obj& obj, rgw_pool *pool); |
672 | bool obj_to_raw(const rgw_placement_rule& placement_rule, const rgw_obj& obj, rgw_raw_obj *raw_obj); | |
7c673cae | 673 | |
11fdf7f2 | 674 | int create_bucket(const RGWUserInfo& owner, rgw_bucket& bucket, |
f67539c2 TL |
675 | const string& zonegroup_id, |
676 | const rgw_placement_rule& placement_rule, | |
677 | const string& swift_ver_location, | |
678 | const RGWQuotaInfo * pquota_info, | |
679 | map<std::string,bufferlist>& attrs, | |
680 | RGWBucketInfo& bucket_info, | |
681 | obj_version *pobjv, | |
682 | obj_version *pep_objv, | |
683 | ceph::real_time creation_time, | |
684 | rgw_bucket *master_bucket, | |
685 | uint32_t *master_num_shards, | |
686 | optional_yield y, | |
b3b6e05e | 687 | const DoutPrefixProvider *dpp, |
f67539c2 | 688 | bool exclusive = true); |
7c673cae FG |
689 | |
690 | RGWCoroutinesManagerRegistry *get_cr_registry() { return cr_registry; } | |
691 | ||
7c673cae FG |
692 | struct BucketShard { |
693 | RGWRados *store; | |
694 | rgw_bucket bucket; | |
695 | int shard_id; | |
9f95a23c | 696 | RGWSI_RADOS::Obj bucket_obj; |
7c673cae FG |
697 | |
698 | explicit BucketShard(RGWRados *_store) : store(_store), shard_id(-1) {} | |
b3b6e05e TL |
699 | int init(const rgw_bucket& _bucket, const rgw_obj& obj, RGWBucketInfo* out, const DoutPrefixProvider *dpp); |
700 | int init(const rgw_bucket& _bucket, int sid, const rgw::bucket_index_layout_generation& idx_layout, RGWBucketInfo* out, const DoutPrefixProvider *dpp); | |
701 | int init(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj); | |
702 | int init(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout, int sid); | |
7c673cae FG |
703 | }; |
704 | ||
705 | class Object { | |
706 | RGWRados *store; | |
707 | RGWBucketInfo bucket_info; | |
708 | RGWObjectCtx& ctx; | |
709 | rgw_obj obj; | |
710 | ||
711 | BucketShard bs; | |
712 | ||
713 | RGWObjState *state; | |
714 | ||
715 | bool versioning_disabled; | |
716 | ||
717 | bool bs_initialized; | |
718 | ||
719 | protected: | |
b3b6e05e | 720 | int get_state(const DoutPrefixProvider *dpp, RGWObjState **pstate, bool follow_olh, optional_yield y, bool assume_noent = false); |
7c673cae FG |
721 | void invalidate_state(); |
722 | ||
b3b6e05e | 723 | int prepare_atomic_modification(const DoutPrefixProvider *dpp, librados::ObjectWriteOperation& op, bool reset_obj, const string *ptag, |
9f95a23c | 724 | const char *ifmatch, const char *ifnomatch, bool removal_op, bool modify_tail, optional_yield y); |
b3b6e05e | 725 | int complete_atomic_modification(const DoutPrefixProvider *dpp); |
7c673cae FG |
726 | |
727 | public: | |
728 | Object(RGWRados *_store, const RGWBucketInfo& _bucket_info, RGWObjectCtx& _ctx, const rgw_obj& _obj) : store(_store), bucket_info(_bucket_info), | |
729 | ctx(_ctx), obj(_obj), bs(store), | |
730 | state(NULL), versioning_disabled(false), | |
731 | bs_initialized(false) {} | |
732 | ||
733 | RGWRados *get_store() { return store; } | |
734 | rgw_obj& get_obj() { return obj; } | |
735 | RGWObjectCtx& get_ctx() { return ctx; } | |
736 | RGWBucketInfo& get_bucket_info() { return bucket_info; } | |
b3b6e05e | 737 | int get_manifest(const DoutPrefixProvider *dpp, RGWObjManifest **pmanifest, optional_yield y); |
7c673cae | 738 | |
b3b6e05e | 739 | int get_bucket_shard(BucketShard **pbs, const DoutPrefixProvider *dpp) { |
7c673cae | 740 | if (!bs_initialized) { |
f64942e4 | 741 | int r = |
b3b6e05e | 742 | bs.init(bucket_info.bucket, obj, nullptr /* no RGWBucketInfo */, dpp); |
7c673cae FG |
743 | if (r < 0) { |
744 | return r; | |
745 | } | |
746 | bs_initialized = true; | |
747 | } | |
748 | *pbs = &bs; | |
749 | return 0; | |
750 | } | |
751 | ||
752 | void set_versioning_disabled(bool status) { | |
753 | versioning_disabled = status; | |
754 | } | |
755 | ||
756 | bool versioning_enabled() { | |
757 | return (!versioning_disabled && bucket_info.versioning_enabled()); | |
758 | } | |
759 | ||
760 | struct Read { | |
761 | RGWRados::Object *source; | |
762 | ||
763 | struct GetObjState { | |
11fdf7f2 TL |
764 | map<rgw_pool, librados::IoCtx> io_ctxs; |
765 | rgw_pool cur_pool; | |
766 | librados::IoCtx *cur_ioctx{nullptr}; | |
7c673cae FG |
767 | rgw_obj obj; |
768 | rgw_raw_obj head_obj; | |
769 | } state; | |
770 | ||
771 | struct ConditionParams { | |
772 | const ceph::real_time *mod_ptr; | |
773 | const ceph::real_time *unmod_ptr; | |
774 | bool high_precision_time; | |
775 | uint32_t mod_zone_id; | |
776 | uint64_t mod_pg_ver; | |
777 | const char *if_match; | |
778 | const char *if_nomatch; | |
779 | ||
780 | ConditionParams() : | |
781 | mod_ptr(NULL), unmod_ptr(NULL), high_precision_time(false), mod_zone_id(0), mod_pg_ver(0), | |
782 | if_match(NULL), if_nomatch(NULL) {} | |
783 | } conds; | |
784 | ||
785 | struct Params { | |
786 | ceph::real_time *lastmod; | |
787 | uint64_t *obj_size; | |
788 | map<string, bufferlist> *attrs; | |
eafe8130 | 789 | rgw_obj *target_obj; |
7c673cae | 790 | |
eafe8130 TL |
791 | Params() : lastmod(nullptr), obj_size(nullptr), attrs(nullptr), |
792 | target_obj(nullptr) {} | |
7c673cae FG |
793 | } params; |
794 | ||
795 | explicit Read(RGWRados::Object *_source) : source(_source) {} | |
796 | ||
b3b6e05e | 797 | int prepare(optional_yield y, const DoutPrefixProvider *dpp); |
7c673cae | 798 | static int range_to_ofs(uint64_t obj_size, int64_t &ofs, int64_t &end); |
b3b6e05e TL |
799 | int read(int64_t ofs, int64_t end, bufferlist& bl, optional_yield y, const DoutPrefixProvider *dpp); |
800 | int iterate(const DoutPrefixProvider *dpp, int64_t ofs, int64_t end, RGWGetDataCB *cb, optional_yield y); | |
801 | int get_attr(const DoutPrefixProvider *dpp, const char *name, bufferlist& dest, optional_yield y); | |
7c673cae FG |
802 | }; |
803 | ||
804 | struct Write { | |
805 | RGWRados::Object *target; | |
806 | ||
807 | struct MetaParams { | |
808 | ceph::real_time *mtime; | |
809 | map<std::string, bufferlist>* rmattrs; | |
810 | const bufferlist *data; | |
811 | RGWObjManifest *manifest; | |
812 | const string *ptag; | |
813 | list<rgw_obj_index_key> *remove_objs; | |
814 | ceph::real_time set_mtime; | |
815 | rgw_user owner; | |
816 | RGWObjCategory category; | |
817 | int flags; | |
818 | const char *if_match; | |
819 | const char *if_nomatch; | |
11fdf7f2 | 820 | std::optional<uint64_t> olh_epoch; |
7c673cae FG |
821 | ceph::real_time delete_at; |
822 | bool canceled; | |
823 | const string *user_data; | |
31f18b77 | 824 | rgw_zone_set *zones_trace; |
181888fb | 825 | bool modify_tail; |
3efd9988 | 826 | bool completeMultipart; |
11fdf7f2 | 827 | bool appendable; |
7c673cae FG |
828 | |
829 | MetaParams() : mtime(NULL), rmattrs(NULL), data(NULL), manifest(NULL), ptag(NULL), | |
11fdf7f2 | 830 | remove_objs(NULL), category(RGWObjCategory::Main), flags(0), |
91327a77 | 831 | if_match(NULL), if_nomatch(NULL), canceled(false), user_data(nullptr), zones_trace(nullptr), |
11fdf7f2 | 832 | modify_tail(false), completeMultipart(false), appendable(false) {} |
7c673cae FG |
833 | } meta; |
834 | ||
835 | explicit Write(RGWRados::Object *_target) : target(_target) {} | |
836 | ||
b3b6e05e TL |
837 | int _do_write_meta(const DoutPrefixProvider *dpp, |
838 | uint64_t size, uint64_t accounted_size, | |
7c673cae | 839 | map<std::string, bufferlist>& attrs, |
181888fb | 840 | bool modify_tail, bool assume_noent, |
9f95a23c | 841 | void *index_op, optional_yield y); |
b3b6e05e | 842 | int write_meta(const DoutPrefixProvider *dpp, uint64_t size, uint64_t accounted_size, |
9f95a23c | 843 | map<std::string, bufferlist>& attrs, optional_yield y); |
7c673cae | 844 | int write_data(const char *data, uint64_t ofs, uint64_t len, bool exclusive); |
11fdf7f2 TL |
845 | const req_state* get_req_state() { |
846 | return (req_state *)target->get_ctx().get_private(); | |
847 | } | |
7c673cae FG |
848 | }; |
849 | ||
850 | struct Delete { | |
851 | RGWRados::Object *target; | |
852 | ||
853 | struct DeleteParams { | |
854 | rgw_user bucket_owner; | |
b3b6e05e TL |
855 | int versioning_status; // versioning flags in enum RGWBucketFlags |
856 | ACLOwner obj_owner; // needed for creation of deletion marker | |
7c673cae FG |
857 | uint64_t olh_epoch; |
858 | string marker_version_id; | |
859 | uint32_t bilog_flags; | |
860 | list<rgw_obj_index_key> *remove_objs; | |
861 | ceph::real_time expiration_time; | |
862 | ceph::real_time unmod_since; | |
863 | ceph::real_time mtime; /* for setting delete marker mtime */ | |
864 | bool high_precision_time; | |
31f18b77 | 865 | rgw_zone_set *zones_trace; |
9f95a23c TL |
866 | bool abortmp; |
867 | uint64_t parts_accounted_size; | |
7c673cae | 868 | |
9f95a23c | 869 | DeleteParams() : versioning_status(0), olh_epoch(0), bilog_flags(0), remove_objs(NULL), high_precision_time(false), zones_trace(nullptr), abortmp(false), parts_accounted_size(0) {} |
7c673cae FG |
870 | } params; |
871 | ||
872 | struct DeleteResult { | |
873 | bool delete_marker; | |
874 | string version_id; | |
875 | ||
876 | DeleteResult() : delete_marker(false) {} | |
877 | } result; | |
878 | ||
879 | explicit Delete(RGWRados::Object *_target) : target(_target) {} | |
880 | ||
b3b6e05e | 881 | int delete_obj(optional_yield y, const DoutPrefixProvider *dpp); |
7c673cae FG |
882 | }; |
883 | ||
884 | struct Stat { | |
885 | RGWRados::Object *source; | |
886 | ||
887 | struct Result { | |
888 | rgw_obj obj; | |
9f95a23c TL |
889 | std::optional<RGWObjManifest> manifest; |
890 | uint64_t size{0}; | |
891 | struct timespec mtime {}; | |
7c673cae | 892 | map<string, bufferlist> attrs; |
7c673cae FG |
893 | } result; |
894 | ||
895 | struct State { | |
896 | librados::IoCtx io_ctx; | |
897 | librados::AioCompletion *completion; | |
898 | int ret; | |
899 | ||
900 | State() : completion(NULL), ret(0) {} | |
901 | } state; | |
902 | ||
903 | ||
904 | explicit Stat(RGWRados::Object *_source) : source(_source) {} | |
905 | ||
b3b6e05e | 906 | int stat_async(const DoutPrefixProvider *dpp); |
7c673cae FG |
907 | int wait(); |
908 | int stat(); | |
909 | private: | |
910 | int finish(); | |
911 | }; | |
912 | }; | |
913 | ||
914 | class Bucket { | |
915 | RGWRados *store; | |
916 | RGWBucketInfo bucket_info; | |
917 | rgw_bucket& bucket; | |
918 | int shard_id; | |
919 | ||
920 | public: | |
921 | Bucket(RGWRados *_store, const RGWBucketInfo& _bucket_info) : store(_store), bucket_info(_bucket_info), bucket(bucket_info.bucket), | |
922 | shard_id(RGW_NO_SHARD) {} | |
923 | RGWRados *get_store() { return store; } | |
924 | rgw_bucket& get_bucket() { return bucket; } | |
925 | RGWBucketInfo& get_bucket_info() { return bucket_info; } | |
926 | ||
b3b6e05e | 927 | int update_bucket_id(const string& new_bucket_id, const DoutPrefixProvider *dpp); |
31f18b77 | 928 | |
7c673cae FG |
929 | int get_shard_id() { return shard_id; } |
930 | void set_shard_id(int id) { | |
931 | shard_id = id; | |
932 | } | |
933 | ||
934 | class UpdateIndex { | |
935 | RGWRados::Bucket *target; | |
936 | string optag; | |
937 | rgw_obj obj; | |
938 | uint16_t bilog_flags{0}; | |
939 | BucketShard bs; | |
940 | bool bs_initialized{false}; | |
941 | bool blind; | |
942 | bool prepared{false}; | |
31f18b77 FG |
943 | rgw_zone_set *zones_trace{nullptr}; |
944 | ||
b3b6e05e | 945 | int init_bs(const DoutPrefixProvider *dpp) { |
f64942e4 | 946 | int r = |
b3b6e05e | 947 | bs.init(target->get_bucket(), obj, nullptr /* no RGWBucketInfo */, dpp); |
31f18b77 FG |
948 | if (r < 0) { |
949 | return r; | |
950 | } | |
951 | bs_initialized = true; | |
952 | return 0; | |
953 | } | |
954 | ||
955 | void invalidate_bs() { | |
956 | bs_initialized = false; | |
957 | } | |
958 | ||
b3b6e05e | 959 | int guard_reshard(const DoutPrefixProvider *dpp, BucketShard **pbs, std::function<int(BucketShard *)> call); |
7c673cae FG |
960 | public: |
961 | ||
962 | UpdateIndex(RGWRados::Bucket *_target, const rgw_obj& _obj) : target(_target), obj(_obj), | |
963 | bs(target->get_store()) { | |
f67539c2 | 964 | blind = (target->get_bucket_info().layout.current_index.layout.type == rgw::BucketIndexType::Indexless); |
7c673cae FG |
965 | } |
966 | ||
b3b6e05e | 967 | int get_bucket_shard(BucketShard **pbs, const DoutPrefixProvider *dpp) { |
7c673cae | 968 | if (!bs_initialized) { |
b3b6e05e | 969 | int r = init_bs(dpp); |
7c673cae FG |
970 | if (r < 0) { |
971 | return r; | |
972 | } | |
7c673cae FG |
973 | } |
974 | *pbs = &bs; | |
975 | return 0; | |
976 | } | |
977 | ||
978 | void set_bilog_flags(uint16_t flags) { | |
979 | bilog_flags = flags; | |
980 | } | |
31f18b77 FG |
981 | |
982 | void set_zones_trace(rgw_zone_set *_zones_trace) { | |
983 | zones_trace = _zones_trace; | |
984 | } | |
7c673cae | 985 | |
b3b6e05e TL |
986 | int prepare(const DoutPrefixProvider *dpp, RGWModifyOp, const string *write_tag, optional_yield y); |
987 | int complete(const DoutPrefixProvider *dpp, int64_t poolid, uint64_t epoch, uint64_t size, | |
7c673cae FG |
988 | uint64_t accounted_size, ceph::real_time& ut, |
989 | const string& etag, const string& content_type, | |
11fdf7f2 | 990 | const string& storage_class, |
7c673cae | 991 | bufferlist *acl_bl, RGWObjCategory category, |
11fdf7f2 | 992 | list<rgw_obj_index_key> *remove_objs, const string *user_data = nullptr, bool appendable = false); |
b3b6e05e TL |
993 | int complete_del(const DoutPrefixProvider *dpp, |
994 | int64_t poolid, uint64_t epoch, | |
7c673cae FG |
995 | ceph::real_time& removed_mtime, /* mtime of removed object */ |
996 | list<rgw_obj_index_key> *remove_objs); | |
b3b6e05e | 997 | int cancel(const DoutPrefixProvider *dpp); |
7c673cae FG |
998 | |
999 | const string *get_optag() { return &optag; } | |
1000 | ||
1001 | bool is_prepared() { return prepared; } | |
1adf2230 AA |
1002 | }; // class UpdateIndex |
1003 | ||
1004 | class List { | |
1005 | protected: | |
eafe8130 TL |
1006 | // absolute maximum number of objects that |
1007 | // list_objects_(un)ordered can return | |
1008 | static constexpr int64_t bucket_list_objects_absolute_max = 25000; | |
7c673cae | 1009 | |
7c673cae FG |
1010 | RGWRados::Bucket *target; |
1011 | rgw_obj_key next_marker; | |
1012 | ||
b3b6e05e TL |
1013 | int list_objects_ordered(const DoutPrefixProvider *dpp, |
1014 | int64_t max, | |
1adf2230 AA |
1015 | vector<rgw_bucket_dir_entry> *result, |
1016 | map<string, bool> *common_prefixes, | |
9f95a23c TL |
1017 | bool *is_truncated, |
1018 | optional_yield y); | |
b3b6e05e TL |
1019 | int list_objects_unordered(const DoutPrefixProvider *dpp, |
1020 | int64_t max, | |
1adf2230 AA |
1021 | vector<rgw_bucket_dir_entry> *result, |
1022 | map<string, bool> *common_prefixes, | |
9f95a23c TL |
1023 | bool *is_truncated, |
1024 | optional_yield y); | |
1adf2230 AA |
1025 | |
1026 | public: | |
1027 | ||
7c673cae FG |
1028 | struct Params { |
1029 | string prefix; | |
1030 | string delim; | |
1031 | rgw_obj_key marker; | |
1032 | rgw_obj_key end_marker; | |
1033 | string ns; | |
1034 | bool enforce_ns; | |
1035 | RGWAccessListFilter *filter; | |
1036 | bool list_versions; | |
1adf2230 AA |
1037 | bool allow_unordered; |
1038 | ||
1039 | Params() : | |
1040 | enforce_ns(true), | |
1041 | filter(NULL), | |
1042 | list_versions(false), | |
1043 | allow_unordered(false) | |
1044 | {} | |
7c673cae FG |
1045 | } params; |
1046 | ||
7c673cae FG |
1047 | explicit List(RGWRados::Bucket *_target) : target(_target) {} |
1048 | ||
b3b6e05e | 1049 | int list_objects(const DoutPrefixProvider *dpp, int64_t max, |
1adf2230 AA |
1050 | vector<rgw_bucket_dir_entry> *result, |
1051 | map<string, bool> *common_prefixes, | |
9f95a23c TL |
1052 | bool *is_truncated, |
1053 | optional_yield y) { | |
1adf2230 | 1054 | if (params.allow_unordered) { |
b3b6e05e | 1055 | return list_objects_unordered(dpp, max, result, common_prefixes, |
9f95a23c | 1056 | is_truncated, y); |
1adf2230 | 1057 | } else { |
b3b6e05e | 1058 | return list_objects_ordered(dpp, max, result, common_prefixes, |
9f95a23c | 1059 | is_truncated, y); |
1adf2230 AA |
1060 | } |
1061 | } | |
7c673cae FG |
1062 | rgw_obj_key& get_next_marker() { |
1063 | return next_marker; | |
1064 | } | |
1adf2230 AA |
1065 | }; // class List |
1066 | }; // class Bucket | |
7c673cae | 1067 | |
b3b6e05e TL |
1068 | int on_last_entry_in_listing(const DoutPrefixProvider *dpp, |
1069 | RGWBucketInfo& bucket_info, | |
7c673cae FG |
1070 | const std::string& obj_prefix, |
1071 | const std::string& obj_delim, | |
1072 | std::function<int(const rgw_bucket_dir_entry&)> handler); | |
1073 | ||
f67539c2 | 1074 | bool swift_versioning_enabled(rgw::sal::RGWBucket* bucket) const; |
7c673cae FG |
1075 | |
1076 | int swift_versioning_copy(RGWObjectCtx& obj_ctx, /* in/out */ | |
1077 | const rgw_user& user, /* in */ | |
f67539c2 TL |
1078 | rgw::sal::RGWBucket* bucket, /* in */ |
1079 | rgw::sal::RGWObject* obj, /* in */ | |
9f95a23c TL |
1080 | const DoutPrefixProvider *dpp, /* in/out */ |
1081 | optional_yield y); /* in */ | |
1082 | int swift_versioning_restore(RGWObjectCtx& obj_ctx, /* in/out */ | |
7c673cae | 1083 | const rgw_user& user, /* in */ |
f67539c2 TL |
1084 | rgw::sal::RGWBucket* bucket, /* in */ |
1085 | rgw::sal::RGWObject* obj, /* in */ | |
9f95a23c TL |
1086 | bool& restored, /* out */ |
1087 | const DoutPrefixProvider *dpp); /* in/out */ | |
b3b6e05e TL |
1088 | int copy_obj_to_remote_dest(const DoutPrefixProvider *dpp, |
1089 | RGWObjState *astate, | |
7c673cae FG |
1090 | map<string, bufferlist>& src_attrs, |
1091 | RGWRados::Object::Read& read_op, | |
1092 | const rgw_user& user_id, | |
f67539c2 | 1093 | rgw::sal::RGWObject* dest_obj, |
7c673cae FG |
1094 | ceph::real_time *mtime); |
1095 | ||
1096 | enum AttrsMod { | |
1097 | ATTRSMOD_NONE = 0, | |
1098 | ATTRSMOD_REPLACE = 1, | |
1099 | ATTRSMOD_MERGE = 2 | |
1100 | }; | |
1101 | ||
f67539c2 | 1102 | int rewrite_obj(RGWBucketInfo& dest_bucket_info, rgw::sal::RGWObject* obj, const DoutPrefixProvider *dpp, optional_yield y); |
7c673cae | 1103 | |
b3b6e05e TL |
1104 | int stat_remote_obj(const DoutPrefixProvider *dpp, |
1105 | RGWObjectCtx& obj_ctx, | |
7c673cae | 1106 | const rgw_user& user_id, |
7c673cae | 1107 | req_info *info, |
9f95a23c | 1108 | const rgw_zone_id& source_zone, |
f67539c2 | 1109 | rgw::sal::RGWObject* src_obj, |
9f95a23c | 1110 | const RGWBucketInfo *src_bucket_info, |
7c673cae FG |
1111 | real_time *src_mtime, |
1112 | uint64_t *psize, | |
1113 | const real_time *mod_ptr, | |
1114 | const real_time *unmod_ptr, | |
1115 | bool high_precision_time, | |
1116 | const char *if_match, | |
1117 | const char *if_nomatch, | |
1118 | map<string, bufferlist> *pattrs, | |
11fdf7f2 | 1119 | map<string, string> *pheaders, |
7c673cae FG |
1120 | string *version_id, |
1121 | string *ptag, | |
1122 | string *petag); | |
1123 | ||
1124 | int fetch_remote_obj(RGWObjectCtx& obj_ctx, | |
1125 | const rgw_user& user_id, | |
7c673cae | 1126 | req_info *info, |
9f95a23c | 1127 | const rgw_zone_id& source_zone, |
f67539c2 TL |
1128 | rgw::sal::RGWObject* dest_obj, |
1129 | rgw::sal::RGWObject* src_obj, | |
1130 | rgw::sal::RGWBucket* dest_bucket, | |
1131 | rgw::sal::RGWBucket* src_bucket, | |
11fdf7f2 | 1132 | std::optional<rgw_placement_rule> dest_placement, |
7c673cae FG |
1133 | ceph::real_time *src_mtime, |
1134 | ceph::real_time *mtime, | |
1135 | const ceph::real_time *mod_ptr, | |
1136 | const ceph::real_time *unmod_ptr, | |
1137 | bool high_precision_time, | |
1138 | const char *if_match, | |
1139 | const char *if_nomatch, | |
1140 | AttrsMod attrs_mod, | |
1141 | bool copy_if_newer, | |
f67539c2 | 1142 | rgw::sal::RGWAttrs& attrs, |
7c673cae | 1143 | RGWObjCategory category, |
11fdf7f2 | 1144 | std::optional<uint64_t> olh_epoch, |
7c673cae | 1145 | ceph::real_time delete_at, |
7c673cae | 1146 | string *ptag, |
11fdf7f2 | 1147 | string *petag, |
7c673cae | 1148 | void (*progress_cb)(off_t, void *), |
31f18b77 | 1149 | void *progress_data, |
9f95a23c TL |
1150 | const DoutPrefixProvider *dpp, |
1151 | RGWFetchObjFilter *filter, | |
81eedcae TL |
1152 | rgw_zone_set *zones_trace= nullptr, |
1153 | std::optional<uint64_t>* bytes_transferred = 0); | |
7c673cae FG |
1154 | /** |
1155 | * Copy an object. | |
1156 | * dest_obj: the object to copy into | |
1157 | * src_obj: the object to copy from | |
1158 | * attrs: usage depends on attrs_mod parameter | |
1159 | * attrs_mod: the modification mode of the attrs, may have the following values: | |
1160 | * ATTRSMOD_NONE - the attributes of the source object will be | |
1161 | * copied without modifications, attrs parameter is ignored; | |
1162 | * ATTRSMOD_REPLACE - new object will have the attributes provided by attrs | |
1163 | * parameter, source object attributes are not copied; | |
1164 | * ATTRSMOD_MERGE - any conflicting meta keys on the source object's attributes | |
1165 | * are overwritten by values contained in attrs parameter. | |
7c673cae FG |
1166 | * Returns: 0 on success, -ERR# otherwise. |
1167 | */ | |
1168 | int copy_obj(RGWObjectCtx& obj_ctx, | |
1169 | const rgw_user& user_id, | |
7c673cae | 1170 | req_info *info, |
9f95a23c | 1171 | const rgw_zone_id& source_zone, |
f67539c2 TL |
1172 | rgw::sal::RGWObject* dest_obj, |
1173 | rgw::sal::RGWObject* src_obj, | |
1174 | rgw::sal::RGWBucket* dest_bucket, | |
1175 | rgw::sal::RGWBucket* src_bucket, | |
11fdf7f2 | 1176 | const rgw_placement_rule& dest_placement, |
7c673cae FG |
1177 | ceph::real_time *src_mtime, |
1178 | ceph::real_time *mtime, | |
1179 | const ceph::real_time *mod_ptr, | |
1180 | const ceph::real_time *unmod_ptr, | |
1181 | bool high_precision_time, | |
1182 | const char *if_match, | |
1183 | const char *if_nomatch, | |
1184 | AttrsMod attrs_mod, | |
1185 | bool copy_if_newer, | |
1186 | map<std::string, bufferlist>& attrs, | |
1187 | RGWObjCategory category, | |
1188 | uint64_t olh_epoch, | |
1189 | ceph::real_time delete_at, | |
1190 | string *version_id, | |
1191 | string *ptag, | |
11fdf7f2 | 1192 | string *petag, |
7c673cae | 1193 | void (*progress_cb)(off_t, void *), |
9f95a23c TL |
1194 | void *progress_data, |
1195 | const DoutPrefixProvider *dpp, | |
1196 | optional_yield y); | |
7c673cae FG |
1197 | |
1198 | int copy_obj_data(RGWObjectCtx& obj_ctx, | |
f67539c2 | 1199 | rgw::sal::RGWBucket* bucket, |
11fdf7f2 | 1200 | const rgw_placement_rule& dest_placement, |
7c673cae | 1201 | RGWRados::Object::Read& read_op, off_t end, |
f67539c2 | 1202 | rgw::sal::RGWObject* dest_obj, |
7c673cae FG |
1203 | ceph::real_time *mtime, |
1204 | ceph::real_time set_mtime, | |
1205 | map<string, bufferlist>& attrs, | |
7c673cae FG |
1206 | uint64_t olh_epoch, |
1207 | ceph::real_time delete_at, | |
9f95a23c TL |
1208 | string *petag, |
1209 | const DoutPrefixProvider *dpp, | |
1210 | optional_yield y); | |
7c673cae | 1211 | |
11fdf7f2 | 1212 | int transition_obj(RGWObjectCtx& obj_ctx, |
f67539c2 TL |
1213 | rgw::sal::RGWBucket* bucket, |
1214 | rgw::sal::RGWObject& obj, | |
11fdf7f2 TL |
1215 | const rgw_placement_rule& placement_rule, |
1216 | const real_time& mtime, | |
9f95a23c TL |
1217 | uint64_t olh_epoch, |
1218 | const DoutPrefixProvider *dpp, | |
1219 | optional_yield y); | |
11fdf7f2 | 1220 | |
b3b6e05e | 1221 | int check_bucket_empty(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, optional_yield y); |
7c673cae FG |
1222 | |
1223 | /** | |
1224 | * Delete a bucket. | |
1225 | * bucket: the name of the bucket to delete | |
1226 | * Returns 0 on success, -ERR# otherwise. | |
1227 | */ | |
b3b6e05e | 1228 | int delete_bucket(RGWBucketInfo& bucket_info, RGWObjVersionTracker& objv_tracker, optional_yield y, const DoutPrefixProvider *dpp, bool check_empty = true); |
7c673cae | 1229 | |
7c673cae | 1230 | void wakeup_meta_sync_shards(set<int>& shard_ids); |
9f95a23c | 1231 | void wakeup_data_sync_shards(const rgw_zone_id& source_zone, map<int, set<string> >& shard_ids); |
7c673cae FG |
1232 | |
1233 | RGWMetaSyncStatusManager* get_meta_sync_manager(); | |
9f95a23c | 1234 | RGWDataSyncStatusManager* get_data_sync_manager(const rgw_zone_id& source_zone); |
7c673cae | 1235 | |
b3b6e05e TL |
1236 | int set_bucket_owner(rgw_bucket& bucket, ACLOwner& owner, const DoutPrefixProvider *dpp); |
1237 | int set_buckets_enabled(std::vector<rgw_bucket>& buckets, bool enabled, const DoutPrefixProvider *dpp); | |
1238 | int bucket_suspended(const DoutPrefixProvider *dpp, rgw_bucket& bucket, bool *suspended); | |
7c673cae FG |
1239 | |
1240 | /** Delete an object.*/ | |
b3b6e05e TL |
1241 | int delete_obj(const DoutPrefixProvider *dpp, |
1242 | RGWObjectCtx& obj_ctx, | |
f67539c2 TL |
1243 | const RGWBucketInfo& bucket_owner, |
1244 | const rgw_obj& src_obj, | |
b3b6e05e | 1245 | int versioning_status, // versioning flags in enum RGWBucketFlags |
f67539c2 TL |
1246 | uint16_t bilog_flags = 0, |
1247 | const ceph::real_time& expiration_time = ceph::real_time(), | |
1248 | rgw_zone_set *zones_trace = nullptr); | |
7c673cae | 1249 | |
b3b6e05e | 1250 | int delete_raw_obj(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj); |
7c673cae | 1251 | |
7c673cae | 1252 | /** Remove an object from the bucket index */ |
b3b6e05e | 1253 | int delete_obj_index(const rgw_obj& obj, ceph::real_time mtime, const DoutPrefixProvider *dpp); |
7c673cae | 1254 | |
7c673cae FG |
1255 | /** |
1256 | * Set an attr on an object. | |
1257 | * bucket: name of the bucket holding the object | |
1258 | * obj: name of the object to set the attr on | |
1259 | * name: the attr to set | |
1260 | * bl: the contents of the attr | |
1261 | * Returns: 0 on success, -ERR# otherwise. | |
1262 | */ | |
b3b6e05e | 1263 | int set_attr(const DoutPrefixProvider *dpp, void *ctx, const RGWBucketInfo& bucket_info, rgw_obj& obj, const char *name, bufferlist& bl); |
7c673cae | 1264 | |
b3b6e05e | 1265 | int set_attrs(const DoutPrefixProvider *dpp, void *ctx, const RGWBucketInfo& bucket_info, rgw_obj& obj, |
7c673cae | 1266 | map<string, bufferlist>& attrs, |
9f95a23c TL |
1267 | map<string, bufferlist>* rmattrs, |
1268 | optional_yield y); | |
7c673cae | 1269 | |
b3b6e05e | 1270 | int get_obj_state(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWObjState **state, |
9f95a23c | 1271 | bool follow_olh, optional_yield y, bool assume_noent = false); |
b3b6e05e TL |
1272 | int get_obj_state(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWObjState **state, optional_yield y) { |
1273 | return get_obj_state(dpp, rctx, bucket_info, obj, state, true, y); | |
7c673cae FG |
1274 | } |
1275 | ||
b3b6e05e | 1276 | using iterate_obj_cb = int (*)(const DoutPrefixProvider*, const rgw_raw_obj&, off_t, off_t, |
11fdf7f2 TL |
1277 | off_t, bool, RGWObjState*, void*); |
1278 | ||
b3b6e05e | 1279 | int iterate_obj(const DoutPrefixProvider *dpp, RGWObjectCtx& ctx, const RGWBucketInfo& bucket_info, |
11fdf7f2 | 1280 | const rgw_obj& obj, off_t ofs, off_t end, |
9f95a23c TL |
1281 | uint64_t max_chunk_size, iterate_obj_cb cb, void *arg, |
1282 | optional_yield y); | |
7c673cae | 1283 | |
b3b6e05e TL |
1284 | int get_obj_iterate_cb(const DoutPrefixProvider *dpp, |
1285 | const rgw_raw_obj& read_obj, off_t obj_ofs, | |
11fdf7f2 TL |
1286 | off_t read_ofs, off_t len, bool is_head_obj, |
1287 | RGWObjState *astate, void *arg); | |
7c673cae FG |
1288 | |
1289 | void get_obj_aio_completion_cb(librados::completion_t cb, void *arg); | |
1290 | ||
1291 | /** | |
1292 | * a simple object read without keeping state | |
1293 | */ | |
1294 | ||
b3b6e05e TL |
1295 | int raw_obj_stat(const DoutPrefixProvider *dpp, |
1296 | rgw_raw_obj& obj, uint64_t *psize, ceph::real_time *pmtime, uint64_t *epoch, | |
11fdf7f2 | 1297 | map<string, bufferlist> *attrs, bufferlist *first_chunk, |
9f95a23c | 1298 | RGWObjVersionTracker *objv_tracker, optional_yield y); |
7c673cae | 1299 | |
b3b6e05e TL |
1300 | int obj_operate(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, librados::ObjectWriteOperation *op); |
1301 | int obj_operate(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, librados::ObjectReadOperation *op); | |
7c673cae | 1302 | |
b3b6e05e TL |
1303 | int guard_reshard(const DoutPrefixProvider *dpp, |
1304 | BucketShard *bs, | |
f64942e4 AA |
1305 | const rgw_obj& obj_instance, |
1306 | const RGWBucketInfo& bucket_info, | |
1307 | std::function<int(BucketShard *)> call); | |
1308 | int block_while_resharding(RGWRados::BucketShard *bs, | |
1309 | string *new_bucket_id, | |
11fdf7f2 | 1310 | const RGWBucketInfo& bucket_info, |
b3b6e05e TL |
1311 | optional_yield y, |
1312 | const DoutPrefixProvider *dpp); | |
1313 | ||
1314 | void bucket_index_guard_olh_op(const DoutPrefixProvider *dpp, RGWObjState& olh_state, librados::ObjectOperation& op); | |
1315 | int olh_init_modification(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, RGWObjState& state, const rgw_obj& olh_obj, string *op_tag); | |
1316 | int olh_init_modification_impl(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, RGWObjState& state, const rgw_obj& olh_obj, string *op_tag); | |
1317 | int bucket_index_link_olh(const DoutPrefixProvider *dpp, | |
1318 | const RGWBucketInfo& bucket_info, RGWObjState& olh_state, | |
7c673cae FG |
1319 | const rgw_obj& obj_instance, bool delete_marker, |
1320 | const string& op_tag, struct rgw_bucket_dir_entry_meta *meta, | |
1321 | uint64_t olh_epoch, | |
91327a77 AA |
1322 | ceph::real_time unmod_since, bool high_precision_time, |
1323 | rgw_zone_set *zones_trace = nullptr, | |
1324 | bool log_data_change = false); | |
b3b6e05e TL |
1325 | int bucket_index_unlink_instance(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj_instance, const string& op_tag, const string& olh_tag, uint64_t olh_epoch, rgw_zone_set *zones_trace = nullptr); |
1326 | int bucket_index_read_olh_log(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, RGWObjState& state, const rgw_obj& obj_instance, uint64_t ver_marker, | |
7c673cae | 1327 | map<uint64_t, vector<rgw_bucket_olh_log_entry> > *log, bool *is_truncated); |
b3b6e05e TL |
1328 | int bucket_index_trim_olh_log(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, RGWObjState& obj_state, const rgw_obj& obj_instance, uint64_t ver); |
1329 | int bucket_index_clear_olh(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, RGWObjState& state, const rgw_obj& obj_instance); | |
1330 | int apply_olh_log(const DoutPrefixProvider *dpp, RGWObjectCtx& ctx, RGWObjState& obj_state, const RGWBucketInfo& bucket_info, const rgw_obj& obj, | |
7c673cae | 1331 | bufferlist& obj_tag, map<uint64_t, vector<rgw_bucket_olh_log_entry> >& log, |
31f18b77 | 1332 | uint64_t *plast_ver, rgw_zone_set *zones_trace = nullptr); |
b3b6e05e TL |
1333 | int update_olh(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, RGWObjState *state, const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_zone_set *zones_trace = nullptr); |
1334 | int set_olh(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, const RGWBucketInfo& bucket_info, const rgw_obj& target_obj, bool delete_marker, rgw_bucket_dir_entry_meta *meta, | |
91327a77 | 1335 | uint64_t olh_epoch, ceph::real_time unmod_since, bool high_precision_time, |
9f95a23c | 1336 | optional_yield y, rgw_zone_set *zones_trace = nullptr, bool log_data_change = false); |
b3b6e05e | 1337 | int repair_olh(const DoutPrefixProvider *dpp, RGWObjState* state, const RGWBucketInfo& bucket_info, |
a8e16298 | 1338 | const rgw_obj& obj); |
b3b6e05e | 1339 | int unlink_obj_instance(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, RGWBucketInfo& bucket_info, const rgw_obj& target_obj, |
9f95a23c | 1340 | uint64_t olh_epoch, optional_yield y, rgw_zone_set *zones_trace = nullptr); |
7c673cae FG |
1341 | |
1342 | void check_pending_olh_entries(map<string, bufferlist>& pending_entries, map<string, bufferlist> *rm_pending_entries); | |
b3b6e05e TL |
1343 | int remove_olh_pending_entries(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, RGWObjState& state, const rgw_obj& olh_obj, map<string, bufferlist>& pending_attrs); |
1344 | int follow_olh(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, RGWObjectCtx& ctx, RGWObjState *state, const rgw_obj& olh_obj, rgw_obj *target); | |
1345 | int get_olh(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWOLHInfo *olh); | |
7c673cae | 1346 | |
11fdf7f2 | 1347 | void gen_rand_obj_instance_name(rgw_obj_key *target_key); |
7c673cae FG |
1348 | void gen_rand_obj_instance_name(rgw_obj *target); |
1349 | ||
b3b6e05e TL |
1350 | int update_containers_stats(map<string, RGWBucketEnt>& m, const DoutPrefixProvider *dpp); |
1351 | int append_async(const DoutPrefixProvider *dpp, rgw_raw_obj& obj, size_t size, bufferlist& bl); | |
7c673cae | 1352 | |
11fdf7f2 | 1353 | public: |
7c673cae FG |
1354 | void set_atomic(void *ctx, rgw_obj& obj) { |
1355 | RGWObjectCtx *rctx = static_cast<RGWObjectCtx *>(ctx); | |
11fdf7f2 | 1356 | rctx->set_atomic(obj); |
7c673cae | 1357 | } |
11fdf7f2 | 1358 | void set_prefetch_data(void *ctx, const rgw_obj& obj) { |
7c673cae | 1359 | RGWObjectCtx *rctx = static_cast<RGWObjectCtx *>(ctx); |
11fdf7f2 | 1360 | rctx->set_prefetch_data(obj); |
7c673cae | 1361 | } |
7c673cae | 1362 | int decode_policy(bufferlist& bl, ACLOwner *owner); |
b3b6e05e | 1363 | int get_bucket_stats(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, int shard_id, string *bucket_ver, string *master_ver, |
c07f9fc5 | 1364 | map<RGWObjCategory, RGWStorageStats>& stats, string *max_marker, bool* syncstopped = NULL); |
b3b6e05e | 1365 | int get_bucket_stats_async(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, int shard_id, RGWGetBucketStats_CB *cb); |
3a9019d9 | 1366 | |
b3b6e05e | 1367 | int put_bucket_instance_info(RGWBucketInfo& info, bool exclusive, ceph::real_time mtime, map<string, bufferlist> *pattrs, const DoutPrefixProvider *dpp); |
9f95a23c | 1368 | /* xxx dang obj_ctx -> svc */ |
b3b6e05e TL |
1369 | int get_bucket_instance_info(RGWSysObjectCtx& obj_ctx, const string& meta_key, RGWBucketInfo& info, ceph::real_time *pmtime, map<string, bufferlist> *pattrs, optional_yield y, const DoutPrefixProvider *dpp); |
1370 | int get_bucket_instance_info(RGWSysObjectCtx& obj_ctx, const rgw_bucket& bucket, RGWBucketInfo& info, ceph::real_time *pmtime, map<string, bufferlist> *pattrs, optional_yield y, const DoutPrefixProvider *dpp); | |
3a9019d9 | 1371 | |
9f95a23c | 1372 | static void make_bucket_entry_name(const string& tenant_name, const string& bucket_name, string& bucket_entry); |
b32b8144 | 1373 | |
9f95a23c | 1374 | int get_bucket_info(RGWServices *svc, |
b32b8144 FG |
1375 | const string& tenant_name, const string& bucket_name, |
1376 | RGWBucketInfo& info, | |
b3b6e05e TL |
1377 | ceph::real_time *pmtime, optional_yield y, |
1378 | const DoutPrefixProvider *dpp, map<string, bufferlist> *pattrs = NULL); | |
b32b8144 | 1379 | |
81eedcae TL |
1380 | // Returns 0 on successful refresh. Returns error code if there was |
1381 | // an error or the version stored on the OSD is the same as that | |
b32b8144 FG |
1382 | // presented in the BucketInfo structure. |
1383 | // | |
1384 | int try_refresh_bucket_info(RGWBucketInfo& info, | |
1385 | ceph::real_time *pmtime, | |
b3b6e05e | 1386 | const DoutPrefixProvider *dpp, |
b32b8144 FG |
1387 | map<string, bufferlist> *pattrs = nullptr); |
1388 | ||
7c673cae | 1389 | int put_linked_bucket_info(RGWBucketInfo& info, bool exclusive, ceph::real_time mtime, obj_version *pep_objv, |
b3b6e05e TL |
1390 | map<string, bufferlist> *pattrs, bool create_entry_point, |
1391 | const DoutPrefixProvider *dpp); | |
7c673cae | 1392 | |
b3b6e05e | 1393 | int cls_obj_prepare_op(const DoutPrefixProvider *dpp, BucketShard& bs, RGWModifyOp op, string& tag, rgw_obj& obj, uint16_t bilog_flags, optional_yield y, rgw_zone_set *zones_trace = nullptr); |
31f18b77 FG |
1394 | int cls_obj_complete_op(BucketShard& bs, const rgw_obj& obj, RGWModifyOp op, string& tag, int64_t pool, uint64_t epoch, |
1395 | rgw_bucket_dir_entry& ent, RGWObjCategory category, list<rgw_obj_index_key> *remove_objs, uint16_t bilog_flags, rgw_zone_set *zones_trace = nullptr); | |
1396 | int cls_obj_complete_add(BucketShard& bs, const rgw_obj& obj, string& tag, int64_t pool, uint64_t epoch, rgw_bucket_dir_entry& ent, | |
1397 | RGWObjCategory category, list<rgw_obj_index_key> *remove_objs, uint16_t bilog_flags, rgw_zone_set *zones_trace = nullptr); | |
7c673cae | 1398 | int cls_obj_complete_del(BucketShard& bs, string& tag, int64_t pool, uint64_t epoch, rgw_obj& obj, |
31f18b77 FG |
1399 | ceph::real_time& removed_mtime, list<rgw_obj_index_key> *remove_objs, uint16_t bilog_flags, rgw_zone_set *zones_trace = nullptr); |
1400 | int cls_obj_complete_cancel(BucketShard& bs, string& tag, rgw_obj& obj, uint16_t bilog_flags, rgw_zone_set *zones_trace = nullptr); | |
b3b6e05e | 1401 | int cls_obj_set_bucket_tag_timeout(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, uint64_t timeout); |
9f95a23c TL |
1402 | |
1403 | using ent_map_t = | |
1404 | boost::container::flat_map<std::string, rgw_bucket_dir_entry>; | |
1405 | ||
1406 | using check_filter_t = bool (*)(const std::string&); | |
1407 | ||
b3b6e05e TL |
1408 | int cls_bucket_list_ordered(const DoutPrefixProvider *dpp, |
1409 | RGWBucketInfo& bucket_info, | |
9f95a23c TL |
1410 | const int shard_id, |
1411 | const rgw_obj_index_key& start_after, | |
11fdf7f2 | 1412 | const string& prefix, |
9f95a23c TL |
1413 | const string& delimiter, |
1414 | const uint32_t num_entries, | |
1415 | const bool list_versions, | |
1416 | const uint16_t exp_factor, // 0 means ignore | |
1417 | ent_map_t& m, | |
1418 | bool* is_truncated, | |
1419 | bool* cls_filtered, | |
1adf2230 | 1420 | rgw_obj_index_key *last_entry, |
9f95a23c TL |
1421 | optional_yield y, |
1422 | check_filter_t force_check_filter = nullptr); | |
b3b6e05e TL |
1423 | int cls_bucket_list_unordered(const DoutPrefixProvider *dpp, |
1424 | RGWBucketInfo& bucket_info, | |
9f95a23c TL |
1425 | int shard_id, |
1426 | const rgw_obj_index_key& start_after, | |
11fdf7f2 | 1427 | const string& prefix, |
9f95a23c TL |
1428 | uint32_t num_entries, |
1429 | bool list_versions, | |
1adf2230 | 1430 | vector<rgw_bucket_dir_entry>& ent_list, |
9f95a23c TL |
1431 | bool *is_truncated, |
1432 | rgw_obj_index_key *last_entry, | |
1433 | optional_yield y, | |
1434 | check_filter_t = nullptr); | |
b3b6e05e TL |
1435 | int cls_bucket_head(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, int shard_id, vector<rgw_bucket_dir_header>& headers, map<int, string> *bucket_instance_ids = NULL); |
1436 | int cls_bucket_head_async(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, int shard_id, RGWGetDirHeader_CB *ctx, int *num_aio); | |
7c673cae | 1437 | |
b3b6e05e TL |
1438 | int bi_get_instance(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_bucket_dir_entry *dirent); |
1439 | int bi_get_olh(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_bucket_olh_entry *olh); | |
1440 | int bi_get(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, BIIndexType index_type, rgw_cls_bi_entry *entry); | |
7c673cae FG |
1441 | void bi_put(librados::ObjectWriteOperation& op, BucketShard& bs, rgw_cls_bi_entry& entry); |
1442 | int bi_put(BucketShard& bs, rgw_cls_bi_entry& entry); | |
b3b6e05e TL |
1443 | int bi_put(const DoutPrefixProvider *dpp, rgw_bucket& bucket, rgw_obj& obj, rgw_cls_bi_entry& entry); |
1444 | int bi_list(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, int shard_id, const string& filter_obj, const string& marker, uint32_t max, list<rgw_cls_bi_entry> *entries, bool *is_truncated); | |
7c673cae | 1445 | int bi_list(BucketShard& bs, const string& filter_obj, const string& marker, uint32_t max, list<rgw_cls_bi_entry> *entries, bool *is_truncated); |
b3b6e05e | 1446 | int bi_list(const DoutPrefixProvider *dpp, rgw_bucket& bucket, const string& obj_name, const string& marker, uint32_t max, |
7c673cae FG |
1447 | list<rgw_cls_bi_entry> *entries, bool *is_truncated); |
1448 | int bi_remove(BucketShard& bs); | |
1449 | ||
b3b6e05e TL |
1450 | int cls_obj_usage_log_add(const DoutPrefixProvider *dpp, const string& oid, rgw_usage_log_info& info); |
1451 | int cls_obj_usage_log_read(const DoutPrefixProvider *dpp, const string& oid, const string& user, const string& bucket, uint64_t start_epoch, | |
11fdf7f2 TL |
1452 | uint64_t end_epoch, uint32_t max_entries, string& read_iter, map<rgw_user_bucket, |
1453 | rgw_usage_log_entry>& usage, bool *is_truncated); | |
b3b6e05e | 1454 | int cls_obj_usage_log_trim(const DoutPrefixProvider *dpp, const string& oid, const string& user, const string& bucket, uint64_t start_epoch, |
11fdf7f2 | 1455 | uint64_t end_epoch); |
b3b6e05e | 1456 | int cls_obj_usage_log_clear(const DoutPrefixProvider *dpp, string& oid); |
7c673cae | 1457 | |
f67539c2 | 1458 | int get_target_shard_id(const rgw::bucket_index_normal_layout& layout, const string& obj_key, int *shard_id); |
9f95a23c TL |
1459 | |
1460 | int lock_exclusive(const rgw_pool& pool, const string& oid, ceph::timespan& duration, rgw_zone_id& zone_id, string& owner_id); | |
1461 | int unlock(const rgw_pool& pool, const string& oid, rgw_zone_id& zone_id, string& owner_id); | |
7c673cae | 1462 | |
b3b6e05e | 1463 | void update_gc_chain(const DoutPrefixProvider *dpp, rgw_obj& head_obj, RGWObjManifest& manifest, cls_rgw_obj_chain *chain); |
9f95a23c | 1464 | int send_chain_to_gc(cls_rgw_obj_chain& chain, const string& tag); |
b3b6e05e TL |
1465 | void delete_objs_inline(const DoutPrefixProvider *dpp, cls_rgw_obj_chain& chain, const string& tag); |
1466 | int gc_operate(const DoutPrefixProvider *dpp, string& oid, librados::ObjectWriteOperation *op); | |
9f95a23c TL |
1467 | int gc_aio_operate(const std::string& oid, librados::AioCompletion *c, |
1468 | librados::ObjectWriteOperation *op); | |
b3b6e05e | 1469 | int gc_operate(const DoutPrefixProvider *dpp, string& oid, librados::ObjectReadOperation *op, bufferlist *pbl); |
7c673cae | 1470 | |
9f95a23c | 1471 | int list_gc_objs(int *index, string& marker, uint32_t max, bool expired_only, std::list<cls_rgw_gc_obj_info>& result, bool *truncated, bool& processing_queue); |
11fdf7f2 | 1472 | int process_gc(bool expired_only); |
b3b6e05e TL |
1473 | bool process_expire_objects(const DoutPrefixProvider *dpp); |
1474 | int defer_gc(const DoutPrefixProvider *dpp, void *ctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, optional_yield y); | |
7c673cae FG |
1475 | |
1476 | int process_lc(); | |
f6b5b4d7 | 1477 | int list_lc_progress(string& marker, uint32_t max_entries, |
f67539c2 | 1478 | vector<rgw::sal::Lifecycle::LCEntry>& progress_map, int& index); |
f6b5b4d7 | 1479 | |
b3b6e05e | 1480 | int bucket_check_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, |
7c673cae FG |
1481 | map<RGWObjCategory, RGWStorageStats> *existing_stats, |
1482 | map<RGWObjCategory, RGWStorageStats> *calculated_stats); | |
b3b6e05e TL |
1483 | int bucket_rebuild_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info); |
1484 | int bucket_set_reshard(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const cls_rgw_bucket_instance_entry& entry); | |
1485 | int remove_objs_from_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, list<rgw_obj_index_key>& oid_list); | |
1486 | int move_rados_obj(const DoutPrefixProvider *dpp, | |
1487 | librados::IoCtx& src_ioctx, | |
7c673cae FG |
1488 | const string& src_oid, const string& src_locator, |
1489 | librados::IoCtx& dst_ioctx, | |
1490 | const string& dst_oid, const string& dst_locator); | |
b3b6e05e TL |
1491 | int fix_head_obj_locator(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, bool copy_obj, bool remove_bad, rgw_obj_key& key); |
1492 | int fix_tail_obj_locator(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, rgw_obj_key& key, bool fix, bool *need_fix, optional_yield y); | |
7c673cae FG |
1493 | |
1494 | int check_quota(const rgw_user& bucket_owner, rgw_bucket& bucket, | |
f67539c2 TL |
1495 | RGWQuotaInfo& user_quota, RGWQuotaInfo& bucket_quota, uint64_t obj_size, |
1496 | optional_yield y, bool check_size_only = false); | |
7c673cae | 1497 | |
224ce89b | 1498 | int check_bucket_shards(const RGWBucketInfo& bucket_info, const rgw_bucket& bucket, |
b3b6e05e | 1499 | uint64_t num_objs, const DoutPrefixProvider *dpp); |
31f18b77 | 1500 | |
b3b6e05e | 1501 | int add_bucket_to_reshard(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, uint32_t new_num_shards); |
31f18b77 | 1502 | |
7c673cae | 1503 | uint64_t instance_id(); |
3efd9988 | 1504 | |
7c673cae FG |
1505 | librados::Rados* get_rados_handle(); |
1506 | ||
b3b6e05e TL |
1507 | int delete_raw_obj_aio(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, list<librados::AioCompletion *>& handles); |
1508 | int delete_obj_aio(const DoutPrefixProvider *dpp, const rgw_obj& obj, RGWBucketInfo& info, RGWObjState *astate, | |
9f95a23c TL |
1509 | list<librados::AioCompletion *>& handles, bool keep_index_consistent, |
1510 | optional_yield y); | |
11fdf7f2 | 1511 | |
11fdf7f2 | 1512 | private: |
7c673cae FG |
1513 | /** |
1514 | * Check the actual on-disk state of the object specified | |
1515 | * by list_state, and fill in the time and size of object. | |
1516 | * Then append any changes to suggested_updates for | |
1517 | * the rgw class' dir_suggest_changes function. | |
1518 | * | |
1519 | * Note that this can maul list_state; don't use it afterwards. Also | |
1520 | * it expects object to already be filled in from list_state; it only | |
1521 | * sets the size and mtime. | |
1522 | * | |
1523 | * Returns 0 on success, -ENOENT if the object doesn't exist on disk, | |
1524 | * and -errno on other failures. (-ENOENT is not a failure, and it | |
1525 | * will encode that info as a suggested update.) | |
1526 | */ | |
b3b6e05e TL |
1527 | int check_disk_state(const DoutPrefixProvider *dpp, |
1528 | librados::IoCtx io_ctx, | |
7c673cae FG |
1529 | const RGWBucketInfo& bucket_info, |
1530 | rgw_bucket_dir_entry& list_state, | |
1531 | rgw_bucket_dir_entry& object, | |
9f95a23c TL |
1532 | bufferlist& suggested_updates, |
1533 | optional_yield y); | |
7c673cae FG |
1534 | |
1535 | /** | |
1536 | * Init pool iteration | |
31f18b77 | 1537 | * pool: pool to use for the ctx initialization |
7c673cae FG |
1538 | * ctx: context object to use for the iteration |
1539 | * Returns: 0 on success, -ERR# otherwise. | |
1540 | */ | |
b3b6e05e | 1541 | int pool_iterate_begin(const DoutPrefixProvider *dpp, const rgw_pool& pool, RGWPoolIterCtx& ctx); |
31f18b77 | 1542 | |
181888fb FG |
1543 | /** |
1544 | * Init pool iteration | |
1545 | * pool: pool to use | |
1546 | * cursor: position to start iteration | |
1547 | * ctx: context object to use for the iteration | |
1548 | * Returns: 0 on success, -ERR# otherwise. | |
1549 | */ | |
b3b6e05e | 1550 | int pool_iterate_begin(const DoutPrefixProvider *dpp, const rgw_pool& pool, const string& cursor, RGWPoolIterCtx& ctx); |
181888fb FG |
1551 | |
1552 | /** | |
1553 | * Get pool iteration position | |
1554 | * ctx: context object to use for the iteration | |
1555 | * Returns: string representation of position | |
1556 | */ | |
1557 | string pool_iterate_get_cursor(RGWPoolIterCtx& ctx); | |
1558 | ||
7c673cae FG |
1559 | /** |
1560 | * Iterate over pool return object names, use optional filter | |
1561 | * ctx: iteration context, initialized with pool_iterate_begin() | |
1562 | * num: max number of objects to return | |
1563 | * objs: a vector that the results will append into | |
1564 | * is_truncated: if not NULL, will hold true iff iteration is complete | |
1565 | * filter: if not NULL, will be used to filter returned objects | |
1566 | * Returns: 0 on success, -ERR# otherwise. | |
1567 | */ | |
1568 | int pool_iterate(RGWPoolIterCtx& ctx, uint32_t num, vector<rgw_bucket_dir_entry>& objs, | |
1569 | bool *is_truncated, RGWAccessListFilter *filter); | |
1570 | ||
1571 | uint64_t next_bucket_id(); | |
11fdf7f2 | 1572 | |
9f95a23c TL |
1573 | /** |
1574 | * This is broken out to facilitate unit testing. | |
1575 | */ | |
1576 | static uint32_t calc_ordered_bucket_list_per_shard(uint32_t num_entries, | |
1577 | uint32_t num_shards); | |
11fdf7f2 TL |
1578 | }; |
1579 | ||
7c673cae | 1580 | #endif |