]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
9f95a23c | 2 | // vim: ts=8 sw=2 smarttab ft=cpp |
7c673cae FG |
3 | |
4 | #ifndef CEPH_RGWRADOS_H | |
5 | #define CEPH_RGWRADOS_H | |
6 | ||
7 | #include <functional> | |
9f95a23c | 8 | #include <boost/container/flat_map.hpp> |
7c673cae FG |
9 | |
10 | #include "include/rados/librados.hpp" | |
11 | #include "include/Context.h" | |
a4b75251 | 12 | #include "include/random.h" |
7c673cae FG |
13 | #include "common/RefCountedObj.h" |
14 | #include "common/RWLock.h" | |
15 | #include "common/ceph_time.h" | |
a4b75251 | 16 | #include "common/Timer.h" |
7c673cae FG |
17 | #include "rgw_common.h" |
18 | #include "cls/rgw/cls_rgw_types.h" | |
19 | #include "cls/version/cls_version_types.h" | |
20 | #include "cls/log/cls_log_types.h" | |
7c673cae | 21 | #include "cls/timeindex/cls_timeindex_types.h" |
11fdf7f2 | 22 | #include "cls/otp/cls_otp_types.h" |
7c673cae FG |
23 | #include "rgw_log.h" |
24 | #include "rgw_metadata.h" | |
25 | #include "rgw_meta_sync_status.h" | |
26 | #include "rgw_period_puller.h" | |
9f95a23c | 27 | #include "rgw_obj_manifest.h" |
7c673cae | 28 | #include "rgw_sync_module.h" |
9f95a23c | 29 | #include "rgw_trim_bilog.h" |
11fdf7f2 | 30 | #include "rgw_service.h" |
f67539c2 | 31 | #include "rgw_sal.h" |
11fdf7f2 TL |
32 | |
33 | #include "services/svc_rados.h" | |
9f95a23c | 34 | #include "services/svc_bi_rados.h" |
7c673cae FG |
35 | |
36 | class RGWWatcher; | |
7c673cae FG |
37 | class ACLOwner; |
38 | class RGWGC; | |
39 | class RGWMetaNotifier; | |
40 | class RGWDataNotifier; | |
41 | class RGWLC; | |
42 | class RGWObjectExpirer; | |
43 | class RGWMetaSyncProcessorThread; | |
44 | class RGWDataSyncProcessorThread; | |
45 | class RGWSyncLogTrimThread; | |
11fdf7f2 | 46 | class RGWSyncTraceManager; |
7c673cae FG |
47 | struct RGWZoneGroup; |
48 | struct RGWZoneParams; | |
31f18b77 FG |
49 | class RGWReshard; |
50 | class RGWReshardWait; | |
7c673cae | 51 | |
11fdf7f2 TL |
52 | class RGWSysObjectCtx; |
53 | ||
7c673cae FG |
54 | /* flags for put_obj_meta() */ |
55 | #define PUT_OBJ_CREATE 0x01 | |
56 | #define PUT_OBJ_EXCL 0x02 | |
57 | #define PUT_OBJ_CREATE_EXCL (PUT_OBJ_CREATE | PUT_OBJ_EXCL) | |
58 | ||
59 | #define RGW_OBJ_NS_MULTIPART "multipart" | |
60 | #define RGW_OBJ_NS_SHADOW "shadow" | |
61 | ||
7c673cae FG |
62 | static inline void prepend_bucket_marker(const rgw_bucket& bucket, const string& orig_oid, string& oid) |
63 | { | |
64 | if (bucket.marker.empty() || orig_oid.empty()) { | |
65 | oid = orig_oid; | |
66 | } else { | |
67 | oid = bucket.marker; | |
68 | oid.append("_"); | |
69 | oid.append(orig_oid); | |
70 | } | |
71 | } | |
72 | ||
73 | static inline void get_obj_bucket_and_oid_loc(const rgw_obj& obj, string& oid, string& locator) | |
74 | { | |
75 | const rgw_bucket& bucket = obj.bucket; | |
76 | prepend_bucket_marker(bucket, obj.get_oid(), oid); | |
77 | const string& loc = obj.key.get_loc(); | |
78 | if (!loc.empty()) { | |
79 | prepend_bucket_marker(bucket, loc, locator); | |
80 | } else { | |
81 | locator.clear(); | |
82 | } | |
83 | } | |
84 | ||
b3b6e05e | 85 | int rgw_policy_from_attrset(const DoutPrefixProvider *dpp, CephContext *cct, map<string, bufferlist>& attrset, RGWAccessControlPolicy *policy); |
7c673cae | 86 | |
7c673cae FG |
87 | struct RGWOLHInfo { |
88 | rgw_obj target; | |
89 | bool removed; | |
90 | ||
91 | RGWOLHInfo() : removed(false) {} | |
92 | ||
93 | void encode(bufferlist& bl) const { | |
94 | ENCODE_START(1, 1, bl); | |
11fdf7f2 TL |
95 | encode(target, bl); |
96 | encode(removed, bl); | |
7c673cae FG |
97 | ENCODE_FINISH(bl); |
98 | } | |
99 | ||
11fdf7f2 | 100 | void decode(bufferlist::const_iterator& bl) { |
7c673cae | 101 | DECODE_START(1, bl); |
11fdf7f2 TL |
102 | decode(target, bl); |
103 | decode(removed, bl); | |
7c673cae FG |
104 | DECODE_FINISH(bl); |
105 | } | |
106 | static void generate_test_instances(list<RGWOLHInfo*>& o); | |
107 | void dump(Formatter *f) const; | |
108 | }; | |
109 | WRITE_CLASS_ENCODER(RGWOLHInfo) | |
110 | ||
111 | struct RGWOLHPendingInfo { | |
112 | ceph::real_time time; | |
113 | ||
114 | RGWOLHPendingInfo() {} | |
115 | ||
116 | void encode(bufferlist& bl) const { | |
117 | ENCODE_START(1, 1, bl); | |
11fdf7f2 | 118 | encode(time, bl); |
7c673cae FG |
119 | ENCODE_FINISH(bl); |
120 | } | |
121 | ||
11fdf7f2 | 122 | void decode(bufferlist::const_iterator& bl) { |
7c673cae | 123 | DECODE_START(1, bl); |
11fdf7f2 | 124 | decode(time, bl); |
7c673cae FG |
125 | DECODE_FINISH(bl); |
126 | } | |
127 | ||
128 | void dump(Formatter *f) const; | |
129 | }; | |
130 | WRITE_CLASS_ENCODER(RGWOLHPendingInfo) | |
131 | ||
132 | struct RGWUsageBatch { | |
133 | map<ceph::real_time, rgw_usage_log_entry> m; | |
134 | ||
135 | void insert(ceph::real_time& t, rgw_usage_log_entry& entry, bool *account) { | |
136 | bool exists = m.find(t) != m.end(); | |
137 | *account = !exists; | |
138 | m[t].aggregate(entry); | |
139 | } | |
140 | }; | |
141 | ||
7c673cae | 142 | class RGWGetDataCB { |
7c673cae FG |
143 | public: |
144 | virtual int handle_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) = 0; | |
11fdf7f2 | 145 | RGWGetDataCB() {} |
7c673cae | 146 | virtual ~RGWGetDataCB() {} |
7c673cae FG |
147 | }; |
148 | ||
149 | struct RGWCloneRangeInfo { | |
150 | rgw_obj src; | |
151 | off_t src_ofs; | |
152 | off_t dst_ofs; | |
153 | uint64_t len; | |
154 | }; | |
155 | ||
7c673cae FG |
156 | struct RGWObjState { |
157 | rgw_obj obj; | |
9f95a23c TL |
158 | bool is_atomic{false}; |
159 | bool has_attrs{false}; | |
160 | bool exists{false}; | |
161 | uint64_t size{0}; //< size of raw object | |
7c673cae FG |
162 | uint64_t accounted_size{0}; //< size before compression, encryption |
163 | ceph::real_time mtime; | |
9f95a23c | 164 | uint64_t epoch{0}; |
7c673cae | 165 | bufferlist obj_tag; |
181888fb | 166 | bufferlist tail_tag; |
7c673cae | 167 | string write_tag; |
9f95a23c TL |
168 | bool fake_tag{false}; |
169 | std::optional<RGWObjManifest> manifest; | |
7c673cae | 170 | string shadow_obj; |
9f95a23c | 171 | bool has_data{false}; |
7c673cae | 172 | bufferlist data; |
9f95a23c TL |
173 | bool prefetch_data{false}; |
174 | bool keep_tail{false}; | |
175 | bool is_olh{false}; | |
7c673cae | 176 | bufferlist olh_tag; |
9f95a23c TL |
177 | uint64_t pg_ver{false}; |
178 | uint32_t zone_short_id{0}; | |
7c673cae FG |
179 | |
180 | /* important! don't forget to update copy constructor */ | |
181 | ||
182 | RGWObjVersionTracker objv_tracker; | |
183 | ||
184 | map<string, bufferlist> attrset; | |
9f95a23c TL |
185 | |
186 | RGWObjState(); | |
187 | RGWObjState(const RGWObjState& rhs); | |
188 | ~RGWObjState(); | |
7c673cae FG |
189 | |
190 | bool get_attr(string name, bufferlist& dest) { | |
191 | map<string, bufferlist>::iterator iter = attrset.find(name); | |
192 | if (iter != attrset.end()) { | |
193 | dest = iter->second; | |
194 | return true; | |
195 | } | |
196 | return false; | |
197 | } | |
198 | }; | |
199 | ||
9f95a23c TL |
200 | class RGWFetchObjFilter { |
201 | public: | |
202 | virtual ~RGWFetchObjFilter() {} | |
203 | ||
204 | virtual int filter(CephContext *cct, | |
205 | const rgw_obj_key& source_key, | |
206 | const RGWBucketInfo& dest_bucket_info, | |
207 | std::optional<rgw_placement_rule> dest_placement_rule, | |
208 | const map<string, bufferlist>& obj_attrs, | |
209 | std::optional<rgw_user> *poverride_owner, | |
210 | const rgw_placement_rule **prule) = 0; | |
211 | }; | |
212 | ||
213 | class RGWFetchObjFilter_Default : public RGWFetchObjFilter { | |
214 | protected: | |
215 | rgw_placement_rule dest_rule; | |
216 | public: | |
217 | RGWFetchObjFilter_Default() {} | |
218 | ||
219 | int filter(CephContext *cct, | |
220 | const rgw_obj_key& source_key, | |
221 | const RGWBucketInfo& dest_bucket_info, | |
222 | std::optional<rgw_placement_rule> dest_placement_rule, | |
223 | const map<string, bufferlist>& obj_attrs, | |
224 | std::optional<rgw_user> *poverride_owner, | |
225 | const rgw_placement_rule **prule) override; | |
226 | }; | |
227 | ||
228 | class RGWObjectCtx { | |
229 | rgw::sal::RGWRadosStore *store; | |
230 | ceph::shared_mutex lock = ceph::make_shared_mutex("RGWObjectCtx"); | |
231 | void *s{nullptr}; | |
232 | ||
233 | std::map<rgw_obj, RGWObjState> objs_state; | |
234 | public: | |
235 | explicit RGWObjectCtx(rgw::sal::RGWRadosStore *_store) : store(_store) {} | |
236 | explicit RGWObjectCtx(rgw::sal::RGWRadosStore *_store, void *_s) : store(_store), s(_s) {} | |
237 | ||
238 | void *get_private() { | |
239 | return s; | |
240 | } | |
241 | ||
242 | rgw::sal::RGWRadosStore *get_store() { | |
243 | return store; | |
244 | } | |
245 | ||
246 | RGWObjState *get_state(const rgw_obj& obj); | |
247 | ||
248 | void set_atomic(rgw_obj& obj); | |
249 | void set_prefetch_data(const rgw_obj& obj); | |
250 | void invalidate(const rgw_obj& obj); | |
251 | }; | |
252 | ||
253 | ||
7c673cae FG |
254 | struct RGWRawObjState { |
255 | rgw_raw_obj obj; | |
256 | bool has_attrs{false}; | |
257 | bool exists{false}; | |
258 | uint64_t size{0}; | |
259 | ceph::real_time mtime; | |
11fdf7f2 | 260 | uint64_t epoch{0}; |
7c673cae FG |
261 | bufferlist obj_tag; |
262 | bool has_data{false}; | |
263 | bufferlist data; | |
264 | bool prefetch_data{false}; | |
265 | uint64_t pg_ver{0}; | |
266 | ||
267 | /* important! don't forget to update copy constructor */ | |
268 | ||
269 | RGWObjVersionTracker objv_tracker; | |
270 | ||
271 | map<string, bufferlist> attrset; | |
272 | RGWRawObjState() {} | |
273 | RGWRawObjState(const RGWRawObjState& rhs) : obj (rhs.obj) { | |
274 | has_attrs = rhs.has_attrs; | |
275 | exists = rhs.exists; | |
276 | size = rhs.size; | |
277 | mtime = rhs.mtime; | |
278 | epoch = rhs.epoch; | |
279 | if (rhs.obj_tag.length()) { | |
280 | obj_tag = rhs.obj_tag; | |
281 | } | |
282 | has_data = rhs.has_data; | |
283 | if (rhs.data.length()) { | |
284 | data = rhs.data; | |
285 | } | |
286 | prefetch_data = rhs.prefetch_data; | |
287 | pg_ver = rhs.pg_ver; | |
288 | objv_tracker = rhs.objv_tracker; | |
289 | } | |
290 | }; | |
291 | ||
292 | struct RGWPoolIterCtx { | |
293 | librados::IoCtx io_ctx; | |
294 | librados::NObjectIterator iter; | |
295 | }; | |
296 | ||
297 | struct RGWListRawObjsCtx { | |
298 | bool initialized; | |
299 | RGWPoolIterCtx iter_ctx; | |
300 | ||
301 | RGWListRawObjsCtx() : initialized(false) {} | |
302 | }; | |
303 | ||
7c673cae FG |
304 | struct objexp_hint_entry { |
305 | string tenant; | |
306 | string bucket_name; | |
307 | string bucket_id; | |
308 | rgw_obj_key obj_key; | |
309 | ceph::real_time exp_time; | |
310 | ||
311 | void encode(bufferlist& bl) const { | |
312 | ENCODE_START(2, 1, bl); | |
11fdf7f2 TL |
313 | encode(bucket_name, bl); |
314 | encode(bucket_id, bl); | |
315 | encode(obj_key, bl); | |
316 | encode(exp_time, bl); | |
317 | encode(tenant, bl); | |
7c673cae FG |
318 | ENCODE_FINISH(bl); |
319 | } | |
320 | ||
11fdf7f2 | 321 | void decode(bufferlist::const_iterator& bl) { |
7c673cae FG |
322 | // XXX Do we want DECODE_START_LEGACY_COMPAT_LEN(2, 1, 1, bl); ? |
323 | DECODE_START(2, bl); | |
11fdf7f2 TL |
324 | decode(bucket_name, bl); |
325 | decode(bucket_id, bl); | |
326 | decode(obj_key, bl); | |
327 | decode(exp_time, bl); | |
7c673cae | 328 | if (struct_v >= 2) { |
11fdf7f2 | 329 | decode(tenant, bl); |
7c673cae FG |
330 | } else { |
331 | tenant.clear(); | |
332 | } | |
333 | DECODE_FINISH(bl); | |
334 | } | |
9f95a23c TL |
335 | |
336 | void dump(Formatter *f) const; | |
337 | static void generate_test_instances(list<objexp_hint_entry*>& o); | |
7c673cae FG |
338 | }; |
339 | WRITE_CLASS_ENCODER(objexp_hint_entry) | |
340 | ||
7c673cae FG |
341 | class RGWDataChangesLog; |
342 | class RGWMetaSyncStatusManager; | |
343 | class RGWDataSyncStatusManager; | |
7c673cae | 344 | class RGWCoroutinesManagerRegistry; |
7c673cae FG |
345 | |
346 | class RGWGetBucketStats_CB : public RefCountedObject { | |
347 | protected: | |
348 | rgw_bucket bucket; | |
349 | map<RGWObjCategory, RGWStorageStats> *stats; | |
350 | public: | |
224ce89b | 351 | explicit RGWGetBucketStats_CB(const rgw_bucket& _bucket) : bucket(_bucket), stats(NULL) {} |
7c673cae FG |
352 | ~RGWGetBucketStats_CB() override {} |
353 | virtual void handle_response(int r) = 0; | |
354 | virtual void set_response(map<RGWObjCategory, RGWStorageStats> *_stats) { | |
355 | stats = _stats; | |
356 | } | |
357 | }; | |
358 | ||
359 | class RGWGetUserStats_CB : public RefCountedObject { | |
360 | protected: | |
361 | rgw_user user; | |
362 | RGWStorageStats stats; | |
363 | public: | |
364 | explicit RGWGetUserStats_CB(const rgw_user& _user) : user(_user) {} | |
365 | ~RGWGetUserStats_CB() override {} | |
366 | virtual void handle_response(int r) = 0; | |
367 | virtual void set_response(RGWStorageStats& _stats) { | |
368 | stats = _stats; | |
369 | } | |
370 | }; | |
371 | ||
372 | class RGWGetDirHeader_CB; | |
373 | class RGWGetUserHeader_CB; | |
f67539c2 TL |
374 | namespace rgw { namespace sal { |
375 | class RGWRadosStore; | |
376 | class MPRadosSerializer; | |
377 | class LCRadosSerializer; | |
378 | } } | |
7c673cae | 379 | |
7c673cae FG |
380 | class RGWAsyncRadosProcessor; |
381 | ||
382 | template <class T> | |
383 | class RGWChainedCacheImpl; | |
384 | ||
385 | struct bucket_info_entry { | |
386 | RGWBucketInfo info; | |
387 | real_time mtime; | |
388 | map<string, bufferlist> attrs; | |
389 | }; | |
390 | ||
9f95a23c | 391 | struct tombstone_entry; |
7c673cae | 392 | |
9f95a23c TL |
393 | template <class K, class V> |
394 | class lru_map; | |
395 | using tombstone_cache_t = lru_map<rgw_obj, tombstone_entry>; | |
7c673cae | 396 | |
31f18b77 FG |
397 | class RGWIndexCompletionManager; |
398 | ||
9f95a23c | 399 | class RGWRados |
7c673cae FG |
400 | { |
401 | friend class RGWGC; | |
402 | friend class RGWMetaNotifier; | |
403 | friend class RGWDataNotifier; | |
7c673cae FG |
404 | friend class RGWObjectExpirer; |
405 | friend class RGWMetaSyncProcessorThread; | |
406 | friend class RGWDataSyncProcessorThread; | |
31f18b77 FG |
407 | friend class RGWReshard; |
408 | friend class RGWBucketReshard; | |
f64942e4 | 409 | friend class RGWBucketReshardLock; |
31f18b77 | 410 | friend class BucketIndexLockGuard; |
f67539c2 TL |
411 | friend class rgw::sal::MPRadosSerializer; |
412 | friend class rgw::sal::LCRadosSerializer; | |
413 | friend class rgw::sal::RGWRadosStore; | |
7c673cae FG |
414 | |
415 | /** Open the pool used as root for this gateway */ | |
b3b6e05e TL |
416 | int open_root_pool_ctx(const DoutPrefixProvider *dpp); |
417 | int open_gc_pool_ctx(const DoutPrefixProvider *dpp); | |
418 | int open_lc_pool_ctx(const DoutPrefixProvider *dpp); | |
419 | int open_objexp_pool_ctx(const DoutPrefixProvider *dpp); | |
420 | int open_reshard_pool_ctx(const DoutPrefixProvider *dpp); | |
421 | int open_notif_pool_ctx(const DoutPrefixProvider *dpp); | |
422 | ||
423 | int open_pool_ctx(const DoutPrefixProvider *dpp, const rgw_pool& pool, librados::IoCtx& io_ctx, | |
494da23a | 424 | bool mostly_omap); |
7c673cae | 425 | |
a4b75251 | 426 | |
9f95a23c | 427 | ceph::mutex lock = ceph::make_mutex("rados_timer_lock"); |
7c673cae FG |
428 | SafeTimer *timer; |
429 | ||
9f95a23c | 430 | rgw::sal::RGWRadosStore *store; |
522d829b | 431 | RGWGC *gc = nullptr; |
7c673cae FG |
432 | RGWLC *lc; |
433 | RGWObjectExpirer *obj_expirer; | |
434 | bool use_gc_thread; | |
435 | bool use_lc_thread; | |
436 | bool quota_threads; | |
437 | bool run_sync_thread; | |
31f18b77 | 438 | bool run_reshard_thread; |
7c673cae | 439 | |
7c673cae FG |
440 | RGWMetaNotifier *meta_notifier; |
441 | RGWDataNotifier *data_notifier; | |
442 | RGWMetaSyncProcessorThread *meta_sync_processor_thread; | |
11fdf7f2 | 443 | RGWSyncTraceManager *sync_tracer = nullptr; |
9f95a23c | 444 | map<rgw_zone_id, RGWDataSyncProcessorThread *> data_sync_processor_threads; |
7c673cae | 445 | |
b32b8144 | 446 | boost::optional<rgw::BucketTrimManager> bucket_trim; |
7c673cae FG |
447 | RGWSyncLogTrimThread *sync_log_trimmer{nullptr}; |
448 | ||
9f95a23c TL |
449 | ceph::mutex meta_sync_thread_lock = ceph::make_mutex("meta_sync_thread_lock"); |
450 | ceph::mutex data_sync_thread_lock = ceph::make_mutex("data_sync_thread_lock"); | |
7c673cae | 451 | |
7c673cae | 452 | librados::IoCtx root_pool_ctx; // .rgw |
11fdf7f2 TL |
453 | |
454 | double inject_notify_timeout_probability = 0; | |
455 | unsigned max_notify_retries = 0; | |
7c673cae FG |
456 | |
457 | friend class RGWWatcher; | |
458 | ||
9f95a23c | 459 | ceph::mutex bucket_id_lock = ceph::make_mutex("rados_bucket_id"); |
7c673cae FG |
460 | |
461 | // This field represents the number of bucket index object shards | |
462 | uint32_t bucket_index_max_shards; | |
463 | ||
b3b6e05e TL |
464 | int get_obj_head_ref(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_rados_ref *ref); |
465 | int get_system_obj_ref(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, rgw_rados_ref *ref); | |
7c673cae FG |
466 | uint64_t max_bucket_id; |
467 | ||
b3b6e05e | 468 | int get_olh_target_state(const DoutPrefixProvider *dpp, RGWObjectCtx& rctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, |
9f95a23c | 469 | RGWObjState *olh_state, RGWObjState **target_state, optional_yield y); |
b3b6e05e | 470 | int get_obj_state_impl(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWObjState **state, |
9f95a23c | 471 | bool follow_olh, optional_yield y, bool assume_noent = false); |
b3b6e05e | 472 | int append_atomic_test(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, |
9f95a23c | 473 | librados::ObjectOperation& op, RGWObjState **state, optional_yield y); |
b3b6e05e | 474 | int append_atomic_test(const DoutPrefixProvider *dpp, const RGWObjState* astate, librados::ObjectOperation& op); |
7c673cae FG |
475 | |
476 | int update_placement_map(); | |
477 | int store_bucket_info(RGWBucketInfo& info, map<string, bufferlist> *pattrs, RGWObjVersionTracker *objv_tracker, bool exclusive); | |
478 | ||
479 | void remove_rgw_head_obj(librados::ObjectWriteOperation& op); | |
480 | void cls_obj_check_prefix_exist(librados::ObjectOperation& op, const string& prefix, bool fail_if_exist); | |
481 | void cls_obj_check_mtime(librados::ObjectOperation& op, const real_time& mtime, bool high_precision_time, RGWCheckMTimeType type); | |
482 | protected: | |
483 | CephContext *cct; | |
484 | ||
494da23a | 485 | librados::Rados rados; |
7c673cae FG |
486 | |
487 | using RGWChainedCacheImpl_bucket_info_entry = RGWChainedCacheImpl<bucket_info_entry>; | |
488 | RGWChainedCacheImpl_bucket_info_entry *binfo_cache; | |
489 | ||
7c673cae FG |
490 | tombstone_cache_t *obj_tombstone_cache; |
491 | ||
492 | librados::IoCtx gc_pool_ctx; // .rgw.gc | |
493 | librados::IoCtx lc_pool_ctx; // .rgw.lc | |
494 | librados::IoCtx objexp_pool_ctx; | |
31f18b77 | 495 | librados::IoCtx reshard_pool_ctx; |
f67539c2 | 496 | librados::IoCtx notif_pool_ctx; // .rgw.notif |
7c673cae | 497 | |
11fdf7f2 | 498 | bool pools_initialized; |
7c673cae | 499 | |
11fdf7f2 | 500 | RGWQuotaHandler *quota_handler; |
7c673cae | 501 | |
11fdf7f2 | 502 | RGWCoroutinesManagerRegistry *cr_registry; |
7c673cae | 503 | |
11fdf7f2 TL |
504 | RGWSyncModuleInstanceRef sync_module; |
505 | bool writeable_zone{false}; | |
7c673cae | 506 | |
11fdf7f2 | 507 | RGWIndexCompletionManager *index_completion_manager{nullptr}; |
7c673cae | 508 | |
11fdf7f2 | 509 | bool use_cache{false}; |
522d829b | 510 | bool use_gc{true}; |
f67539c2 | 511 | |
b3b6e05e | 512 | int get_obj_head_ioctx(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, librados::IoCtx *ioctx); |
11fdf7f2 | 513 | public: |
9f95a23c | 514 | RGWRados(): timer(NULL), |
11fdf7f2 | 515 | gc(NULL), lc(NULL), obj_expirer(NULL), use_gc_thread(false), use_lc_thread(false), quota_threads(false), |
9f95a23c | 516 | run_sync_thread(false), run_reshard_thread(false), meta_notifier(NULL), |
11fdf7f2 | 517 | data_notifier(NULL), meta_sync_processor_thread(NULL), |
11fdf7f2 TL |
518 | bucket_index_max_shards(0), |
519 | max_bucket_id(0), cct(NULL), | |
11fdf7f2 TL |
520 | binfo_cache(NULL), obj_tombstone_cache(nullptr), |
521 | pools_initialized(false), | |
522 | quota_handler(NULL), | |
523 | cr_registry(NULL), | |
9f95a23c TL |
524 | pctl(&ctl), |
525 | reshard(NULL) {} | |
7c673cae | 526 | |
11fdf7f2 TL |
527 | RGWRados& set_use_cache(bool status) { |
528 | use_cache = status; | |
529 | return *this; | |
7c673cae FG |
530 | } |
531 | ||
522d829b TL |
532 | RGWRados& set_use_gc(bool status) { |
533 | use_gc = status; | |
534 | return *this; | |
535 | } | |
536 | ||
11fdf7f2 TL |
537 | RGWLC *get_lc() { |
538 | return lc; | |
7c673cae FG |
539 | } |
540 | ||
11fdf7f2 TL |
541 | RGWRados& set_run_gc_thread(bool _use_gc_thread) { |
542 | use_gc_thread = _use_gc_thread; | |
543 | return *this; | |
7c673cae FG |
544 | } |
545 | ||
11fdf7f2 TL |
546 | RGWRados& set_run_lc_thread(bool _use_lc_thread) { |
547 | use_lc_thread = _use_lc_thread; | |
548 | return *this; | |
7c673cae FG |
549 | } |
550 | ||
11fdf7f2 TL |
551 | RGWRados& set_run_quota_threads(bool _run_quota_threads) { |
552 | quota_threads = _run_quota_threads; | |
553 | return *this; | |
7c673cae FG |
554 | } |
555 | ||
11fdf7f2 TL |
556 | RGWRados& set_run_sync_thread(bool _run_sync_thread) { |
557 | run_sync_thread = _run_sync_thread; | |
558 | return *this; | |
7c673cae FG |
559 | } |
560 | ||
11fdf7f2 TL |
561 | RGWRados& set_run_reshard_thread(bool _run_reshard_thread) { |
562 | run_reshard_thread = _run_reshard_thread; | |
563 | return *this; | |
7c673cae FG |
564 | } |
565 | ||
11fdf7f2 | 566 | uint64_t get_new_req_id() { |
a4b75251 | 567 | return ceph::util::generate_random_number<uint64_t>(); |
7c673cae FG |
568 | } |
569 | ||
11fdf7f2 TL |
570 | librados::IoCtx* get_lc_pool_ctx() { |
571 | return &lc_pool_ctx; | |
7c673cae | 572 | } |
f67539c2 TL |
573 | |
574 | librados::IoCtx& get_notif_pool_ctx() { | |
575 | return notif_pool_ctx; | |
576 | } | |
577 | ||
11fdf7f2 TL |
578 | void set_context(CephContext *_cct) { |
579 | cct = _cct; | |
7c673cae | 580 | } |
9f95a23c TL |
581 | void set_store(rgw::sal::RGWRadosStore *_store) { |
582 | store = _store; | |
583 | } | |
31f18b77 | 584 | |
11fdf7f2 | 585 | RGWServices svc; |
9f95a23c TL |
586 | RGWCtl ctl; |
587 | ||
588 | RGWCtl *pctl{nullptr}; | |
11fdf7f2 TL |
589 | |
590 | /** | |
591 | * AmazonS3 errors contain a HostId string, but is an opaque base64 blob; we | |
592 | * try to be more transparent. This has a wrapper so we can update it when zonegroup/zone are changed. | |
593 | */ | |
594 | string host_id; | |
31f18b77 | 595 | |
31f18b77 FG |
596 | RGWReshard *reshard; |
597 | std::shared_ptr<RGWReshardWait> reshard_wait; | |
598 | ||
7c673cae FG |
599 | virtual ~RGWRados() = default; |
600 | ||
601 | tombstone_cache_t *get_tombstone_cache() { | |
602 | return obj_tombstone_cache; | |
603 | } | |
7c673cae FG |
604 | const RGWSyncModuleInstanceRef& get_sync_module() { |
605 | return sync_module; | |
606 | } | |
11fdf7f2 TL |
607 | RGWSyncTraceManager *get_sync_tracer() { |
608 | return sync_tracer; | |
609 | } | |
7c673cae | 610 | |
b3b6e05e | 611 | int get_required_alignment(const DoutPrefixProvider *dpp, const rgw_pool& pool, uint64_t *alignment); |
11fdf7f2 | 612 | void get_max_aligned_size(uint64_t size, uint64_t alignment, uint64_t *max_size); |
b3b6e05e TL |
613 | int get_max_chunk_size(const rgw_pool& pool, uint64_t *max_chunk_size, const DoutPrefixProvider *dpp, uint64_t *palignment = nullptr); |
614 | int get_max_chunk_size(const rgw_placement_rule& placement_rule, const rgw_obj& obj, uint64_t *max_chunk_size, const DoutPrefixProvider *dpp, uint64_t *palignment = nullptr); | |
7c673cae FG |
615 | |
616 | uint32_t get_max_bucket_shards() { | |
9f95a23c | 617 | return RGWSI_BucketIndex_RADOS::shards_max(); |
7c673cae FG |
618 | } |
619 | ||
181888fb | 620 | |
b3b6e05e | 621 | int get_raw_obj_ref(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, rgw_rados_ref *ref); |
7c673cae | 622 | |
b3b6e05e TL |
623 | int list_raw_objects_init(const DoutPrefixProvider *dpp, const rgw_pool& pool, const string& marker, RGWListRawObjsCtx *ctx); |
624 | int list_raw_objects_next(const DoutPrefixProvider *dpp, const string& prefix_filter, int max, | |
181888fb FG |
625 | RGWListRawObjsCtx& ctx, list<string>& oids, |
626 | bool *is_truncated); | |
b3b6e05e | 627 | int list_raw_objects(const DoutPrefixProvider *dpp, const rgw_pool& pool, const string& prefix_filter, int max, |
7c673cae FG |
628 | RGWListRawObjsCtx& ctx, list<string>& oids, |
629 | bool *is_truncated); | |
181888fb | 630 | string list_raw_objs_get_cursor(RGWListRawObjsCtx& ctx); |
7c673cae | 631 | |
7c673cae FG |
632 | CephContext *ctx() { return cct; } |
633 | /** do all necessary setup of the storage device */ | |
b3b6e05e | 634 | int initialize(CephContext *_cct, const DoutPrefixProvider *dpp) { |
7c673cae | 635 | set_context(_cct); |
b3b6e05e | 636 | return initialize(dpp); |
7c673cae FG |
637 | } |
638 | /** Initialize the RADOS instance and prepare to do other ops */ | |
b3b6e05e TL |
639 | int init_svc(bool raw, const DoutPrefixProvider *dpp); |
640 | int init_ctl(const DoutPrefixProvider *dpp); | |
11fdf7f2 | 641 | int init_rados(); |
b3b6e05e TL |
642 | int init_complete(const DoutPrefixProvider *dpp); |
643 | int initialize(const DoutPrefixProvider *dpp); | |
7c673cae FG |
644 | void finalize(); |
645 | ||
224ce89b | 646 | int register_to_service_map(const string& daemon_type, const map<string, string>& meta); |
11fdf7f2 | 647 | int update_service_map(std::map<std::string, std::string>&& status); |
7c673cae FG |
648 | |
649 | /// list logs | |
b3b6e05e | 650 | int log_list_init(const DoutPrefixProvider *dpp, const string& prefix, RGWAccessHandle *handle); |
7c673cae FG |
651 | int log_list_next(RGWAccessHandle handle, string *name); |
652 | ||
653 | /// remove log | |
b3b6e05e | 654 | int log_remove(const DoutPrefixProvider *dpp, const string& name); |
7c673cae FG |
655 | |
656 | /// show log | |
b3b6e05e | 657 | int log_show_init(const DoutPrefixProvider *dpp, const string& name, RGWAccessHandle *handle); |
7c673cae FG |
658 | int log_show_next(RGWAccessHandle handle, rgw_log_entry *entry); |
659 | ||
660 | // log bandwidth info | |
b3b6e05e TL |
661 | int log_usage(const DoutPrefixProvider *dpp, map<rgw_user_bucket, RGWUsageBatch>& usage_info); |
662 | int read_usage(const DoutPrefixProvider *dpp, const rgw_user& user, const string& bucket_name, uint64_t start_epoch, uint64_t end_epoch, | |
11fdf7f2 TL |
663 | uint32_t max_entries, bool *is_truncated, RGWUsageIter& read_iter, map<rgw_user_bucket, |
664 | rgw_usage_log_entry>& usage); | |
b3b6e05e TL |
665 | int trim_usage(const DoutPrefixProvider *dpp, const rgw_user& user, const string& bucket_name, uint64_t start_epoch, uint64_t end_epoch); |
666 | int clear_usage(const DoutPrefixProvider *dpp); | |
7c673cae | 667 | |
b3b6e05e | 668 | int create_pool(const DoutPrefixProvider *dpp, const rgw_pool& pool); |
7c673cae | 669 | |
7c673cae FG |
670 | void create_bucket_id(string *bucket_id); |
671 | ||
11fdf7f2 TL |
672 | bool get_obj_data_pool(const rgw_placement_rule& placement_rule, const rgw_obj& obj, rgw_pool *pool); |
673 | bool obj_to_raw(const rgw_placement_rule& placement_rule, const rgw_obj& obj, rgw_raw_obj *raw_obj); | |
7c673cae | 674 | |
11fdf7f2 | 675 | int create_bucket(const RGWUserInfo& owner, rgw_bucket& bucket, |
f67539c2 TL |
676 | const string& zonegroup_id, |
677 | const rgw_placement_rule& placement_rule, | |
678 | const string& swift_ver_location, | |
679 | const RGWQuotaInfo * pquota_info, | |
680 | map<std::string,bufferlist>& attrs, | |
681 | RGWBucketInfo& bucket_info, | |
682 | obj_version *pobjv, | |
683 | obj_version *pep_objv, | |
684 | ceph::real_time creation_time, | |
685 | rgw_bucket *master_bucket, | |
686 | uint32_t *master_num_shards, | |
687 | optional_yield y, | |
b3b6e05e | 688 | const DoutPrefixProvider *dpp, |
f67539c2 | 689 | bool exclusive = true); |
7c673cae FG |
690 | |
691 | RGWCoroutinesManagerRegistry *get_cr_registry() { return cr_registry; } | |
692 | ||
7c673cae FG |
693 | struct BucketShard { |
694 | RGWRados *store; | |
695 | rgw_bucket bucket; | |
696 | int shard_id; | |
9f95a23c | 697 | RGWSI_RADOS::Obj bucket_obj; |
7c673cae FG |
698 | |
699 | explicit BucketShard(RGWRados *_store) : store(_store), shard_id(-1) {} | |
b3b6e05e TL |
700 | int init(const rgw_bucket& _bucket, const rgw_obj& obj, RGWBucketInfo* out, const DoutPrefixProvider *dpp); |
701 | int init(const rgw_bucket& _bucket, int sid, const rgw::bucket_index_layout_generation& idx_layout, RGWBucketInfo* out, const DoutPrefixProvider *dpp); | |
702 | int init(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj); | |
703 | int init(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout, int sid); | |
7c673cae FG |
704 | }; |
705 | ||
706 | class Object { | |
707 | RGWRados *store; | |
708 | RGWBucketInfo bucket_info; | |
709 | RGWObjectCtx& ctx; | |
710 | rgw_obj obj; | |
711 | ||
712 | BucketShard bs; | |
713 | ||
714 | RGWObjState *state; | |
715 | ||
716 | bool versioning_disabled; | |
717 | ||
718 | bool bs_initialized; | |
719 | ||
720 | protected: | |
b3b6e05e | 721 | int get_state(const DoutPrefixProvider *dpp, RGWObjState **pstate, bool follow_olh, optional_yield y, bool assume_noent = false); |
7c673cae FG |
722 | void invalidate_state(); |
723 | ||
b3b6e05e | 724 | int prepare_atomic_modification(const DoutPrefixProvider *dpp, librados::ObjectWriteOperation& op, bool reset_obj, const string *ptag, |
9f95a23c | 725 | const char *ifmatch, const char *ifnomatch, bool removal_op, bool modify_tail, optional_yield y); |
b3b6e05e | 726 | int complete_atomic_modification(const DoutPrefixProvider *dpp); |
7c673cae FG |
727 | |
728 | public: | |
729 | Object(RGWRados *_store, const RGWBucketInfo& _bucket_info, RGWObjectCtx& _ctx, const rgw_obj& _obj) : store(_store), bucket_info(_bucket_info), | |
730 | ctx(_ctx), obj(_obj), bs(store), | |
731 | state(NULL), versioning_disabled(false), | |
732 | bs_initialized(false) {} | |
733 | ||
734 | RGWRados *get_store() { return store; } | |
735 | rgw_obj& get_obj() { return obj; } | |
736 | RGWObjectCtx& get_ctx() { return ctx; } | |
737 | RGWBucketInfo& get_bucket_info() { return bucket_info; } | |
b3b6e05e | 738 | int get_manifest(const DoutPrefixProvider *dpp, RGWObjManifest **pmanifest, optional_yield y); |
7c673cae | 739 | |
b3b6e05e | 740 | int get_bucket_shard(BucketShard **pbs, const DoutPrefixProvider *dpp) { |
7c673cae | 741 | if (!bs_initialized) { |
f64942e4 | 742 | int r = |
b3b6e05e | 743 | bs.init(bucket_info.bucket, obj, nullptr /* no RGWBucketInfo */, dpp); |
7c673cae FG |
744 | if (r < 0) { |
745 | return r; | |
746 | } | |
747 | bs_initialized = true; | |
748 | } | |
749 | *pbs = &bs; | |
750 | return 0; | |
751 | } | |
752 | ||
753 | void set_versioning_disabled(bool status) { | |
754 | versioning_disabled = status; | |
755 | } | |
756 | ||
757 | bool versioning_enabled() { | |
758 | return (!versioning_disabled && bucket_info.versioning_enabled()); | |
759 | } | |
760 | ||
761 | struct Read { | |
762 | RGWRados::Object *source; | |
763 | ||
764 | struct GetObjState { | |
11fdf7f2 TL |
765 | map<rgw_pool, librados::IoCtx> io_ctxs; |
766 | rgw_pool cur_pool; | |
767 | librados::IoCtx *cur_ioctx{nullptr}; | |
7c673cae FG |
768 | rgw_obj obj; |
769 | rgw_raw_obj head_obj; | |
770 | } state; | |
771 | ||
772 | struct ConditionParams { | |
773 | const ceph::real_time *mod_ptr; | |
774 | const ceph::real_time *unmod_ptr; | |
775 | bool high_precision_time; | |
776 | uint32_t mod_zone_id; | |
777 | uint64_t mod_pg_ver; | |
778 | const char *if_match; | |
779 | const char *if_nomatch; | |
780 | ||
781 | ConditionParams() : | |
782 | mod_ptr(NULL), unmod_ptr(NULL), high_precision_time(false), mod_zone_id(0), mod_pg_ver(0), | |
783 | if_match(NULL), if_nomatch(NULL) {} | |
784 | } conds; | |
785 | ||
786 | struct Params { | |
787 | ceph::real_time *lastmod; | |
788 | uint64_t *obj_size; | |
789 | map<string, bufferlist> *attrs; | |
eafe8130 | 790 | rgw_obj *target_obj; |
7c673cae | 791 | |
eafe8130 TL |
792 | Params() : lastmod(nullptr), obj_size(nullptr), attrs(nullptr), |
793 | target_obj(nullptr) {} | |
7c673cae FG |
794 | } params; |
795 | ||
796 | explicit Read(RGWRados::Object *_source) : source(_source) {} | |
797 | ||
b3b6e05e | 798 | int prepare(optional_yield y, const DoutPrefixProvider *dpp); |
7c673cae | 799 | static int range_to_ofs(uint64_t obj_size, int64_t &ofs, int64_t &end); |
b3b6e05e TL |
800 | int read(int64_t ofs, int64_t end, bufferlist& bl, optional_yield y, const DoutPrefixProvider *dpp); |
801 | int iterate(const DoutPrefixProvider *dpp, int64_t ofs, int64_t end, RGWGetDataCB *cb, optional_yield y); | |
802 | int get_attr(const DoutPrefixProvider *dpp, const char *name, bufferlist& dest, optional_yield y); | |
7c673cae FG |
803 | }; |
804 | ||
805 | struct Write { | |
806 | RGWRados::Object *target; | |
807 | ||
808 | struct MetaParams { | |
809 | ceph::real_time *mtime; | |
810 | map<std::string, bufferlist>* rmattrs; | |
811 | const bufferlist *data; | |
812 | RGWObjManifest *manifest; | |
813 | const string *ptag; | |
814 | list<rgw_obj_index_key> *remove_objs; | |
815 | ceph::real_time set_mtime; | |
816 | rgw_user owner; | |
817 | RGWObjCategory category; | |
818 | int flags; | |
819 | const char *if_match; | |
820 | const char *if_nomatch; | |
11fdf7f2 | 821 | std::optional<uint64_t> olh_epoch; |
7c673cae FG |
822 | ceph::real_time delete_at; |
823 | bool canceled; | |
824 | const string *user_data; | |
31f18b77 | 825 | rgw_zone_set *zones_trace; |
181888fb | 826 | bool modify_tail; |
3efd9988 | 827 | bool completeMultipart; |
11fdf7f2 | 828 | bool appendable; |
7c673cae FG |
829 | |
830 | MetaParams() : mtime(NULL), rmattrs(NULL), data(NULL), manifest(NULL), ptag(NULL), | |
11fdf7f2 | 831 | remove_objs(NULL), category(RGWObjCategory::Main), flags(0), |
91327a77 | 832 | if_match(NULL), if_nomatch(NULL), canceled(false), user_data(nullptr), zones_trace(nullptr), |
11fdf7f2 | 833 | modify_tail(false), completeMultipart(false), appendable(false) {} |
7c673cae FG |
834 | } meta; |
835 | ||
836 | explicit Write(RGWRados::Object *_target) : target(_target) {} | |
837 | ||
b3b6e05e TL |
838 | int _do_write_meta(const DoutPrefixProvider *dpp, |
839 | uint64_t size, uint64_t accounted_size, | |
7c673cae | 840 | map<std::string, bufferlist>& attrs, |
181888fb | 841 | bool modify_tail, bool assume_noent, |
9f95a23c | 842 | void *index_op, optional_yield y); |
b3b6e05e | 843 | int write_meta(const DoutPrefixProvider *dpp, uint64_t size, uint64_t accounted_size, |
9f95a23c | 844 | map<std::string, bufferlist>& attrs, optional_yield y); |
7c673cae | 845 | int write_data(const char *data, uint64_t ofs, uint64_t len, bool exclusive); |
11fdf7f2 TL |
846 | const req_state* get_req_state() { |
847 | return (req_state *)target->get_ctx().get_private(); | |
848 | } | |
7c673cae FG |
849 | }; |
850 | ||
851 | struct Delete { | |
852 | RGWRados::Object *target; | |
853 | ||
854 | struct DeleteParams { | |
855 | rgw_user bucket_owner; | |
b3b6e05e TL |
856 | int versioning_status; // versioning flags in enum RGWBucketFlags |
857 | ACLOwner obj_owner; // needed for creation of deletion marker | |
7c673cae FG |
858 | uint64_t olh_epoch; |
859 | string marker_version_id; | |
860 | uint32_t bilog_flags; | |
861 | list<rgw_obj_index_key> *remove_objs; | |
862 | ceph::real_time expiration_time; | |
863 | ceph::real_time unmod_since; | |
864 | ceph::real_time mtime; /* for setting delete marker mtime */ | |
865 | bool high_precision_time; | |
31f18b77 | 866 | rgw_zone_set *zones_trace; |
9f95a23c TL |
867 | bool abortmp; |
868 | uint64_t parts_accounted_size; | |
7c673cae | 869 | |
9f95a23c | 870 | DeleteParams() : versioning_status(0), olh_epoch(0), bilog_flags(0), remove_objs(NULL), high_precision_time(false), zones_trace(nullptr), abortmp(false), parts_accounted_size(0) {} |
7c673cae FG |
871 | } params; |
872 | ||
873 | struct DeleteResult { | |
874 | bool delete_marker; | |
875 | string version_id; | |
876 | ||
877 | DeleteResult() : delete_marker(false) {} | |
878 | } result; | |
879 | ||
880 | explicit Delete(RGWRados::Object *_target) : target(_target) {} | |
881 | ||
b3b6e05e | 882 | int delete_obj(optional_yield y, const DoutPrefixProvider *dpp); |
7c673cae FG |
883 | }; |
884 | ||
885 | struct Stat { | |
886 | RGWRados::Object *source; | |
887 | ||
888 | struct Result { | |
889 | rgw_obj obj; | |
9f95a23c TL |
890 | std::optional<RGWObjManifest> manifest; |
891 | uint64_t size{0}; | |
892 | struct timespec mtime {}; | |
7c673cae | 893 | map<string, bufferlist> attrs; |
7c673cae FG |
894 | } result; |
895 | ||
896 | struct State { | |
897 | librados::IoCtx io_ctx; | |
898 | librados::AioCompletion *completion; | |
899 | int ret; | |
900 | ||
901 | State() : completion(NULL), ret(0) {} | |
902 | } state; | |
903 | ||
904 | ||
905 | explicit Stat(RGWRados::Object *_source) : source(_source) {} | |
906 | ||
b3b6e05e | 907 | int stat_async(const DoutPrefixProvider *dpp); |
7c673cae FG |
908 | int wait(); |
909 | int stat(); | |
910 | private: | |
911 | int finish(); | |
912 | }; | |
913 | }; | |
914 | ||
915 | class Bucket { | |
916 | RGWRados *store; | |
917 | RGWBucketInfo bucket_info; | |
918 | rgw_bucket& bucket; | |
919 | int shard_id; | |
920 | ||
921 | public: | |
922 | Bucket(RGWRados *_store, const RGWBucketInfo& _bucket_info) : store(_store), bucket_info(_bucket_info), bucket(bucket_info.bucket), | |
923 | shard_id(RGW_NO_SHARD) {} | |
924 | RGWRados *get_store() { return store; } | |
925 | rgw_bucket& get_bucket() { return bucket; } | |
926 | RGWBucketInfo& get_bucket_info() { return bucket_info; } | |
927 | ||
b3b6e05e | 928 | int update_bucket_id(const string& new_bucket_id, const DoutPrefixProvider *dpp); |
31f18b77 | 929 | |
7c673cae FG |
930 | int get_shard_id() { return shard_id; } |
931 | void set_shard_id(int id) { | |
932 | shard_id = id; | |
933 | } | |
934 | ||
935 | class UpdateIndex { | |
936 | RGWRados::Bucket *target; | |
937 | string optag; | |
938 | rgw_obj obj; | |
939 | uint16_t bilog_flags{0}; | |
940 | BucketShard bs; | |
941 | bool bs_initialized{false}; | |
942 | bool blind; | |
943 | bool prepared{false}; | |
31f18b77 FG |
944 | rgw_zone_set *zones_trace{nullptr}; |
945 | ||
b3b6e05e | 946 | int init_bs(const DoutPrefixProvider *dpp) { |
f64942e4 | 947 | int r = |
b3b6e05e | 948 | bs.init(target->get_bucket(), obj, nullptr /* no RGWBucketInfo */, dpp); |
31f18b77 FG |
949 | if (r < 0) { |
950 | return r; | |
951 | } | |
952 | bs_initialized = true; | |
953 | return 0; | |
954 | } | |
955 | ||
956 | void invalidate_bs() { | |
957 | bs_initialized = false; | |
958 | } | |
959 | ||
b3b6e05e | 960 | int guard_reshard(const DoutPrefixProvider *dpp, BucketShard **pbs, std::function<int(BucketShard *)> call); |
7c673cae FG |
961 | public: |
962 | ||
963 | UpdateIndex(RGWRados::Bucket *_target, const rgw_obj& _obj) : target(_target), obj(_obj), | |
964 | bs(target->get_store()) { | |
f67539c2 | 965 | blind = (target->get_bucket_info().layout.current_index.layout.type == rgw::BucketIndexType::Indexless); |
7c673cae FG |
966 | } |
967 | ||
b3b6e05e | 968 | int get_bucket_shard(BucketShard **pbs, const DoutPrefixProvider *dpp) { |
7c673cae | 969 | if (!bs_initialized) { |
b3b6e05e | 970 | int r = init_bs(dpp); |
7c673cae FG |
971 | if (r < 0) { |
972 | return r; | |
973 | } | |
7c673cae FG |
974 | } |
975 | *pbs = &bs; | |
976 | return 0; | |
977 | } | |
978 | ||
979 | void set_bilog_flags(uint16_t flags) { | |
980 | bilog_flags = flags; | |
981 | } | |
31f18b77 FG |
982 | |
983 | void set_zones_trace(rgw_zone_set *_zones_trace) { | |
984 | zones_trace = _zones_trace; | |
985 | } | |
7c673cae | 986 | |
b3b6e05e TL |
987 | int prepare(const DoutPrefixProvider *dpp, RGWModifyOp, const string *write_tag, optional_yield y); |
988 | int complete(const DoutPrefixProvider *dpp, int64_t poolid, uint64_t epoch, uint64_t size, | |
7c673cae FG |
989 | uint64_t accounted_size, ceph::real_time& ut, |
990 | const string& etag, const string& content_type, | |
11fdf7f2 | 991 | const string& storage_class, |
7c673cae | 992 | bufferlist *acl_bl, RGWObjCategory category, |
11fdf7f2 | 993 | list<rgw_obj_index_key> *remove_objs, const string *user_data = nullptr, bool appendable = false); |
b3b6e05e TL |
994 | int complete_del(const DoutPrefixProvider *dpp, |
995 | int64_t poolid, uint64_t epoch, | |
7c673cae FG |
996 | ceph::real_time& removed_mtime, /* mtime of removed object */ |
997 | list<rgw_obj_index_key> *remove_objs); | |
b3b6e05e | 998 | int cancel(const DoutPrefixProvider *dpp); |
7c673cae FG |
999 | |
1000 | const string *get_optag() { return &optag; } | |
1001 | ||
1002 | bool is_prepared() { return prepared; } | |
1adf2230 AA |
1003 | }; // class UpdateIndex |
1004 | ||
1005 | class List { | |
1006 | protected: | |
eafe8130 TL |
1007 | // absolute maximum number of objects that |
1008 | // list_objects_(un)ordered can return | |
1009 | static constexpr int64_t bucket_list_objects_absolute_max = 25000; | |
7c673cae | 1010 | |
7c673cae FG |
1011 | RGWRados::Bucket *target; |
1012 | rgw_obj_key next_marker; | |
1013 | ||
b3b6e05e TL |
1014 | int list_objects_ordered(const DoutPrefixProvider *dpp, |
1015 | int64_t max, | |
1adf2230 AA |
1016 | vector<rgw_bucket_dir_entry> *result, |
1017 | map<string, bool> *common_prefixes, | |
9f95a23c TL |
1018 | bool *is_truncated, |
1019 | optional_yield y); | |
b3b6e05e TL |
1020 | int list_objects_unordered(const DoutPrefixProvider *dpp, |
1021 | int64_t max, | |
1adf2230 AA |
1022 | vector<rgw_bucket_dir_entry> *result, |
1023 | map<string, bool> *common_prefixes, | |
9f95a23c TL |
1024 | bool *is_truncated, |
1025 | optional_yield y); | |
1adf2230 AA |
1026 | |
1027 | public: | |
1028 | ||
7c673cae FG |
1029 | struct Params { |
1030 | string prefix; | |
1031 | string delim; | |
1032 | rgw_obj_key marker; | |
1033 | rgw_obj_key end_marker; | |
1034 | string ns; | |
1035 | bool enforce_ns; | |
1036 | RGWAccessListFilter *filter; | |
1037 | bool list_versions; | |
1adf2230 AA |
1038 | bool allow_unordered; |
1039 | ||
1040 | Params() : | |
1041 | enforce_ns(true), | |
1042 | filter(NULL), | |
1043 | list_versions(false), | |
1044 | allow_unordered(false) | |
1045 | {} | |
7c673cae FG |
1046 | } params; |
1047 | ||
7c673cae FG |
1048 | explicit List(RGWRados::Bucket *_target) : target(_target) {} |
1049 | ||
b3b6e05e | 1050 | int list_objects(const DoutPrefixProvider *dpp, int64_t max, |
1adf2230 AA |
1051 | vector<rgw_bucket_dir_entry> *result, |
1052 | map<string, bool> *common_prefixes, | |
9f95a23c TL |
1053 | bool *is_truncated, |
1054 | optional_yield y) { | |
1adf2230 | 1055 | if (params.allow_unordered) { |
b3b6e05e | 1056 | return list_objects_unordered(dpp, max, result, common_prefixes, |
9f95a23c | 1057 | is_truncated, y); |
1adf2230 | 1058 | } else { |
b3b6e05e | 1059 | return list_objects_ordered(dpp, max, result, common_prefixes, |
9f95a23c | 1060 | is_truncated, y); |
1adf2230 AA |
1061 | } |
1062 | } | |
7c673cae FG |
1063 | rgw_obj_key& get_next_marker() { |
1064 | return next_marker; | |
1065 | } | |
1adf2230 AA |
1066 | }; // class List |
1067 | }; // class Bucket | |
7c673cae | 1068 | |
b3b6e05e TL |
1069 | int on_last_entry_in_listing(const DoutPrefixProvider *dpp, |
1070 | RGWBucketInfo& bucket_info, | |
7c673cae FG |
1071 | const std::string& obj_prefix, |
1072 | const std::string& obj_delim, | |
1073 | std::function<int(const rgw_bucket_dir_entry&)> handler); | |
1074 | ||
f67539c2 | 1075 | bool swift_versioning_enabled(rgw::sal::RGWBucket* bucket) const; |
7c673cae FG |
1076 | |
1077 | int swift_versioning_copy(RGWObjectCtx& obj_ctx, /* in/out */ | |
1078 | const rgw_user& user, /* in */ | |
f67539c2 TL |
1079 | rgw::sal::RGWBucket* bucket, /* in */ |
1080 | rgw::sal::RGWObject* obj, /* in */ | |
9f95a23c TL |
1081 | const DoutPrefixProvider *dpp, /* in/out */ |
1082 | optional_yield y); /* in */ | |
1083 | int swift_versioning_restore(RGWObjectCtx& obj_ctx, /* in/out */ | |
7c673cae | 1084 | const rgw_user& user, /* in */ |
f67539c2 TL |
1085 | rgw::sal::RGWBucket* bucket, /* in */ |
1086 | rgw::sal::RGWObject* obj, /* in */ | |
9f95a23c TL |
1087 | bool& restored, /* out */ |
1088 | const DoutPrefixProvider *dpp); /* in/out */ | |
b3b6e05e TL |
1089 | int copy_obj_to_remote_dest(const DoutPrefixProvider *dpp, |
1090 | RGWObjState *astate, | |
7c673cae FG |
1091 | map<string, bufferlist>& src_attrs, |
1092 | RGWRados::Object::Read& read_op, | |
1093 | const rgw_user& user_id, | |
f67539c2 | 1094 | rgw::sal::RGWObject* dest_obj, |
7c673cae FG |
1095 | ceph::real_time *mtime); |
1096 | ||
1097 | enum AttrsMod { | |
1098 | ATTRSMOD_NONE = 0, | |
1099 | ATTRSMOD_REPLACE = 1, | |
1100 | ATTRSMOD_MERGE = 2 | |
1101 | }; | |
1102 | ||
f67539c2 | 1103 | int rewrite_obj(RGWBucketInfo& dest_bucket_info, rgw::sal::RGWObject* obj, const DoutPrefixProvider *dpp, optional_yield y); |
7c673cae | 1104 | |
b3b6e05e TL |
1105 | int stat_remote_obj(const DoutPrefixProvider *dpp, |
1106 | RGWObjectCtx& obj_ctx, | |
7c673cae | 1107 | const rgw_user& user_id, |
7c673cae | 1108 | req_info *info, |
9f95a23c | 1109 | const rgw_zone_id& source_zone, |
f67539c2 | 1110 | rgw::sal::RGWObject* src_obj, |
9f95a23c | 1111 | const RGWBucketInfo *src_bucket_info, |
7c673cae FG |
1112 | real_time *src_mtime, |
1113 | uint64_t *psize, | |
1114 | const real_time *mod_ptr, | |
1115 | const real_time *unmod_ptr, | |
1116 | bool high_precision_time, | |
1117 | const char *if_match, | |
1118 | const char *if_nomatch, | |
1119 | map<string, bufferlist> *pattrs, | |
11fdf7f2 | 1120 | map<string, string> *pheaders, |
7c673cae FG |
1121 | string *version_id, |
1122 | string *ptag, | |
1123 | string *petag); | |
1124 | ||
1125 | int fetch_remote_obj(RGWObjectCtx& obj_ctx, | |
1126 | const rgw_user& user_id, | |
7c673cae | 1127 | req_info *info, |
9f95a23c | 1128 | const rgw_zone_id& source_zone, |
f67539c2 TL |
1129 | rgw::sal::RGWObject* dest_obj, |
1130 | rgw::sal::RGWObject* src_obj, | |
1131 | rgw::sal::RGWBucket* dest_bucket, | |
1132 | rgw::sal::RGWBucket* src_bucket, | |
11fdf7f2 | 1133 | std::optional<rgw_placement_rule> dest_placement, |
7c673cae FG |
1134 | ceph::real_time *src_mtime, |
1135 | ceph::real_time *mtime, | |
1136 | const ceph::real_time *mod_ptr, | |
1137 | const ceph::real_time *unmod_ptr, | |
1138 | bool high_precision_time, | |
1139 | const char *if_match, | |
1140 | const char *if_nomatch, | |
1141 | AttrsMod attrs_mod, | |
1142 | bool copy_if_newer, | |
f67539c2 | 1143 | rgw::sal::RGWAttrs& attrs, |
7c673cae | 1144 | RGWObjCategory category, |
11fdf7f2 | 1145 | std::optional<uint64_t> olh_epoch, |
7c673cae | 1146 | ceph::real_time delete_at, |
7c673cae | 1147 | string *ptag, |
11fdf7f2 | 1148 | string *petag, |
7c673cae | 1149 | void (*progress_cb)(off_t, void *), |
31f18b77 | 1150 | void *progress_data, |
9f95a23c TL |
1151 | const DoutPrefixProvider *dpp, |
1152 | RGWFetchObjFilter *filter, | |
81eedcae TL |
1153 | rgw_zone_set *zones_trace= nullptr, |
1154 | std::optional<uint64_t>* bytes_transferred = 0); | |
7c673cae FG |
1155 | /** |
1156 | * Copy an object. | |
1157 | * dest_obj: the object to copy into | |
1158 | * src_obj: the object to copy from | |
1159 | * attrs: usage depends on attrs_mod parameter | |
1160 | * attrs_mod: the modification mode of the attrs, may have the following values: | |
1161 | * ATTRSMOD_NONE - the attributes of the source object will be | |
1162 | * copied without modifications, attrs parameter is ignored; | |
1163 | * ATTRSMOD_REPLACE - new object will have the attributes provided by attrs | |
1164 | * parameter, source object attributes are not copied; | |
1165 | * ATTRSMOD_MERGE - any conflicting meta keys on the source object's attributes | |
1166 | * are overwritten by values contained in attrs parameter. | |
7c673cae FG |
1167 | * Returns: 0 on success, -ERR# otherwise. |
1168 | */ | |
1169 | int copy_obj(RGWObjectCtx& obj_ctx, | |
1170 | const rgw_user& user_id, | |
7c673cae | 1171 | req_info *info, |
9f95a23c | 1172 | const rgw_zone_id& source_zone, |
f67539c2 TL |
1173 | rgw::sal::RGWObject* dest_obj, |
1174 | rgw::sal::RGWObject* src_obj, | |
1175 | rgw::sal::RGWBucket* dest_bucket, | |
1176 | rgw::sal::RGWBucket* src_bucket, | |
11fdf7f2 | 1177 | const rgw_placement_rule& dest_placement, |
7c673cae FG |
1178 | ceph::real_time *src_mtime, |
1179 | ceph::real_time *mtime, | |
1180 | const ceph::real_time *mod_ptr, | |
1181 | const ceph::real_time *unmod_ptr, | |
1182 | bool high_precision_time, | |
1183 | const char *if_match, | |
1184 | const char *if_nomatch, | |
1185 | AttrsMod attrs_mod, | |
1186 | bool copy_if_newer, | |
1187 | map<std::string, bufferlist>& attrs, | |
1188 | RGWObjCategory category, | |
1189 | uint64_t olh_epoch, | |
1190 | ceph::real_time delete_at, | |
1191 | string *version_id, | |
1192 | string *ptag, | |
11fdf7f2 | 1193 | string *petag, |
7c673cae | 1194 | void (*progress_cb)(off_t, void *), |
9f95a23c TL |
1195 | void *progress_data, |
1196 | const DoutPrefixProvider *dpp, | |
1197 | optional_yield y); | |
7c673cae FG |
1198 | |
1199 | int copy_obj_data(RGWObjectCtx& obj_ctx, | |
f67539c2 | 1200 | rgw::sal::RGWBucket* bucket, |
11fdf7f2 | 1201 | const rgw_placement_rule& dest_placement, |
7c673cae | 1202 | RGWRados::Object::Read& read_op, off_t end, |
f67539c2 | 1203 | rgw::sal::RGWObject* dest_obj, |
7c673cae FG |
1204 | ceph::real_time *mtime, |
1205 | ceph::real_time set_mtime, | |
1206 | map<string, bufferlist>& attrs, | |
7c673cae FG |
1207 | uint64_t olh_epoch, |
1208 | ceph::real_time delete_at, | |
9f95a23c TL |
1209 | string *petag, |
1210 | const DoutPrefixProvider *dpp, | |
1211 | optional_yield y); | |
7c673cae | 1212 | |
11fdf7f2 | 1213 | int transition_obj(RGWObjectCtx& obj_ctx, |
f67539c2 TL |
1214 | rgw::sal::RGWBucket* bucket, |
1215 | rgw::sal::RGWObject& obj, | |
11fdf7f2 TL |
1216 | const rgw_placement_rule& placement_rule, |
1217 | const real_time& mtime, | |
9f95a23c TL |
1218 | uint64_t olh_epoch, |
1219 | const DoutPrefixProvider *dpp, | |
1220 | optional_yield y); | |
11fdf7f2 | 1221 | |
b3b6e05e | 1222 | int check_bucket_empty(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, optional_yield y); |
7c673cae FG |
1223 | |
1224 | /** | |
1225 | * Delete a bucket. | |
1226 | * bucket: the name of the bucket to delete | |
1227 | * Returns 0 on success, -ERR# otherwise. | |
1228 | */ | |
b3b6e05e | 1229 | int delete_bucket(RGWBucketInfo& bucket_info, RGWObjVersionTracker& objv_tracker, optional_yield y, const DoutPrefixProvider *dpp, bool check_empty = true); |
7c673cae | 1230 | |
7c673cae | 1231 | void wakeup_meta_sync_shards(set<int>& shard_ids); |
9f95a23c | 1232 | void wakeup_data_sync_shards(const rgw_zone_id& source_zone, map<int, set<string> >& shard_ids); |
7c673cae FG |
1233 | |
1234 | RGWMetaSyncStatusManager* get_meta_sync_manager(); | |
9f95a23c | 1235 | RGWDataSyncStatusManager* get_data_sync_manager(const rgw_zone_id& source_zone); |
7c673cae | 1236 | |
b3b6e05e TL |
1237 | int set_bucket_owner(rgw_bucket& bucket, ACLOwner& owner, const DoutPrefixProvider *dpp); |
1238 | int set_buckets_enabled(std::vector<rgw_bucket>& buckets, bool enabled, const DoutPrefixProvider *dpp); | |
1239 | int bucket_suspended(const DoutPrefixProvider *dpp, rgw_bucket& bucket, bool *suspended); | |
7c673cae FG |
1240 | |
1241 | /** Delete an object.*/ | |
b3b6e05e TL |
1242 | int delete_obj(const DoutPrefixProvider *dpp, |
1243 | RGWObjectCtx& obj_ctx, | |
f67539c2 TL |
1244 | const RGWBucketInfo& bucket_owner, |
1245 | const rgw_obj& src_obj, | |
b3b6e05e | 1246 | int versioning_status, // versioning flags in enum RGWBucketFlags |
f67539c2 TL |
1247 | uint16_t bilog_flags = 0, |
1248 | const ceph::real_time& expiration_time = ceph::real_time(), | |
1249 | rgw_zone_set *zones_trace = nullptr); | |
7c673cae | 1250 | |
b3b6e05e | 1251 | int delete_raw_obj(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj); |
7c673cae | 1252 | |
7c673cae | 1253 | /** Remove an object from the bucket index */ |
b3b6e05e | 1254 | int delete_obj_index(const rgw_obj& obj, ceph::real_time mtime, const DoutPrefixProvider *dpp); |
7c673cae | 1255 | |
7c673cae FG |
1256 | /** |
1257 | * Set an attr on an object. | |
1258 | * bucket: name of the bucket holding the object | |
1259 | * obj: name of the object to set the attr on | |
1260 | * name: the attr to set | |
1261 | * bl: the contents of the attr | |
1262 | * Returns: 0 on success, -ERR# otherwise. | |
1263 | */ | |
b3b6e05e | 1264 | int set_attr(const DoutPrefixProvider *dpp, void *ctx, const RGWBucketInfo& bucket_info, rgw_obj& obj, const char *name, bufferlist& bl); |
7c673cae | 1265 | |
b3b6e05e | 1266 | int set_attrs(const DoutPrefixProvider *dpp, void *ctx, const RGWBucketInfo& bucket_info, rgw_obj& obj, |
7c673cae | 1267 | map<string, bufferlist>& attrs, |
9f95a23c TL |
1268 | map<string, bufferlist>* rmattrs, |
1269 | optional_yield y); | |
7c673cae | 1270 | |
b3b6e05e | 1271 | int get_obj_state(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWObjState **state, |
9f95a23c | 1272 | bool follow_olh, optional_yield y, bool assume_noent = false); |
b3b6e05e TL |
1273 | int get_obj_state(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWObjState **state, optional_yield y) { |
1274 | return get_obj_state(dpp, rctx, bucket_info, obj, state, true, y); | |
7c673cae FG |
1275 | } |
1276 | ||
b3b6e05e | 1277 | using iterate_obj_cb = int (*)(const DoutPrefixProvider*, const rgw_raw_obj&, off_t, off_t, |
11fdf7f2 TL |
1278 | off_t, bool, RGWObjState*, void*); |
1279 | ||
b3b6e05e | 1280 | int iterate_obj(const DoutPrefixProvider *dpp, RGWObjectCtx& ctx, const RGWBucketInfo& bucket_info, |
11fdf7f2 | 1281 | const rgw_obj& obj, off_t ofs, off_t end, |
9f95a23c TL |
1282 | uint64_t max_chunk_size, iterate_obj_cb cb, void *arg, |
1283 | optional_yield y); | |
7c673cae | 1284 | |
b3b6e05e TL |
1285 | int get_obj_iterate_cb(const DoutPrefixProvider *dpp, |
1286 | const rgw_raw_obj& read_obj, off_t obj_ofs, | |
11fdf7f2 TL |
1287 | off_t read_ofs, off_t len, bool is_head_obj, |
1288 | RGWObjState *astate, void *arg); | |
7c673cae FG |
1289 | |
1290 | void get_obj_aio_completion_cb(librados::completion_t cb, void *arg); | |
1291 | ||
1292 | /** | |
1293 | * a simple object read without keeping state | |
1294 | */ | |
1295 | ||
b3b6e05e TL |
1296 | int raw_obj_stat(const DoutPrefixProvider *dpp, |
1297 | rgw_raw_obj& obj, uint64_t *psize, ceph::real_time *pmtime, uint64_t *epoch, | |
11fdf7f2 | 1298 | map<string, bufferlist> *attrs, bufferlist *first_chunk, |
9f95a23c | 1299 | RGWObjVersionTracker *objv_tracker, optional_yield y); |
7c673cae | 1300 | |
b3b6e05e TL |
1301 | int obj_operate(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, librados::ObjectWriteOperation *op); |
1302 | int obj_operate(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, librados::ObjectReadOperation *op); | |
7c673cae | 1303 | |
b3b6e05e TL |
1304 | int guard_reshard(const DoutPrefixProvider *dpp, |
1305 | BucketShard *bs, | |
f64942e4 AA |
1306 | const rgw_obj& obj_instance, |
1307 | const RGWBucketInfo& bucket_info, | |
1308 | std::function<int(BucketShard *)> call); | |
1309 | int block_while_resharding(RGWRados::BucketShard *bs, | |
1310 | string *new_bucket_id, | |
11fdf7f2 | 1311 | const RGWBucketInfo& bucket_info, |
b3b6e05e TL |
1312 | optional_yield y, |
1313 | const DoutPrefixProvider *dpp); | |
1314 | ||
1315 | void bucket_index_guard_olh_op(const DoutPrefixProvider *dpp, RGWObjState& olh_state, librados::ObjectOperation& op); | |
1316 | int olh_init_modification(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, RGWObjState& state, const rgw_obj& olh_obj, string *op_tag); | |
1317 | int olh_init_modification_impl(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, RGWObjState& state, const rgw_obj& olh_obj, string *op_tag); | |
1318 | int bucket_index_link_olh(const DoutPrefixProvider *dpp, | |
1319 | const RGWBucketInfo& bucket_info, RGWObjState& olh_state, | |
7c673cae FG |
1320 | const rgw_obj& obj_instance, bool delete_marker, |
1321 | const string& op_tag, struct rgw_bucket_dir_entry_meta *meta, | |
1322 | uint64_t olh_epoch, | |
91327a77 AA |
1323 | ceph::real_time unmod_since, bool high_precision_time, |
1324 | rgw_zone_set *zones_trace = nullptr, | |
1325 | bool log_data_change = false); | |
b3b6e05e TL |
1326 | int bucket_index_unlink_instance(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj_instance, const string& op_tag, const string& olh_tag, uint64_t olh_epoch, rgw_zone_set *zones_trace = nullptr); |
1327 | int bucket_index_read_olh_log(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, RGWObjState& state, const rgw_obj& obj_instance, uint64_t ver_marker, | |
7c673cae | 1328 | map<uint64_t, vector<rgw_bucket_olh_log_entry> > *log, bool *is_truncated); |
b3b6e05e TL |
1329 | int bucket_index_trim_olh_log(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, RGWObjState& obj_state, const rgw_obj& obj_instance, uint64_t ver); |
1330 | int bucket_index_clear_olh(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, RGWObjState& state, const rgw_obj& obj_instance); | |
1331 | int apply_olh_log(const DoutPrefixProvider *dpp, RGWObjectCtx& ctx, RGWObjState& obj_state, const RGWBucketInfo& bucket_info, const rgw_obj& obj, | |
7c673cae | 1332 | bufferlist& obj_tag, map<uint64_t, vector<rgw_bucket_olh_log_entry> >& log, |
31f18b77 | 1333 | uint64_t *plast_ver, rgw_zone_set *zones_trace = nullptr); |
b3b6e05e TL |
1334 | int update_olh(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, RGWObjState *state, const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_zone_set *zones_trace = nullptr); |
1335 | int set_olh(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, const RGWBucketInfo& bucket_info, const rgw_obj& target_obj, bool delete_marker, rgw_bucket_dir_entry_meta *meta, | |
91327a77 | 1336 | uint64_t olh_epoch, ceph::real_time unmod_since, bool high_precision_time, |
9f95a23c | 1337 | optional_yield y, rgw_zone_set *zones_trace = nullptr, bool log_data_change = false); |
b3b6e05e | 1338 | int repair_olh(const DoutPrefixProvider *dpp, RGWObjState* state, const RGWBucketInfo& bucket_info, |
a8e16298 | 1339 | const rgw_obj& obj); |
b3b6e05e | 1340 | int unlink_obj_instance(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, RGWBucketInfo& bucket_info, const rgw_obj& target_obj, |
9f95a23c | 1341 | uint64_t olh_epoch, optional_yield y, rgw_zone_set *zones_trace = nullptr); |
7c673cae FG |
1342 | |
1343 | void check_pending_olh_entries(map<string, bufferlist>& pending_entries, map<string, bufferlist> *rm_pending_entries); | |
b3b6e05e TL |
1344 | int remove_olh_pending_entries(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, RGWObjState& state, const rgw_obj& olh_obj, map<string, bufferlist>& pending_attrs); |
1345 | int follow_olh(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, RGWObjectCtx& ctx, RGWObjState *state, const rgw_obj& olh_obj, rgw_obj *target); | |
1346 | int get_olh(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWOLHInfo *olh); | |
7c673cae | 1347 | |
11fdf7f2 | 1348 | void gen_rand_obj_instance_name(rgw_obj_key *target_key); |
7c673cae FG |
1349 | void gen_rand_obj_instance_name(rgw_obj *target); |
1350 | ||
b3b6e05e TL |
1351 | int update_containers_stats(map<string, RGWBucketEnt>& m, const DoutPrefixProvider *dpp); |
1352 | int append_async(const DoutPrefixProvider *dpp, rgw_raw_obj& obj, size_t size, bufferlist& bl); | |
7c673cae | 1353 | |
11fdf7f2 | 1354 | public: |
7c673cae FG |
1355 | void set_atomic(void *ctx, rgw_obj& obj) { |
1356 | RGWObjectCtx *rctx = static_cast<RGWObjectCtx *>(ctx); | |
11fdf7f2 | 1357 | rctx->set_atomic(obj); |
7c673cae | 1358 | } |
11fdf7f2 | 1359 | void set_prefetch_data(void *ctx, const rgw_obj& obj) { |
7c673cae | 1360 | RGWObjectCtx *rctx = static_cast<RGWObjectCtx *>(ctx); |
11fdf7f2 | 1361 | rctx->set_prefetch_data(obj); |
7c673cae | 1362 | } |
7c673cae | 1363 | int decode_policy(bufferlist& bl, ACLOwner *owner); |
b3b6e05e | 1364 | int get_bucket_stats(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, int shard_id, string *bucket_ver, string *master_ver, |
c07f9fc5 | 1365 | map<RGWObjCategory, RGWStorageStats>& stats, string *max_marker, bool* syncstopped = NULL); |
b3b6e05e | 1366 | int get_bucket_stats_async(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, int shard_id, RGWGetBucketStats_CB *cb); |
3a9019d9 | 1367 | |
b3b6e05e | 1368 | int put_bucket_instance_info(RGWBucketInfo& info, bool exclusive, ceph::real_time mtime, map<string, bufferlist> *pattrs, const DoutPrefixProvider *dpp); |
9f95a23c | 1369 | /* xxx dang obj_ctx -> svc */ |
b3b6e05e TL |
1370 | int get_bucket_instance_info(RGWSysObjectCtx& obj_ctx, const string& meta_key, RGWBucketInfo& info, ceph::real_time *pmtime, map<string, bufferlist> *pattrs, optional_yield y, const DoutPrefixProvider *dpp); |
1371 | int get_bucket_instance_info(RGWSysObjectCtx& obj_ctx, const rgw_bucket& bucket, RGWBucketInfo& info, ceph::real_time *pmtime, map<string, bufferlist> *pattrs, optional_yield y, const DoutPrefixProvider *dpp); | |
3a9019d9 | 1372 | |
9f95a23c | 1373 | static void make_bucket_entry_name(const string& tenant_name, const string& bucket_name, string& bucket_entry); |
b32b8144 | 1374 | |
9f95a23c | 1375 | int get_bucket_info(RGWServices *svc, |
b32b8144 FG |
1376 | const string& tenant_name, const string& bucket_name, |
1377 | RGWBucketInfo& info, | |
b3b6e05e TL |
1378 | ceph::real_time *pmtime, optional_yield y, |
1379 | const DoutPrefixProvider *dpp, map<string, bufferlist> *pattrs = NULL); | |
b32b8144 | 1380 | |
81eedcae TL |
1381 | // Returns 0 on successful refresh. Returns error code if there was |
1382 | // an error or the version stored on the OSD is the same as that | |
b32b8144 FG |
1383 | // presented in the BucketInfo structure. |
1384 | // | |
1385 | int try_refresh_bucket_info(RGWBucketInfo& info, | |
1386 | ceph::real_time *pmtime, | |
b3b6e05e | 1387 | const DoutPrefixProvider *dpp, |
b32b8144 FG |
1388 | map<string, bufferlist> *pattrs = nullptr); |
1389 | ||
7c673cae | 1390 | int put_linked_bucket_info(RGWBucketInfo& info, bool exclusive, ceph::real_time mtime, obj_version *pep_objv, |
b3b6e05e TL |
1391 | map<string, bufferlist> *pattrs, bool create_entry_point, |
1392 | const DoutPrefixProvider *dpp); | |
7c673cae | 1393 | |
b3b6e05e | 1394 | int cls_obj_prepare_op(const DoutPrefixProvider *dpp, BucketShard& bs, RGWModifyOp op, string& tag, rgw_obj& obj, uint16_t bilog_flags, optional_yield y, rgw_zone_set *zones_trace = nullptr); |
31f18b77 FG |
1395 | int cls_obj_complete_op(BucketShard& bs, const rgw_obj& obj, RGWModifyOp op, string& tag, int64_t pool, uint64_t epoch, |
1396 | rgw_bucket_dir_entry& ent, RGWObjCategory category, list<rgw_obj_index_key> *remove_objs, uint16_t bilog_flags, rgw_zone_set *zones_trace = nullptr); | |
1397 | int cls_obj_complete_add(BucketShard& bs, const rgw_obj& obj, string& tag, int64_t pool, uint64_t epoch, rgw_bucket_dir_entry& ent, | |
1398 | RGWObjCategory category, list<rgw_obj_index_key> *remove_objs, uint16_t bilog_flags, rgw_zone_set *zones_trace = nullptr); | |
7c673cae | 1399 | int cls_obj_complete_del(BucketShard& bs, string& tag, int64_t pool, uint64_t epoch, rgw_obj& obj, |
31f18b77 FG |
1400 | ceph::real_time& removed_mtime, list<rgw_obj_index_key> *remove_objs, uint16_t bilog_flags, rgw_zone_set *zones_trace = nullptr); |
1401 | int cls_obj_complete_cancel(BucketShard& bs, string& tag, rgw_obj& obj, uint16_t bilog_flags, rgw_zone_set *zones_trace = nullptr); | |
b3b6e05e | 1402 | int cls_obj_set_bucket_tag_timeout(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, uint64_t timeout); |
9f95a23c TL |
1403 | |
1404 | using ent_map_t = | |
1405 | boost::container::flat_map<std::string, rgw_bucket_dir_entry>; | |
1406 | ||
1407 | using check_filter_t = bool (*)(const std::string&); | |
1408 | ||
b3b6e05e TL |
1409 | int cls_bucket_list_ordered(const DoutPrefixProvider *dpp, |
1410 | RGWBucketInfo& bucket_info, | |
9f95a23c TL |
1411 | const int shard_id, |
1412 | const rgw_obj_index_key& start_after, | |
11fdf7f2 | 1413 | const string& prefix, |
9f95a23c TL |
1414 | const string& delimiter, |
1415 | const uint32_t num_entries, | |
1416 | const bool list_versions, | |
1417 | const uint16_t exp_factor, // 0 means ignore | |
1418 | ent_map_t& m, | |
1419 | bool* is_truncated, | |
1420 | bool* cls_filtered, | |
1adf2230 | 1421 | rgw_obj_index_key *last_entry, |
9f95a23c TL |
1422 | optional_yield y, |
1423 | check_filter_t force_check_filter = nullptr); | |
b3b6e05e TL |
1424 | int cls_bucket_list_unordered(const DoutPrefixProvider *dpp, |
1425 | RGWBucketInfo& bucket_info, | |
9f95a23c TL |
1426 | int shard_id, |
1427 | const rgw_obj_index_key& start_after, | |
11fdf7f2 | 1428 | const string& prefix, |
9f95a23c TL |
1429 | uint32_t num_entries, |
1430 | bool list_versions, | |
1adf2230 | 1431 | vector<rgw_bucket_dir_entry>& ent_list, |
9f95a23c TL |
1432 | bool *is_truncated, |
1433 | rgw_obj_index_key *last_entry, | |
1434 | optional_yield y, | |
1435 | check_filter_t = nullptr); | |
b3b6e05e TL |
1436 | int cls_bucket_head(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, int shard_id, vector<rgw_bucket_dir_header>& headers, map<int, string> *bucket_instance_ids = NULL); |
1437 | int cls_bucket_head_async(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, int shard_id, RGWGetDirHeader_CB *ctx, int *num_aio); | |
7c673cae | 1438 | |
b3b6e05e TL |
1439 | int bi_get_instance(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_bucket_dir_entry *dirent); |
1440 | int bi_get_olh(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_bucket_olh_entry *olh); | |
1441 | int bi_get(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, BIIndexType index_type, rgw_cls_bi_entry *entry); | |
7c673cae FG |
1442 | void bi_put(librados::ObjectWriteOperation& op, BucketShard& bs, rgw_cls_bi_entry& entry); |
1443 | int bi_put(BucketShard& bs, rgw_cls_bi_entry& entry); | |
b3b6e05e TL |
1444 | int bi_put(const DoutPrefixProvider *dpp, rgw_bucket& bucket, rgw_obj& obj, rgw_cls_bi_entry& entry); |
1445 | int bi_list(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, int shard_id, const string& filter_obj, const string& marker, uint32_t max, list<rgw_cls_bi_entry> *entries, bool *is_truncated); | |
7c673cae | 1446 | int bi_list(BucketShard& bs, const string& filter_obj, const string& marker, uint32_t max, list<rgw_cls_bi_entry> *entries, bool *is_truncated); |
b3b6e05e | 1447 | int bi_list(const DoutPrefixProvider *dpp, rgw_bucket& bucket, const string& obj_name, const string& marker, uint32_t max, |
7c673cae FG |
1448 | list<rgw_cls_bi_entry> *entries, bool *is_truncated); |
1449 | int bi_remove(BucketShard& bs); | |
1450 | ||
b3b6e05e TL |
1451 | int cls_obj_usage_log_add(const DoutPrefixProvider *dpp, const string& oid, rgw_usage_log_info& info); |
1452 | int cls_obj_usage_log_read(const DoutPrefixProvider *dpp, const string& oid, const string& user, const string& bucket, uint64_t start_epoch, | |
11fdf7f2 TL |
1453 | uint64_t end_epoch, uint32_t max_entries, string& read_iter, map<rgw_user_bucket, |
1454 | rgw_usage_log_entry>& usage, bool *is_truncated); | |
b3b6e05e | 1455 | int cls_obj_usage_log_trim(const DoutPrefixProvider *dpp, const string& oid, const string& user, const string& bucket, uint64_t start_epoch, |
11fdf7f2 | 1456 | uint64_t end_epoch); |
b3b6e05e | 1457 | int cls_obj_usage_log_clear(const DoutPrefixProvider *dpp, string& oid); |
7c673cae | 1458 | |
f67539c2 | 1459 | int get_target_shard_id(const rgw::bucket_index_normal_layout& layout, const string& obj_key, int *shard_id); |
9f95a23c TL |
1460 | |
1461 | int lock_exclusive(const rgw_pool& pool, const string& oid, ceph::timespan& duration, rgw_zone_id& zone_id, string& owner_id); | |
1462 | int unlock(const rgw_pool& pool, const string& oid, rgw_zone_id& zone_id, string& owner_id); | |
7c673cae | 1463 | |
b3b6e05e | 1464 | void update_gc_chain(const DoutPrefixProvider *dpp, rgw_obj& head_obj, RGWObjManifest& manifest, cls_rgw_obj_chain *chain); |
9f95a23c | 1465 | int send_chain_to_gc(cls_rgw_obj_chain& chain, const string& tag); |
b3b6e05e TL |
1466 | void delete_objs_inline(const DoutPrefixProvider *dpp, cls_rgw_obj_chain& chain, const string& tag); |
1467 | int gc_operate(const DoutPrefixProvider *dpp, string& oid, librados::ObjectWriteOperation *op); | |
9f95a23c TL |
1468 | int gc_aio_operate(const std::string& oid, librados::AioCompletion *c, |
1469 | librados::ObjectWriteOperation *op); | |
b3b6e05e | 1470 | int gc_operate(const DoutPrefixProvider *dpp, string& oid, librados::ObjectReadOperation *op, bufferlist *pbl); |
7c673cae | 1471 | |
9f95a23c | 1472 | int list_gc_objs(int *index, string& marker, uint32_t max, bool expired_only, std::list<cls_rgw_gc_obj_info>& result, bool *truncated, bool& processing_queue); |
11fdf7f2 | 1473 | int process_gc(bool expired_only); |
b3b6e05e TL |
1474 | bool process_expire_objects(const DoutPrefixProvider *dpp); |
1475 | int defer_gc(const DoutPrefixProvider *dpp, void *ctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, optional_yield y); | |
7c673cae FG |
1476 | |
1477 | int process_lc(); | |
f6b5b4d7 | 1478 | int list_lc_progress(string& marker, uint32_t max_entries, |
f67539c2 | 1479 | vector<rgw::sal::Lifecycle::LCEntry>& progress_map, int& index); |
f6b5b4d7 | 1480 | |
b3b6e05e | 1481 | int bucket_check_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, |
7c673cae FG |
1482 | map<RGWObjCategory, RGWStorageStats> *existing_stats, |
1483 | map<RGWObjCategory, RGWStorageStats> *calculated_stats); | |
b3b6e05e TL |
1484 | int bucket_rebuild_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info); |
1485 | int bucket_set_reshard(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const cls_rgw_bucket_instance_entry& entry); | |
1486 | int remove_objs_from_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, list<rgw_obj_index_key>& oid_list); | |
1487 | int move_rados_obj(const DoutPrefixProvider *dpp, | |
1488 | librados::IoCtx& src_ioctx, | |
7c673cae FG |
1489 | const string& src_oid, const string& src_locator, |
1490 | librados::IoCtx& dst_ioctx, | |
1491 | const string& dst_oid, const string& dst_locator); | |
b3b6e05e TL |
1492 | int fix_head_obj_locator(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, bool copy_obj, bool remove_bad, rgw_obj_key& key); |
1493 | int fix_tail_obj_locator(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, rgw_obj_key& key, bool fix, bool *need_fix, optional_yield y); | |
7c673cae FG |
1494 | |
1495 | int check_quota(const rgw_user& bucket_owner, rgw_bucket& bucket, | |
f67539c2 TL |
1496 | RGWQuotaInfo& user_quota, RGWQuotaInfo& bucket_quota, uint64_t obj_size, |
1497 | optional_yield y, bool check_size_only = false); | |
7c673cae | 1498 | |
224ce89b | 1499 | int check_bucket_shards(const RGWBucketInfo& bucket_info, const rgw_bucket& bucket, |
b3b6e05e | 1500 | uint64_t num_objs, const DoutPrefixProvider *dpp); |
31f18b77 | 1501 | |
b3b6e05e | 1502 | int add_bucket_to_reshard(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, uint32_t new_num_shards); |
31f18b77 | 1503 | |
7c673cae | 1504 | uint64_t instance_id(); |
3efd9988 | 1505 | |
7c673cae FG |
1506 | librados::Rados* get_rados_handle(); |
1507 | ||
b3b6e05e TL |
1508 | int delete_raw_obj_aio(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, list<librados::AioCompletion *>& handles); |
1509 | int delete_obj_aio(const DoutPrefixProvider *dpp, const rgw_obj& obj, RGWBucketInfo& info, RGWObjState *astate, | |
9f95a23c TL |
1510 | list<librados::AioCompletion *>& handles, bool keep_index_consistent, |
1511 | optional_yield y); | |
11fdf7f2 | 1512 | |
11fdf7f2 | 1513 | private: |
7c673cae FG |
1514 | /** |
1515 | * Check the actual on-disk state of the object specified | |
1516 | * by list_state, and fill in the time and size of object. | |
1517 | * Then append any changes to suggested_updates for | |
1518 | * the rgw class' dir_suggest_changes function. | |
1519 | * | |
1520 | * Note that this can maul list_state; don't use it afterwards. Also | |
1521 | * it expects object to already be filled in from list_state; it only | |
1522 | * sets the size and mtime. | |
1523 | * | |
1524 | * Returns 0 on success, -ENOENT if the object doesn't exist on disk, | |
1525 | * and -errno on other failures. (-ENOENT is not a failure, and it | |
1526 | * will encode that info as a suggested update.) | |
1527 | */ | |
b3b6e05e TL |
1528 | int check_disk_state(const DoutPrefixProvider *dpp, |
1529 | librados::IoCtx io_ctx, | |
7c673cae FG |
1530 | const RGWBucketInfo& bucket_info, |
1531 | rgw_bucket_dir_entry& list_state, | |
1532 | rgw_bucket_dir_entry& object, | |
9f95a23c TL |
1533 | bufferlist& suggested_updates, |
1534 | optional_yield y); | |
7c673cae FG |
1535 | |
1536 | /** | |
1537 | * Init pool iteration | |
31f18b77 | 1538 | * pool: pool to use for the ctx initialization |
7c673cae FG |
1539 | * ctx: context object to use for the iteration |
1540 | * Returns: 0 on success, -ERR# otherwise. | |
1541 | */ | |
b3b6e05e | 1542 | int pool_iterate_begin(const DoutPrefixProvider *dpp, const rgw_pool& pool, RGWPoolIterCtx& ctx); |
31f18b77 | 1543 | |
181888fb FG |
1544 | /** |
1545 | * Init pool iteration | |
1546 | * pool: pool to use | |
1547 | * cursor: position to start iteration | |
1548 | * ctx: context object to use for the iteration | |
1549 | * Returns: 0 on success, -ERR# otherwise. | |
1550 | */ | |
b3b6e05e | 1551 | int pool_iterate_begin(const DoutPrefixProvider *dpp, const rgw_pool& pool, const string& cursor, RGWPoolIterCtx& ctx); |
181888fb FG |
1552 | |
1553 | /** | |
1554 | * Get pool iteration position | |
1555 | * ctx: context object to use for the iteration | |
1556 | * Returns: string representation of position | |
1557 | */ | |
1558 | string pool_iterate_get_cursor(RGWPoolIterCtx& ctx); | |
1559 | ||
7c673cae FG |
1560 | /** |
1561 | * Iterate over pool return object names, use optional filter | |
1562 | * ctx: iteration context, initialized with pool_iterate_begin() | |
1563 | * num: max number of objects to return | |
1564 | * objs: a vector that the results will append into | |
1565 | * is_truncated: if not NULL, will hold true iff iteration is complete | |
1566 | * filter: if not NULL, will be used to filter returned objects | |
1567 | * Returns: 0 on success, -ERR# otherwise. | |
1568 | */ | |
1569 | int pool_iterate(RGWPoolIterCtx& ctx, uint32_t num, vector<rgw_bucket_dir_entry>& objs, | |
1570 | bool *is_truncated, RGWAccessListFilter *filter); | |
1571 | ||
1572 | uint64_t next_bucket_id(); | |
11fdf7f2 | 1573 | |
9f95a23c TL |
1574 | /** |
1575 | * This is broken out to facilitate unit testing. | |
1576 | */ | |
1577 | static uint32_t calc_ordered_bucket_list_per_shard(uint32_t num_entries, | |
1578 | uint32_t num_shards); | |
11fdf7f2 TL |
1579 | }; |
1580 | ||
7c673cae | 1581 | #endif |