]>
Commit | Line | Data |
---|---|---|
1e59de90 TL |
1 | |
2 | // vim: ts=2 sw=2 expandtab ft=cpp | |
3 | ||
4 | /* | |
5 | * Ceph - scalable distributed file system | |
6 | * | |
7 | * SAL implementation for the CORTX Motr backend | |
8 | * | |
9 | * Copyright (C) 2021 Seagate Technology LLC and/or its Affiliates | |
10 | * | |
11 | * This is free software; you can redistribute it and/or | |
12 | * modify it under the terms of the GNU Lesser General Public | |
13 | * License version 2.1, as published by the Free Software | |
14 | * Foundation. See file COPYING. | |
15 | * | |
16 | */ | |
17 | ||
18 | #pragma once | |
19 | ||
20 | extern "C" { | |
21 | #pragma clang diagnostic push | |
22 | #pragma clang diagnostic ignored "-Wextern-c-compat" | |
23 | #pragma clang diagnostic ignored "-Wdeprecated-anon-enum-enum-conversion" | |
24 | #include "motr/config.h" | |
25 | #include "motr/client.h" | |
26 | #pragma clang diagnostic pop | |
27 | } | |
28 | ||
29 | #include "rgw_sal_store.h" | |
30 | #include "rgw_rados.h" | |
31 | #include "rgw_notify.h" | |
32 | #include "rgw_oidc_provider.h" | |
33 | #include "rgw_role.h" | |
34 | #include "rgw_multi.h" | |
35 | #include "rgw_putobj_processor.h" | |
36 | ||
37 | namespace rgw::sal { | |
38 | ||
39 | class MotrStore; | |
40 | ||
41 | // Global Motr indices | |
42 | #define RGW_MOTR_USERS_IDX_NAME "motr.rgw.users" | |
43 | #define RGW_MOTR_BUCKET_INST_IDX_NAME "motr.rgw.bucket.instances" | |
44 | #define RGW_MOTR_BUCKET_HD_IDX_NAME "motr.rgw.bucket.headers" | |
45 | #define RGW_IAM_MOTR_ACCESS_KEY "motr.rgw.accesskeys" | |
46 | #define RGW_IAM_MOTR_EMAIL_KEY "motr.rgw.emails" | |
47 | ||
48 | //#define RGW_MOTR_BUCKET_ACL_IDX_NAME "motr.rgw.bucket.acls" | |
49 | ||
50 | // A simplified metadata cache implementation. | |
51 | // Note: MotrObjMetaCache doesn't handle the IO operations to Motr. A proxy | |
52 | // class can be added to handle cache and 'real' ops. | |
53 | class MotrMetaCache | |
54 | { | |
55 | protected: | |
56 | // MGW re-uses ObjectCache to cache object's metadata as it has already | |
57 | // implemented a lru cache: (1) ObjectCache internally uses a map and lru | |
58 | // list to manage cache entry. POC uses object name, user name or bucket | |
59 | // name as the key to lookup and insert an entry. (2) ObjectCache::data is | |
60 | // a bufferlist and can be used to store any metadata structure, such as | |
61 | // object's bucket dir entry, user info or bucket instance. | |
62 | // | |
63 | // Note from RGW: | |
64 | // The Rados Gateway stores metadata and objects in an internal cache. This | |
65 | // should be kept consistent by the OSD's relaying notify events between | |
66 | // multiple watching RGW processes. In the event that this notification | |
67 | // protocol fails, bounding the length of time that any data in the cache will | |
68 | // be assumed valid will ensure that any RGW instance that falls out of sync | |
69 | // will eventually recover. This seems to be an issue mostly for large numbers | |
70 | // of RGW instances under heavy use. If you would like to turn off cache expiry, | |
71 | // set this value to zero. | |
72 | // | |
73 | // Currently POC hasn't implemented the watch-notify menchanism yet. So the | |
74 | // current implementation is similar to cortx-s3server which is based on expiry | |
75 | // time. TODO: see comments on distribute_cache). | |
76 | // | |
77 | // Beaware: Motr object data is not cached in current POC as RGW! | |
78 | // RGW caches the first chunk (4MB by default). | |
79 | ObjectCache cache; | |
80 | ||
81 | public: | |
82 | // Lookup a cache entry. | |
83 | int get(const DoutPrefixProvider *dpp, const std::string& name, bufferlist& data); | |
84 | ||
85 | // Insert a cache entry. | |
86 | int put(const DoutPrefixProvider *dpp, const std::string& name, const bufferlist& data); | |
87 | ||
88 | // Called when an object is deleted. Notification should be sent to other | |
89 | // RGW instances. | |
90 | int remove(const DoutPrefixProvider *dpp, const std::string& name); | |
91 | ||
92 | // Make the local cache entry invalid. | |
93 | void invalid(const DoutPrefixProvider *dpp, const std::string& name); | |
94 | ||
95 | // TODO: Distribute_cache() and watch_cb() now are only place holder functions. | |
96 | // Checkout services/svc_sys_obj_cache.h/cc for reference. | |
97 | // These 2 functions are designed to notify or to act on cache notification. | |
98 | // It is feasible to implement the functionality using Motr's FDMI after discussing | |
99 | // with Hua. | |
100 | int distribute_cache(const DoutPrefixProvider *dpp, | |
101 | const std::string& normal_name, | |
102 | ObjectCacheInfo& obj_info, int op); | |
103 | int watch_cb(const DoutPrefixProvider *dpp, | |
104 | uint64_t notify_id, | |
105 | uint64_t cookie, | |
106 | uint64_t notifier_id, | |
107 | bufferlist& bl); | |
108 | ||
109 | void set_enabled(bool status); | |
110 | ||
111 | MotrMetaCache(const DoutPrefixProvider *dpp, CephContext *cct) { | |
112 | cache.set_ctx(cct); | |
113 | } | |
114 | }; | |
115 | ||
116 | struct MotrUserInfo { | |
117 | RGWUserInfo info; | |
118 | obj_version user_version; | |
119 | rgw::sal::Attrs attrs; | |
120 | ||
121 | void encode(bufferlist& bl) const | |
122 | { | |
123 | ENCODE_START(3, 3, bl); | |
124 | encode(info, bl); | |
125 | encode(user_version, bl); | |
126 | encode(attrs, bl); | |
127 | ENCODE_FINISH(bl); | |
128 | } | |
129 | ||
130 | void decode(bufferlist::const_iterator& bl) | |
131 | { | |
132 | DECODE_START(3, bl); | |
133 | decode(info, bl); | |
134 | decode(user_version, bl); | |
135 | decode(attrs, bl); | |
136 | DECODE_FINISH(bl); | |
137 | } | |
138 | }; | |
139 | WRITE_CLASS_ENCODER(MotrUserInfo); | |
140 | ||
141 | struct MotrEmailInfo { | |
142 | std::string user_id; | |
143 | std::string email_id; | |
144 | ||
145 | MotrEmailInfo() {} | |
146 | MotrEmailInfo(std::string _user_id, std::string _email_id ) | |
147 | : user_id(std::move(_user_id)), email_id(std::move(_email_id)) {} | |
148 | ||
149 | void encode(bufferlist& bl) const { | |
150 | ENCODE_START(2, 2, bl); | |
151 | encode(user_id, bl); | |
152 | encode(email_id, bl); | |
153 | ENCODE_FINISH(bl); | |
154 | } | |
155 | ||
156 | void decode(bufferlist::const_iterator& bl) { | |
157 | DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl); | |
158 | decode(user_id, bl); | |
159 | decode(email_id, bl); | |
160 | DECODE_FINISH(bl); | |
161 | } | |
162 | }; | |
163 | WRITE_CLASS_ENCODER(MotrEmailInfo); | |
164 | ||
165 | struct MotrAccessKey { | |
166 | std::string id; // AccessKey | |
167 | std::string key; // SecretKey | |
168 | std::string user_id; // UserID | |
169 | ||
170 | MotrAccessKey() {} | |
171 | MotrAccessKey(std::string _id, std::string _key, std::string _user_id) | |
172 | : id(std::move(_id)), key(std::move(_key)), user_id(std::move(_user_id)) {} | |
173 | ||
174 | void encode(bufferlist& bl) const { | |
175 | ENCODE_START(2, 2, bl); | |
176 | encode(id, bl); | |
177 | encode(key, bl); | |
178 | encode(user_id, bl); | |
179 | ENCODE_FINISH(bl); | |
180 | } | |
181 | ||
182 | void decode(bufferlist::const_iterator& bl) { | |
183 | DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl); | |
184 | decode(id, bl); | |
185 | decode(key, bl); | |
186 | decode(user_id, bl); | |
187 | DECODE_FINISH(bl); | |
188 | } | |
189 | }; | |
190 | WRITE_CLASS_ENCODER(MotrAccessKey); | |
191 | ||
192 | class MotrNotification : public StoreNotification { | |
193 | public: | |
194 | MotrNotification(Object* _obj, Object* _src_obj, rgw::notify::EventType _type) : | |
195 | StoreNotification(_obj, _src_obj, _type) {} | |
196 | ~MotrNotification() = default; | |
197 | ||
198 | virtual int publish_reserve(const DoutPrefixProvider *dpp, RGWObjTags* obj_tags = nullptr) override { return 0;} | |
199 | virtual int publish_commit(const DoutPrefixProvider* dpp, uint64_t size, | |
200 | const ceph::real_time& mtime, const std::string& etag, const std::string& version) override { return 0; } | |
201 | }; | |
202 | ||
203 | class MotrUser : public StoreUser { | |
204 | private: | |
205 | MotrStore *store; | |
206 | struct m0_uint128 idxID = {0xe5ecb53640d4ecce, 0x6a156cd5a74aa3b8}; // MD5 of “motr.rgw.users“ | |
207 | struct m0_idx idx; | |
208 | ||
209 | public: | |
210 | std::set<std::string> access_key_tracker; | |
211 | MotrUser(MotrStore *_st, const rgw_user& _u) : StoreUser(_u), store(_st) { } | |
212 | MotrUser(MotrStore *_st, const RGWUserInfo& _i) : StoreUser(_i), store(_st) { } | |
213 | MotrUser(MotrStore *_st) : store(_st) { } | |
214 | MotrUser(MotrUser& _o) = default; | |
215 | MotrUser() {} | |
216 | ||
217 | virtual std::unique_ptr<User> clone() override { | |
218 | return std::unique_ptr<User>(new MotrUser(*this)); | |
219 | } | |
220 | int list_buckets(const DoutPrefixProvider *dpp, const std::string& marker, const std::string& end_marker, | |
221 | uint64_t max, bool need_stats, BucketList& buckets, optional_yield y) override; | |
222 | virtual int create_bucket(const DoutPrefixProvider* dpp, | |
223 | const rgw_bucket& b, | |
224 | const std::string& zonegroup_id, | |
225 | rgw_placement_rule& placement_rule, | |
226 | std::string& swift_ver_location, | |
227 | const RGWQuotaInfo* pquota_info, | |
228 | const RGWAccessControlPolicy& policy, | |
229 | Attrs& attrs, | |
230 | RGWBucketInfo& info, | |
231 | obj_version& ep_objv, | |
232 | bool exclusive, | |
233 | bool obj_lock_enabled, | |
234 | bool* existed, | |
235 | req_info& req_info, | |
236 | std::unique_ptr<Bucket>* bucket, | |
237 | optional_yield y) override; | |
238 | virtual int read_attrs(const DoutPrefixProvider* dpp, optional_yield y) override; | |
239 | virtual int merge_and_store_attrs(const DoutPrefixProvider* dpp, Attrs& new_attrs, optional_yield y) override; | |
240 | virtual int read_stats(const DoutPrefixProvider *dpp, | |
241 | optional_yield y, RGWStorageStats* stats, | |
242 | ceph::real_time *last_stats_sync = nullptr, | |
243 | ceph::real_time *last_stats_update = nullptr) override; | |
244 | virtual int read_stats_async(const DoutPrefixProvider *dpp, RGWGetUserStats_CB* cb) override; | |
245 | virtual int complete_flush_stats(const DoutPrefixProvider *dpp, optional_yield y) override; | |
246 | virtual int read_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch, uint64_t end_epoch, uint32_t max_entries, | |
247 | bool* is_truncated, RGWUsageIter& usage_iter, | |
248 | std::map<rgw_user_bucket, rgw_usage_log_entry>& usage) override; | |
249 | virtual int trim_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch, uint64_t end_epoch) override; | |
250 | ||
251 | virtual int load_user(const DoutPrefixProvider* dpp, optional_yield y) override; | |
252 | virtual int store_user(const DoutPrefixProvider* dpp, optional_yield y, bool exclusive, RGWUserInfo* old_info = nullptr) override; | |
253 | virtual int remove_user(const DoutPrefixProvider* dpp, optional_yield y) override; | |
254 | virtual int verify_mfa(const std::string& mfa_str, bool* verified, const DoutPrefixProvider* dpp, optional_yield y) override; | |
255 | ||
256 | int create_user_info_idx(); | |
257 | int load_user_from_idx(const DoutPrefixProvider *dpp, MotrStore *store, RGWUserInfo& info, std::map<std::string, | |
258 | bufferlist> *attrs, RGWObjVersionTracker *objv_tr); | |
259 | ||
260 | friend class MotrBucket; | |
261 | }; | |
262 | ||
263 | class MotrBucket : public StoreBucket { | |
264 | private: | |
265 | MotrStore *store; | |
266 | RGWAccessControlPolicy acls; | |
267 | ||
268 | // RGWBucketInfo and other information that are shown when listing a bucket is | |
269 | // represented in struct MotrBucketInfo. The structure is encoded and stored | |
270 | // as the value of the global bucket instance index. | |
271 | // TODO: compare pros and cons of separating the bucket_attrs (ACLs, tag etc.) | |
272 | // into a different index. | |
273 | struct MotrBucketInfo { | |
274 | RGWBucketInfo info; | |
275 | ||
276 | obj_version bucket_version; | |
277 | ceph::real_time mtime; | |
278 | ||
279 | rgw::sal::Attrs bucket_attrs; | |
280 | ||
281 | void encode(bufferlist& bl) const | |
282 | { | |
283 | ENCODE_START(4, 4, bl); | |
284 | encode(info, bl); | |
285 | encode(bucket_version, bl); | |
286 | encode(mtime, bl); | |
287 | encode(bucket_attrs, bl); //rgw_cache.h example for a map | |
288 | ENCODE_FINISH(bl); | |
289 | } | |
290 | ||
291 | void decode(bufferlist::const_iterator& bl) | |
292 | { | |
293 | DECODE_START(4, bl); | |
294 | decode(info, bl); | |
295 | decode(bucket_version, bl); | |
296 | decode(mtime, bl); | |
297 | decode(bucket_attrs, bl); | |
298 | DECODE_FINISH(bl); | |
299 | } | |
300 | }; | |
301 | WRITE_CLASS_ENCODER(MotrBucketInfo); | |
302 | ||
303 | public: | |
304 | MotrBucket(MotrStore *_st) | |
305 | : store(_st), | |
306 | acls() { | |
307 | } | |
308 | ||
309 | MotrBucket(MotrStore *_st, User* _u) | |
310 | : StoreBucket(_u), | |
311 | store(_st), | |
312 | acls() { | |
313 | } | |
314 | ||
315 | MotrBucket(MotrStore *_st, const rgw_bucket& _b) | |
316 | : StoreBucket(_b), | |
317 | store(_st), | |
318 | acls() { | |
319 | } | |
320 | ||
321 | MotrBucket(MotrStore *_st, const RGWBucketEnt& _e) | |
322 | : StoreBucket(_e), | |
323 | store(_st), | |
324 | acls() { | |
325 | } | |
326 | ||
327 | MotrBucket(MotrStore *_st, const RGWBucketInfo& _i) | |
328 | : StoreBucket(_i), | |
329 | store(_st), | |
330 | acls() { | |
331 | } | |
332 | ||
333 | MotrBucket(MotrStore *_st, const rgw_bucket& _b, User* _u) | |
334 | : StoreBucket(_b, _u), | |
335 | store(_st), | |
336 | acls() { | |
337 | } | |
338 | ||
339 | MotrBucket(MotrStore *_st, const RGWBucketEnt& _e, User* _u) | |
340 | : StoreBucket(_e, _u), | |
341 | store(_st), | |
342 | acls() { | |
343 | } | |
344 | ||
345 | MotrBucket(MotrStore *_st, const RGWBucketInfo& _i, User* _u) | |
346 | : StoreBucket(_i, _u), | |
347 | store(_st), | |
348 | acls() { | |
349 | } | |
350 | ||
351 | ~MotrBucket() { } | |
352 | ||
353 | virtual std::unique_ptr<Object> get_object(const rgw_obj_key& k) override; | |
354 | virtual int list(const DoutPrefixProvider *dpp, ListParams&, int, ListResults&, optional_yield y) override; | |
355 | virtual int remove_bucket(const DoutPrefixProvider *dpp, bool delete_children, bool forward_to_master, req_info* req_info, optional_yield y) override; | |
356 | virtual int remove_bucket_bypass_gc(int concurrent_max, bool | |
357 | keep_index_consistent, | |
358 | optional_yield y, const | |
359 | DoutPrefixProvider *dpp) override; | |
360 | virtual RGWAccessControlPolicy& get_acl(void) override { return acls; } | |
361 | virtual int set_acl(const DoutPrefixProvider *dpp, RGWAccessControlPolicy& acl, optional_yield y) override; | |
362 | virtual int load_bucket(const DoutPrefixProvider *dpp, optional_yield y, bool get_stats = false) override; | |
363 | int link_user(const DoutPrefixProvider* dpp, User* new_user, optional_yield y); | |
364 | int unlink_user(const DoutPrefixProvider* dpp, User* new_user, optional_yield y); | |
365 | int create_bucket_index(); | |
366 | int create_multipart_indices(); | |
367 | virtual int read_stats(const DoutPrefixProvider *dpp, | |
368 | const bucket_index_layout_generation& idx_layout, int shard_id, | |
369 | std::string *bucket_ver, std::string *master_ver, | |
370 | std::map<RGWObjCategory, RGWStorageStats>& stats, | |
371 | std::string *max_marker = nullptr, | |
372 | bool *syncstopped = nullptr) override; | |
373 | virtual int read_stats_async(const DoutPrefixProvider *dpp, | |
374 | const bucket_index_layout_generation& idx_layout, | |
375 | int shard_id, RGWGetBucketStats_CB* ctx) override; | |
376 | virtual int sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y) override; | |
377 | virtual int update_container_stats(const DoutPrefixProvider *dpp) override; | |
378 | virtual int check_bucket_shards(const DoutPrefixProvider *dpp) override; | |
379 | virtual int chown(const DoutPrefixProvider *dpp, User& new_user, optional_yield y) override; | |
380 | virtual int put_info(const DoutPrefixProvider *dpp, bool exclusive, ceph::real_time mtime) override; | |
381 | virtual bool is_owner(User* user) override; | |
382 | virtual int check_empty(const DoutPrefixProvider *dpp, optional_yield y) override; | |
383 | virtual int check_quota(const DoutPrefixProvider *dpp, RGWQuota& quota, uint64_t obj_size, optional_yield y, bool check_size_only = false) override; | |
384 | virtual int merge_and_store_attrs(const DoutPrefixProvider *dpp, Attrs& attrs, optional_yield y) override; | |
385 | virtual int try_refresh_info(const DoutPrefixProvider *dpp, ceph::real_time *pmtime) override; | |
386 | virtual int read_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch, uint64_t end_epoch, uint32_t max_entries, | |
387 | bool *is_truncated, RGWUsageIter& usage_iter, | |
388 | std::map<rgw_user_bucket, rgw_usage_log_entry>& usage) override; | |
389 | virtual int trim_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch, uint64_t end_epoch) override; | |
390 | virtual int remove_objs_from_index(const DoutPrefixProvider *dpp, std::list<rgw_obj_index_key>& objs_to_unlink) override; | |
391 | virtual int check_index(const DoutPrefixProvider *dpp, std::map<RGWObjCategory, RGWStorageStats>& existing_stats, std::map<RGWObjCategory, RGWStorageStats>& calculated_stats) override; | |
392 | virtual int rebuild_index(const DoutPrefixProvider *dpp) override; | |
393 | virtual int set_tag_timeout(const DoutPrefixProvider *dpp, uint64_t timeout) override; | |
394 | virtual int purge_instance(const DoutPrefixProvider *dpp) override; | |
395 | virtual std::unique_ptr<Bucket> clone() override { | |
396 | return std::make_unique<MotrBucket>(*this); | |
397 | } | |
398 | virtual std::unique_ptr<MultipartUpload> get_multipart_upload(const std::string& oid, | |
399 | std::optional<std::string> upload_id=std::nullopt, | |
400 | ACLOwner owner={}, ceph::real_time mtime=real_clock::now()) override; | |
401 | virtual int list_multiparts(const DoutPrefixProvider *dpp, | |
402 | const std::string& prefix, | |
403 | std::string& marker, | |
404 | const std::string& delim, | |
405 | const int& max_uploads, | |
406 | std::vector<std::unique_ptr<MultipartUpload>>& uploads, | |
407 | std::map<std::string, bool> *common_prefixes, | |
408 | bool *is_truncated) override; | |
409 | virtual int abort_multiparts(const DoutPrefixProvider *dpp, CephContext *cct) override; | |
410 | ||
411 | friend class MotrStore; | |
412 | }; | |
413 | ||
414 | class MotrPlacementTier: public StorePlacementTier { | |
415 | MotrStore* store; | |
416 | RGWZoneGroupPlacementTier tier; | |
417 | public: | |
418 | MotrPlacementTier(MotrStore* _store, const RGWZoneGroupPlacementTier& _tier) : store(_store), tier(_tier) {} | |
419 | virtual ~MotrPlacementTier() = default; | |
420 | ||
421 | virtual const std::string& get_tier_type() { return tier.tier_type; } | |
422 | virtual const std::string& get_storage_class() { return tier.storage_class; } | |
423 | virtual bool retain_head_object() { return tier.retain_head_object; } | |
424 | RGWZoneGroupPlacementTier& get_rt() { return tier; } | |
425 | }; | |
426 | ||
427 | class MotrZoneGroup : public StoreZoneGroup { | |
428 | MotrStore* store; | |
429 | const RGWZoneGroup group; | |
430 | std::string empty; | |
431 | public: | |
432 | MotrZoneGroup(MotrStore* _store) : store(_store), group() {} | |
433 | MotrZoneGroup(MotrStore* _store, const RGWZoneGroup& _group) : store(_store), group(_group) {} | |
434 | virtual ~MotrZoneGroup() = default; | |
435 | ||
436 | virtual const std::string& get_id() const override { return group.get_id(); }; | |
437 | virtual const std::string& get_name() const override { return group.get_name(); }; | |
438 | virtual int equals(const std::string& other_zonegroup) const override { | |
439 | return group.equals(other_zonegroup); | |
440 | }; | |
441 | /** Get the endpoint from zonegroup, or from master zone if not set */ | |
442 | virtual const std::string& get_endpoint() const override; | |
443 | virtual bool placement_target_exists(std::string& target) const override; | |
444 | virtual bool is_master_zonegroup() const override { | |
445 | return group.is_master_zonegroup(); | |
446 | }; | |
447 | virtual const std::string& get_api_name() const override { return group.api_name; }; | |
448 | virtual int get_placement_target_names(std::set<std::string>& names) const override; | |
449 | virtual const std::string& get_default_placement_name() const override { | |
450 | return group.default_placement.name; }; | |
451 | virtual int get_hostnames(std::list<std::string>& names) const override { | |
452 | names = group.hostnames; | |
453 | return 0; | |
454 | }; | |
455 | virtual int get_s3website_hostnames(std::list<std::string>& names) const override { | |
456 | names = group.hostnames_s3website; | |
457 | return 0; | |
458 | }; | |
459 | virtual int get_zone_count() const override { | |
460 | return group.zones.size(); | |
461 | } | |
462 | virtual int get_placement_tier(const rgw_placement_rule& rule, std::unique_ptr<PlacementTier>* tier); | |
463 | virtual int get_zone_by_id(const std::string& id, std::unique_ptr<Zone>* zone) override { | |
464 | return -1; | |
465 | } | |
466 | virtual int get_zone_by_name(const std::string& name, std::unique_ptr<Zone>* zone) override { | |
467 | return -1; | |
468 | } | |
469 | virtual int list_zones(std::list<std::string>& zone_ids) override { | |
470 | zone_ids.clear(); | |
471 | return 0; | |
472 | } | |
473 | const RGWZoneGroup& get_group() { return group; } | |
474 | virtual std::unique_ptr<ZoneGroup> clone() override { | |
475 | return std::make_unique<MotrZoneGroup>(store, group); | |
476 | } | |
477 | }; | |
478 | ||
479 | class MotrZone : public StoreZone { | |
480 | protected: | |
481 | MotrStore* store; | |
482 | RGWRealm *realm{nullptr}; | |
483 | MotrZoneGroup zonegroup; | |
484 | RGWZone *zone_public_config{nullptr}; /* external zone params, e.g., entrypoints, log flags, etc. */ | |
485 | RGWZoneParams *zone_params{nullptr}; /* internal zone params, e.g., rados pools */ | |
486 | RGWPeriod *current_period{nullptr}; | |
487 | ||
488 | public: | |
489 | MotrZone(MotrStore* _store) : store(_store), zonegroup(_store) { | |
490 | realm = new RGWRealm(); | |
491 | zone_public_config = new RGWZone(); | |
492 | zone_params = new RGWZoneParams(); | |
493 | current_period = new RGWPeriod(); | |
494 | ||
495 | // XXX: only default and STANDARD supported for now | |
496 | RGWZonePlacementInfo info; | |
497 | RGWZoneStorageClasses sc; | |
498 | sc.set_storage_class("STANDARD", nullptr, nullptr); | |
499 | info.storage_classes = sc; | |
500 | zone_params->placement_pools["default"] = info; | |
501 | } | |
502 | MotrZone(MotrStore* _store, MotrZoneGroup _zg) : store(_store), zonegroup(_zg) { | |
503 | realm = new RGWRealm(); | |
504 | // TODO: fetch zonegroup params (eg. id) from provisioner config. | |
505 | zonegroup.set_id("0956b174-fe14-4f97-8b50-bb7ec5e1cf62"); | |
506 | zonegroup.api_name = "default"; | |
507 | zone_public_config = new RGWZone(); | |
508 | zone_params = new RGWZoneParams(); | |
509 | current_period = new RGWPeriod(); | |
510 | ||
511 | // XXX: only default and STANDARD supported for now | |
512 | RGWZonePlacementInfo info; | |
513 | RGWZoneStorageClasses sc; | |
514 | sc.set_storage_class("STANDARD", nullptr, nullptr); | |
515 | info.storage_classes = sc; | |
516 | zone_params->placement_pools["default"] = info; | |
517 | } | |
518 | ~MotrZone() = default; | |
519 | ||
520 | virtual std::unique_ptr<Zone> clone() override { | |
521 | return std::make_unique<MotrZone>(store); | |
522 | } | |
523 | virtual ZoneGroup& get_zonegroup() override; | |
524 | virtual const std::string& get_id() override; | |
525 | virtual const std::string& get_name() const override; | |
526 | virtual bool is_writeable() override; | |
527 | virtual bool get_redirect_endpoint(std::string* endpoint) override; | |
528 | virtual bool has_zonegroup_api(const std::string& api) const override; | |
529 | virtual const std::string& get_current_period_id() override; | |
530 | virtual const RGWAccessKey& get_system_key() { return zone_params->system_key; } | |
531 | virtual const std::string& get_realm_name() { return realm->get_name(); } | |
532 | virtual const std::string& get_realm_id() { return realm->get_id(); } | |
533 | virtual const std::string_view get_tier_type() { return "rgw"; } | |
534 | virtual RGWBucketSyncPolicyHandlerRef get_sync_policy_handler() { return nullptr; } | |
535 | friend class MotrStore; | |
536 | }; | |
537 | ||
538 | class MotrLuaManager : public StoreLuaManager { | |
539 | MotrStore* store; | |
540 | ||
541 | public: | |
542 | MotrLuaManager(MotrStore* _s) : store(_s) | |
543 | { | |
544 | } | |
545 | virtual ~MotrLuaManager() = default; | |
546 | ||
547 | /** Get a script named with the given key from the backing store */ | |
548 | virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) override; | |
549 | /** Put a script named with the given key to the backing store */ | |
550 | virtual int put_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script) override; | |
551 | /** Delete a script named with the given key from the backing store */ | |
552 | virtual int del_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) override; | |
553 | /** Add a lua package */ | |
554 | virtual int add_package(const DoutPrefixProvider* dpp, optional_yield y, const std::string& package_name) override; | |
555 | /** Remove a lua package */ | |
556 | virtual int remove_package(const DoutPrefixProvider* dpp, optional_yield y, const std::string& package_name) override; | |
557 | /** List lua packages */ | |
558 | virtual int list_packages(const DoutPrefixProvider* dpp, optional_yield y, rgw::lua::packages_t& packages) override; | |
559 | }; | |
560 | ||
561 | class MotrOIDCProvider : public RGWOIDCProvider { | |
562 | MotrStore* store; | |
563 | public: | |
564 | MotrOIDCProvider(MotrStore* _store) : store(_store) {} | |
565 | ~MotrOIDCProvider() = default; | |
566 | ||
567 | virtual int store_url(const DoutPrefixProvider *dpp, const std::string& url, bool exclusive, optional_yield y) override { return 0; } | |
568 | virtual int read_url(const DoutPrefixProvider *dpp, const std::string& url, const std::string& tenant) override { return 0; } | |
569 | virtual int delete_obj(const DoutPrefixProvider *dpp, optional_yield y) override { return 0;} | |
570 | ||
571 | void encode(bufferlist& bl) const { | |
572 | RGWOIDCProvider::encode(bl); | |
573 | } | |
574 | void decode(bufferlist::const_iterator& bl) { | |
575 | RGWOIDCProvider::decode(bl); | |
576 | } | |
577 | }; | |
578 | ||
579 | class MotrObject : public StoreObject { | |
580 | private: | |
581 | MotrStore *store; | |
582 | RGWAccessControlPolicy acls; | |
583 | RGWObjCategory category; | |
584 | ||
585 | // If this object is pat of a multipart uploaded one. | |
586 | // TODO: do it in another class? MotrPartObject : public MotrObject | |
587 | uint64_t part_off; | |
588 | uint64_t part_size; | |
589 | uint64_t part_num; | |
590 | ||
591 | public: | |
592 | ||
593 | // motr object metadata stored in index | |
594 | struct Meta { | |
595 | struct m0_uint128 oid = {}; | |
596 | struct m0_fid pver = {}; | |
597 | uint64_t layout_id = 0; | |
598 | ||
599 | void encode(bufferlist& bl) const | |
600 | { | |
601 | ENCODE_START(5, 5, bl); | |
602 | encode(oid.u_hi, bl); | |
603 | encode(oid.u_lo, bl); | |
604 | encode(pver.f_container, bl); | |
605 | encode(pver.f_key, bl); | |
606 | encode(layout_id, bl); | |
607 | ENCODE_FINISH(bl); | |
608 | } | |
609 | ||
610 | void decode(bufferlist::const_iterator& bl) | |
611 | { | |
612 | DECODE_START(5, bl); | |
613 | decode(oid.u_hi, bl); | |
614 | decode(oid.u_lo, bl); | |
615 | decode(pver.f_container, bl); | |
616 | decode(pver.f_key, bl); | |
617 | decode(layout_id, bl); | |
618 | DECODE_FINISH(bl); | |
619 | } | |
620 | }; | |
621 | ||
622 | struct m0_obj *mobj = NULL; | |
623 | Meta meta; | |
624 | ||
625 | struct MotrReadOp : public ReadOp { | |
626 | private: | |
627 | MotrObject* source; | |
628 | ||
629 | // The set of part objects if the source is | |
630 | // a multipart uploaded object. | |
631 | std::map<int, std::unique_ptr<MotrObject>> part_objs; | |
632 | ||
633 | public: | |
634 | MotrReadOp(MotrObject *_source); | |
635 | ||
636 | virtual int prepare(optional_yield y, const DoutPrefixProvider* dpp) override; | |
637 | ||
638 | /* | |
639 | * Both `read` and `iterate` read up through index `end` | |
640 | * *inclusive*. The number of bytes that could be returned is | |
641 | * `end - ofs + 1`. | |
642 | */ | |
643 | virtual int read(int64_t off, int64_t end, bufferlist& bl, | |
644 | optional_yield y, | |
645 | const DoutPrefixProvider* dpp) override; | |
646 | virtual int iterate(const DoutPrefixProvider* dpp, int64_t off, | |
647 | int64_t end, RGWGetDataCB* cb, | |
648 | optional_yield y) override; | |
649 | ||
650 | virtual int get_attr(const DoutPrefixProvider* dpp, const char* name, bufferlist& dest, optional_yield y) override; | |
651 | }; | |
652 | ||
653 | struct MotrDeleteOp : public DeleteOp { | |
654 | private: | |
655 | MotrObject* source; | |
656 | ||
657 | public: | |
658 | MotrDeleteOp(MotrObject* _source); | |
659 | ||
660 | virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y) override; | |
661 | }; | |
662 | ||
663 | MotrObject() = default; | |
664 | ||
665 | MotrObject(MotrStore *_st, const rgw_obj_key& _k) | |
666 | : StoreObject(_k), store(_st), acls() {} | |
667 | MotrObject(MotrStore *_st, const rgw_obj_key& _k, Bucket* _b) | |
668 | : StoreObject(_k, _b), store(_st), acls() {} | |
669 | ||
670 | MotrObject(MotrObject& _o) = default; | |
671 | ||
672 | virtual ~MotrObject(); | |
673 | ||
674 | virtual int delete_object(const DoutPrefixProvider* dpp, | |
675 | optional_yield y, | |
676 | bool prevent_versioning = false) override; | |
677 | virtual int delete_obj_aio(const DoutPrefixProvider* dpp, RGWObjState* astate, Completions* aio, | |
678 | bool keep_index_consistent, optional_yield y) override; | |
679 | virtual int copy_object(User* user, | |
680 | req_info* info, const rgw_zone_id& source_zone, | |
681 | rgw::sal::Object* dest_object, rgw::sal::Bucket* dest_bucket, | |
682 | rgw::sal::Bucket* src_bucket, | |
683 | const rgw_placement_rule& dest_placement, | |
684 | ceph::real_time* src_mtime, ceph::real_time* mtime, | |
685 | const ceph::real_time* mod_ptr, const ceph::real_time* unmod_ptr, | |
686 | bool high_precision_time, | |
687 | const char* if_match, const char* if_nomatch, | |
688 | AttrsMod attrs_mod, bool copy_if_newer, Attrs& attrs, | |
689 | RGWObjCategory category, uint64_t olh_epoch, | |
690 | boost::optional<ceph::real_time> delete_at, | |
691 | std::string* version_id, std::string* tag, std::string* etag, | |
692 | void (*progress_cb)(off_t, void *), void* progress_data, | |
693 | const DoutPrefixProvider* dpp, optional_yield y) override; | |
694 | virtual RGWAccessControlPolicy& get_acl(void) override { return acls; } | |
695 | virtual int set_acl(const RGWAccessControlPolicy& acl) override { acls = acl; return 0; } | |
696 | virtual int get_obj_state(const DoutPrefixProvider* dpp, RGWObjState **state, optional_yield y, bool follow_olh = true) override; | |
697 | virtual int set_obj_attrs(const DoutPrefixProvider* dpp, Attrs* setattrs, Attrs* delattrs, optional_yield y) override; | |
698 | virtual int get_obj_attrs(optional_yield y, const DoutPrefixProvider* dpp, rgw_obj* target_obj = NULL) override; | |
699 | virtual int modify_obj_attrs(const char* attr_name, bufferlist& attr_val, optional_yield y, const DoutPrefixProvider* dpp) override; | |
700 | virtual int delete_obj_attrs(const DoutPrefixProvider* dpp, const char* attr_name, optional_yield y) override; | |
701 | virtual bool is_expired() override; | |
702 | virtual void gen_rand_obj_instance_name() override; | |
703 | virtual std::unique_ptr<Object> clone() override { | |
704 | return std::unique_ptr<Object>(new MotrObject(*this)); | |
705 | } | |
706 | virtual std::unique_ptr<MPSerializer> get_serializer(const DoutPrefixProvider *dpp, const std::string& lock_name) override; | |
707 | virtual int transition(Bucket* bucket, | |
708 | const rgw_placement_rule& placement_rule, | |
709 | const real_time& mtime, | |
710 | uint64_t olh_epoch, | |
711 | const DoutPrefixProvider* dpp, | |
712 | optional_yield y) override; | |
713 | virtual bool placement_rules_match(rgw_placement_rule& r1, rgw_placement_rule& r2) override; | |
714 | virtual int dump_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f) override; | |
715 | ||
716 | /* Swift versioning */ | |
717 | virtual int swift_versioning_restore(bool& restored, | |
718 | const DoutPrefixProvider* dpp) override; | |
719 | virtual int swift_versioning_copy(const DoutPrefixProvider* dpp, | |
720 | optional_yield y) override; | |
721 | ||
722 | /* OPs */ | |
723 | virtual std::unique_ptr<ReadOp> get_read_op() override; | |
724 | virtual std::unique_ptr<DeleteOp> get_delete_op() override; | |
725 | ||
726 | /* OMAP */ | |
727 | virtual int omap_get_vals(const DoutPrefixProvider *dpp, const std::string& marker, uint64_t count, | |
728 | std::map<std::string, bufferlist> *m, | |
729 | bool* pmore, optional_yield y) override; | |
730 | virtual int omap_get_all(const DoutPrefixProvider *dpp, std::map<std::string, bufferlist> *m, | |
731 | optional_yield y) override; | |
732 | virtual int omap_get_vals_by_keys(const DoutPrefixProvider *dpp, const std::string& oid, | |
733 | const std::set<std::string>& keys, | |
734 | Attrs* vals) override; | |
735 | virtual int omap_set_val_by_key(const DoutPrefixProvider *dpp, const std::string& key, bufferlist& val, | |
736 | bool must_exist, optional_yield y) override; | |
737 | virtual int chown(User& new_user, const DoutPrefixProvider* dpp, optional_yield y) override; | |
738 | private: | |
739 | //int read_attrs(const DoutPrefixProvider* dpp, Motr::Object::Read &read_op, optional_yield y, rgw_obj* target_obj = nullptr); | |
740 | ||
741 | public: | |
742 | bool is_opened() { return mobj != NULL; } | |
743 | int create_mobj(const DoutPrefixProvider *dpp, uint64_t sz); | |
744 | int open_mobj(const DoutPrefixProvider *dpp); | |
745 | int delete_mobj(const DoutPrefixProvider *dpp); | |
746 | void close_mobj(); | |
747 | int write_mobj(const DoutPrefixProvider *dpp, bufferlist&& data, uint64_t offset); | |
748 | int read_mobj(const DoutPrefixProvider* dpp, int64_t off, int64_t end, RGWGetDataCB* cb); | |
749 | unsigned get_optimal_bs(unsigned len); | |
750 | ||
751 | int get_part_objs(const DoutPrefixProvider *dpp, | |
752 | std::map<int, std::unique_ptr<MotrObject>>& part_objs); | |
753 | int open_part_objs(const DoutPrefixProvider* dpp, | |
754 | std::map<int, std::unique_ptr<MotrObject>>& part_objs); | |
755 | int read_multipart_obj(const DoutPrefixProvider* dpp, | |
756 | int64_t off, int64_t end, RGWGetDataCB* cb, | |
757 | std::map<int, std::unique_ptr<MotrObject>>& part_objs); | |
758 | int delete_part_objs(const DoutPrefixProvider* dpp); | |
759 | void set_category(RGWObjCategory _category) {category = _category;} | |
760 | int get_bucket_dir_ent(const DoutPrefixProvider *dpp, rgw_bucket_dir_entry& ent); | |
761 | int update_version_entries(const DoutPrefixProvider *dpp); | |
762 | }; | |
763 | ||
764 | // A placeholder locking class for multipart upload. | |
765 | // TODO: implement it using Motr object locks. | |
766 | class MPMotrSerializer : public StoreMPSerializer { | |
767 | ||
768 | public: | |
769 | MPMotrSerializer(const DoutPrefixProvider *dpp, MotrStore* store, MotrObject* obj, const std::string& lock_name) {} | |
770 | ||
771 | virtual int try_lock(const DoutPrefixProvider *dpp, utime_t dur, optional_yield y) override {return 0; } | |
772 | virtual int unlock() override { return 0;} | |
773 | }; | |
774 | ||
775 | class MotrAtomicWriter : public StoreWriter { | |
776 | protected: | |
777 | rgw::sal::MotrStore* store; | |
778 | const rgw_user& owner; | |
779 | const rgw_placement_rule *ptail_placement_rule; | |
780 | uint64_t olh_epoch; | |
781 | const std::string& unique_tag; | |
782 | MotrObject obj; | |
783 | MotrObject old_obj; | |
784 | uint64_t total_data_size; // for total data being uploaded | |
785 | bufferlist acc_data; // accumulated data | |
786 | uint64_t acc_off; // accumulated data offset | |
787 | ||
788 | struct m0_bufvec buf; | |
789 | struct m0_bufvec attr; | |
790 | struct m0_indexvec ext; | |
791 | ||
792 | public: | |
793 | MotrAtomicWriter(const DoutPrefixProvider *dpp, | |
794 | optional_yield y, | |
795 | rgw::sal::Object* obj, | |
796 | MotrStore* _store, | |
797 | const rgw_user& _owner, | |
798 | const rgw_placement_rule *_ptail_placement_rule, | |
799 | uint64_t _olh_epoch, | |
800 | const std::string& _unique_tag); | |
801 | ~MotrAtomicWriter() = default; | |
802 | ||
803 | // prepare to start processing object data | |
804 | virtual int prepare(optional_yield y) override; | |
805 | ||
806 | // Process a bufferlist | |
807 | virtual int process(bufferlist&& data, uint64_t offset) override; | |
808 | ||
809 | int write(); | |
810 | ||
811 | // complete the operation and make its result visible to clients | |
812 | virtual int complete(size_t accounted_size, const std::string& etag, | |
813 | ceph::real_time *mtime, ceph::real_time set_mtime, | |
814 | std::map<std::string, bufferlist>& attrs, | |
815 | ceph::real_time delete_at, | |
816 | const char *if_match, const char *if_nomatch, | |
817 | const std::string *user_data, | |
818 | rgw_zone_set *zones_trace, bool *canceled, | |
819 | optional_yield y) override; | |
820 | ||
821 | unsigned populate_bvec(unsigned len, bufferlist::iterator &bi); | |
822 | void cleanup(); | |
823 | }; | |
824 | ||
825 | class MotrMultipartWriter : public StoreWriter { | |
826 | protected: | |
827 | rgw::sal::MotrStore* store; | |
828 | ||
829 | // Head object. | |
830 | rgw::sal::Object* head_obj; | |
831 | ||
832 | // Part parameters. | |
833 | const uint64_t part_num; | |
834 | const std::string part_num_str; | |
835 | std::unique_ptr<MotrObject> part_obj; | |
836 | uint64_t actual_part_size = 0; | |
837 | ||
838 | public: | |
839 | MotrMultipartWriter(const DoutPrefixProvider *dpp, | |
840 | optional_yield y, MultipartUpload* upload, | |
841 | rgw::sal::Object* obj, | |
842 | MotrStore* _store, | |
843 | const rgw_user& owner, | |
844 | const rgw_placement_rule *ptail_placement_rule, | |
845 | uint64_t _part_num, const std::string& part_num_str) : | |
846 | StoreWriter(dpp, y), store(_store), head_obj(obj), | |
847 | part_num(_part_num), part_num_str(part_num_str) | |
848 | { | |
849 | } | |
850 | ~MotrMultipartWriter() = default; | |
851 | ||
852 | // prepare to start processing object data | |
853 | virtual int prepare(optional_yield y) override; | |
854 | ||
855 | // Process a bufferlist | |
856 | virtual int process(bufferlist&& data, uint64_t offset) override; | |
857 | ||
858 | // complete the operation and make its result visible to clients | |
859 | virtual int complete(size_t accounted_size, const std::string& etag, | |
860 | ceph::real_time *mtime, ceph::real_time set_mtime, | |
861 | std::map<std::string, bufferlist>& attrs, | |
862 | ceph::real_time delete_at, | |
863 | const char *if_match, const char *if_nomatch, | |
864 | const std::string *user_data, | |
865 | rgw_zone_set *zones_trace, bool *canceled, | |
866 | optional_yield y) override; | |
867 | }; | |
868 | ||
869 | // The implementation of multipart upload in POC roughly follows the | |
870 | // cortx-s3server's design. Parts are stored in separate Motr objects. | |
871 | // s3server uses a few auxiliary Motr indices to manage multipart | |
872 | // related metadata: (1) Bucket multipart index (bucket_nnn_multipart_index) | |
873 | // which contains metadata that answers questions such as which objects have | |
874 | // started multipart upload and its upload id. This index is created during | |
875 | // bucket creation. (2) Object part index (object_nnn_part_index) which stores | |
876 | // metadata of a part's details (size, pvid, oid...). This index is created in | |
877 | // MotrMultipartUpload::init(). (3) Extended metadata index | |
878 | // (bucket_nnn_extended_metadata): once parts has been uploaded and their | |
879 | // metadata saved in the part index, the user may issue multipart completion | |
880 | // request. When processing the completion request, the parts are read from | |
881 | // object part index and for each part an entry is created in extended index. | |
882 | // The entry for the object is created in bucket (object list) index. The part | |
883 | // index is deleted and an entry removed from bucket_nnn_multipart_index. Like | |
884 | // bucket multipart index, bucket part extened metadata index is created during | |
885 | // bucket creation. | |
886 | // | |
887 | // The extended metadata index is used mainly due to fault tolerant | |
888 | // considerations (how to handle Motr service crash when uploading an object) | |
889 | // and to avoid to create too many Motr indices (I am not sure I understand | |
890 | // why many Motr indices is bad.). In our POC, to keep it simple, only 2 | |
891 | // indices are maintained: bucket multipart index and object_nnn_part_index. | |
892 | // | |
893 | // | |
894 | ||
895 | class MotrMultipartPart : public StoreMultipartPart { | |
896 | protected: | |
897 | RGWUploadPartInfo info; | |
898 | ||
899 | public: | |
900 | MotrObject::Meta meta; | |
901 | ||
902 | MotrMultipartPart(RGWUploadPartInfo _info, MotrObject::Meta _meta) : | |
903 | info(_info), meta(_meta) {} | |
904 | virtual ~MotrMultipartPart() = default; | |
905 | ||
906 | virtual uint32_t get_num() { return info.num; } | |
907 | virtual uint64_t get_size() { return info.accounted_size; } | |
908 | virtual const std::string& get_etag() { return info.etag; } | |
909 | virtual ceph::real_time& get_mtime() { return info.modified; } | |
910 | ||
911 | RGWObjManifest& get_manifest() { return info.manifest; } | |
912 | ||
913 | friend class MotrMultipartUpload; | |
914 | }; | |
915 | ||
916 | class MotrMultipartUpload : public StoreMultipartUpload { | |
917 | MotrStore* store; | |
918 | RGWMPObj mp_obj; | |
919 | ACLOwner owner; | |
920 | ceph::real_time mtime; | |
921 | rgw_placement_rule placement; | |
922 | RGWObjManifest manifest; | |
923 | ||
924 | public: | |
925 | MotrMultipartUpload(MotrStore* _store, Bucket* _bucket, const std::string& oid, | |
926 | std::optional<std::string> upload_id, ACLOwner _owner, ceph::real_time _mtime) : | |
927 | StoreMultipartUpload(_bucket), store(_store), mp_obj(oid, upload_id), owner(_owner), mtime(_mtime) {} | |
928 | virtual ~MotrMultipartUpload() = default; | |
929 | ||
930 | virtual const std::string& get_meta() const { return mp_obj.get_meta(); } | |
931 | virtual const std::string& get_key() const { return mp_obj.get_key(); } | |
932 | virtual const std::string& get_upload_id() const { return mp_obj.get_upload_id(); } | |
933 | virtual const ACLOwner& get_owner() const override { return owner; } | |
934 | virtual ceph::real_time& get_mtime() { return mtime; } | |
935 | virtual std::unique_ptr<rgw::sal::Object> get_meta_obj() override; | |
936 | virtual int init(const DoutPrefixProvider* dpp, optional_yield y, ACLOwner& owner, rgw_placement_rule& dest_placement, rgw::sal::Attrs& attrs) override; | |
937 | virtual int list_parts(const DoutPrefixProvider* dpp, CephContext* cct, | |
938 | int num_parts, int marker, | |
939 | int* next_marker, bool* truncated, | |
940 | bool assume_unsorted = false) override; | |
941 | virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct) override; | |
942 | virtual int complete(const DoutPrefixProvider* dpp, | |
943 | optional_yield y, CephContext* cct, | |
944 | std::map<int, std::string>& part_etags, | |
945 | std::list<rgw_obj_index_key>& remove_objs, | |
946 | uint64_t& accounted_size, bool& compressed, | |
947 | RGWCompressionInfo& cs_info, off_t& off, | |
948 | std::string& tag, ACLOwner& owner, | |
949 | uint64_t olh_epoch, | |
950 | rgw::sal::Object* target_obj) override; | |
951 | virtual int get_info(const DoutPrefixProvider *dpp, optional_yield y, rgw_placement_rule** rule, rgw::sal::Attrs* attrs = nullptr) override; | |
952 | virtual std::unique_ptr<Writer> get_writer(const DoutPrefixProvider *dpp, | |
953 | optional_yield y, | |
954 | rgw::sal::Object* obj, | |
955 | const rgw_user& owner, | |
956 | const rgw_placement_rule *ptail_placement_rule, | |
957 | uint64_t part_num, | |
958 | const std::string& part_num_str) override; | |
959 | int delete_parts(const DoutPrefixProvider *dpp); | |
960 | }; | |
961 | ||
962 | class MotrStore : public StoreDriver { | |
963 | private: | |
964 | MotrZone zone; | |
965 | RGWSyncModuleInstanceRef sync_module; | |
966 | ||
967 | MotrMetaCache* obj_meta_cache; | |
968 | MotrMetaCache* user_cache; | |
969 | MotrMetaCache* bucket_inst_cache; | |
970 | ||
971 | public: | |
972 | CephContext *cctx; | |
973 | struct m0_client *instance; | |
974 | struct m0_container container; | |
975 | struct m0_realm uber_realm; | |
976 | struct m0_config conf = {}; | |
977 | struct m0_idx_dix_config dix_conf = {}; | |
978 | ||
979 | MotrStore(CephContext *c): zone(this), cctx(c) {} | |
980 | ~MotrStore() { | |
981 | delete obj_meta_cache; | |
982 | delete user_cache; | |
983 | delete bucket_inst_cache; | |
984 | } | |
985 | ||
986 | virtual int initialize(CephContext *cct, const DoutPrefixProvider *dpp) { return 0; } | |
987 | virtual const std::string get_name() const override { | |
988 | return "motr"; | |
989 | } | |
990 | ||
991 | virtual std::unique_ptr<User> get_user(const rgw_user& u) override; | |
992 | virtual std::string get_cluster_id(const DoutPrefixProvider* dpp, optional_yield y) override; | |
993 | virtual int get_user_by_access_key(const DoutPrefixProvider *dpp, const std::string& key, optional_yield y, std::unique_ptr<User>* user) override; | |
994 | virtual int get_user_by_email(const DoutPrefixProvider *dpp, const std::string& email, optional_yield y, std::unique_ptr<User>* user) override; | |
995 | virtual int get_user_by_swift(const DoutPrefixProvider *dpp, const std::string& user_str, optional_yield y, std::unique_ptr<User>* user) override; | |
996 | virtual std::unique_ptr<Object> get_object(const rgw_obj_key& k) override; | |
997 | virtual int get_bucket(const DoutPrefixProvider *dpp, User* u, const rgw_bucket& b, std::unique_ptr<Bucket>* bucket, optional_yield y) override; | |
998 | virtual int get_bucket(User* u, const RGWBucketInfo& i, std::unique_ptr<Bucket>* bucket) override; | |
999 | virtual int get_bucket(const DoutPrefixProvider *dpp, User* u, const std::string& tenant, const std::string&name, std::unique_ptr<Bucket>* bucket, optional_yield y) override; | |
1000 | virtual bool is_meta_master() override; | |
1001 | virtual int forward_request_to_master(const DoutPrefixProvider *dpp, User* user, obj_version* objv, | |
1002 | bufferlist& in_data, JSONParser *jp, req_info& info, | |
1003 | optional_yield y) override; | |
1004 | virtual int forward_iam_request_to_master(const DoutPrefixProvider *dpp, const RGWAccessKey& key, obj_version* objv, | |
1005 | bufferlist& in_data, | |
1006 | RGWXMLDecoder::XMLParser* parser, req_info& info, | |
1007 | optional_yield y) override; | |
1008 | virtual Zone* get_zone() { return &zone; } | |
1009 | virtual std::string zone_unique_id(uint64_t unique_num) override; | |
1010 | virtual std::string zone_unique_trans_id(const uint64_t unique_num) override; | |
1011 | virtual int get_zonegroup(const std::string& id, std::unique_ptr<ZoneGroup>* zonegroup) override; | |
1012 | virtual int list_all_zones(const DoutPrefixProvider* dpp, std::list<std::string>& zone_ids) override; | |
1013 | virtual int cluster_stat(RGWClusterStat& stats) override; | |
1014 | virtual std::unique_ptr<Lifecycle> get_lifecycle(void) override; | |
1015 | virtual std::unique_ptr<Completions> get_completions(void) override; | |
1016 | virtual std::unique_ptr<Notification> get_notification(rgw::sal::Object* obj, rgw::sal::Object* src_obj, | |
1017 | req_state* s, rgw::notify::EventType event_type, optional_yield y, const std::string* object_name=nullptr) override; | |
1018 | virtual std::unique_ptr<Notification> get_notification(const DoutPrefixProvider* dpp, rgw::sal::Object* obj, | |
1019 | rgw::sal::Object* src_obj, rgw::notify::EventType event_type, rgw::sal::Bucket* _bucket, | |
1020 | std::string& _user_id, std::string& _user_tenant, std::string& _req_id, optional_yield y) override; | |
1021 | virtual RGWLC* get_rgwlc(void) override { return NULL; } | |
1022 | virtual RGWCoroutinesManagerRegistry* get_cr_registry() override { return NULL; } | |
1023 | ||
1024 | virtual int log_usage(const DoutPrefixProvider *dpp, std::map<rgw_user_bucket, RGWUsageBatch>& usage_info) override; | |
1025 | virtual int log_op(const DoutPrefixProvider *dpp, std::string& oid, bufferlist& bl) override; | |
1026 | virtual int register_to_service_map(const DoutPrefixProvider *dpp, const std::string& daemon_type, | |
1027 | const std::map<std::string, std::string>& meta) override; | |
1028 | virtual void get_ratelimit(RGWRateLimitInfo& bucket_ratelimit, RGWRateLimitInfo& user_ratelimit, RGWRateLimitInfo& anon_ratelimit) override; | |
1029 | virtual void get_quota(RGWQuota& quota) override; | |
1030 | virtual int set_buckets_enabled(const DoutPrefixProvider *dpp, std::vector<rgw_bucket>& buckets, bool enabled) override; | |
1031 | virtual int get_sync_policy_handler(const DoutPrefixProvider *dpp, | |
1032 | std::optional<rgw_zone_id> zone, | |
1033 | std::optional<rgw_bucket> bucket, | |
1034 | RGWBucketSyncPolicyHandlerRef *phandler, | |
1035 | optional_yield y) override; | |
1036 | virtual RGWDataSyncStatusManager* get_data_sync_manager(const rgw_zone_id& source_zone) override; | |
1037 | virtual void wakeup_meta_sync_shards(std::set<int>& shard_ids) override { return; } | |
1038 | virtual void wakeup_data_sync_shards(const DoutPrefixProvider *dpp, const rgw_zone_id& source_zone, boost::container::flat_map<int, boost::container::flat_set<rgw_data_notify_entry>>& shard_ids) override {} | |
1039 | virtual int clear_usage(const DoutPrefixProvider *dpp) override { return 0; } | |
1040 | virtual int read_all_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch, uint64_t end_epoch, | |
1041 | uint32_t max_entries, bool *is_truncated, | |
1042 | RGWUsageIter& usage_iter, | |
1043 | std::map<rgw_user_bucket, rgw_usage_log_entry>& usage) override; | |
1044 | virtual int trim_all_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch, uint64_t end_epoch) override; | |
1045 | virtual int get_config_key_val(std::string name, bufferlist* bl) override; | |
1046 | virtual int meta_list_keys_init(const DoutPrefixProvider *dpp, const std::string& section, const std::string& marker, void** phandle) override; | |
1047 | virtual int meta_list_keys_next(const DoutPrefixProvider *dpp, void* handle, int max, std::list<std::string>& keys, bool* truncated) override; | |
1048 | virtual void meta_list_keys_complete(void* handle) override; | |
1049 | virtual std::string meta_get_marker(void *handle) override; | |
1050 | virtual int meta_remove(const DoutPrefixProvider *dpp, std::string& metadata_key, optional_yield y) override; | |
1051 | ||
1052 | virtual const RGWSyncModuleInstanceRef& get_sync_module() { return sync_module; } | |
1053 | virtual std::string get_host_id() { return ""; } | |
1054 | ||
1055 | virtual std::unique_ptr<LuaManager> get_lua_manager() override; | |
1056 | virtual std::unique_ptr<RGWRole> get_role(std::string name, | |
1057 | std::string tenant, | |
1058 | std::string path="", | |
1059 | std::string trust_policy="", | |
1060 | std::string max_session_duration_str="", | |
1061 | std::multimap<std::string, std::string> tags={}) override; | |
1062 | virtual std::unique_ptr<RGWRole> get_role(const RGWRoleInfo& info) override; | |
1063 | virtual std::unique_ptr<RGWRole> get_role(std::string id) override; | |
1064 | virtual int get_roles(const DoutPrefixProvider *dpp, | |
1065 | optional_yield y, | |
1066 | const std::string& path_prefix, | |
1067 | const std::string& tenant, | |
1068 | std::vector<std::unique_ptr<RGWRole>>& roles) override; | |
1069 | virtual std::unique_ptr<RGWOIDCProvider> get_oidc_provider() override; | |
1070 | virtual int get_oidc_providers(const DoutPrefixProvider *dpp, | |
1071 | const std::string& tenant, | |
1072 | std::vector<std::unique_ptr<RGWOIDCProvider>>& providers) override; | |
1073 | virtual std::unique_ptr<Writer> get_append_writer(const DoutPrefixProvider *dpp, | |
1074 | optional_yield y, | |
1075 | rgw::sal::Object* obj, | |
1076 | const rgw_user& owner, | |
1077 | const rgw_placement_rule *ptail_placement_rule, | |
1078 | const std::string& unique_tag, | |
1079 | uint64_t position, | |
1080 | uint64_t *cur_accounted_size) override; | |
1081 | virtual std::unique_ptr<Writer> get_atomic_writer(const DoutPrefixProvider *dpp, | |
1082 | optional_yield y, | |
1083 | rgw::sal::Object* obj, | |
1084 | const rgw_user& owner, | |
1085 | const rgw_placement_rule *ptail_placement_rule, | |
1086 | uint64_t olh_epoch, | |
1087 | const std::string& unique_tag) override; | |
1088 | virtual const std::string& get_compression_type(const rgw_placement_rule& rule) override; | |
1089 | virtual bool valid_placement(const rgw_placement_rule& rule) override; | |
1090 | ||
1091 | virtual void finalize(void) override; | |
1092 | ||
1093 | virtual CephContext *ctx(void) override { | |
1094 | return cctx; | |
1095 | } | |
1096 | ||
1097 | virtual void register_admin_apis(RGWRESTMgr* mgr) override { }; | |
1098 | ||
1099 | int open_idx(struct m0_uint128 *id, bool create, struct m0_idx *out); | |
1100 | void close_idx(struct m0_idx *idx) { m0_idx_fini(idx); } | |
1101 | int do_idx_op(struct m0_idx *, enum m0_idx_opcode opcode, | |
1102 | std::vector<uint8_t>& key, std::vector<uint8_t>& val, bool update = false); | |
1103 | ||
1104 | int do_idx_next_op(struct m0_idx *idx, | |
1105 | std::vector<std::vector<uint8_t>>& key_vec, | |
1106 | std::vector<std::vector<uint8_t>>& val_vec); | |
1107 | int next_query_by_name(std::string idx_name, std::vector<std::string>& key_str_vec, | |
1108 | std::vector<bufferlist>& val_bl_vec, | |
1109 | std::string prefix="", std::string delim=""); | |
1110 | ||
1111 | void index_name_to_motr_fid(std::string iname, struct m0_uint128 *fid); | |
1112 | int open_motr_idx(struct m0_uint128 *id, struct m0_idx *idx); | |
1113 | int create_motr_idx_by_name(std::string iname); | |
1114 | int delete_motr_idx_by_name(std::string iname); | |
1115 | int do_idx_op_by_name(std::string idx_name, enum m0_idx_opcode opcode, | |
1116 | std::string key_str, bufferlist &bl, bool update=true); | |
1117 | int check_n_create_global_indices(); | |
1118 | int store_access_key(const DoutPrefixProvider *dpp, optional_yield y, MotrAccessKey access_key); | |
1119 | int delete_access_key(const DoutPrefixProvider *dpp, optional_yield y, std::string access_key); | |
1120 | int store_email_info(const DoutPrefixProvider *dpp, optional_yield y, MotrEmailInfo& email_info); | |
1121 | ||
1122 | int init_metadata_cache(const DoutPrefixProvider *dpp, CephContext *cct); | |
1123 | MotrMetaCache* get_obj_meta_cache() {return obj_meta_cache;} | |
1124 | MotrMetaCache* get_user_cache() {return user_cache;} | |
1125 | MotrMetaCache* get_bucket_inst_cache() {return bucket_inst_cache;} | |
1126 | }; | |
1127 | ||
1128 | struct obj_time_weight { | |
1129 | real_time mtime; | |
1130 | uint32_t zone_short_id; | |
1131 | uint64_t pg_ver; | |
1132 | bool high_precision; | |
1133 | ||
1134 | obj_time_weight() : zone_short_id(0), pg_ver(0), high_precision(false) {} | |
1135 | ||
1136 | bool compare_low_precision(const obj_time_weight& rhs) { | |
1137 | struct timespec l = ceph::real_clock::to_timespec(mtime); | |
1138 | struct timespec r = ceph::real_clock::to_timespec(rhs.mtime); | |
1139 | l.tv_nsec = 0; | |
1140 | r.tv_nsec = 0; | |
1141 | if (l > r) { | |
1142 | return false; | |
1143 | } | |
1144 | if (l < r) { | |
1145 | return true; | |
1146 | } | |
1147 | if (!zone_short_id || !rhs.zone_short_id) { | |
1148 | /* don't compare zone ids, if one wasn't provided */ | |
1149 | return false; | |
1150 | } | |
1151 | if (zone_short_id != rhs.zone_short_id) { | |
1152 | return (zone_short_id < rhs.zone_short_id); | |
1153 | } | |
1154 | return (pg_ver < rhs.pg_ver); | |
1155 | ||
1156 | } | |
1157 | ||
1158 | bool operator<(const obj_time_weight& rhs) { | |
1159 | if (!high_precision || !rhs.high_precision) { | |
1160 | return compare_low_precision(rhs); | |
1161 | } | |
1162 | if (mtime > rhs.mtime) { | |
1163 | return false; | |
1164 | } | |
1165 | if (mtime < rhs.mtime) { | |
1166 | return true; | |
1167 | } | |
1168 | if (!zone_short_id || !rhs.zone_short_id) { | |
1169 | /* don't compare zone ids, if one wasn't provided */ | |
1170 | return false; | |
1171 | } | |
1172 | if (zone_short_id != rhs.zone_short_id) { | |
1173 | return (zone_short_id < rhs.zone_short_id); | |
1174 | } | |
1175 | return (pg_ver < rhs.pg_ver); | |
1176 | } | |
1177 | ||
1178 | void init(const real_time& _mtime, uint32_t _short_id, uint64_t _pg_ver) { | |
1179 | mtime = _mtime; | |
1180 | zone_short_id = _short_id; | |
1181 | pg_ver = _pg_ver; | |
1182 | } | |
1183 | ||
1184 | void init(RGWObjState *state) { | |
1185 | mtime = state->mtime; | |
1186 | zone_short_id = state->zone_short_id; | |
1187 | pg_ver = state->pg_ver; | |
1188 | } | |
1189 | }; | |
1190 | ||
1191 | inline std::ostream& operator<<(std::ostream& out, const obj_time_weight &o) { | |
1192 | out << o.mtime; | |
1193 | ||
1194 | if (o.zone_short_id != 0 || o.pg_ver != 0) { | |
1195 | out << "[zid=" << o.zone_short_id << ", pgv=" << o.pg_ver << "]"; | |
1196 | } | |
1197 | ||
1198 | return out; | |
1199 | } | |
1200 | ||
1201 | } // namespace rgw::sal |