ceph/src/rgw/rgw_sal_motr.h

   1
   2 // vim: ts=2 sw=2 expandtab ft=cpp
   3
   4 /*
   5  * Ceph - scalable distributed file system
   6  *
   7  * SAL implementation for the CORTX Motr backend
   8  *
   9  * Copyright (C) 2021 Seagate Technology LLC and/or its Affiliates
  10  *
  11  * This is free software; you can redistribute it and/or
  12  * modify it under the terms of the GNU Lesser General Public
  13  * License version 2.1, as published by the Free Software
  14  * Foundation. See file COPYING.
  15  *
  16  */
  17
  18 #pragma once
  19
  20 extern "C" {
  21 #pragma clang diagnostic push
  22 #pragma clang diagnostic ignored "-Wextern-c-compat"
  23 #pragma clang diagnostic ignored "-Wdeprecated-anon-enum-enum-conversion"
  24 #include "motr/config.h"
  25 #include "motr/client.h"
  26 #pragma clang diagnostic pop
  27 }
  28
  29 #include "rgw_sal_store.h"
  30 #include "rgw_rados.h"
  31 #include "rgw_notify.h"
  32 #include "rgw_oidc_provider.h"
  33 #include "rgw_role.h"
  34 #include "rgw_multi.h"
  35 #include "rgw_putobj_processor.h"
  36
  37 namespace rgw::sal {
  38
  39 class MotrStore;
  40
  41 // Global Motr indices
  42 #define RGW_MOTR_USERS_IDX_NAME       "motr.rgw.users"
  43 #define RGW_MOTR_BUCKET_INST_IDX_NAME "motr.rgw.bucket.instances"
  44 #define RGW_MOTR_BUCKET_HD_IDX_NAME   "motr.rgw.bucket.headers"
  45 #define RGW_IAM_MOTR_ACCESS_KEY       "motr.rgw.accesskeys"
  46 #define RGW_IAM_MOTR_EMAIL_KEY        "motr.rgw.emails"
  47
  48 //#define RGW_MOTR_BUCKET_ACL_IDX_NAME  "motr.rgw.bucket.acls"
  49
  50 // A simplified metadata cache implementation.
  51 // Note: MotrObjMetaCache doesn't handle the IO operations to Motr. A proxy
  52 // class can be added to handle cache and 'real' ops.
  53 class MotrMetaCache
  54 {
  55 protected:
  56   // MGW re-uses ObjectCache to cache object's metadata as it has already
  57   // implemented a lru cache: (1) ObjectCache internally uses a map and lru
  58   // list to manage cache entry. POC uses object name, user name or bucket
  59   // name as the key to lookup and insert an entry. (2) ObjectCache::data is
  60   // a bufferlist and can be used to store any metadata structure, such as
  61   // object's bucket dir entry, user info or bucket instance.
  62   //
  63   // Note from RGW:
  64   // The Rados Gateway stores metadata and objects in an internal cache. This
  65   // should be kept consistent by the OSD's relaying notify events between
  66   // multiple watching RGW processes. In the event that this notification
  67   // protocol fails, bounding the length of time that any data in the cache will
  68   // be assumed valid will ensure that any RGW instance that falls out of sync
  69   // will eventually recover. This seems to be an issue mostly for large numbers
  70   // of RGW instances under heavy use. If you would like to turn off cache expiry,
  71   // set this value to zero.
  72   //
  73   // Currently POC hasn't implemented the watch-notify menchanism yet. So the
  74   // current implementation is similar to cortx-s3server which is based on expiry
  75   // time. TODO: see comments on distribute_cache).
  76   //
  77   // Beaware: Motr object data is not cached in current POC as RGW!
  78   // RGW caches the first chunk (4MB by default).
  79   ObjectCache cache;
  80
  81 public:
  82   // Lookup a cache entry.
  83   int get(const DoutPrefixProvider *dpp, const std::string& name, bufferlist& data);
  84
  85   // Insert a cache entry.
  86   int put(const DoutPrefixProvider *dpp, const std::string& name, const bufferlist& data);
  87
  88   // Called when an object is deleted. Notification should be sent to other
  89   // RGW instances.
  90   int remove(const DoutPrefixProvider *dpp, const std::string& name);
  91
  92   // Make the local cache entry invalid.
  93   void invalid(const DoutPrefixProvider *dpp, const std::string& name);
  94
  95   // TODO: Distribute_cache() and watch_cb() now are only place holder functions.
  96   // Checkout services/svc_sys_obj_cache.h/cc for reference.
  97   // These 2 functions are designed to notify or to act on cache notification.
  98   // It is feasible to implement the functionality using Motr's FDMI after discussing
  99   // with Hua.
 100   int distribute_cache(const DoutPrefixProvider *dpp,
 101                        const std::string& normal_name,
 102                        ObjectCacheInfo& obj_info, int op);
 103   int watch_cb(const DoutPrefixProvider *dpp,
 104                uint64_t notify_id,
 105                uint64_t cookie,
 106                uint64_t notifier_id,
 107                bufferlist& bl);
 108
 109   void set_enabled(bool status);
 110
 111   MotrMetaCache(const DoutPrefixProvider *dpp, CephContext *cct) {
 112     cache.set_ctx(cct);
 113   }
 114 };
 115
 116 struct MotrUserInfo {
 117   RGWUserInfo info;
 118   obj_version user_version;
 119   rgw::sal::Attrs attrs;
 120
 121   void encode(bufferlist& bl)  const
 122   {
 123     ENCODE_START(3, 3, bl);
 124     encode(info, bl);
 125     encode(user_version, bl);
 126     encode(attrs, bl);
 127     ENCODE_FINISH(bl);
 128   }
 129
 130   void decode(bufferlist::const_iterator& bl)
 131   {
 132     DECODE_START(3, bl);
 133     decode(info, bl);
 134     decode(user_version, bl);
 135     decode(attrs, bl);
 136     DECODE_FINISH(bl);
 137   }
 138 };
 139 WRITE_CLASS_ENCODER(MotrUserInfo);
 140
 141 struct MotrEmailInfo {
 142   std::string user_id;
 143   std::string email_id;
 144
 145   MotrEmailInfo() {}
 146   MotrEmailInfo(std::string _user_id, std::string _email_id )
 147     : user_id(std::move(_user_id)), email_id(std::move(_email_id)) {}
 148
 149   void encode(bufferlist& bl) const {
 150     ENCODE_START(2, 2, bl);
 151     encode(user_id, bl);
 152     encode(email_id, bl);
 153     ENCODE_FINISH(bl);
 154   }
 155
 156   void decode(bufferlist::const_iterator& bl) {
 157      DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl);
 158      decode(user_id, bl);
 159      decode(email_id, bl);
 160       DECODE_FINISH(bl);
 161   }
 162 };
 163 WRITE_CLASS_ENCODER(MotrEmailInfo);
 164
 165 struct MotrAccessKey {
 166   std::string id; // AccessKey
 167   std::string key; // SecretKey
 168   std::string user_id; // UserID
 169
 170   MotrAccessKey() {}
 171   MotrAccessKey(std::string _id, std::string _key, std::string _user_id)
 172     : id(std::move(_id)), key(std::move(_key)), user_id(std::move(_user_id)) {}
 173
 174   void encode(bufferlist& bl) const {
 175     ENCODE_START(2, 2, bl);
 176     encode(id, bl);
 177     encode(key, bl);
 178     encode(user_id, bl);
 179     ENCODE_FINISH(bl);
 180   }
 181
 182   void decode(bufferlist::const_iterator& bl) {
 183      DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl);
 184      decode(id, bl);
 185      decode(key, bl);
 186      decode(user_id, bl);
 187      DECODE_FINISH(bl);
 188   }
 189 };
 190 WRITE_CLASS_ENCODER(MotrAccessKey);
 191
 192 class MotrNotification : public StoreNotification {
 193   public:
 194     MotrNotification(Object* _obj, Object* _src_obj, rgw::notify::EventType _type) :
 195         StoreNotification(_obj, _src_obj, _type) {}
 196     ~MotrNotification() = default;
 197
 198     virtual int publish_reserve(const DoutPrefixProvider *dpp, RGWObjTags* obj_tags = nullptr) override { return 0;}
 199     virtual int publish_commit(const DoutPrefixProvider* dpp, uint64_t size,
 200                                const ceph::real_time& mtime, const std::string& etag, const std::string& version) override { return 0; }
 201 };
 202
 203 class MotrUser : public StoreUser {
 204   private:
 205     MotrStore         *store;
 206     struct m0_uint128  idxID = {0xe5ecb53640d4ecce, 0x6a156cd5a74aa3b8}; // MD5 of “motr.rgw.users“
 207     struct m0_idx      idx;
 208
 209   public:
 210     std::set<std::string> access_key_tracker;
 211     MotrUser(MotrStore *_st, const rgw_user& _u) : StoreUser(_u), store(_st) { }
 212     MotrUser(MotrStore *_st, const RGWUserInfo& _i) : StoreUser(_i), store(_st) { }
 213     MotrUser(MotrStore *_st) : store(_st) { }
 214     MotrUser(MotrUser& _o) = default;
 215     MotrUser() {}
 216
 217     virtual std::unique_ptr<User> clone() override {
 218       return std::unique_ptr<User>(new MotrUser(*this));
 219     }
 220     int list_buckets(const DoutPrefixProvider *dpp, const std::string& marker, const std::string& end_marker,
 221         uint64_t max, bool need_stats, BucketList& buckets, optional_yield y) override;
 222     virtual int create_bucket(const DoutPrefixProvider* dpp,
 223                             const rgw_bucket& b,
 224                             const std::string& zonegroup_id,
 225                             rgw_placement_rule& placement_rule,
 226                             std::string& swift_ver_location,
 227                             const RGWQuotaInfo* pquota_info,
 228                             const RGWAccessControlPolicy& policy,
 229                             Attrs& attrs,
 230                             RGWBucketInfo& info,
 231                             obj_version& ep_objv,
 232                             bool exclusive,
 233                             bool obj_lock_enabled,
 234                             bool* existed,
 235                             req_info& req_info,
 236                             std::unique_ptr<Bucket>* bucket,
 237                             optional_yield y) override;
 238     virtual int read_attrs(const DoutPrefixProvider* dpp, optional_yield y) override;
 239     virtual int merge_and_store_attrs(const DoutPrefixProvider* dpp, Attrs& new_attrs, optional_yield y) override;
 240     virtual int read_stats(const DoutPrefixProvider *dpp,
 241         optional_yield y, RGWStorageStats* stats,
 242         ceph::real_time *last_stats_sync = nullptr,
 243         ceph::real_time *last_stats_update = nullptr) override;
 244     virtual int read_stats_async(const DoutPrefixProvider *dpp, RGWGetUserStats_CB* cb) override;
 245     virtual int complete_flush_stats(const DoutPrefixProvider *dpp, optional_yield y) override;
 246     virtual int read_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch, uint64_t end_epoch, uint32_t max_entries,
 247         bool* is_truncated, RGWUsageIter& usage_iter,
 248         std::map<rgw_user_bucket, rgw_usage_log_entry>& usage) override;
 249     virtual int trim_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch, uint64_t end_epoch) override;
 250
 251     virtual int load_user(const DoutPrefixProvider* dpp, optional_yield y) override;
 252     virtual int store_user(const DoutPrefixProvider* dpp, optional_yield y, bool exclusive, RGWUserInfo* old_info = nullptr) override;
 253     virtual int remove_user(const DoutPrefixProvider* dpp, optional_yield y) override;
 254     virtual int verify_mfa(const std::string& mfa_str, bool* verified, const DoutPrefixProvider* dpp, optional_yield y) override;
 255
 256     int create_user_info_idx();
 257     int load_user_from_idx(const DoutPrefixProvider *dpp, MotrStore *store, RGWUserInfo& info, std::map<std::string,
 258                               bufferlist> *attrs, RGWObjVersionTracker *objv_tr);
 259
 260     friend class MotrBucket;
 261 };
 262
 263 class MotrBucket : public StoreBucket {
 264   private:
 265     MotrStore *store;
 266     RGWAccessControlPolicy acls;
 267
 268   // RGWBucketInfo and other information that are shown when listing a bucket is
 269   // represented in struct MotrBucketInfo. The structure is encoded and stored
 270   // as the value of the global bucket instance index.
 271   // TODO: compare pros and cons of separating the bucket_attrs (ACLs, tag etc.)
 272   // into a different index.
 273   struct MotrBucketInfo {
 274     RGWBucketInfo info;
 275
 276     obj_version bucket_version;
 277     ceph::real_time mtime;
 278
 279     rgw::sal::Attrs bucket_attrs;
 280
 281     void encode(bufferlist& bl)  const
 282     {
 283       ENCODE_START(4, 4, bl);
 284       encode(info, bl);
 285       encode(bucket_version, bl);
 286       encode(mtime, bl);
 287       encode(bucket_attrs, bl); //rgw_cache.h example for a map
 288       ENCODE_FINISH(bl);
 289     }
 290
 291     void decode(bufferlist::const_iterator& bl)
 292     {
 293       DECODE_START(4, bl);
 294       decode(info, bl);
 295       decode(bucket_version, bl);
 296       decode(mtime, bl);
 297       decode(bucket_attrs, bl);
 298       DECODE_FINISH(bl);
 299     }
 300   };
 301   WRITE_CLASS_ENCODER(MotrBucketInfo);
 302
 303   public:
 304     MotrBucket(MotrStore *_st)
 305       : store(_st),
 306       acls() {
 307       }
 308
 309     MotrBucket(MotrStore *_st, User* _u)
 310       : StoreBucket(_u),
 311       store(_st),
 312       acls() {
 313       }
 314
 315     MotrBucket(MotrStore *_st, const rgw_bucket& _b)
 316       : StoreBucket(_b),
 317       store(_st),
 318       acls() {
 319       }
 320
 321     MotrBucket(MotrStore *_st, const RGWBucketEnt& _e)
 322       : StoreBucket(_e),
 323       store(_st),
 324       acls() {
 325       }
 326
 327     MotrBucket(MotrStore *_st, const RGWBucketInfo& _i)
 328       : StoreBucket(_i),
 329       store(_st),
 330       acls() {
 331       }
 332
 333     MotrBucket(MotrStore *_st, const rgw_bucket& _b, User* _u)
 334       : StoreBucket(_b, _u),
 335       store(_st),
 336       acls() {
 337       }
 338
 339     MotrBucket(MotrStore *_st, const RGWBucketEnt& _e, User* _u)
 340       : StoreBucket(_e, _u),
 341       store(_st),
 342       acls() {
 343       }
 344
 345     MotrBucket(MotrStore *_st, const RGWBucketInfo& _i, User* _u)
 346       : StoreBucket(_i, _u),
 347       store(_st),
 348       acls() {
 349       }
 350
 351     ~MotrBucket() { }
 352
 353     virtual std::unique_ptr<Object> get_object(const rgw_obj_key& k) override;
 354     virtual int list(const DoutPrefixProvider *dpp, ListParams&, int, ListResults&, optional_yield y) override;
 355     virtual int remove_bucket(const DoutPrefixProvider *dpp, bool delete_children, bool forward_to_master, req_info* req_info, optional_yield y) override;
 356     virtual int remove_bucket_bypass_gc(int concurrent_max, bool
 357         keep_index_consistent,
 358         optional_yield y, const
 359         DoutPrefixProvider *dpp) override;
 360     virtual RGWAccessControlPolicy& get_acl(void) override { return acls; }
 361     virtual int set_acl(const DoutPrefixProvider *dpp, RGWAccessControlPolicy& acl, optional_yield y) override;
 362     virtual int load_bucket(const DoutPrefixProvider *dpp, optional_yield y, bool get_stats = false) override;
 363     int link_user(const DoutPrefixProvider* dpp, User* new_user, optional_yield y);
 364     int unlink_user(const DoutPrefixProvider* dpp, User* new_user, optional_yield y);
 365     int create_bucket_index();
 366     int create_multipart_indices();
 367     virtual int read_stats(const DoutPrefixProvider *dpp,
 368         const bucket_index_layout_generation& idx_layout, int shard_id,
 369         std::string *bucket_ver, std::string *master_ver,
 370         std::map<RGWObjCategory, RGWStorageStats>& stats,
 371         std::string *max_marker = nullptr,
 372         bool *syncstopped = nullptr) override;
 373     virtual int read_stats_async(const DoutPrefixProvider *dpp,
 374                                  const bucket_index_layout_generation& idx_layout,
 375                                  int shard_id, RGWGetBucketStats_CB* ctx) override;
 376     virtual int sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y) override;
 377     virtual int update_container_stats(const DoutPrefixProvider *dpp) override;
 378     virtual int check_bucket_shards(const DoutPrefixProvider *dpp) override;
 379     virtual int chown(const DoutPrefixProvider *dpp, User& new_user, optional_yield y) override;
 380     virtual int put_info(const DoutPrefixProvider *dpp, bool exclusive, ceph::real_time mtime) override;
 381     virtual bool is_owner(User* user) override;
 382     virtual int check_empty(const DoutPrefixProvider *dpp, optional_yield y) override;
 383     virtual int check_quota(const DoutPrefixProvider *dpp, RGWQuota& quota, uint64_t obj_size, optional_yield y, bool check_size_only = false) override;
 384     virtual int merge_and_store_attrs(const DoutPrefixProvider *dpp, Attrs& attrs, optional_yield y) override;
 385     virtual int try_refresh_info(const DoutPrefixProvider *dpp, ceph::real_time *pmtime) override;
 386     virtual int read_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch, uint64_t end_epoch, uint32_t max_entries,
 387         bool *is_truncated, RGWUsageIter& usage_iter,
 388         std::map<rgw_user_bucket, rgw_usage_log_entry>& usage) override;
 389     virtual int trim_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch, uint64_t end_epoch) override;
 390     virtual int remove_objs_from_index(const DoutPrefixProvider *dpp, std::list<rgw_obj_index_key>& objs_to_unlink) override;
 391     virtual int check_index(const DoutPrefixProvider *dpp, std::map<RGWObjCategory, RGWStorageStats>& existing_stats, std::map<RGWObjCategory, RGWStorageStats>& calculated_stats) override;
 392     virtual int rebuild_index(const DoutPrefixProvider *dpp) override;
 393     virtual int set_tag_timeout(const DoutPrefixProvider *dpp, uint64_t timeout) override;
 394     virtual int purge_instance(const DoutPrefixProvider *dpp) override;
 395     virtual std::unique_ptr<Bucket> clone() override {
 396       return std::make_unique<MotrBucket>(*this);
 397     }
 398     virtual std::unique_ptr<MultipartUpload> get_multipart_upload(const std::string& oid,
 399                                 std::optional<std::string> upload_id=std::nullopt,
 400                                 ACLOwner owner={}, ceph::real_time mtime=real_clock::now()) override;
 401     virtual int list_multiparts(const DoutPrefixProvider *dpp,
 402       const std::string& prefix,
 403       std::string& marker,
 404       const std::string& delim,
 405       const int& max_uploads,
 406       std::vector<std::unique_ptr<MultipartUpload>>& uploads,
 407       std::map<std::string, bool> *common_prefixes,
 408       bool *is_truncated) override;
 409     virtual int abort_multiparts(const DoutPrefixProvider *dpp, CephContext *cct) override;
 410
 411     friend class MotrStore;
 412 };
 413
 414 class MotrPlacementTier: public StorePlacementTier {
 415   MotrStore* store;
 416   RGWZoneGroupPlacementTier tier;
 417 public:
 418   MotrPlacementTier(MotrStore* _store, const RGWZoneGroupPlacementTier& _tier) : store(_store), tier(_tier) {}
 419   virtual ~MotrPlacementTier() = default;
 420
 421   virtual const std::string& get_tier_type() { return tier.tier_type; }
 422   virtual const std::string& get_storage_class() { return tier.storage_class; }
 423   virtual bool retain_head_object() { return tier.retain_head_object; }
 424   RGWZoneGroupPlacementTier& get_rt() { return tier; }
 425 };
 426
 427 class MotrZoneGroup : public StoreZoneGroup {
 428   MotrStore* store;
 429   const RGWZoneGroup group;
 430   std::string empty;
 431 public:
 432   MotrZoneGroup(MotrStore* _store) : store(_store), group() {}
 433   MotrZoneGroup(MotrStore* _store, const RGWZoneGroup& _group) : store(_store), group(_group) {}
 434   virtual ~MotrZoneGroup() = default;
 435
 436   virtual const std::string& get_id() const override { return group.get_id(); };
 437   virtual const std::string& get_name() const override { return group.get_name(); };
 438   virtual int equals(const std::string& other_zonegroup) const override {
 439     return group.equals(other_zonegroup);
 440   };
 441   /** Get the endpoint from zonegroup, or from master zone if not set */
 442   virtual const std::string& get_endpoint() const override;
 443   virtual bool placement_target_exists(std::string& target) const override;
 444   virtual bool is_master_zonegroup() const override {
 445     return group.is_master_zonegroup();
 446   };
 447   virtual const std::string& get_api_name() const override { return group.api_name; };
 448   virtual int get_placement_target_names(std::set<std::string>& names) const override;
 449   virtual const std::string& get_default_placement_name() const override {
 450     return group.default_placement.name; };
 451   virtual int get_hostnames(std::list<std::string>& names) const override {
 452     names = group.hostnames;
 453     return 0;
 454   };
 455   virtual int get_s3website_hostnames(std::list<std::string>& names) const override {
 456    names = group.hostnames_s3website;
 457     return 0;
 458   };
 459   virtual int get_zone_count() const override {
 460     return group.zones.size();
 461   }
 462   virtual int get_placement_tier(const rgw_placement_rule& rule, std::unique_ptr<PlacementTier>* tier);
 463   virtual int get_zone_by_id(const std::string& id, std::unique_ptr<Zone>* zone) override {
 464     return -1;
 465   }
 466   virtual int get_zone_by_name(const std::string& name, std::unique_ptr<Zone>* zone) override {
 467     return -1;
 468   }
 469   virtual int list_zones(std::list<std::string>& zone_ids) override {
 470     zone_ids.clear();
 471     return 0;
 472   }
 473   const RGWZoneGroup& get_group() { return group; }
 474   bool supports(std::string_view feature) const override {
 475     return group.supports(features);
 476   }
 477   virtual std::unique_ptr<ZoneGroup> clone() override {
 478     return std::make_unique<MotrZoneGroup>(store, group);
 479   }
 480 };
 481
 482 class MotrZone : public StoreZone {
 483   protected:
 484     MotrStore* store;
 485     RGWRealm *realm{nullptr};
 486     MotrZoneGroup zonegroup;
 487     RGWZone *zone_public_config{nullptr}; /* external zone params, e.g., entrypoints, log flags, etc. */
 488     RGWZoneParams *zone_params{nullptr}; /* internal zone params, e.g., rados pools */
 489     RGWPeriod *current_period{nullptr};
 490
 491   public:
 492     MotrZone(MotrStore* _store) : store(_store), zonegroup(_store) {
 493       realm = new RGWRealm();
 494       zone_public_config = new RGWZone();
 495       zone_params = new RGWZoneParams();
 496       current_period = new RGWPeriod();
 497
 498       // XXX: only default and STANDARD supported for now
 499       RGWZonePlacementInfo info;
 500       RGWZoneStorageClasses sc;
 501       sc.set_storage_class("STANDARD", nullptr, nullptr);
 502       info.storage_classes = sc;
 503       zone_params->placement_pools["default"] = info;
 504     }
 505     MotrZone(MotrStore* _store, MotrZoneGroup _zg) : store(_store), zonegroup(_zg) {
 506       realm = new RGWRealm();
 507       // TODO: fetch zonegroup params (eg. id) from provisioner config.
 508       zonegroup.set_id("0956b174-fe14-4f97-8b50-bb7ec5e1cf62");
 509       zonegroup.api_name = "default";
 510       zone_public_config = new RGWZone();
 511       zone_params = new RGWZoneParams();
 512       current_period = new RGWPeriod();
 513
 514       // XXX: only default and STANDARD supported for now
 515       RGWZonePlacementInfo info;
 516       RGWZoneStorageClasses sc;
 517       sc.set_storage_class("STANDARD", nullptr, nullptr);
 518       info.storage_classes = sc;
 519       zone_params->placement_pools["default"] = info;
 520     }
 521     ~MotrZone() = default;
 522
 523     virtual std::unique_ptr<Zone> clone() override {
 524       return std::make_unique<MotrZone>(store);
 525     }
 526     virtual ZoneGroup& get_zonegroup() override;
 527     virtual const std::string& get_id() override;
 528     virtual const std::string& get_name() const override;
 529     virtual bool is_writeable() override;
 530     virtual bool get_redirect_endpoint(std::string* endpoint) override;
 531     virtual bool has_zonegroup_api(const std::string& api) const override;
 532     virtual const std::string& get_current_period_id() override;
 533     virtual const RGWAccessKey& get_system_key() { return zone_params->system_key; }
 534     virtual const std::string& get_realm_name() { return realm->get_name(); }
 535     virtual const std::string& get_realm_id() { return realm->get_id(); }
 536     virtual const std::string_view get_tier_type() { return "rgw"; }
 537     virtual RGWBucketSyncPolicyHandlerRef get_sync_policy_handler() { return nullptr; }
 538     friend class MotrStore;
 539 };
 540
 541 class MotrLuaManager : public StoreLuaManager {
 542   MotrStore* store;
 543
 544   public:
 545   MotrLuaManager(MotrStore* _s) : store(_s)
 546   {
 547   }
 548   virtual ~MotrLuaManager() = default;
 549
 550   /** Get a script named with the given key from the backing store */
 551   virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) override;
 552   /** Put a script named with the given key to the backing store */
 553   virtual int put_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script) override;
 554   /** Delete a script named with the given key from the backing store */
 555   virtual int del_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) override;
 556   /** Add a lua package */
 557   virtual int add_package(const DoutPrefixProvider* dpp, optional_yield y, const std::string& package_name) override;
 558   /** Remove a lua package */
 559   virtual int remove_package(const DoutPrefixProvider* dpp, optional_yield y, const std::string& package_name) override;
 560   /** List lua packages */
 561   virtual int list_packages(const DoutPrefixProvider* dpp, optional_yield y, rgw::lua::packages_t& packages) override;
 562 };
 563
 564 class MotrOIDCProvider : public RGWOIDCProvider {
 565   MotrStore* store;
 566   public:
 567   MotrOIDCProvider(MotrStore* _store) : store(_store) {}
 568   ~MotrOIDCProvider() = default;
 569
 570   virtual int store_url(const DoutPrefixProvider *dpp, const std::string& url, bool exclusive, optional_yield y) override { return 0; }
 571   virtual int read_url(const DoutPrefixProvider *dpp, const std::string& url, const std::string& tenant) override { return 0; }
 572   virtual int delete_obj(const DoutPrefixProvider *dpp, optional_yield y) override { return 0;}
 573
 574   void encode(bufferlist& bl) const {
 575     RGWOIDCProvider::encode(bl);
 576   }
 577   void decode(bufferlist::const_iterator& bl) {
 578     RGWOIDCProvider::decode(bl);
 579   }
 580 };
 581
 582 class MotrObject : public StoreObject {
 583   private:
 584     MotrStore *store;
 585     RGWAccessControlPolicy acls;
 586     RGWObjCategory category;
 587
 588     // If this object is pat of a multipart uploaded one.
 589     // TODO: do it in another class? MotrPartObject : public MotrObject
 590     uint64_t part_off;
 591     uint64_t part_size;
 592     uint64_t part_num;
 593
 594   public:
 595
 596     // motr object metadata stored in index
 597     struct Meta {
 598       struct m0_uint128 oid = {};
 599       struct m0_fid pver = {};
 600       uint64_t layout_id = 0;
 601
 602       void encode(bufferlist& bl) const
 603       {
 604         ENCODE_START(5, 5, bl);
 605         encode(oid.u_hi, bl);
 606         encode(oid.u_lo, bl);
 607         encode(pver.f_container, bl);
 608         encode(pver.f_key, bl);
 609         encode(layout_id, bl);
 610         ENCODE_FINISH(bl);
 611       }
 612
 613       void decode(bufferlist::const_iterator& bl)
 614       {
 615         DECODE_START(5, bl);
 616         decode(oid.u_hi, bl);
 617         decode(oid.u_lo, bl);
 618         decode(pver.f_container, bl);
 619         decode(pver.f_key, bl);
 620         decode(layout_id, bl);
 621         DECODE_FINISH(bl);
 622       }
 623     };
 624
 625     struct m0_obj     *mobj = NULL;
 626     Meta               meta;
 627
 628     struct MotrReadOp : public ReadOp {
 629       private:
 630         MotrObject* source;
 631
 632         // The set of part objects if the source is
 633         // a multipart uploaded object.
 634         std::map<int, std::unique_ptr<MotrObject>> part_objs;
 635
 636       public:
 637         MotrReadOp(MotrObject *_source);
 638
 639         virtual int prepare(optional_yield y, const DoutPrefixProvider* dpp) override;
 640
 641         /*
 642          * Both `read` and `iterate` read up through index `end`
 643          * *inclusive*. The number of bytes that could be returned is
 644          * `end - ofs + 1`.
 645          */
 646         virtual int read(int64_t off, int64_t end, bufferlist& bl,
 647                          optional_yield y,
 648                          const DoutPrefixProvider* dpp) override;
 649         virtual int iterate(const DoutPrefixProvider* dpp, int64_t off,
 650                             int64_t end, RGWGetDataCB* cb,
 651                             optional_yield y) override;
 652
 653         virtual int get_attr(const DoutPrefixProvider* dpp, const char* name, bufferlist& dest, optional_yield y) override;
 654     };
 655
 656     struct MotrDeleteOp : public DeleteOp {
 657       private:
 658         MotrObject* source;
 659
 660       public:
 661         MotrDeleteOp(MotrObject* _source);
 662
 663         virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y) override;
 664     };
 665
 666     MotrObject() = default;
 667
 668     MotrObject(MotrStore *_st, const rgw_obj_key& _k)
 669       : StoreObject(_k), store(_st), acls() {}
 670     MotrObject(MotrStore *_st, const rgw_obj_key& _k, Bucket* _b)
 671       : StoreObject(_k, _b), store(_st), acls() {}
 672
 673     MotrObject(MotrObject& _o) = default;
 674
 675     virtual ~MotrObject();
 676
 677     virtual int delete_object(const DoutPrefixProvider* dpp,
 678         optional_yield y,
 679         bool prevent_versioning = false) override;
 680     virtual int delete_obj_aio(const DoutPrefixProvider* dpp, RGWObjState* astate, Completions* aio,
 681         bool keep_index_consistent, optional_yield y) override;
 682     virtual int copy_object(User* user,
 683         req_info* info, const rgw_zone_id& source_zone,
 684         rgw::sal::Object* dest_object, rgw::sal::Bucket* dest_bucket,
 685         rgw::sal::Bucket* src_bucket,
 686         const rgw_placement_rule& dest_placement,
 687         ceph::real_time* src_mtime, ceph::real_time* mtime,
 688         const ceph::real_time* mod_ptr, const ceph::real_time* unmod_ptr,
 689         bool high_precision_time,
 690         const char* if_match, const char* if_nomatch,
 691         AttrsMod attrs_mod, bool copy_if_newer, Attrs& attrs,
 692         RGWObjCategory category, uint64_t olh_epoch,
 693         boost::optional<ceph::real_time> delete_at,
 694         std::string* version_id, std::string* tag, std::string* etag,
 695         void (*progress_cb)(off_t, void *), void* progress_data,
 696         const DoutPrefixProvider* dpp, optional_yield y) override;
 697     virtual RGWAccessControlPolicy& get_acl(void) override { return acls; }
 698     virtual int set_acl(const RGWAccessControlPolicy& acl) override { acls = acl; return 0; }
 699     virtual int get_obj_state(const DoutPrefixProvider* dpp, RGWObjState **state, optional_yield y, bool follow_olh = true) override;
 700     virtual int set_obj_attrs(const DoutPrefixProvider* dpp, Attrs* setattrs, Attrs* delattrs, optional_yield y) override;
 701     virtual int get_obj_attrs(optional_yield y, const DoutPrefixProvider* dpp, rgw_obj* target_obj = NULL) override;
 702     virtual int modify_obj_attrs(const char* attr_name, bufferlist& attr_val, optional_yield y, const DoutPrefixProvider* dpp) override;
 703     virtual int delete_obj_attrs(const DoutPrefixProvider* dpp, const char* attr_name, optional_yield y) override;
 704     virtual bool is_expired() override;
 705     virtual void gen_rand_obj_instance_name() override;
 706     virtual std::unique_ptr<Object> clone() override {
 707       return std::unique_ptr<Object>(new MotrObject(*this));
 708     }
 709     virtual std::unique_ptr<MPSerializer> get_serializer(const DoutPrefixProvider *dpp, const std::string& lock_name) override;
 710     virtual int transition(Bucket* bucket,
 711         const rgw_placement_rule& placement_rule,
 712         const real_time& mtime,
 713         uint64_t olh_epoch,
 714         const DoutPrefixProvider* dpp,
 715         optional_yield y) override;
 716     virtual bool placement_rules_match(rgw_placement_rule& r1, rgw_placement_rule& r2) override;
 717     virtual int dump_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f) override;
 718
 719     /* Swift versioning */
 720     virtual int swift_versioning_restore(bool& restored,
 721         const DoutPrefixProvider* dpp) override;
 722     virtual int swift_versioning_copy(const DoutPrefixProvider* dpp,
 723         optional_yield y) override;
 724
 725     /* OPs */
 726     virtual std::unique_ptr<ReadOp> get_read_op() override;
 727     virtual std::unique_ptr<DeleteOp> get_delete_op() override;
 728
 729     /* OMAP */
 730     virtual int omap_get_vals(const DoutPrefixProvider *dpp, const std::string& marker, uint64_t count,
 731         std::map<std::string, bufferlist> *m,
 732         bool* pmore, optional_yield y) override;
 733     virtual int omap_get_all(const DoutPrefixProvider *dpp, std::map<std::string, bufferlist> *m,
 734         optional_yield y) override;
 735     virtual int omap_get_vals_by_keys(const DoutPrefixProvider *dpp, const std::string& oid,
 736         const std::set<std::string>& keys,
 737         Attrs* vals) override;
 738     virtual int omap_set_val_by_key(const DoutPrefixProvider *dpp, const std::string& key, bufferlist& val,
 739         bool must_exist, optional_yield y) override;
 740     virtual int chown(User& new_user, const DoutPrefixProvider* dpp, optional_yield y) override;
 741   private:
 742     //int read_attrs(const DoutPrefixProvider* dpp, Motr::Object::Read &read_op, optional_yield y, rgw_obj* target_obj = nullptr);
 743
 744   public:
 745     bool is_opened() { return mobj != NULL; }
 746     int create_mobj(const DoutPrefixProvider *dpp, uint64_t sz);
 747     int open_mobj(const DoutPrefixProvider *dpp);
 748     int delete_mobj(const DoutPrefixProvider *dpp);
 749     void close_mobj();
 750     int write_mobj(const DoutPrefixProvider *dpp, bufferlist&& data, uint64_t offset);
 751     int read_mobj(const DoutPrefixProvider* dpp, int64_t off, int64_t end, RGWGetDataCB* cb);
 752     unsigned get_optimal_bs(unsigned len);
 753
 754     int get_part_objs(const DoutPrefixProvider *dpp,
 755                       std::map<int, std::unique_ptr<MotrObject>>& part_objs);
 756     int open_part_objs(const DoutPrefixProvider* dpp,
 757                        std::map<int, std::unique_ptr<MotrObject>>& part_objs);
 758     int read_multipart_obj(const DoutPrefixProvider* dpp,
 759                            int64_t off, int64_t end, RGWGetDataCB* cb,
 760                            std::map<int, std::unique_ptr<MotrObject>>& part_objs);
 761     int delete_part_objs(const DoutPrefixProvider* dpp);
 762     void set_category(RGWObjCategory _category) {category = _category;}
 763     int get_bucket_dir_ent(const DoutPrefixProvider *dpp, rgw_bucket_dir_entry& ent);
 764     int update_version_entries(const DoutPrefixProvider *dpp);
 765 };
 766
 767 // A placeholder locking class for multipart upload.
 768 // TODO: implement it using Motr object locks.
 769 class MPMotrSerializer : public StoreMPSerializer {
 770
 771   public:
 772     MPMotrSerializer(const DoutPrefixProvider *dpp, MotrStore* store, MotrObject* obj, const std::string& lock_name) {}
 773
 774     virtual int try_lock(const DoutPrefixProvider *dpp, utime_t dur, optional_yield y) override {return 0; }
 775     virtual int unlock() override { return 0;}
 776 };
 777
 778 class MotrAtomicWriter : public StoreWriter {
 779   protected:
 780   rgw::sal::MotrStore* store;
 781   const rgw_user& owner;
 782   const rgw_placement_rule *ptail_placement_rule;
 783   uint64_t olh_epoch;
 784   const std::string& unique_tag;
 785   MotrObject obj;
 786   MotrObject old_obj;
 787   uint64_t total_data_size; // for total data being uploaded
 788   bufferlist acc_data;  // accumulated data
 789   uint64_t   acc_off; // accumulated data offset
 790
 791   struct m0_bufvec buf;
 792   struct m0_bufvec attr;
 793   struct m0_indexvec ext;
 794
 795   public:
 796   MotrAtomicWriter(const DoutPrefixProvider *dpp,
 797           optional_yield y,
 798           rgw::sal::Object* obj,
 799           MotrStore* _store,
 800           const rgw_user& _owner,
 801           const rgw_placement_rule *_ptail_placement_rule,
 802           uint64_t _olh_epoch,
 803           const std::string& _unique_tag);
 804   ~MotrAtomicWriter() = default;
 805
 806   // prepare to start processing object data
 807   virtual int prepare(optional_yield y) override;
 808
 809   // Process a bufferlist
 810   virtual int process(bufferlist&& data, uint64_t offset) override;
 811
 812   int write();
 813
 814   // complete the operation and make its result visible to clients
 815   virtual int complete(size_t accounted_size, const std::string& etag,
 816                        ceph::real_time *mtime, ceph::real_time set_mtime,
 817                        std::map<std::string, bufferlist>& attrs,
 818                        ceph::real_time delete_at,
 819                        const char *if_match, const char *if_nomatch,
 820                        const std::string *user_data,
 821                        rgw_zone_set *zones_trace, bool *canceled,
 822                        optional_yield y) override;
 823
 824   unsigned populate_bvec(unsigned len, bufferlist::iterator &bi);
 825   void cleanup();
 826 };
 827
 828 class MotrMultipartWriter : public StoreWriter {
 829 protected:
 830   rgw::sal::MotrStore* store;
 831
 832   // Head object.
 833   rgw::sal::Object* head_obj;
 834
 835   // Part parameters.
 836   const uint64_t part_num;
 837   const std::string part_num_str;
 838   std::unique_ptr<MotrObject> part_obj;
 839   uint64_t actual_part_size = 0;
 840
 841 public:
 842   MotrMultipartWriter(const DoutPrefixProvider *dpp,
 843                        optional_yield y, MultipartUpload* upload,
 844                        rgw::sal::Object* obj,
 845                        MotrStore* _store,
 846                        const rgw_user& owner,
 847                        const rgw_placement_rule *ptail_placement_rule,
 848                        uint64_t _part_num, const std::string& part_num_str) :
 849                                   StoreWriter(dpp, y), store(_store), head_obj(obj),
 850                                   part_num(_part_num), part_num_str(part_num_str)
 851   {
 852   }
 853   ~MotrMultipartWriter() = default;
 854
 855   // prepare to start processing object data
 856   virtual int prepare(optional_yield y) override;
 857
 858   // Process a bufferlist
 859   virtual int process(bufferlist&& data, uint64_t offset) override;
 860
 861   // complete the operation and make its result visible to clients
 862   virtual int complete(size_t accounted_size, const std::string& etag,
 863                        ceph::real_time *mtime, ceph::real_time set_mtime,
 864                        std::map<std::string, bufferlist>& attrs,
 865                        ceph::real_time delete_at,
 866                        const char *if_match, const char *if_nomatch,
 867                        const std::string *user_data,
 868                        rgw_zone_set *zones_trace, bool *canceled,
 869                        optional_yield y) override;
 870 };
 871
 872 // The implementation of multipart upload in POC roughly follows the
 873 // cortx-s3server's design. Parts are stored in separate Motr objects.
 874 // s3server uses a few auxiliary Motr indices to manage multipart
 875 // related metadata: (1) Bucket multipart index (bucket_nnn_multipart_index)
 876 // which contains metadata that answers questions such as which objects have
 877 // started  multipart upload and its upload id. This index is created during
 878 // bucket creation. (2) Object part index (object_nnn_part_index) which stores
 879 // metadata of a part's details (size, pvid, oid...). This index is created in
 880 // MotrMultipartUpload::init(). (3) Extended metadata index
 881 // (bucket_nnn_extended_metadata): once parts has been uploaded and their
 882 // metadata saved in the part index, the user may issue multipart completion
 883 // request. When processing the completion request, the parts are read from
 884 // object part index and for each part an entry is created in extended index.
 885 // The entry for the object is created in bucket (object list) index. The part
 886 // index is deleted and an entry removed from bucket_nnn_multipart_index. Like
 887 // bucket multipart index, bucket part extened metadata index is created during
 888 // bucket creation.
 889 //
 890 // The extended metadata index is used mainly due to fault tolerant
 891 // considerations (how to handle Motr service crash when uploading an object)
 892 // and to avoid to create too many Motr indices (I am not sure I understand
 893 // why many Motr indices is bad.). In our POC, to keep it simple, only 2
 894 // indices are maintained: bucket multipart index and object_nnn_part_index.
 895 //
 896 //
 897
 898 class MotrMultipartPart : public StoreMultipartPart {
 899 protected:
 900   RGWUploadPartInfo info;
 901
 902 public:
 903   MotrObject::Meta  meta;
 904
 905   MotrMultipartPart(RGWUploadPartInfo _info, MotrObject::Meta _meta) :
 906     info(_info), meta(_meta) {}
 907   virtual ~MotrMultipartPart() = default;
 908
 909   virtual uint32_t get_num() { return info.num; }
 910   virtual uint64_t get_size() { return info.accounted_size; }
 911   virtual const std::string& get_etag() { return info.etag; }
 912   virtual ceph::real_time& get_mtime() { return info.modified; }
 913
 914   RGWObjManifest& get_manifest() { return info.manifest; }
 915
 916   friend class MotrMultipartUpload;
 917 };
 918
 919 class MotrMultipartUpload : public StoreMultipartUpload {
 920   MotrStore* store;
 921   RGWMPObj mp_obj;
 922   ACLOwner owner;
 923   ceph::real_time mtime;
 924   rgw_placement_rule placement;
 925   RGWObjManifest manifest;
 926
 927 public:
 928   MotrMultipartUpload(MotrStore* _store, Bucket* _bucket, const std::string& oid,
 929                       std::optional<std::string> upload_id, ACLOwner _owner, ceph::real_time _mtime) :
 930        StoreMultipartUpload(_bucket), store(_store), mp_obj(oid, upload_id), owner(_owner), mtime(_mtime) {}
 931   virtual ~MotrMultipartUpload() = default;
 932
 933   virtual const std::string& get_meta() const { return mp_obj.get_meta(); }
 934   virtual const std::string& get_key() const { return mp_obj.get_key(); }
 935   virtual const std::string& get_upload_id() const { return mp_obj.get_upload_id(); }
 936   virtual const ACLOwner& get_owner() const override { return owner; }
 937   virtual ceph::real_time& get_mtime() { return mtime; }
 938   virtual std::unique_ptr<rgw::sal::Object> get_meta_obj() override;
 939   virtual int init(const DoutPrefixProvider* dpp, optional_yield y, ACLOwner& owner, rgw_placement_rule& dest_placement, rgw::sal::Attrs& attrs) override;
 940   virtual int list_parts(const DoutPrefixProvider* dpp, CephContext* cct,
 941                          int num_parts, int marker,
 942                          int* next_marker, bool* truncated,
 943                          bool assume_unsorted = false) override;
 944   virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct) override;
 945   virtual int complete(const DoutPrefixProvider* dpp,
 946                        optional_yield y, CephContext* cct,
 947                        std::map<int, std::string>& part_etags,
 948                        std::list<rgw_obj_index_key>& remove_objs,
 949                        uint64_t& accounted_size, bool& compressed,
 950                        RGWCompressionInfo& cs_info, off_t& off,
 951                        std::string& tag, ACLOwner& owner,
 952                        uint64_t olh_epoch,
 953                        rgw::sal::Object* target_obj) override;
 954   virtual int get_info(const DoutPrefixProvider *dpp, optional_yield y, rgw_placement_rule** rule, rgw::sal::Attrs* attrs = nullptr) override;
 955   virtual std::unique_ptr<Writer> get_writer(const DoutPrefixProvider *dpp,
 956                           optional_yield y,
 957                           rgw::sal::Object* obj,
 958                           const rgw_user& owner,
 959                           const rgw_placement_rule *ptail_placement_rule,
 960                           uint64_t part_num,
 961                           const std::string& part_num_str) override;
 962   int delete_parts(const DoutPrefixProvider *dpp);
 963 };
 964
 965 class MotrStore : public StoreDriver {
 966   private:
 967     MotrZone zone;
 968     RGWSyncModuleInstanceRef sync_module;
 969
 970     MotrMetaCache* obj_meta_cache;
 971     MotrMetaCache* user_cache;
 972     MotrMetaCache* bucket_inst_cache;
 973
 974   public:
 975     CephContext *cctx;
 976     struct m0_client   *instance;
 977     struct m0_container container;
 978     struct m0_realm     uber_realm;
 979     struct m0_config    conf = {};
 980     struct m0_idx_dix_config dix_conf = {};
 981
 982     MotrStore(CephContext *c): zone(this), cctx(c) {}
 983     ~MotrStore() {
 984       delete obj_meta_cache;
 985       delete user_cache;
 986       delete bucket_inst_cache;
 987     }
 988
 989     virtual int initialize(CephContext *cct, const DoutPrefixProvider *dpp) { return 0; }
 990     virtual const std::string get_name() const override {
 991       return "motr";
 992     }
 993
 994     virtual std::unique_ptr<User> get_user(const rgw_user& u) override;
 995     virtual std::string get_cluster_id(const DoutPrefixProvider* dpp,  optional_yield y) override;
 996     virtual int get_user_by_access_key(const DoutPrefixProvider *dpp, const std::string& key, optional_yield y, std::unique_ptr<User>* user) override;
 997     virtual int get_user_by_email(const DoutPrefixProvider *dpp, const std::string& email, optional_yield y, std::unique_ptr<User>* user) override;
 998     virtual int get_user_by_swift(const DoutPrefixProvider *dpp, const std::string& user_str, optional_yield y, std::unique_ptr<User>* user) override;
 999     virtual std::unique_ptr<Object> get_object(const rgw_obj_key& k) override;
1000     virtual int get_bucket(const DoutPrefixProvider *dpp, User* u, const rgw_bucket& b, std::unique_ptr<Bucket>* bucket, optional_yield y) override;
1001     virtual int get_bucket(User* u, const RGWBucketInfo& i, std::unique_ptr<Bucket>* bucket) override;
1002     virtual int get_bucket(const DoutPrefixProvider *dpp, User* u, const std::string& tenant, const std::string&name, std::unique_ptr<Bucket>* bucket, optional_yield y) override;
1003     virtual bool is_meta_master() override;
1004     virtual int forward_request_to_master(const DoutPrefixProvider *dpp, User* user, obj_version* objv,
1005         bufferlist& in_data, JSONParser *jp, req_info& info,
1006         optional_yield y) override;
1007     virtual int forward_iam_request_to_master(const DoutPrefixProvider *dpp, const RGWAccessKey& key, obj_version* objv,
1008                                              bufferlist& in_data,
1009                                              RGWXMLDecoder::XMLParser* parser, req_info& info,
1010                                              optional_yield y) override;
1011     virtual Zone* get_zone() { return &zone; }
1012     virtual std::string zone_unique_id(uint64_t unique_num) override;
1013     virtual std::string zone_unique_trans_id(const uint64_t unique_num) override;
1014     virtual int get_zonegroup(const std::string& id, std::unique_ptr<ZoneGroup>* zonegroup) override;
1015     virtual int list_all_zones(const DoutPrefixProvider* dpp, std::list<std::string>& zone_ids) override;
1016     virtual int cluster_stat(RGWClusterStat& stats) override;
1017     virtual std::unique_ptr<Lifecycle> get_lifecycle(void) override;
1018     virtual std::unique_ptr<Completions> get_completions(void) override;
1019     virtual std::unique_ptr<Notification> get_notification(rgw::sal::Object* obj, rgw::sal::Object* src_obj,
1020         req_state* s, rgw::notify::EventType event_type, optional_yield y, const std::string* object_name=nullptr) override;
1021     virtual std::unique_ptr<Notification> get_notification(const DoutPrefixProvider* dpp, rgw::sal::Object* obj,
1022         rgw::sal::Object* src_obj, rgw::notify::EventType event_type, rgw::sal::Bucket* _bucket,
1023         std::string& _user_id, std::string& _user_tenant, std::string& _req_id, optional_yield y) override;
1024     virtual RGWLC* get_rgwlc(void) override { return NULL; }
1025     virtual RGWCoroutinesManagerRegistry* get_cr_registry() override { return NULL; }
1026
1027     virtual int log_usage(const DoutPrefixProvider *dpp, std::map<rgw_user_bucket, RGWUsageBatch>& usage_info) override;
1028     virtual int log_op(const DoutPrefixProvider *dpp, std::string& oid, bufferlist& bl) override;
1029     virtual int register_to_service_map(const DoutPrefixProvider *dpp, const std::string& daemon_type,
1030         const std::map<std::string, std::string>& meta) override;
1031     virtual void get_ratelimit(RGWRateLimitInfo& bucket_ratelimit, RGWRateLimitInfo& user_ratelimit, RGWRateLimitInfo& anon_ratelimit) override;
1032     virtual void get_quota(RGWQuota& quota) override;
1033     virtual int set_buckets_enabled(const DoutPrefixProvider *dpp, std::vector<rgw_bucket>& buckets, bool enabled) override;
1034     virtual int get_sync_policy_handler(const DoutPrefixProvider *dpp,
1035         std::optional<rgw_zone_id> zone,
1036         std::optional<rgw_bucket> bucket,
1037         RGWBucketSyncPolicyHandlerRef *phandler,
1038         optional_yield y) override;
1039     virtual RGWDataSyncStatusManager* get_data_sync_manager(const rgw_zone_id& source_zone) override;
1040     virtual void wakeup_meta_sync_shards(std::set<int>& shard_ids) override { return; }
1041     virtual void wakeup_data_sync_shards(const DoutPrefixProvider *dpp, const rgw_zone_id& source_zone, boost::container::flat_map<int, boost::container::flat_set<rgw_data_notify_entry>>& shard_ids) override {}
1042     virtual int clear_usage(const DoutPrefixProvider *dpp) override { return 0; }
1043     virtual int read_all_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch, uint64_t end_epoch,
1044         uint32_t max_entries, bool *is_truncated,
1045         RGWUsageIter& usage_iter,
1046         std::map<rgw_user_bucket, rgw_usage_log_entry>& usage) override;
1047     virtual int trim_all_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch, uint64_t end_epoch) override;
1048     virtual int get_config_key_val(std::string name, bufferlist* bl) override;
1049     virtual int meta_list_keys_init(const DoutPrefixProvider *dpp, const std::string& section, const std::string& marker, void** phandle) override;
1050     virtual int meta_list_keys_next(const DoutPrefixProvider *dpp, void* handle, int max, std::list<std::string>& keys, bool* truncated) override;
1051     virtual void meta_list_keys_complete(void* handle) override;
1052     virtual std::string meta_get_marker(void *handle) override;
1053     virtual int meta_remove(const DoutPrefixProvider *dpp, std::string& metadata_key, optional_yield y) override;
1054
1055     virtual const RGWSyncModuleInstanceRef& get_sync_module() { return sync_module; }
1056     virtual std::string get_host_id() { return ""; }
1057
1058     virtual std::unique_ptr<LuaManager> get_lua_manager() override;
1059     virtual std::unique_ptr<RGWRole> get_role(std::string name,
1060         std::string tenant,
1061         std::string path="",
1062         std::string trust_policy="",
1063         std::string max_session_duration_str="",
1064         std::multimap<std::string, std::string> tags={}) override;
1065     virtual std::unique_ptr<RGWRole> get_role(const RGWRoleInfo& info) override;
1066     virtual std::unique_ptr<RGWRole> get_role(std::string id) override;
1067     virtual int get_roles(const DoutPrefixProvider *dpp,
1068         optional_yield y,
1069         const std::string& path_prefix,
1070         const std::string& tenant,
1071         std::vector<std::unique_ptr<RGWRole>>& roles) override;
1072     virtual std::unique_ptr<RGWOIDCProvider> get_oidc_provider() override;
1073     virtual int get_oidc_providers(const DoutPrefixProvider *dpp,
1074         const std::string& tenant,
1075         std::vector<std::unique_ptr<RGWOIDCProvider>>& providers) override;
1076     virtual std::unique_ptr<Writer> get_append_writer(const DoutPrefixProvider *dpp,
1077         optional_yield y,
1078         rgw::sal::Object* obj,
1079         const rgw_user& owner,
1080         const rgw_placement_rule *ptail_placement_rule,
1081         const std::string& unique_tag,
1082         uint64_t position,
1083         uint64_t *cur_accounted_size) override;
1084     virtual std::unique_ptr<Writer> get_atomic_writer(const DoutPrefixProvider *dpp,
1085         optional_yield y,
1086         rgw::sal::Object* obj,
1087         const rgw_user& owner,
1088         const rgw_placement_rule *ptail_placement_rule,
1089         uint64_t olh_epoch,
1090         const std::string& unique_tag) override;
1091     virtual const std::string& get_compression_type(const rgw_placement_rule& rule) override;
1092     virtual bool valid_placement(const rgw_placement_rule& rule) override;
1093
1094     virtual void finalize(void) override;
1095
1096     virtual CephContext *ctx(void) override {
1097       return cctx;
1098     }
1099
1100     virtual void register_admin_apis(RGWRESTMgr* mgr) override { };
1101
1102     int open_idx(struct m0_uint128 *id, bool create, struct m0_idx *out);
1103     void close_idx(struct m0_idx *idx) { m0_idx_fini(idx); }
1104     int do_idx_op(struct m0_idx *, enum m0_idx_opcode opcode,
1105       std::vector<uint8_t>& key, std::vector<uint8_t>& val, bool update = false);
1106
1107     int do_idx_next_op(struct m0_idx *idx,
1108                        std::vector<std::vector<uint8_t>>& key_vec,
1109                        std::vector<std::vector<uint8_t>>& val_vec);
1110     int next_query_by_name(std::string idx_name, std::vector<std::string>& key_str_vec,
1111                                             std::vector<bufferlist>& val_bl_vec,
1112                                             std::string prefix="", std::string delim="");
1113
1114     void index_name_to_motr_fid(std::string iname, struct m0_uint128 *fid);
1115     int open_motr_idx(struct m0_uint128 *id, struct m0_idx *idx);
1116     int create_motr_idx_by_name(std::string iname);
1117     int delete_motr_idx_by_name(std::string iname);
1118     int do_idx_op_by_name(std::string idx_name, enum m0_idx_opcode opcode,
1119                           std::string key_str, bufferlist &bl, bool update=true);
1120     int check_n_create_global_indices();
1121     int store_access_key(const DoutPrefixProvider *dpp, optional_yield y, MotrAccessKey access_key);
1122     int delete_access_key(const DoutPrefixProvider *dpp, optional_yield y, std::string access_key);
1123     int store_email_info(const DoutPrefixProvider *dpp, optional_yield y, MotrEmailInfo& email_info);
1124
1125     int init_metadata_cache(const DoutPrefixProvider *dpp, CephContext *cct);
1126     MotrMetaCache* get_obj_meta_cache() {return obj_meta_cache;}
1127     MotrMetaCache* get_user_cache() {return user_cache;}
1128     MotrMetaCache* get_bucket_inst_cache() {return bucket_inst_cache;}
1129 };
1130
1131 struct obj_time_weight {
1132   real_time mtime;
1133   uint32_t zone_short_id;
1134   uint64_t pg_ver;
1135   bool high_precision;
1136
1137   obj_time_weight() : zone_short_id(0), pg_ver(0), high_precision(false) {}
1138
1139   bool compare_low_precision(const obj_time_weight& rhs) {
1140     struct timespec l = ceph::real_clock::to_timespec(mtime);
1141     struct timespec r = ceph::real_clock::to_timespec(rhs.mtime);
1142     l.tv_nsec = 0;
1143     r.tv_nsec = 0;
1144     if (l > r) {
1145       return false;
1146     }
1147     if (l < r) {
1148       return true;
1149     }
1150     if (!zone_short_id || !rhs.zone_short_id) {
1151       /* don't compare zone ids, if one wasn't provided */
1152       return false;
1153     }
1154     if (zone_short_id != rhs.zone_short_id) {
1155       return (zone_short_id < rhs.zone_short_id);
1156     }
1157     return (pg_ver < rhs.pg_ver);
1158
1159   }
1160
1161   bool operator<(const obj_time_weight& rhs) {
1162     if (!high_precision || !rhs.high_precision) {
1163       return compare_low_precision(rhs);
1164     }
1165     if (mtime > rhs.mtime) {
1166       return false;
1167     }
1168     if (mtime < rhs.mtime) {
1169       return true;
1170     }
1171     if (!zone_short_id || !rhs.zone_short_id) {
1172       /* don't compare zone ids, if one wasn't provided */
1173       return false;
1174     }
1175     if (zone_short_id != rhs.zone_short_id) {
1176       return (zone_short_id < rhs.zone_short_id);
1177     }
1178     return (pg_ver < rhs.pg_ver);
1179   }
1180
1181   void init(const real_time& _mtime, uint32_t _short_id, uint64_t _pg_ver) {
1182     mtime = _mtime;
1183     zone_short_id = _short_id;
1184     pg_ver = _pg_ver;
1185   }
1186
1187   void init(RGWObjState *state) {
1188     mtime = state->mtime;
1189     zone_short_id = state->zone_short_id;
1190     pg_ver = state->pg_ver;
1191   }
1192 };
1193
1194 inline std::ostream& operator<<(std::ostream& out, const obj_time_weight &o) {
1195   out << o.mtime;
1196
1197   if (o.zone_short_id != 0 || o.pg_ver != 0) {
1198     out << "[zid=" << o.zone_short_id << ", pgv=" << o.pg_ver << "]";
1199   }
1200
1201   return out;
1202 }
1203
1204 } // namespace rgw::sal