ceph/src/rgw/driver/rados/rgw_obj_manifest.h

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab ft=cpp
   3
   4 /*
   5  * Ceph - scalable distributed file system
   6  *
   7  * Copyright (C) 2019 Red Hat, Inc.
   8  *
   9  * This is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU Lesser General Public
  11  * License version 2.1, as published by the Free Software
  12  * Foundation. See file COPYING.
  13  *
  14  */
  15
  16 /* N.B., this header defines fundamental serialized types.  Do not
  17  * introduce changes or include files which can only be compiled in
  18  * radosgw or OSD contexts (e.g., rgw_sal.h, rgw_common.h)
  19  */
  20
  21 #pragma once
  22
  23 #include "rgw_zone_types.h"
  24 #include "rgw_bucket_types.h"
  25 #include "rgw_obj_types.h"
  26 #include "rgw_placement_types.h"
  27
  28 #include "common/dout.h"
  29 #include "common/Formatter.h"
  30
  31 class RGWSI_Zone;
  32 struct RGWZoneGroup;
  33 struct RGWZoneParams;
  34 class RGWRados;
  35
  36 namespace rgw { namespace sal {
  37   class RadosStore;
  38 } };
  39
  40 class rgw_obj_select {
  41   rgw_placement_rule placement_rule;
  42   rgw_obj obj;
  43   rgw_raw_obj raw_obj;
  44   bool is_raw;
  45
  46 public:
  47   rgw_obj_select() : is_raw(false) {}
  48   explicit rgw_obj_select(const rgw_obj& _obj) : obj(_obj), is_raw(false) {}
  49   explicit rgw_obj_select(const rgw_raw_obj& _raw_obj) : raw_obj(_raw_obj), is_raw(true) {}
  50   rgw_obj_select(const rgw_obj_select& rhs) {
  51     placement_rule = rhs.placement_rule;
  52     is_raw = rhs.is_raw;
  53     if (is_raw) {
  54       raw_obj = rhs.raw_obj;
  55     } else {
  56       obj = rhs.obj;
  57     }
  58   }
  59
  60   rgw_raw_obj get_raw_obj(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params) const;
  61   rgw_raw_obj get_raw_obj(RGWRados* store) const;
  62
  63   rgw_obj_select& operator=(const rgw_obj& rhs) {
  64     obj = rhs;
  65     is_raw = false;
  66     return *this;
  67   }
  68
  69   rgw_obj_select& operator=(const rgw_raw_obj& rhs) {
  70     raw_obj = rhs;
  71     is_raw = true;
  72     return *this;
  73   }
  74
  75   void set_placement_rule(const rgw_placement_rule& rule) {
  76     placement_rule = rule;
  77   }
  78   void dump(Formatter *f) const;
  79 };
  80
  81 struct RGWObjManifestPart {
  82   rgw_obj loc;   /* the object where the data is located */
  83   uint64_t loc_ofs;  /* the offset at that object where the data is located */
  84   uint64_t size;     /* the part size */
  85
  86   RGWObjManifestPart() : loc_ofs(0), size(0) {}
  87
  88   void encode(bufferlist& bl) const {
  89     ENCODE_START(2, 2, bl);
  90     encode(loc, bl);
  91     encode(loc_ofs, bl);
  92     encode(size, bl);
  93     ENCODE_FINISH(bl);
  94   }
  95
  96   void decode(bufferlist::const_iterator& bl) {
  97      DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl);
  98      decode(loc, bl);
  99      decode(loc_ofs, bl);
 100      decode(size, bl);
 101      DECODE_FINISH(bl);
 102   }
 103
 104   void dump(Formatter *f) const;
 105   static void generate_test_instances(std::list<RGWObjManifestPart*>& o);
 106 };
 107 WRITE_CLASS_ENCODER(RGWObjManifestPart)
 108
 109 /*
 110  The manifest defines a set of rules for structuring the object parts.
 111  There are a few terms to note:
 112      - head: the head part of the object, which is the part that contains
 113        the first chunk of data. An object might not have a head (as in the
 114        case of multipart-part objects).
 115      - stripe: data portion of a single rgw object that resides on a single
 116        rados object.
 117      - part: a collection of stripes that make a contiguous part of an
 118        object. A regular object will only have one part (although might have
 119        many stripes), a multipart object might have many parts. Each part
 120        has a fixed stripe size, although the last stripe of a part might
 121        be smaller than that. Consecutive parts may be merged if their stripe
 122        value is the same.
 123 */
 124
 125 struct RGWObjManifestRule {
 126   uint32_t start_part_num;
 127   uint64_t start_ofs;
 128   uint64_t part_size; /* each part size, 0 if there's no part size, meaning it's unlimited */
 129   uint64_t stripe_max_size; /* underlying obj max size */
 130   std::string override_prefix;
 131
 132   RGWObjManifestRule() : start_part_num(0), start_ofs(0), part_size(0), stripe_max_size(0) {}
 133   RGWObjManifestRule(uint32_t _start_part_num, uint64_t _start_ofs, uint64_t _part_size, uint64_t _stripe_max_size) :
 134                        start_part_num(_start_part_num), start_ofs(_start_ofs), part_size(_part_size), stripe_max_size(_stripe_max_size) {}
 135
 136   void encode(bufferlist& bl) const {
 137     ENCODE_START(2, 1, bl);
 138     encode(start_part_num, bl);
 139     encode(start_ofs, bl);
 140     encode(part_size, bl);
 141     encode(stripe_max_size, bl);
 142     encode(override_prefix, bl);
 143     ENCODE_FINISH(bl);
 144   }
 145
 146   void decode(bufferlist::const_iterator& bl) {
 147     DECODE_START(2, bl);
 148     decode(start_part_num, bl);
 149     decode(start_ofs, bl);
 150     decode(part_size, bl);
 151     decode(stripe_max_size, bl);
 152     if (struct_v >= 2)
 153       decode(override_prefix, bl);
 154     DECODE_FINISH(bl);
 155   }
 156   void dump(Formatter *f) const;
 157 };
 158 WRITE_CLASS_ENCODER(RGWObjManifestRule)
 159
 160 struct RGWObjTier {
 161     std::string name;
 162     RGWZoneGroupPlacementTier tier_placement;
 163     bool is_multipart_upload{false};
 164
 165     RGWObjTier(): name("none") {}
 166
 167     void encode(bufferlist& bl) const {
 168       ENCODE_START(2, 2, bl);
 169       encode(name, bl);
 170       encode(tier_placement, bl);
 171       encode(is_multipart_upload, bl);
 172       ENCODE_FINISH(bl);
 173     }
 174
 175     void decode(bufferlist::const_iterator& bl) {
 176       DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
 177       decode(name, bl);
 178       decode(tier_placement, bl);
 179       decode(is_multipart_upload, bl);
 180       DECODE_FINISH(bl);
 181     }
 182     void dump(Formatter *f) const;
 183 };
 184 WRITE_CLASS_ENCODER(RGWObjTier)
 185
 186 class RGWObjManifest {
 187 protected:
 188   bool explicit_objs{false}; /* really old manifest? */
 189   std::map<uint64_t, RGWObjManifestPart> objs;
 190
 191   uint64_t obj_size{0};
 192
 193   rgw_obj obj;
 194   uint64_t head_size{0};
 195   rgw_placement_rule head_placement_rule;
 196
 197   uint64_t max_head_size{0};
 198   std::string prefix;
 199   rgw_bucket_placement tail_placement; /* might be different than the original bucket,
 200                                        as object might have been copied across pools */
 201   std::map<uint64_t, RGWObjManifestRule> rules;
 202
 203   std::string tail_instance; /* tail object's instance */
 204
 205   std::string tier_type;
 206   RGWObjTier tier_config;
 207
 208   void convert_to_explicit(const DoutPrefixProvider *dpp, const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params);
 209   int append_explicit(const DoutPrefixProvider *dpp, RGWObjManifest& m, const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params);
 210   void append_rules(RGWObjManifest& m, std::map<uint64_t, RGWObjManifestRule>::iterator& iter, std::string *override_prefix);
 211
 212 public:
 213
 214   RGWObjManifest() = default;
 215   RGWObjManifest(const RGWObjManifest& rhs) {
 216     *this = rhs;
 217   }
 218   RGWObjManifest& operator=(const RGWObjManifest& rhs) {
 219     explicit_objs = rhs.explicit_objs;
 220     objs = rhs.objs;
 221     obj_size = rhs.obj_size;
 222     obj = rhs.obj;
 223     head_size = rhs.head_size;
 224     max_head_size = rhs.max_head_size;
 225     prefix = rhs.prefix;
 226     tail_placement = rhs.tail_placement;
 227     rules = rhs.rules;
 228     tail_instance = rhs.tail_instance;
 229     tier_type = rhs.tier_type;
 230     tier_config = rhs.tier_config;
 231     return *this;
 232   }
 233
 234   std::map<uint64_t, RGWObjManifestPart>& get_explicit_objs() {
 235     return objs;
 236   }
 237
 238
 239   void set_explicit(uint64_t _size, std::map<uint64_t, RGWObjManifestPart>& _objs) {
 240     explicit_objs = true;
 241     objs.swap(_objs);
 242     set_obj_size(_size);
 243   }
 244
 245   void get_implicit_location(uint64_t cur_part_id, uint64_t cur_stripe, uint64_t ofs,
 246                              std::string *override_prefix, rgw_obj_select *location) const;
 247
 248   void set_trivial_rule(uint64_t tail_ofs, uint64_t stripe_max_size) {
 249     RGWObjManifestRule rule(0, tail_ofs, 0, stripe_max_size);
 250     rules[0] = rule;
 251     max_head_size = tail_ofs;
 252   }
 253
 254   void set_multipart_part_rule(uint64_t stripe_max_size, uint64_t part_num) {
 255     RGWObjManifestRule rule(0, 0, 0, stripe_max_size);
 256     rule.start_part_num = part_num;
 257     rules[0] = rule;
 258     max_head_size = 0;
 259   }
 260
 261   void encode(bufferlist& bl) const {
 262     ENCODE_START(8, 6, bl);
 263     encode(obj_size, bl);
 264     encode(objs, bl);
 265     encode(explicit_objs, bl);
 266     encode(obj, bl);
 267     encode(head_size, bl);
 268     encode(max_head_size, bl);
 269     encode(prefix, bl);
 270     encode(rules, bl);
 271     bool encode_tail_bucket = !(tail_placement.bucket == obj.bucket);
 272     encode(encode_tail_bucket, bl);
 273     if (encode_tail_bucket) {
 274       encode(tail_placement.bucket, bl);
 275     }
 276     bool encode_tail_instance = (tail_instance != obj.key.instance);
 277     encode(encode_tail_instance, bl);
 278     if (encode_tail_instance) {
 279       encode(tail_instance, bl);
 280     }
 281     encode(head_placement_rule, bl);
 282     encode(tail_placement.placement_rule, bl);
 283     encode(tier_type, bl);
 284     encode(tier_config, bl);
 285     ENCODE_FINISH(bl);
 286   }
 287
 288   void decode(bufferlist::const_iterator& bl) {
 289     DECODE_START_LEGACY_COMPAT_LEN_32(7, 2, 2, bl);
 290     decode(obj_size, bl);
 291     decode(objs, bl);
 292     if (struct_v >= 3) {
 293       decode(explicit_objs, bl);
 294       decode(obj, bl);
 295       decode(head_size, bl);
 296       decode(max_head_size, bl);
 297       decode(prefix, bl);
 298       decode(rules, bl);
 299     } else {
 300       explicit_objs = true;
 301       if (!objs.empty()) {
 302         std::map<uint64_t, RGWObjManifestPart>::iterator iter = objs.begin();
 303         obj = iter->second.loc;
 304         head_size = iter->second.size;
 305         max_head_size = head_size;
 306       }
 307     }
 308
 309     if (explicit_objs && head_size > 0 && !objs.empty()) {
 310       /* patch up manifest due to issue 16435:
 311        * the first object in the explicit objs list might not be the one we need to access, use the
 312        * head object instead if set. This would happen if we had an old object that was created
 313        * when the explicit objs manifest was around, and it got copied.
 314        */
 315       rgw_obj& obj_0 = objs[0].loc;
 316       if (!obj_0.get_oid().empty() && obj_0.key.ns.empty()) {
 317         objs[0].loc = obj;
 318         objs[0].size = head_size;
 319       }
 320     }
 321
 322     if (struct_v >= 4) {
 323       if (struct_v < 6) {
 324         decode(tail_placement.bucket, bl);
 325       } else {
 326         bool need_to_decode;
 327         decode(need_to_decode, bl);
 328         if (need_to_decode) {
 329           decode(tail_placement.bucket, bl);
 330         } else {
 331           tail_placement.bucket = obj.bucket;
 332         }
 333       }
 334     }
 335
 336     if (struct_v >= 5) {
 337       if (struct_v < 6) {
 338         decode(tail_instance, bl);
 339       } else {
 340         bool need_to_decode;
 341         decode(need_to_decode, bl);
 342         if (need_to_decode) {
 343           decode(tail_instance, bl);
 344         } else {
 345           tail_instance = obj.key.instance;
 346         }
 347       }
 348     } else { // old object created before 'tail_instance' field added to manifest
 349       tail_instance = obj.key.instance;
 350     }
 351
 352     if (struct_v >= 7) {
 353       decode(head_placement_rule, bl);
 354       decode(tail_placement.placement_rule, bl);
 355     }
 356
 357     if (struct_v >= 8) {
 358       decode(tier_type, bl);
 359       decode(tier_config, bl);
 360     }
 361
 362     DECODE_FINISH(bl);
 363   }
 364
 365   void dump(Formatter *f) const;
 366   static void generate_test_instances(std::list<RGWObjManifest*>& o);
 367
 368   int append(const DoutPrefixProvider *dpp, RGWObjManifest& m, const RGWZoneGroup& zonegroup,
 369              const RGWZoneParams& zone_params);
 370
 371   bool get_rule(uint64_t ofs, RGWObjManifestRule *rule);
 372
 373   bool empty() const {
 374     if (explicit_objs)
 375       return objs.empty();
 376     return rules.empty();
 377   }
 378
 379   bool has_explicit_objs() const {
 380     return explicit_objs;
 381   }
 382
 383   bool has_tail() const {
 384     if (explicit_objs) {
 385       if (objs.size() == 1) {
 386         auto iter = objs.begin();
 387         const rgw_obj& o = iter->second.loc;
 388         return !(obj == o);
 389       }
 390       return (objs.size() >= 2);
 391     }
 392     return (obj_size > head_size);
 393   }
 394
 395   void set_head(const rgw_placement_rule& placement_rule, const rgw_obj& _o, uint64_t _s) {
 396     head_placement_rule = placement_rule;
 397     obj = _o;
 398     head_size = _s;
 399
 400     if (explicit_objs && head_size > 0) {
 401       objs[0].loc = obj;
 402       objs[0].size = head_size;
 403     }
 404   }
 405
 406   const rgw_obj& get_obj() const {
 407     return obj;
 408   }
 409
 410   void set_tail_placement(const rgw_placement_rule& placement_rule, const rgw_bucket& _b) {
 411     tail_placement.placement_rule = placement_rule;
 412     tail_placement.bucket = _b;
 413   }
 414
 415   const rgw_bucket_placement& get_tail_placement() const {
 416     return tail_placement;
 417   }
 418
 419   const rgw_placement_rule& get_head_placement_rule() const {
 420     return head_placement_rule;
 421   }
 422
 423   void set_prefix(const std::string& _p) {
 424     prefix = _p;
 425   }
 426
 427   const std::string& get_prefix() const {
 428     return prefix;
 429   }
 430
 431   void set_tail_instance(const std::string& _ti) {
 432     tail_instance = _ti;
 433   }
 434
 435   const std::string& get_tail_instance() const {
 436     return tail_instance;
 437   }
 438
 439   void set_head_size(uint64_t _s) {
 440     head_size = _s;
 441   }
 442
 443   void set_obj_size(uint64_t s) {
 444     obj_size = s;
 445   }
 446
 447   uint64_t get_obj_size() const {
 448     return obj_size;
 449   }
 450
 451   uint64_t get_head_size() const {
 452     return head_size;
 453   }
 454
 455   uint64_t get_max_head_size() const {
 456     return max_head_size;
 457   }
 458
 459   const std::string& get_tier_type() {
 460       return tier_type;
 461   }
 462
 463   inline void set_tier_type(std::string value) {
 464       /* Only "cloud-s3" tier-type is supported for now */
 465       if (value == "cloud-s3") {
 466         tier_type = value;
 467       }
 468   }
 469
 470   inline void set_tier_config(RGWObjTier t) {
 471       /* Set only if tier_type set to "cloud-s3" */
 472       if (tier_type != "cloud-s3")
 473         return;
 474
 475       tier_config.name = t.name;
 476       tier_config.tier_placement = t.tier_placement;
 477       tier_config.is_multipart_upload = t.is_multipart_upload;
 478   }
 479
 480   inline const void get_tier_config(RGWObjTier* t) {
 481       if (tier_type != "cloud-s3")
 482         return;
 483
 484       t->name = tier_config.name;
 485       t->tier_placement = tier_config.tier_placement;
 486       t->is_multipart_upload = tier_config.is_multipart_upload;
 487   }
 488
 489   class obj_iterator {
 490     const DoutPrefixProvider *dpp;
 491     const RGWObjManifest *manifest = nullptr;
 492     uint64_t part_ofs = 0;   /* where current part starts */
 493     uint64_t stripe_ofs = 0; /* where current stripe starts */
 494     uint64_t ofs = 0;        /* current position within the object */
 495     uint64_t stripe_size = 0;      /* current part size */
 496
 497     int cur_part_id = 0;
 498     int cur_stripe = 0;
 499     std::string cur_override_prefix;
 500
 501     rgw_obj_select location;
 502
 503     std::map<uint64_t, RGWObjManifestRule>::const_iterator rule_iter;
 504     std::map<uint64_t, RGWObjManifestRule>::const_iterator next_rule_iter;
 505     std::map<uint64_t, RGWObjManifestPart>::const_iterator explicit_iter;
 506
 507     void update_explicit_pos();
 508
 509   public:
 510     obj_iterator() = default;
 511     explicit obj_iterator(const DoutPrefixProvider *_dpp, const RGWObjManifest *_m)
 512       : obj_iterator(_dpp, _m, 0)
 513     {}
 514     obj_iterator(const DoutPrefixProvider *_dpp, const RGWObjManifest *_m, uint64_t _ofs) : dpp(_dpp), manifest(_m) {
 515       seek(_ofs);
 516     }
 517     void seek(uint64_t ofs);
 518
 519     void operator++();
 520     bool operator==(const obj_iterator& rhs) const {
 521       return (ofs == rhs.ofs);
 522     }
 523     bool operator!=(const obj_iterator& rhs) const {
 524       return (ofs != rhs.ofs);
 525     }
 526     const rgw_obj_select& get_location() {
 527       return location;
 528     }
 529
 530     /* where current part starts */
 531     uint64_t get_part_ofs() const {
 532       return part_ofs;
 533     }
 534
 535     /* start of current stripe */
 536     uint64_t get_stripe_ofs() {
 537       if (manifest->explicit_objs) {
 538         return explicit_iter->first;
 539       }
 540       return stripe_ofs;
 541     }
 542
 543     /* current ofs relative to start of rgw object */
 544     uint64_t get_ofs() const {
 545       return ofs;
 546     }
 547
 548     int get_cur_part_id() const {
 549       return cur_part_id;
 550     }
 551
 552     /* stripe number */
 553     int get_cur_stripe() const {
 554       return cur_stripe;
 555     }
 556
 557     /* current stripe size */
 558     uint64_t get_stripe_size() {
 559       if (manifest->explicit_objs) {
 560         return explicit_iter->second.size;
 561       }
 562       return stripe_size;
 563     }
 564
 565     /* offset where data starts within current stripe */
 566     uint64_t location_ofs() {
 567       if (manifest->explicit_objs) {
 568         return explicit_iter->second.loc_ofs;
 569       }
 570       return 0; /* all stripes start at zero offset */
 571     }
 572
 573     void update_location();
 574
 575     void dump(Formatter *f) const;
 576   }; // class obj_iterator
 577
 578   obj_iterator obj_begin(const DoutPrefixProvider *dpp) const { return obj_iterator{dpp, this}; }
 579   obj_iterator obj_end(const DoutPrefixProvider *dpp) const { return obj_iterator{dpp, this, obj_size}; }
 580   obj_iterator obj_find(const DoutPrefixProvider *dpp, uint64_t ofs) const {
 581     return obj_iterator{dpp, this, std::min(ofs, obj_size)};
 582   }
 583
 584   /*
 585    * simple object generator. Using a simple single rule manifest.
 586    */
 587   class generator {
 588     RGWObjManifest *manifest;
 589     uint64_t last_ofs;
 590     uint64_t cur_part_ofs;
 591     int cur_part_id;
 592     int cur_stripe;
 593     uint64_t cur_stripe_size;
 594     std::string cur_oid;
 595
 596     std::string oid_prefix;
 597
 598     rgw_obj_select cur_obj;
 599
 600     RGWObjManifestRule rule;
 601
 602   public:
 603     generator() : manifest(NULL), last_ofs(0), cur_part_ofs(0), cur_part_id(0),
 604                   cur_stripe(0), cur_stripe_size(0) {}
 605     int create_begin(CephContext *cct, RGWObjManifest *manifest,
 606                      const rgw_placement_rule& head_placement_rule,
 607                      const rgw_placement_rule *tail_placement_rule,
 608                      const rgw_bucket& bucket,
 609                      const rgw_obj& obj);
 610
 611     int create_next(uint64_t ofs);
 612
 613     rgw_raw_obj get_cur_obj(RGWZoneGroup& zonegroup, RGWZoneParams& zone_params) { return cur_obj.get_raw_obj(zonegroup, zone_params); }
 614     rgw_raw_obj get_cur_obj(RGWRados* store) const { return cur_obj.get_raw_obj(store); }
 615
 616     /* total max size of current stripe (including head obj) */
 617     uint64_t cur_stripe_max_size() const {
 618       return cur_stripe_size;
 619     }
 620   };
 621 };
 622 WRITE_CLASS_ENCODER(RGWObjManifest)