ceph/src/rgw/driver/rados/rgw_obj_manifest.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab ft=cpp
   3
   4 #include "rgw_obj_manifest.h"
   5
   6 #include "services/svc_zone.h"
   7 #include "rgw_rados.h"
   8 #include "rgw_bucket.h"
   9
  10 #define dout_context g_ceph_context
  11 #define dout_subsys ceph_subsys_rgw
  12
  13 using namespace std;
  14
  15 int RGWObjManifest::generator::create_next(uint64_t ofs)
  16 {
  17   if (ofs < last_ofs) /* only going forward */
  18     return -EINVAL;
  19
  20   uint64_t max_head_size = manifest->get_max_head_size();
  21
  22   if (ofs < max_head_size) {
  23     manifest->set_head_size(ofs);
  24   }
  25
  26   if (ofs >= max_head_size) {
  27     manifest->set_head_size(max_head_size);
  28     cur_stripe = (ofs - max_head_size) / rule.stripe_max_size;
  29     cur_stripe_size = rule.stripe_max_size;
  30
  31     if (cur_part_id == 0 && max_head_size > 0) {
  32       cur_stripe++;
  33     }
  34   }
  35
  36   last_ofs = ofs;
  37   manifest->set_obj_size(ofs);
  38
  39   manifest->get_implicit_location(cur_part_id, cur_stripe, ofs, NULL, &cur_obj);
  40
  41   return 0;
  42 }
  43
  44 int RGWObjManifest::append(const DoutPrefixProvider *dpp, RGWObjManifest& m, const RGWZoneGroup& zonegroup,
  45                            const RGWZoneParams& zone_params)
  46 {
  47   if (explicit_objs || m.explicit_objs) {
  48     return append_explicit(dpp, m, zonegroup, zone_params);
  49   }
  50
  51   if (rules.empty()) {
  52     *this = m;
  53     return 0;
  54   }
  55
  56   string override_prefix;
  57
  58   if (prefix.empty()) {
  59     prefix = m.prefix;
  60   }
  61
  62   if (prefix != m.prefix) {
  63     override_prefix = m.prefix;
  64   }
  65
  66   map<uint64_t, RGWObjManifestRule>::iterator miter = m.rules.begin();
  67   if (miter == m.rules.end()) {
  68     return append_explicit(dpp, m, zonegroup, zone_params);
  69   }
  70
  71   for (; miter != m.rules.end(); ++miter) {
  72     map<uint64_t, RGWObjManifestRule>::reverse_iterator last_rule = rules.rbegin();
  73
  74     RGWObjManifestRule& rule = last_rule->second;
  75
  76     if (rule.part_size == 0) {
  77       rule.part_size = obj_size - rule.start_ofs;
  78     }
  79
  80     RGWObjManifestRule& next_rule = miter->second;
  81     if (!next_rule.part_size) {
  82       next_rule.part_size = m.obj_size - next_rule.start_ofs;
  83     }
  84
  85     string rule_prefix = prefix;
  86     if (!rule.override_prefix.empty()) {
  87       rule_prefix = rule.override_prefix;
  88     }
  89
  90     string next_rule_prefix = m.prefix;
  91     if (!next_rule.override_prefix.empty()) {
  92       next_rule_prefix = next_rule.override_prefix;
  93     }
  94
  95     if (rule.part_size != next_rule.part_size ||
  96         rule.stripe_max_size != next_rule.stripe_max_size ||
  97         rule_prefix != next_rule_prefix) {
  98       if (next_rule_prefix != prefix) {
  99         append_rules(m, miter, &next_rule_prefix);
 100       } else {
 101         append_rules(m, miter, NULL);
 102       }
 103       break;
 104     }
 105
 106     uint64_t expected_part_num = rule.start_part_num + 1;
 107     if (rule.part_size > 0) {
 108       expected_part_num = rule.start_part_num + (obj_size + next_rule.start_ofs - rule.start_ofs) / rule.part_size;
 109     }
 110
 111     if (expected_part_num != next_rule.start_part_num) {
 112       append_rules(m, miter, NULL);
 113       break;
 114     }
 115   }
 116
 117   set_obj_size(obj_size + m.obj_size);
 118
 119   return 0;
 120 }
 121
 122 void RGWObjManifest::append_rules(RGWObjManifest& m, map<uint64_t, RGWObjManifestRule>::iterator& miter,
 123                                   string *override_prefix)
 124 {
 125   for (; miter != m.rules.end(); ++miter) {
 126     RGWObjManifestRule rule = miter->second;
 127     rule.start_ofs += obj_size;
 128     if (override_prefix)
 129       rule.override_prefix = *override_prefix;
 130     rules[rule.start_ofs] = rule;
 131   }
 132 }
 133
 134 void RGWObjManifest::convert_to_explicit(const DoutPrefixProvider *dpp, const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params)
 135 {
 136   if (explicit_objs) {
 137     return;
 138   }
 139   obj_iterator iter = obj_begin(dpp);
 140
 141   while (iter != obj_end(dpp)) {
 142     RGWObjManifestPart& part = objs[iter.get_stripe_ofs()];
 143     const rgw_obj_select& os = iter.get_location();
 144     const rgw_raw_obj& raw_loc = os.get_raw_obj(zonegroup, zone_params);
 145     part.loc_ofs = 0;
 146
 147     uint64_t ofs = iter.get_stripe_ofs();
 148
 149     if (ofs == 0) {
 150       part.loc = obj;
 151     } else {
 152       RGWSI_Tier_RADOS::raw_obj_to_obj(tail_placement.bucket, raw_loc, &part.loc);
 153     }
 154     ++iter;
 155     uint64_t next_ofs = iter.get_stripe_ofs();
 156
 157     part.size = next_ofs - ofs;
 158   }
 159
 160   explicit_objs = true;
 161   rules.clear();
 162   prefix.clear();
 163 }
 164
 165 int RGWObjManifest::append_explicit(const DoutPrefixProvider *dpp, RGWObjManifest& m, const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params)
 166 {
 167   if (!explicit_objs) {
 168     convert_to_explicit(dpp, zonegroup, zone_params);
 169   }
 170   if (!m.explicit_objs) {
 171     m.convert_to_explicit(dpp, zonegroup, zone_params);
 172   }
 173   map<uint64_t, RGWObjManifestPart>::iterator iter;
 174   uint64_t base = obj_size;
 175   for (iter = m.objs.begin(); iter != m.objs.end(); ++iter) {
 176     RGWObjManifestPart& part = iter->second;
 177     objs[base + iter->first] = part;
 178   }
 179   obj_size += m.obj_size;
 180
 181   return 0;
 182 }
 183
 184 bool RGWObjManifest::get_rule(uint64_t ofs, RGWObjManifestRule *rule)
 185 {
 186   if (rules.empty()) {
 187     return false;
 188   }
 189
 190   map<uint64_t, RGWObjManifestRule>::iterator iter = rules.upper_bound(ofs);
 191   if (iter != rules.begin()) {
 192     --iter;
 193   }
 194
 195   *rule = iter->second;
 196
 197   return true;
 198 }
 199
 200 int RGWObjManifest::generator::create_begin(CephContext *cct, RGWObjManifest *_m,
 201                                             const rgw_placement_rule& head_placement_rule,
 202                                             const rgw_placement_rule *tail_placement_rule,
 203                                             const rgw_bucket& _b, const rgw_obj& _obj)
 204 {
 205   manifest = _m;
 206
 207   if (!tail_placement_rule) {
 208     manifest->set_tail_placement(head_placement_rule, _b);
 209   } else {
 210     rgw_placement_rule new_tail_rule = *tail_placement_rule;
 211     new_tail_rule.inherit_from(head_placement_rule);
 212     manifest->set_tail_placement(new_tail_rule, _b);
 213   }
 214
 215   manifest->set_head(head_placement_rule, _obj, 0);
 216   last_ofs = 0;
 217
 218   if (manifest->get_prefix().empty()) {
 219     char buf[33];
 220     gen_rand_alphanumeric(cct, buf, sizeof(buf) - 1);
 221
 222     string oid_prefix = ".";
 223     oid_prefix.append(buf);
 224     oid_prefix.append("_");
 225
 226     manifest->set_prefix(oid_prefix);
 227   }
 228
 229   bool found = manifest->get_rule(0, &rule);
 230   if (!found) {
 231     derr << "ERROR: manifest->get_rule() could not find rule" << dendl;
 232     return -EIO;
 233   }
 234
 235   uint64_t head_size = manifest->get_head_size();
 236
 237   if (head_size > 0) {
 238     cur_stripe_size = head_size;
 239   } else {
 240     cur_stripe_size = rule.stripe_max_size;
 241   }
 242
 243   cur_part_id = rule.start_part_num;
 244
 245   manifest->get_implicit_location(cur_part_id, cur_stripe, 0, NULL, &cur_obj);
 246
 247   // Normal object which not generated through copy operation
 248   manifest->set_tail_instance(_obj.key.instance);
 249
 250   return 0;
 251 }
 252
 253 void RGWObjManifestPart::generate_test_instances(std::list<RGWObjManifestPart*>& o)
 254 {
 255   o.push_back(new RGWObjManifestPart);
 256
 257   RGWObjManifestPart *p = new RGWObjManifestPart;
 258   rgw_bucket b;
 259   init_bucket(&b, "tenant", "bucket", ".pool", ".index_pool", "marker_", "12");
 260
 261   p->loc = rgw_obj(b, "object");
 262   p->loc_ofs = 512 * 1024;
 263   p->size = 128 * 1024;
 264   o.push_back(p);
 265 }
 266
 267 void RGWObjManifest::generate_test_instances(std::list<RGWObjManifest*>& o)
 268 {
 269   RGWObjManifest *m = new RGWObjManifest;
 270   map<uint64_t, RGWObjManifestPart> objs;
 271   uint64_t total_size = 0;
 272   for (int i = 0; i<10; i++) {
 273     RGWObjManifestPart p;
 274     rgw_bucket b;
 275     init_bucket(&b, "tenant", "bucket", ".pool", ".index_pool", "marker_", "12");
 276     p.loc = rgw_obj(b, "object");
 277     p.loc_ofs = 0;
 278     p.size = 512 * 1024;
 279     total_size += p.size;
 280     objs[total_size] = p;
 281   }
 282   m->set_explicit(total_size, objs);
 283   o.push_back(m);
 284   o.push_back(new RGWObjManifest);
 285 }
 286
 287 void RGWObjManifestPart::dump(Formatter *f) const
 288 {
 289   f->open_object_section("loc");
 290   loc.dump(f);
 291   f->close_section();
 292   f->dump_unsigned("loc_ofs", loc_ofs);
 293   f->dump_unsigned("size", size);
 294 }
 295
 296 void RGWObjManifest::obj_iterator::dump(Formatter *f) const
 297 {
 298   f->dump_unsigned("part_ofs", part_ofs);
 299   f->dump_unsigned("stripe_ofs", stripe_ofs);
 300   f->dump_unsigned("ofs", ofs);
 301   f->dump_unsigned("stripe_size", stripe_size);
 302   f->dump_int("cur_part_id", cur_part_id);
 303   f->dump_int("cur_stripe", cur_stripe);
 304   f->dump_string("cur_override_prefix", cur_override_prefix);
 305   f->dump_object("location", location);
 306 }
 307
 308 void RGWObjManifest::dump(Formatter *f) const
 309 {
 310   map<uint64_t, RGWObjManifestPart>::const_iterator iter = objs.begin();
 311   f->open_array_section("objs");
 312   for (; iter != objs.end(); ++iter) {
 313     f->dump_unsigned("ofs", iter->first);
 314     f->open_object_section("part");
 315     iter->second.dump(f);
 316     f->close_section();
 317   }
 318   f->close_section();
 319   f->dump_unsigned("obj_size", obj_size);
 320   ::encode_json("explicit_objs", explicit_objs, f);
 321   ::encode_json("head_size", head_size, f);
 322   ::encode_json("max_head_size", max_head_size, f);
 323   ::encode_json("prefix", prefix, f);
 324   ::encode_json("rules", rules, f);
 325   ::encode_json("tail_instance", tail_instance, f);
 326   ::encode_json("tail_placement", tail_placement, f);
 327   ::encode_json("tier_type", tier_type, f);
 328
 329   if (tier_type == "cloud-s3") {
 330     ::encode_json("tier_config", tier_config, f);
 331   }
 332
 333   // nullptr being passed into iterators since there
 334   // is no cct and we aren't doing anything with these
 335   // iterators that would write do the log
 336   f->dump_object("begin_iter", obj_begin(nullptr));
 337   f->dump_object("end_iter", obj_end(nullptr));
 338 }
 339
 340 void RGWObjManifestRule::dump(Formatter *f) const
 341 {
 342   encode_json("start_part_num", start_part_num, f);
 343   encode_json("start_ofs", start_ofs, f);
 344   encode_json("part_size", part_size, f);
 345   encode_json("stripe_max_size", stripe_max_size, f);
 346   encode_json("override_prefix", override_prefix, f);
 347 }
 348
 349 void rgw_obj_select::dump(Formatter *f) const
 350 {
 351   f->dump_string("placement_rule", placement_rule.to_str());
 352   f->dump_object("obj", obj);
 353   f->dump_object("raw_obj", raw_obj);
 354   f->dump_bool("is_raw", is_raw);
 355 }
 356
 357 void RGWObjTier::dump(Formatter *f) const
 358 {
 359   encode_json("name", name, f);
 360   encode_json("tier_placement", tier_placement, f);
 361   encode_json("is_multipart_upload", is_multipart_upload, f);
 362 }
 363
 364 // returns true on success, false on failure
 365 static bool rgw_get_obj_data_pool(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params,
 366                                   const rgw_placement_rule& head_placement_rule,
 367                                   const rgw_obj& obj, rgw_pool *pool)
 368 {
 369   if (!zone_params.get_head_data_pool(head_placement_rule, obj, pool)) {
 370     RGWZonePlacementInfo placement;
 371     if (!zone_params.get_placement(zonegroup.default_placement.name, &placement)) {
 372       return false;
 373     }
 374
 375     if (!obj.in_extra_data) {
 376       *pool = placement.get_data_pool(zonegroup.default_placement.storage_class);
 377     } else {
 378       *pool = placement.get_data_extra_pool();
 379     }
 380   }
 381
 382   return true;
 383 }
 384
 385 static bool rgw_obj_to_raw(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params,
 386                            const rgw_placement_rule& head_placement_rule,
 387                            const rgw_obj& obj, rgw_raw_obj *raw_obj)
 388 {
 389   get_obj_bucket_and_oid_loc(obj, raw_obj->oid, raw_obj->loc);
 390
 391   return rgw_get_obj_data_pool(zonegroup, zone_params, head_placement_rule, obj, &raw_obj->pool);
 392 }
 393
 394 rgw_raw_obj rgw_obj_select::get_raw_obj(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params) const
 395 {
 396   if (!is_raw) {
 397     rgw_raw_obj r;
 398     rgw_obj_to_raw(zonegroup, zone_params, placement_rule, obj, &r);
 399     return r;
 400   }
 401   return raw_obj;
 402 }
 403
 404 // returns true on success, false on failure
 405 bool RGWRados::get_obj_data_pool(const rgw_placement_rule& placement_rule, const rgw_obj& obj, rgw_pool *pool)
 406 {
 407   return rgw_get_obj_data_pool(svc.zone->get_zonegroup(), svc.zone->get_zone_params(), placement_rule, obj, pool);
 408 }
 409