ceph/src/rgw/rgw_obj_manifest.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab ft=cpp
   3
   4 #include "rgw_obj_manifest.h"
   5
   6 #include "services/svc_zone.h"
   7 #include "services/svc_tier_rados.h"
   8
   9 #define dout_context g_ceph_context
  10 #define dout_subsys ceph_subsys_rgw
  11
  12 int RGWObjManifest::generator::create_next(uint64_t ofs)
  13 {
  14   if (ofs < last_ofs) /* only going forward */
  15     return -EINVAL;
  16
  17   uint64_t max_head_size = manifest->get_max_head_size();
  18
  19   if (ofs < max_head_size) {
  20     manifest->set_head_size(ofs);
  21   }
  22
  23   if (ofs >= max_head_size) {
  24     manifest->set_head_size(max_head_size);
  25     cur_stripe = (ofs - max_head_size) / rule.stripe_max_size;
  26     cur_stripe_size = rule.stripe_max_size;
  27
  28     if (cur_part_id == 0 && max_head_size > 0) {
  29       cur_stripe++;
  30     }
  31   }
  32
  33   last_ofs = ofs;
  34   manifest->set_obj_size(ofs);
  35
  36   manifest->get_implicit_location(cur_part_id, cur_stripe, ofs, NULL, &cur_obj);
  37
  38   manifest->update_iterators();
  39
  40   return 0;
  41 }
  42
  43 const RGWObjManifest::obj_iterator& RGWObjManifest::obj_begin()
  44 {
  45   return begin_iter;
  46 }
  47
  48 const RGWObjManifest::obj_iterator& RGWObjManifest::obj_end()
  49 {
  50   return end_iter;
  51 }
  52
  53 RGWObjManifest::obj_iterator RGWObjManifest::obj_find(uint64_t ofs)
  54 {
  55   if (ofs > obj_size) {
  56     ofs = obj_size;
  57   }
  58   RGWObjManifest::obj_iterator iter(this);
  59   iter.seek(ofs);
  60   return iter;
  61 }
  62
  63 int RGWObjManifest::append(RGWObjManifest& m, const RGWZoneGroup& zonegroup,
  64                            const RGWZoneParams& zone_params)
  65 {
  66   if (explicit_objs || m.explicit_objs) {
  67     return append_explicit(m, zonegroup, zone_params);
  68   }
  69
  70   if (rules.empty()) {
  71     *this = m;
  72     return 0;
  73   }
  74
  75   string override_prefix;
  76
  77   if (prefix.empty()) {
  78     prefix = m.prefix;
  79   }
  80
  81   if (prefix != m.prefix) {
  82     override_prefix = m.prefix;
  83   }
  84
  85   map<uint64_t, RGWObjManifestRule>::iterator miter = m.rules.begin();
  86   if (miter == m.rules.end()) {
  87     return append_explicit(m, zonegroup, zone_params);
  88   }
  89
  90   for (; miter != m.rules.end(); ++miter) {
  91     map<uint64_t, RGWObjManifestRule>::reverse_iterator last_rule = rules.rbegin();
  92
  93     RGWObjManifestRule& rule = last_rule->second;
  94
  95     if (rule.part_size == 0) {
  96       rule.part_size = obj_size - rule.start_ofs;
  97     }
  98
  99     RGWObjManifestRule& next_rule = miter->second;
 100     if (!next_rule.part_size) {
 101       next_rule.part_size = m.obj_size - next_rule.start_ofs;
 102     }
 103
 104     string rule_prefix = prefix;
 105     if (!rule.override_prefix.empty()) {
 106       rule_prefix = rule.override_prefix;
 107     }
 108
 109     string next_rule_prefix = m.prefix;
 110     if (!next_rule.override_prefix.empty()) {
 111       next_rule_prefix = next_rule.override_prefix;
 112     }
 113
 114     if (rule.part_size != next_rule.part_size ||
 115         rule.stripe_max_size != next_rule.stripe_max_size ||
 116         rule_prefix != next_rule_prefix) {
 117       if (next_rule_prefix != prefix) {
 118         append_rules(m, miter, &next_rule_prefix);
 119       } else {
 120         append_rules(m, miter, NULL);
 121       }
 122       break;
 123     }
 124
 125     uint64_t expected_part_num = rule.start_part_num + 1;
 126     if (rule.part_size > 0) {
 127       expected_part_num = rule.start_part_num + (obj_size + next_rule.start_ofs - rule.start_ofs) / rule.part_size;
 128     }
 129
 130     if (expected_part_num != next_rule.start_part_num) {
 131       append_rules(m, miter, NULL);
 132       break;
 133     }
 134   }
 135
 136   set_obj_size(obj_size + m.obj_size);
 137
 138   return 0;
 139 }
 140
 141 int RGWObjManifest::append(RGWObjManifest& m, RGWSI_Zone *zone_svc)
 142 {
 143   return append(m, zone_svc->get_zonegroup(), zone_svc->get_zone_params());
 144 }
 145
 146 void RGWObjManifest::append_rules(RGWObjManifest& m, map<uint64_t, RGWObjManifestRule>::iterator& miter,
 147                                   string *override_prefix)
 148 {
 149   for (; miter != m.rules.end(); ++miter) {
 150     RGWObjManifestRule rule = miter->second;
 151     rule.start_ofs += obj_size;
 152     if (override_prefix)
 153       rule.override_prefix = *override_prefix;
 154     rules[rule.start_ofs] = rule;
 155   }
 156 }
 157
 158 void RGWObjManifest::convert_to_explicit(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params)
 159 {
 160   if (explicit_objs) {
 161     return;
 162   }
 163   obj_iterator iter = obj_begin();
 164
 165   while (iter != obj_end()) {
 166     RGWObjManifestPart& part = objs[iter.get_stripe_ofs()];
 167     const rgw_obj_select& os = iter.get_location();
 168     const rgw_raw_obj& raw_loc = os.get_raw_obj(zonegroup, zone_params);
 169     part.loc_ofs = 0;
 170
 171     uint64_t ofs = iter.get_stripe_ofs();
 172
 173     if (ofs == 0) {
 174       part.loc = obj;
 175     } else {
 176       RGWSI_Tier_RADOS::raw_obj_to_obj(tail_placement.bucket, raw_loc, &part.loc);
 177     }
 178     ++iter;
 179     uint64_t next_ofs = iter.get_stripe_ofs();
 180
 181     part.size = next_ofs - ofs;
 182   }
 183
 184   explicit_objs = true;
 185   rules.clear();
 186   prefix.clear();
 187 }
 188
 189 int RGWObjManifest::append_explicit(RGWObjManifest& m, const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params)
 190 {
 191   if (!explicit_objs) {
 192     convert_to_explicit(zonegroup, zone_params);
 193   }
 194   if (!m.explicit_objs) {
 195     m.convert_to_explicit(zonegroup, zone_params);
 196   }
 197   map<uint64_t, RGWObjManifestPart>::iterator iter;
 198   uint64_t base = obj_size;
 199   for (iter = m.objs.begin(); iter != m.objs.end(); ++iter) {
 200     RGWObjManifestPart& part = iter->second;
 201     objs[base + iter->first] = part;
 202   }
 203   obj_size += m.obj_size;
 204
 205   return 0;
 206 }
 207
 208 bool RGWObjManifest::get_rule(uint64_t ofs, RGWObjManifestRule *rule)
 209 {
 210   if (rules.empty()) {
 211     return false;
 212   }
 213
 214   map<uint64_t, RGWObjManifestRule>::iterator iter = rules.upper_bound(ofs);
 215   if (iter != rules.begin()) {
 216     --iter;
 217   }
 218
 219   *rule = iter->second;
 220
 221   return true;
 222 }
 223
 224 void RGWObjManifest::obj_iterator::operator++()
 225 {
 226   if (manifest->explicit_objs) {
 227     ++explicit_iter;
 228
 229     update_explicit_pos();
 230
 231     update_location();
 232     return;
 233   }
 234
 235   uint64_t obj_size = manifest->get_obj_size();
 236   uint64_t head_size = manifest->get_head_size();
 237
 238   if (ofs == obj_size) {
 239     return;
 240   }
 241
 242   if (manifest->rules.empty()) {
 243     return;
 244   }
 245
 246   /* are we still pointing at the head? */
 247   if (ofs < head_size) {
 248     rule_iter = manifest->rules.begin();
 249     RGWObjManifestRule *rule = &rule_iter->second;
 250     ofs = std::min(head_size, obj_size);
 251     stripe_ofs = ofs;
 252     cur_stripe = 1;
 253     stripe_size = std::min(obj_size - ofs, rule->stripe_max_size);
 254     if (rule->part_size > 0) {
 255       stripe_size = std::min(stripe_size, rule->part_size);
 256     }
 257     update_location();
 258     return;
 259   }
 260
 261   RGWObjManifestRule *rule = &rule_iter->second;
 262
 263   stripe_ofs += rule->stripe_max_size;
 264   cur_stripe++;
 265   dout(20) << "RGWObjManifest::operator++(): rule->part_size=" << rule->part_size << " rules.size()=" << manifest->rules.size() << dendl;
 266
 267   if (rule->part_size > 0) {
 268     /* multi part, multi stripes object */
 269
 270     dout(20) << "RGWObjManifest::operator++(): stripe_ofs=" << stripe_ofs << " part_ofs=" << part_ofs << " rule->part_size=" << rule->part_size << dendl;
 271
 272     if (stripe_ofs >= part_ofs + rule->part_size) {
 273       /* moved to the next part */
 274       cur_stripe = 0;
 275       part_ofs += rule->part_size;
 276       stripe_ofs = part_ofs;
 277
 278       bool last_rule = (next_rule_iter == manifest->rules.end());
 279       /* move to the next rule? */
 280       if (!last_rule && stripe_ofs >= next_rule_iter->second.start_ofs) {
 281         rule_iter = next_rule_iter;
 282         last_rule = (next_rule_iter == manifest->rules.end());
 283         if (!last_rule) {
 284           ++next_rule_iter;
 285         }
 286         cur_part_id = rule_iter->second.start_part_num;
 287       } else {
 288         cur_part_id++;
 289       }
 290
 291       rule = &rule_iter->second;
 292     }
 293
 294     stripe_size = std::min(rule->part_size - (stripe_ofs - part_ofs), rule->stripe_max_size);
 295   }
 296
 297   cur_override_prefix = rule->override_prefix;
 298
 299   ofs = stripe_ofs;
 300   if (ofs > obj_size) {
 301     ofs = obj_size;
 302     stripe_ofs = ofs;
 303     stripe_size = 0;
 304   }
 305
 306   dout(20) << "RGWObjManifest::operator++(): result: ofs=" << ofs << " stripe_ofs=" << stripe_ofs << " part_ofs=" << part_ofs << " rule->part_size=" << rule->part_size << dendl;
 307   update_location();
 308 }
 309
 310 int RGWObjManifest::generator::create_begin(CephContext *cct, RGWObjManifest *_m,
 311                                             const rgw_placement_rule& head_placement_rule,
 312                                             const rgw_placement_rule *tail_placement_rule,
 313                                             const rgw_bucket& _b, const rgw_obj& _obj)
 314 {
 315   manifest = _m;
 316
 317   if (!tail_placement_rule) {
 318     manifest->set_tail_placement(head_placement_rule, _b);
 319   } else {
 320     rgw_placement_rule new_tail_rule = *tail_placement_rule;
 321     new_tail_rule.inherit_from(head_placement_rule);
 322     manifest->set_tail_placement(new_tail_rule, _b);
 323   }
 324
 325   manifest->set_head(head_placement_rule, _obj, 0);
 326   last_ofs = 0;
 327
 328   if (manifest->get_prefix().empty()) {
 329     char buf[33];
 330     gen_rand_alphanumeric(cct, buf, sizeof(buf) - 1);
 331
 332     string oid_prefix = ".";
 333     oid_prefix.append(buf);
 334     oid_prefix.append("_");
 335
 336     manifest->set_prefix(oid_prefix);
 337   }
 338
 339   bool found = manifest->get_rule(0, &rule);
 340   if (!found) {
 341     derr << "ERROR: manifest->get_rule() could not find rule" << dendl;
 342     return -EIO;
 343   }
 344
 345   uint64_t head_size = manifest->get_head_size();
 346
 347   if (head_size > 0) {
 348     cur_stripe_size = head_size;
 349   } else {
 350     cur_stripe_size = rule.stripe_max_size;
 351   }
 352
 353   cur_part_id = rule.start_part_num;
 354
 355   manifest->get_implicit_location(cur_part_id, cur_stripe, 0, NULL, &cur_obj);
 356
 357   // Normal object which not generated through copy operation
 358   manifest->set_tail_instance(_obj.key.instance);
 359
 360   manifest->update_iterators();
 361
 362   return 0;
 363 }
 364