ceph/src/rgw/rgw_dencoder.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab ft=cpp
   3
   4 #include "rgw_common.h"
   5 #include "rgw_rados.h"
   6 #include "rgw_zone.h"
   7 #include "rgw_log.h"
   8 #include "rgw_acl.h"
   9 #include "rgw_acl_s3.h"
  10 #include "rgw_cache.h"
  11 #include "rgw_meta_sync_status.h"
  12 #include "rgw_data_sync.h"
  13 #include "rgw_multi.h"
  14
  15 #include "common/Formatter.h"
  16
  17 static string shadow_ns = RGW_OBJ_NS_SHADOW;
  18
  19 static void init_bucket(rgw_bucket *b, const char *t, const char *n, const char *dp, const char *ip, const char *m, const char *id)
  20 {
  21   b->tenant = t;
  22   b->name = n;
  23   b->marker = m;
  24   b->bucket_id = id;
  25   b->explicit_placement.data_pool = rgw_pool(dp);
  26   b->explicit_placement.index_pool = rgw_pool(ip);
  27 }
  28
  29 void RGWObjManifestPart::generate_test_instances(std::list<RGWObjManifestPart*>& o)
  30 {
  31   o.push_back(new RGWObjManifestPart);
  32
  33   RGWObjManifestPart *p = new RGWObjManifestPart;
  34   rgw_bucket b;
  35   init_bucket(&b, "tenant", "bucket", ".pool", ".index_pool", "marker_", "12");
  36
  37   p->loc = rgw_obj(b, "object");
  38   p->loc_ofs = 512 * 1024;
  39   p->size = 128 * 1024;
  40   o.push_back(p);
  41 }
  42
  43 void RGWObjManifest::obj_iterator::seek(uint64_t o)
  44 {
  45   ofs = o;
  46   if (manifest->explicit_objs) {
  47     explicit_iter = manifest->objs.upper_bound(ofs);
  48     if (explicit_iter != manifest->objs.begin()) {
  49       --explicit_iter;
  50     }
  51     if (ofs < manifest->obj_size) {
  52       update_explicit_pos();
  53     } else {
  54       ofs = manifest->obj_size;
  55     }
  56     update_location();
  57     return;
  58   }
  59   if (o < manifest->get_head_size()) {
  60     rule_iter = manifest->rules.begin();
  61     stripe_ofs = 0;
  62     stripe_size = manifest->get_head_size();
  63     if (rule_iter != manifest->rules.end()) {
  64       cur_part_id = rule_iter->second.start_part_num;
  65       cur_override_prefix = rule_iter->second.override_prefix;
  66     }
  67     update_location();
  68     return;
  69   }
  70
  71   rule_iter = manifest->rules.upper_bound(ofs);
  72   next_rule_iter = rule_iter;
  73   if (rule_iter != manifest->rules.begin()) {
  74     --rule_iter;
  75   }
  76
  77   if (rule_iter == manifest->rules.end()) {
  78     update_location();
  79     return;
  80   }
  81
  82   RGWObjManifestRule& rule = rule_iter->second;
  83
  84   if (rule.part_size > 0) {
  85     cur_part_id = rule.start_part_num + (ofs - rule.start_ofs) / rule.part_size;
  86   } else {
  87     cur_part_id = rule.start_part_num;
  88   }
  89   part_ofs = rule.start_ofs + (cur_part_id - rule.start_part_num) * rule.part_size;
  90
  91   if (rule.stripe_max_size > 0) {
  92     cur_stripe = (ofs - part_ofs) / rule.stripe_max_size;
  93
  94     stripe_ofs = part_ofs + cur_stripe * rule.stripe_max_size;
  95     if (!cur_part_id && manifest->get_head_size() > 0) {
  96       cur_stripe++;
  97     }
  98   } else {
  99     cur_stripe = 0;
 100     stripe_ofs = part_ofs;
 101   }
 102
 103   if (!rule.part_size) {
 104     stripe_size = rule.stripe_max_size;
 105     stripe_size = std::min(manifest->get_obj_size() - stripe_ofs, stripe_size);
 106   } else {
 107     uint64_t next = std::min(stripe_ofs + rule.stripe_max_size, part_ofs + rule.part_size);
 108     stripe_size = next - stripe_ofs;
 109   }
 110
 111   cur_override_prefix = rule.override_prefix;
 112
 113   update_location();
 114 }
 115
 116 void RGWObjManifest::obj_iterator::update_location()
 117 {
 118   if (manifest->explicit_objs) {
 119     if (manifest->empty()) {
 120       location = rgw_obj_select{};
 121     } else {
 122       location = explicit_iter->second.loc;
 123     }
 124     return;
 125   }
 126
 127   if (ofs < manifest->get_head_size()) {
 128     location = manifest->get_obj();
 129     location.set_placement_rule(manifest->get_head_placement_rule());
 130     return;
 131   }
 132
 133   manifest->get_implicit_location(cur_part_id, cur_stripe, ofs, &cur_override_prefix, &location);
 134 }
 135
 136 void RGWObjManifest::obj_iterator::update_explicit_pos()
 137 {
 138   ofs = explicit_iter->first;
 139   stripe_ofs = ofs;
 140
 141   map<uint64_t, RGWObjManifestPart>::iterator next_iter = explicit_iter;
 142   ++next_iter;
 143   if (next_iter != manifest->objs.end()) {
 144     stripe_size = next_iter->first - ofs;
 145   } else {
 146     stripe_size = manifest->obj_size - ofs;
 147   }
 148 }
 149
 150 void RGWObjManifest::generate_test_instances(std::list<RGWObjManifest*>& o)
 151 {
 152   RGWObjManifest *m = new RGWObjManifest;
 153   map<uint64_t, RGWObjManifestPart> objs;
 154   uint64_t total_size = 0;
 155   for (int i = 0; i<10; i++) {
 156     RGWObjManifestPart p;
 157     rgw_bucket b;
 158     init_bucket(&b, "tenant", "bucket", ".pool", ".index_pool", "marker_", "12");
 159     p.loc = rgw_obj(b, "object");
 160     p.loc_ofs = 0;
 161     p.size = 512 * 1024;
 162     total_size += p.size;
 163     objs[total_size] = p;
 164   }
 165   m->set_explicit(total_size, objs);
 166   o.push_back(m);
 167   o.push_back(new RGWObjManifest);
 168 }
 169
 170 void RGWObjManifest::get_implicit_location(uint64_t cur_part_id, uint64_t cur_stripe, uint64_t ofs, string *override_prefix, rgw_obj_select *location)
 171 {
 172   rgw_obj loc;
 173
 174   string& oid = loc.key.name;
 175   string& ns = loc.key.ns;
 176
 177   if (!override_prefix || override_prefix->empty()) {
 178     oid = prefix;
 179   } else {
 180     oid = *override_prefix;
 181   }
 182
 183   if (!cur_part_id) {
 184     if (ofs < max_head_size) {
 185       location->set_placement_rule(head_placement_rule);
 186       *location = obj;
 187       return;
 188     } else {
 189       char buf[16];
 190       snprintf(buf, sizeof(buf), "%d", (int)cur_stripe);
 191       oid += buf;
 192       ns = shadow_ns;
 193     }
 194   } else {
 195     char buf[32];
 196     if (cur_stripe == 0) {
 197       snprintf(buf, sizeof(buf), ".%d", (int)cur_part_id);
 198       oid += buf;
 199       ns= RGW_OBJ_NS_MULTIPART;
 200     } else {
 201       snprintf(buf, sizeof(buf), ".%d_%d", (int)cur_part_id, (int)cur_stripe);
 202       oid += buf;
 203       ns = shadow_ns;
 204     }
 205   }
 206
 207   if (!tail_placement.bucket.name.empty()) {
 208     loc.bucket = tail_placement.bucket;
 209   } else {
 210     loc.bucket = obj.bucket;
 211   }
 212
 213   // Always overwrite instance with tail_instance
 214   // to get the right shadow object location
 215   loc.key.set_instance(tail_instance);
 216
 217   location->set_placement_rule(tail_placement.placement_rule);
 218   *location = loc;
 219 }
 220
 221
 222
 223 void rgw_log_entry::generate_test_instances(list<rgw_log_entry*>& o)
 224 {
 225   rgw_log_entry *e = new rgw_log_entry;
 226   e->object_owner = "object_owner";
 227   e->bucket_owner = "bucket_owner";
 228   e->bucket = "bucket";
 229   e->remote_addr = "1.2.3.4";
 230   e->user = "user";
 231   e->obj = rgw_obj_key("obj");
 232   e->uri = "http://uri/bucket/obj";
 233   e->http_status = "200";
 234   e->error_code = "error_code";
 235   e->bytes_sent = 1024;
 236   e->bytes_received = 512;
 237   e->obj_size = 2048;
 238   e->user_agent = "user_agent";
 239   e->referrer = "referrer";
 240   e->bucket_id = "10";
 241   e->trans_id = "trans_id";
 242   o.push_back(e);
 243   o.push_back(new rgw_log_entry);
 244 }
 245
 246 void ACLPermission::generate_test_instances(list<ACLPermission*>& o)
 247 {
 248   ACLPermission *p = new ACLPermission;
 249   p->set_permissions(RGW_PERM_WRITE_ACP);
 250   o.push_back(p);
 251   o.push_back(new ACLPermission);
 252 }
 253
 254 void ACLGranteeType::generate_test_instances(list<ACLGranteeType*>& o)
 255 {
 256   ACLGranteeType *t = new ACLGranteeType;
 257   t->set(ACL_TYPE_CANON_USER);
 258   o.push_back(t);
 259   o.push_back(new ACLGranteeType);
 260 }
 261
 262 /* the following is copied here from rgw_acl_s3.cc, to avoid having to have excessive linking
 263    with everything it needs */
 264
 265 #define RGW_URI_ALL_USERS       "http://acs.amazonaws.com/groups/global/AllUsers"
 266 #define RGW_URI_AUTH_USERS      "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"
 267
 268 static string rgw_uri_all_users = RGW_URI_ALL_USERS;
 269 static string rgw_uri_auth_users = RGW_URI_AUTH_USERS;
 270
 271 ACLGroupTypeEnum ACLGrant::uri_to_group(string& uri)
 272 {
 273   // this is required for backward compatibility
 274   return ACLGrant_S3::uri_to_group(uri);
 275 }
 276
 277 ACLGroupTypeEnum ACLGrant_S3::uri_to_group(string& uri)
 278 {
 279   if (uri.compare(rgw_uri_all_users) == 0)
 280     return ACL_GROUP_ALL_USERS;
 281   else if (uri.compare(rgw_uri_auth_users) == 0)
 282     return ACL_GROUP_AUTHENTICATED_USERS;
 283
 284   return ACL_GROUP_NONE;
 285 }
 286
 287 void ACLGrant::generate_test_instances(list<ACLGrant*>& o)
 288 {
 289   rgw_user id("rgw");
 290   string name, email;
 291   name = "Mr. RGW";
 292   email = "r@gw";
 293
 294   ACLGrant *g1 = new ACLGrant;
 295   g1->set_canon(id, name, RGW_PERM_READ);
 296   g1->email = email;
 297   o.push_back(g1);
 298
 299   ACLGrant *g2 = new ACLGrant;
 300   g1->set_group(ACL_GROUP_AUTHENTICATED_USERS, RGW_PERM_WRITE);
 301   o.push_back(g2);
 302
 303   o.push_back(new ACLGrant);
 304 }
 305
 306 void RGWAccessControlList::generate_test_instances(list<RGWAccessControlList*>& o)
 307 {
 308   RGWAccessControlList *acl = new RGWAccessControlList(NULL);
 309
 310   list<ACLGrant *> glist;
 311   list<ACLGrant *>::iterator iter;
 312
 313   ACLGrant::generate_test_instances(glist);
 314   for (iter = glist.begin(); iter != glist.end(); ++iter) {
 315     ACLGrant *grant = *iter;
 316     acl->add_grant(grant);
 317
 318     delete grant;
 319   }
 320   o.push_back(acl);
 321   o.push_back(new RGWAccessControlList(NULL));
 322 }
 323
 324 void ACLOwner::generate_test_instances(list<ACLOwner*>& o)
 325 {
 326   ACLOwner *owner = new ACLOwner;
 327   owner->id = "rgw";
 328   owner->display_name = "Mr. RGW";
 329   o.push_back(owner);
 330   o.push_back(new ACLOwner);
 331 }
 332
 333 void RGWAccessControlPolicy::generate_test_instances(list<RGWAccessControlPolicy*>& o)
 334 {
 335   list<RGWAccessControlList *> acl_list;
 336   list<RGWAccessControlList *>::iterator iter;
 337   for (iter = acl_list.begin(); iter != acl_list.end(); ++iter) {
 338     RGWAccessControlList::generate_test_instances(acl_list);
 339     iter = acl_list.begin();
 340
 341     RGWAccessControlPolicy *p = new RGWAccessControlPolicy(NULL);
 342     RGWAccessControlList *l = *iter;
 343     p->acl = *l;
 344
 345     string name = "radosgw";
 346     rgw_user id("rgw");
 347     p->owner.set_name(name);
 348     p->owner.set_id(id);
 349
 350     o.push_back(p);
 351
 352     delete l;
 353   }
 354
 355   o.push_back(new RGWAccessControlPolicy(NULL));
 356 }
 357
 358
 359 void ObjectMetaInfo::generate_test_instances(list<ObjectMetaInfo*>& o)
 360 {
 361   ObjectMetaInfo *m = new ObjectMetaInfo;
 362   m->size = 1024 * 1024;
 363   o.push_back(m);
 364   o.push_back(new ObjectMetaInfo);
 365 }
 366
 367 void ObjectCacheInfo::generate_test_instances(list<ObjectCacheInfo*>& o)
 368 {
 369   using ceph::encode;
 370   ObjectCacheInfo *i = new ObjectCacheInfo;
 371   i->status = 0;
 372   i->flags = CACHE_FLAG_MODIFY_XATTRS;
 373   string s = "this is a string";
 374   string s2 = "this is a another string";
 375   bufferlist data, data2;
 376   encode(s, data);
 377   encode(s2, data2);
 378   i->data = data;
 379   i->xattrs["x1"] = data;
 380   i->xattrs["x2"] = data2;
 381   i->rm_xattrs["r2"] = data2;
 382   i->rm_xattrs["r3"] = data;
 383   i->meta.size = 512 * 1024;
 384   o.push_back(i);
 385   o.push_back(new ObjectCacheInfo);
 386 }
 387
 388 void RGWCacheNotifyInfo::generate_test_instances(list<RGWCacheNotifyInfo*>& o)
 389 {
 390   o.push_back(new RGWCacheNotifyInfo);
 391 }
 392
 393 void RGWAccessKey::generate_test_instances(list<RGWAccessKey*>& o)
 394 {
 395   RGWAccessKey *k = new RGWAccessKey;
 396   k->id = "id";
 397   k->key = "key";
 398   k->subuser = "subuser";
 399   o.push_back(k);
 400   o.push_back(new RGWAccessKey);
 401 }
 402
 403 void RGWSubUser::generate_test_instances(list<RGWSubUser*>& o)
 404 {
 405   RGWSubUser *u = new RGWSubUser;
 406   u->name = "name";
 407   u->perm_mask = 0xf;
 408   o.push_back(u);
 409   o.push_back(new RGWSubUser);
 410 }
 411
 412 void RGWUserInfo::generate_test_instances(list<RGWUserInfo*>& o)
 413 {
 414   RGWUserInfo *i = new RGWUserInfo;
 415   i->user_id = "user_id";
 416   i->display_name =  "display_name";
 417   i->user_email = "user@email";
 418   RGWAccessKey k1, k2;
 419   k1.id = "id1";
 420   k1.key = "key1";
 421   k2.id = "id2";
 422   k2.subuser = "subuser";
 423   RGWSubUser u;
 424   u.name = "id2";
 425   u.perm_mask = 0x1;
 426   i->access_keys[k1.id] = k1;
 427   i->swift_keys[k2.id] = k2;
 428   i->subusers[u.name] = u;
 429   o.push_back(i);
 430
 431   o.push_back(new RGWUserInfo);
 432 }
 433
 434 void rgw_bucket::generate_test_instances(list<rgw_bucket*>& o)
 435 {
 436   rgw_bucket *b = new rgw_bucket;
 437   init_bucket(b, "tenant", "name", "pool", ".index_pool", "marker", "123");
 438   o.push_back(b);
 439   o.push_back(new rgw_bucket);
 440 }
 441
 442 void RGWBucketInfo::generate_test_instances(list<RGWBucketInfo*>& o)
 443 {
 444   RGWBucketInfo *i = new RGWBucketInfo;
 445   init_bucket(&i->bucket, "tenant", "bucket", "pool", ".index_pool", "marker", "10");
 446   i->owner = "owner";
 447   i->flags = BUCKET_SUSPENDED;
 448   o.push_back(i);
 449   o.push_back(new RGWBucketInfo);
 450 }
 451
 452 void RGWZoneGroup::generate_test_instances(list<RGWZoneGroup*>& o)
 453 {
 454   RGWZoneGroup *r = new RGWZoneGroup;
 455   o.push_back(r);
 456   o.push_back(new RGWZoneGroup);
 457 }
 458
 459 void RGWZone::generate_test_instances(list<RGWZone*> &o)
 460 {
 461   RGWZone *z = new RGWZone;
 462   o.push_back(z);
 463   o.push_back(new RGWZone);
 464 }
 465
 466 void RGWRealm::generate_test_instances(list<RGWRealm*> &o)
 467 {
 468   RGWRealm *z = new RGWRealm;
 469   o.push_back(z);
 470   o.push_back(new RGWRealm);
 471 }
 472
 473 void RGWPeriod::generate_test_instances(list<RGWPeriod*> &o)
 474 {
 475   RGWPeriod *z = new RGWPeriod;
 476   o.push_back(z);
 477   o.push_back(new RGWPeriod);
 478 }
 479
 480 void RGWPeriodLatestEpochInfo::generate_test_instances(list<RGWPeriodLatestEpochInfo*> &o)
 481 {
 482   RGWPeriodLatestEpochInfo *z = new RGWPeriodLatestEpochInfo;
 483   o.push_back(z);
 484   o.push_back(new RGWPeriodLatestEpochInfo);
 485 }
 486
 487 void RGWZoneParams::generate_test_instances(list<RGWZoneParams*> &o)
 488 {
 489   o.push_back(new RGWZoneParams);
 490   o.push_back(new RGWZoneParams);
 491 }
 492
 493 void RGWOLHInfo::generate_test_instances(list<RGWOLHInfo*> &o)
 494 {
 495   RGWOLHInfo *olh = new RGWOLHInfo;
 496   olh->removed = false;
 497   o.push_back(olh);
 498   o.push_back(new RGWOLHInfo);
 499 }
 500
 501 void RGWBucketEnt::generate_test_instances(list<RGWBucketEnt*>& o)
 502 {
 503   RGWBucketEnt *e = new RGWBucketEnt;
 504   init_bucket(&e->bucket, "tenant", "bucket", "pool", ".index_pool", "marker", "10");
 505   e->size = 1024;
 506   e->size_rounded = 4096;
 507   e->count = 1;
 508   o.push_back(e);
 509   o.push_back(new RGWBucketEnt);
 510 }
 511
 512 void RGWUploadPartInfo::generate_test_instances(list<RGWUploadPartInfo*>& o)
 513 {
 514   RGWUploadPartInfo *i = new RGWUploadPartInfo;
 515   i->num = 1;
 516   i->size = 10 * 1024 * 1024;
 517   i->etag = "etag";
 518   o.push_back(i);
 519   o.push_back(new RGWUploadPartInfo);
 520 }
 521
 522 void rgw_obj::generate_test_instances(list<rgw_obj*>& o)
 523 {
 524   rgw_bucket b;
 525   init_bucket(&b, "tenant", "bucket", "pool", ".index_pool", "marker", "10");
 526   rgw_obj *obj = new rgw_obj(b, "object");
 527   o.push_back(obj);
 528   o.push_back(new rgw_obj);
 529 }
 530
 531 void rgw_meta_sync_info::generate_test_instances(list<rgw_meta_sync_info*>& o)
 532 {
 533   auto info = new rgw_meta_sync_info;
 534   info->state = rgw_meta_sync_info::StateBuildingFullSyncMaps;
 535   info->period = "periodid";
 536   info->realm_epoch = 5;
 537   o.push_back(info);
 538   o.push_back(new rgw_meta_sync_info);
 539 }
 540
 541 void rgw_meta_sync_marker::generate_test_instances(list<rgw_meta_sync_marker*>& o)
 542 {
 543   auto marker = new rgw_meta_sync_marker;
 544   marker->state = rgw_meta_sync_marker::IncrementalSync;
 545   marker->marker = "01234";
 546   marker->realm_epoch = 5;
 547   o.push_back(marker);
 548   o.push_back(new rgw_meta_sync_marker);
 549 }
 550
 551 void rgw_meta_sync_status::generate_test_instances(list<rgw_meta_sync_status*>& o)
 552 {
 553   o.push_back(new rgw_meta_sync_status);
 554 }
 555
 556 void rgw_data_sync_info::generate_test_instances(list<rgw_data_sync_info*>& o)
 557 {
 558   auto info = new rgw_data_sync_info;
 559   info->state = rgw_data_sync_info::StateBuildingFullSyncMaps;
 560   info->num_shards = 8;
 561   o.push_back(info);
 562   o.push_back(new rgw_data_sync_info);
 563 }
 564
 565 void rgw_data_sync_marker::generate_test_instances(list<rgw_data_sync_marker*>& o)
 566 {
 567   auto marker = new rgw_data_sync_marker;
 568   marker->state = rgw_data_sync_marker::IncrementalSync;
 569   marker->marker = "01234";
 570   marker->pos = 5;
 571   o.push_back(marker);
 572   o.push_back(new rgw_data_sync_marker);
 573 }
 574
 575 void rgw_data_sync_status::generate_test_instances(list<rgw_data_sync_status*>& o)
 576 {
 577   o.push_back(new rgw_data_sync_status);
 578 }
 579
 580 void objexp_hint_entry::generate_test_instances(list<objexp_hint_entry*>& o)
 581 {
 582   auto it = new objexp_hint_entry;
 583   it->tenant = "tenant1";
 584   it->bucket_name = "bucket1";
 585   it->bucket_id = "1234";
 586   it->obj_key = rgw_obj_key("obj");
 587   o.push_back(it);
 588   o.push_back(new objexp_hint_entry);
 589 }
 590
 591 void RGWBucketEntryPoint::generate_test_instances(list<RGWBucketEntryPoint*>& o)
 592 {
 593   RGWBucketEntryPoint *bp = new RGWBucketEntryPoint();
 594   init_bucket(&bp->bucket, "tenant", "bucket", "pool", ".index.pool", "marker", "10");
 595   bp->owner = "owner";
 596   bp->creation_time = ceph::real_clock::from_ceph_timespec({init_le32(2), init_le32(3)});
 597
 598   o.push_back(bp);
 599   o.push_back(new RGWBucketEntryPoint);
 600 }
 601
 602 void rgw_user::generate_test_instances(list<rgw_user*>& o)
 603 {
 604   rgw_user *u = new rgw_user("tenant", "user");
 605
 606   o.push_back(u);
 607   o.push_back(new rgw_user);
 608 }
 609
 610 void obj_version::generate_test_instances(list<obj_version*>& o)
 611 {
 612   obj_version *v = new obj_version;
 613   v->ver = 5;
 614   v->tag = "tag";
 615
 616   o.push_back(v);
 617   o.push_back(new obj_version);
 618 }