ceph/src/rgw/rgw_dencoder.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "rgw_common.h"
   5 #include "rgw_rados.h"
   6 #include "rgw_log.h"
   7 #include "rgw_acl.h"
   8 #include "rgw_acl_s3.h"
   9 #include "rgw_cache.h"
  10 #include "rgw_meta_sync_status.h"
  11 #include "rgw_data_sync.h"
  12
  13 #include "common/Formatter.h"
  14
  15 static string shadow_ns = RGW_OBJ_NS_SHADOW;
  16
  17 static void init_bucket(rgw_bucket *b, const char *t, const char *n, const char *dp, const char *ip, const char *m, const char *id)
  18 {
  19   b->tenant = t;
  20   b->name = n;
  21   b->marker = m;
  22   b->bucket_id = id;
  23   b->explicit_placement.data_pool = rgw_pool(dp);
  24   b->explicit_placement.index_pool = rgw_pool(ip);
  25 }
  26
  27 void RGWObjManifestPart::generate_test_instances(std::list<RGWObjManifestPart*>& o)
  28 {
  29   o.push_back(new RGWObjManifestPart);
  30
  31   RGWObjManifestPart *p = new RGWObjManifestPart;
  32   rgw_bucket b;
  33   init_bucket(&b, "tenant", "bucket", ".pool", ".index_pool", "marker_", "12");
  34
  35   p->loc = rgw_obj(b, "object");
  36   p->loc_ofs = 512 * 1024;
  37   p->size = 128 * 1024;
  38   o.push_back(p);
  39 }
  40
  41 void RGWObjManifest::obj_iterator::seek(uint64_t o)
  42 {
  43   ofs = o;
  44   if (manifest->explicit_objs) {
  45     explicit_iter = manifest->objs.upper_bound(ofs);
  46     if (explicit_iter != manifest->objs.begin()) {
  47       --explicit_iter;
  48     }
  49     if (ofs >= manifest->obj_size) {
  50       ofs = manifest->obj_size;
  51       return;
  52     }
  53     update_explicit_pos();
  54     update_location();
  55     return;
  56   }
  57   if (o < manifest->get_head_size()) {
  58     rule_iter = manifest->rules.begin();
  59     stripe_ofs = 0;
  60     stripe_size = manifest->get_head_size();
  61     if (rule_iter != manifest->rules.end()) {
  62       cur_part_id = rule_iter->second.start_part_num;
  63       cur_override_prefix = rule_iter->second.override_prefix;
  64     }
  65     update_location();
  66     return;
  67   }
  68
  69   rule_iter = manifest->rules.upper_bound(ofs);
  70   next_rule_iter = rule_iter;
  71   if (rule_iter != manifest->rules.begin()) {
  72     --rule_iter;
  73   }
  74
  75   if (rule_iter == manifest->rules.end()) {
  76     update_location();
  77     return;
  78   }
  79
  80   RGWObjManifestRule& rule = rule_iter->second;
  81
  82   if (rule.part_size > 0) {
  83     cur_part_id = rule.start_part_num + (ofs - rule.start_ofs) / rule.part_size;
  84   } else {
  85     cur_part_id = rule.start_part_num;
  86   }
  87   part_ofs = rule.start_ofs + (cur_part_id - rule.start_part_num) * rule.part_size;
  88
  89   if (rule.stripe_max_size > 0) {
  90     cur_stripe = (ofs - part_ofs) / rule.stripe_max_size;
  91
  92     stripe_ofs = part_ofs + cur_stripe * rule.stripe_max_size;
  93     if (!cur_part_id && manifest->get_head_size() > 0) {
  94       cur_stripe++;
  95     }
  96   } else {
  97     cur_stripe = 0;
  98     stripe_ofs = part_ofs;
  99   }
 100
 101   if (!rule.part_size) {
 102     stripe_size = rule.stripe_max_size;
 103     stripe_size = MIN(manifest->get_obj_size() - stripe_ofs, stripe_size);
 104   } else {
 105     uint64_t next = MIN(stripe_ofs + rule.stripe_max_size, part_ofs + rule.part_size);
 106     stripe_size = next - stripe_ofs;
 107   }
 108
 109   cur_override_prefix = rule.override_prefix;
 110
 111   update_location();
 112 }
 113
 114 void RGWObjManifest::obj_iterator::update_location()
 115 {
 116   if (manifest->explicit_objs) {
 117     location = explicit_iter->second.loc;
 118     return;
 119   }
 120
 121   if (ofs < manifest->get_head_size()) {
 122     location = manifest->get_obj();
 123     location.set_placement_rule(manifest->get_head_placement_rule());
 124     return;
 125   }
 126
 127   manifest->get_implicit_location(cur_part_id, cur_stripe, ofs, &cur_override_prefix, &location);
 128 }
 129
 130 void RGWObjManifest::obj_iterator::update_explicit_pos()
 131 {
 132   ofs = explicit_iter->first;
 133   stripe_ofs = ofs;
 134
 135   map<uint64_t, RGWObjManifestPart>::iterator next_iter = explicit_iter;
 136   ++next_iter;
 137   if (next_iter != manifest->objs.end()) {
 138     stripe_size = next_iter->first - ofs;
 139   } else {
 140     stripe_size = manifest->obj_size - ofs;
 141   }
 142 }
 143
 144 void RGWObjManifest::generate_test_instances(std::list<RGWObjManifest*>& o)
 145 {
 146   RGWObjManifest *m = new RGWObjManifest;
 147   for (int i = 0; i<10; i++) {
 148     RGWObjManifestPart p;
 149     rgw_bucket b;
 150     init_bucket(&b, "tenant", "bucket", ".pool", ".index_pool", "marker_", "12");
 151     p.loc = rgw_obj(b, "object");
 152     p.loc_ofs = 0;
 153     p.size = 512 * 1024;
 154     m->objs[(uint64_t)i * 512 * 1024] = p;
 155   }
 156   m->obj_size = 5 * 1024 * 1024;
 157
 158   o.push_back(m);
 159
 160   o.push_back(new RGWObjManifest);
 161 }
 162
 163 void RGWObjManifest::get_implicit_location(uint64_t cur_part_id, uint64_t cur_stripe, uint64_t ofs, string *override_prefix, rgw_obj_select *location)
 164 {
 165   rgw_obj loc;
 166
 167   string& oid = loc.key.name;
 168   string& ns = loc.key.ns;
 169
 170   if (!override_prefix || override_prefix->empty()) {
 171     oid = prefix;
 172   } else {
 173     oid = *override_prefix;
 174   }
 175
 176   if (!cur_part_id) {
 177     if (ofs < max_head_size) {
 178       location->set_placement_rule(head_placement_rule);
 179       *location = obj;
 180       return;
 181     } else {
 182       char buf[16];
 183       snprintf(buf, sizeof(buf), "%d", (int)cur_stripe);
 184       oid += buf;
 185       ns = shadow_ns;
 186     }
 187   } else {
 188     char buf[32];
 189     if (cur_stripe == 0) {
 190       snprintf(buf, sizeof(buf), ".%d", (int)cur_part_id);
 191       oid += buf;
 192       ns= RGW_OBJ_NS_MULTIPART;
 193     } else {
 194       snprintf(buf, sizeof(buf), ".%d_%d", (int)cur_part_id, (int)cur_stripe);
 195       oid += buf;
 196       ns = shadow_ns;
 197     }
 198   }
 199
 200   if (!tail_placement.bucket.name.empty()) {
 201     loc.bucket = tail_placement.bucket;
 202   } else {
 203     loc.bucket = obj.bucket;
 204   }
 205
 206   // Always overwrite instance with tail_instance
 207   // to get the right shadow object location
 208   loc.key.set_instance(tail_instance);
 209
 210   location->set_placement_rule(tail_placement.placement_rule);
 211   *location = loc;
 212 }
 213
 214
 215
 216 void rgw_log_entry::generate_test_instances(list<rgw_log_entry*>& o)
 217 {
 218   rgw_log_entry *e = new rgw_log_entry;
 219   e->object_owner = "object_owner";
 220   e->bucket_owner = "bucket_owner";
 221   e->bucket = "bucket";
 222   e->remote_addr = "1.2.3.4";
 223   e->user = "user";
 224   e->obj = rgw_obj_key("obj");
 225   e->uri = "http://uri/bucket/obj";
 226   e->http_status = "200";
 227   e->error_code = "error_code";
 228   e->bytes_sent = 1024;
 229   e->bytes_received = 512;
 230   e->obj_size = 2048;
 231   e->user_agent = "user_agent";
 232   e->referrer = "referrer";
 233   e->bucket_id = "10";
 234   o.push_back(e);
 235   o.push_back(new rgw_log_entry);
 236 }
 237
 238 void ACLPermission::generate_test_instances(list<ACLPermission*>& o)
 239 {
 240   ACLPermission *p = new ACLPermission;
 241   p->set_permissions(RGW_PERM_WRITE_ACP);
 242   o.push_back(p);
 243   o.push_back(new ACLPermission);
 244 }
 245
 246 void ACLGranteeType::generate_test_instances(list<ACLGranteeType*>& o)
 247 {
 248   ACLGranteeType *t = new ACLGranteeType;
 249   t->set(ACL_TYPE_CANON_USER);
 250   o.push_back(t);
 251   o.push_back(new ACLGranteeType);
 252 }
 253
 254 /* the following is copied here from rgw_acl_s3.cc, to avoid having to have excessive linking
 255    with everything it needs */
 256
 257 #define RGW_URI_ALL_USERS       "http://acs.amazonaws.com/groups/global/AllUsers"
 258 #define RGW_URI_AUTH_USERS      "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"
 259
 260 static string rgw_uri_all_users = RGW_URI_ALL_USERS;
 261 static string rgw_uri_auth_users = RGW_URI_AUTH_USERS;
 262
 263 ACLGroupTypeEnum ACLGrant::uri_to_group(string& uri)
 264 {
 265   // this is required for backward compatibility
 266   return ACLGrant_S3::uri_to_group(uri);
 267 }
 268
 269 ACLGroupTypeEnum ACLGrant_S3::uri_to_group(string& uri)
 270 {
 271   if (uri.compare(rgw_uri_all_users) == 0)
 272     return ACL_GROUP_ALL_USERS;
 273   else if (uri.compare(rgw_uri_auth_users) == 0)
 274     return ACL_GROUP_AUTHENTICATED_USERS;
 275
 276   return ACL_GROUP_NONE;
 277 }
 278
 279 void ACLGrant::generate_test_instances(list<ACLGrant*>& o)
 280 {
 281   rgw_user id("rgw");
 282   string name, email;
 283   name = "Mr. RGW";
 284   email = "r@gw";
 285
 286   ACLGrant *g1 = new ACLGrant;
 287   g1->set_canon(id, name, RGW_PERM_READ);
 288   g1->email = email;
 289   o.push_back(g1);
 290
 291   ACLGrant *g2 = new ACLGrant;
 292   g1->set_group(ACL_GROUP_AUTHENTICATED_USERS, RGW_PERM_WRITE);
 293   o.push_back(g2);
 294
 295   o.push_back(new ACLGrant);
 296 }
 297
 298 void RGWAccessControlList::generate_test_instances(list<RGWAccessControlList*>& o)
 299 {
 300   RGWAccessControlList *acl = new RGWAccessControlList(NULL);
 301
 302   list<ACLGrant *> glist;
 303   list<ACLGrant *>::iterator iter;
 304
 305   ACLGrant::generate_test_instances(glist);
 306   for (iter = glist.begin(); iter != glist.end(); ++iter) {
 307     ACLGrant *grant = *iter;
 308     acl->add_grant(grant);
 309
 310     delete grant;
 311   }
 312   o.push_back(acl);
 313   o.push_back(new RGWAccessControlList(NULL));
 314 }
 315
 316 void ACLOwner::generate_test_instances(list<ACLOwner*>& o)
 317 {
 318   ACLOwner *owner = new ACLOwner;
 319   owner->id = "rgw";
 320   owner->display_name = "Mr. RGW";
 321   o.push_back(owner);
 322   o.push_back(new ACLOwner);
 323 }
 324
 325 void RGWAccessControlPolicy::generate_test_instances(list<RGWAccessControlPolicy*>& o)
 326 {
 327   list<RGWAccessControlList *> acl_list;
 328   list<RGWAccessControlList *>::iterator iter;
 329   for (iter = acl_list.begin(); iter != acl_list.end(); ++iter) {
 330     RGWAccessControlList::generate_test_instances(acl_list);
 331     iter = acl_list.begin();
 332
 333     RGWAccessControlPolicy *p = new RGWAccessControlPolicy(NULL);
 334     RGWAccessControlList *l = *iter;
 335     p->acl = *l;
 336
 337     string name = "radosgw";
 338     rgw_user id("rgw");
 339     p->owner.set_name(name);
 340     p->owner.set_id(id);
 341
 342     o.push_back(p);
 343
 344     delete l;
 345   }
 346
 347   o.push_back(new RGWAccessControlPolicy(NULL));
 348 }
 349
 350
 351 void ObjectMetaInfo::generate_test_instances(list<ObjectMetaInfo*>& o)
 352 {
 353   ObjectMetaInfo *m = new ObjectMetaInfo;
 354   m->size = 1024 * 1024;
 355   o.push_back(m);
 356   o.push_back(new ObjectMetaInfo);
 357 }
 358
 359 void ObjectCacheInfo::generate_test_instances(list<ObjectCacheInfo*>& o)
 360 {
 361   ObjectCacheInfo *i = new ObjectCacheInfo;
 362   i->status = 0;
 363   i->flags = CACHE_FLAG_MODIFY_XATTRS;
 364   string s = "this is a string";
 365   string s2 = "this is a another string";
 366   bufferlist data, data2;
 367   ::encode(s, data);
 368   ::encode(s2, data2);
 369   i->data = data;
 370   i->xattrs["x1"] = data;
 371   i->xattrs["x2"] = data2;
 372   i->rm_xattrs["r2"] = data2;
 373   i->rm_xattrs["r3"] = data;
 374   i->meta.size = 512 * 1024;
 375   o.push_back(i);
 376   o.push_back(new ObjectCacheInfo);
 377 }
 378
 379 void RGWCacheNotifyInfo::generate_test_instances(list<RGWCacheNotifyInfo*>& o)
 380 {
 381   o.push_back(new RGWCacheNotifyInfo);
 382 }
 383
 384 void RGWAccessKey::generate_test_instances(list<RGWAccessKey*>& o)
 385 {
 386   RGWAccessKey *k = new RGWAccessKey;
 387   k->id = "id";
 388   k->key = "key";
 389   k->subuser = "subuser";
 390   o.push_back(k);
 391   o.push_back(new RGWAccessKey);
 392 }
 393
 394 void RGWSubUser::generate_test_instances(list<RGWSubUser*>& o)
 395 {
 396   RGWSubUser *u = new RGWSubUser;
 397   u->name = "name";
 398   u->perm_mask = 0xf;
 399   o.push_back(u);
 400   o.push_back(new RGWSubUser);
 401 }
 402
 403 void RGWUserInfo::generate_test_instances(list<RGWUserInfo*>& o)
 404 {
 405   RGWUserInfo *i = new RGWUserInfo;
 406   i->auid = 1;
 407   i->user_id = "user_id";
 408   i->display_name =  "display_name";
 409   i->user_email = "user@email";
 410   RGWAccessKey k1, k2;
 411   k1.id = "id1";
 412   k1.key = "key1";
 413   k2.id = "id2";
 414   k2.subuser = "subuser";
 415   RGWSubUser u;
 416   u.name = "id2";
 417   u.perm_mask = 0x1;
 418   i->access_keys[k1.id] = k1;
 419   i->swift_keys[k2.id] = k2;
 420   i->subusers[u.name] = u;
 421   o.push_back(i);
 422
 423   o.push_back(new RGWUserInfo);
 424 }
 425
 426 void rgw_bucket::generate_test_instances(list<rgw_bucket*>& o)
 427 {
 428   rgw_bucket *b = new rgw_bucket;
 429   init_bucket(b, "tenant", "name", "pool", ".index_pool", "marker", "123");
 430   o.push_back(b);
 431   o.push_back(new rgw_bucket);
 432 }
 433
 434 void RGWBucketInfo::generate_test_instances(list<RGWBucketInfo*>& o)
 435 {
 436   RGWBucketInfo *i = new RGWBucketInfo;
 437   init_bucket(&i->bucket, "tenant", "bucket", "pool", ".index_pool", "marker", "10");
 438   i->owner = "owner";
 439   i->flags = BUCKET_SUSPENDED;
 440   o.push_back(i);
 441   o.push_back(new RGWBucketInfo);
 442 }
 443
 444 void RGWZoneGroup::generate_test_instances(list<RGWZoneGroup*>& o)
 445 {
 446   RGWZoneGroup *r = new RGWZoneGroup;
 447   o.push_back(r);
 448   o.push_back(new RGWZoneGroup);
 449 }
 450
 451 void RGWZone::generate_test_instances(list<RGWZone*> &o)
 452 {
 453   RGWZone *z = new RGWZone;
 454   o.push_back(z);
 455   o.push_back(new RGWZone);
 456 }
 457
 458 void RGWZoneParams::generate_test_instances(list<RGWZoneParams*> &o)
 459 {
 460   o.push_back(new RGWZoneParams);
 461   o.push_back(new RGWZoneParams);
 462 }
 463
 464 void RGWOLHInfo::generate_test_instances(list<RGWOLHInfo*> &o)
 465 {
 466   RGWOLHInfo *olh = new RGWOLHInfo;
 467   olh->removed = false;
 468   o.push_back(olh);
 469   o.push_back(new RGWOLHInfo);
 470 }
 471
 472 void RGWBucketEnt::generate_test_instances(list<RGWBucketEnt*>& o)
 473 {
 474   RGWBucketEnt *e = new RGWBucketEnt;
 475   init_bucket(&e->bucket, "tenant", "bucket", "pool", ".index_pool", "marker", "10");
 476   e->size = 1024;
 477   e->size_rounded = 4096;
 478   e->count = 1;
 479   o.push_back(e);
 480   o.push_back(new RGWBucketEnt);
 481 }
 482
 483 void RGWUploadPartInfo::generate_test_instances(list<RGWUploadPartInfo*>& o)
 484 {
 485   RGWUploadPartInfo *i = new RGWUploadPartInfo;
 486   i->num = 1;
 487   i->size = 10 * 1024 * 1024;
 488   i->etag = "etag";
 489   o.push_back(i);
 490   o.push_back(new RGWUploadPartInfo);
 491 }
 492
 493 void rgw_obj::generate_test_instances(list<rgw_obj*>& o)
 494 {
 495   rgw_bucket b;
 496   init_bucket(&b, "tenant", "bucket", "pool", ".index_pool", "marker", "10");
 497   rgw_obj *obj = new rgw_obj(b, "object");
 498   o.push_back(obj);
 499   o.push_back(new rgw_obj);
 500 }
 501
 502 void rgw_meta_sync_info::generate_test_instances(list<rgw_meta_sync_info*>& o)
 503 {
 504   auto info = new rgw_meta_sync_info;
 505   info->state = rgw_meta_sync_info::StateBuildingFullSyncMaps;
 506   info->period = "periodid";
 507   info->realm_epoch = 5;
 508   o.push_back(info);
 509   o.push_back(new rgw_meta_sync_info);
 510 }
 511
 512 void rgw_meta_sync_marker::generate_test_instances(list<rgw_meta_sync_marker*>& o)
 513 {
 514   auto marker = new rgw_meta_sync_marker;
 515   marker->state = rgw_meta_sync_marker::IncrementalSync;
 516   marker->marker = "01234";
 517   marker->realm_epoch = 5;
 518   o.push_back(marker);
 519   o.push_back(new rgw_meta_sync_marker);
 520 }
 521
 522 void rgw_meta_sync_status::generate_test_instances(list<rgw_meta_sync_status*>& o)
 523 {
 524   o.push_back(new rgw_meta_sync_status);
 525 }
 526
 527 void rgw_data_sync_info::generate_test_instances(list<rgw_data_sync_info*>& o)
 528 {
 529   auto info = new rgw_data_sync_info;
 530   info->state = rgw_data_sync_info::StateBuildingFullSyncMaps;
 531   info->num_shards = 8;
 532   o.push_back(info);
 533   o.push_back(new rgw_data_sync_info);
 534 }
 535
 536 void rgw_data_sync_marker::generate_test_instances(list<rgw_data_sync_marker*>& o)
 537 {
 538   auto marker = new rgw_data_sync_marker;
 539   marker->state = rgw_data_sync_marker::IncrementalSync;
 540   marker->marker = "01234";
 541   marker->pos = 5;
 542   o.push_back(marker);
 543   o.push_back(new rgw_data_sync_marker);
 544 }
 545
 546 void rgw_data_sync_status::generate_test_instances(list<rgw_data_sync_status*>& o)
 547 {
 548   o.push_back(new rgw_data_sync_status);
 549 }