ceph/src/test/rgw/test_rgw_manifest.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 /*
   4  * Ceph - scalable distributed file system
   5  *
   6  * Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
   7  *
   8  * This is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License version 2.1, as published by the Free Software
  11  * Foundation. See file COPYING.
  12  *
  13  */
  14 #include <iostream>
  15 #include "global/global_init.h"
  16 #include "common/ceph_argparse.h"
  17 #include "rgw/rgw_common.h"
  18 #include "rgw/rgw_rados.h"
  19 #include "test_rgw_common.h"
  20 #include <gtest/gtest.h>
  21
  22 using namespace std;
  23
  24 auto cct = new CephContext(CEPH_ENTITY_TYPE_CLIENT);
  25 const DoutPrefix dp(cct, 1, "test rgw manifest: ");
  26
  27 struct OldObjManifestPart {
  28   old_rgw_obj loc;   /* the object where the data is located */
  29   uint64_t loc_ofs;  /* the offset at that object where the data is located */
  30   uint64_t size;     /* the part size */
  31
  32   OldObjManifestPart() : loc_ofs(0), size(0) {}
  33
  34   void encode(bufferlist& bl) const {
  35     ENCODE_START(2, 2, bl);
  36     encode(loc, bl);
  37     encode(loc_ofs, bl);
  38     encode(size, bl);
  39     ENCODE_FINISH(bl);
  40   }
  41
  42   void decode(bufferlist::const_iterator& bl) {
  43      DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl);
  44      decode(loc, bl);
  45      decode(loc_ofs, bl);
  46      decode(size, bl);
  47      DECODE_FINISH(bl);
  48   }
  49
  50   void dump(Formatter *f) const;
  51   static void generate_test_instances(list<OldObjManifestPart*>& o);
  52 };
  53 WRITE_CLASS_ENCODER(OldObjManifestPart)
  54
  55 class OldObjManifest {
  56 protected:
  57   map<uint64_t, OldObjManifestPart> objs;
  58
  59   uint64_t obj_size;
  60 public:
  61
  62   OldObjManifest() : obj_size(0) {}
  63   OldObjManifest(const OldObjManifest& rhs) {
  64     *this = rhs;
  65   }
  66   OldObjManifest& operator=(const OldObjManifest& rhs) {
  67     objs = rhs.objs;
  68     obj_size = rhs.obj_size;
  69     return *this;
  70   }
  71
  72   const map<uint64_t, OldObjManifestPart>& get_objs() {
  73     return objs;
  74   }
  75
  76   void append(uint64_t ofs, const OldObjManifestPart& part) {
  77     objs[ofs] = part;
  78     obj_size = std::max(obj_size, ofs + part.size);
  79   }
  80
  81   void encode(bufferlist& bl) const {
  82     ENCODE_START(2, 2, bl);
  83     encode(obj_size, bl);
  84     encode(objs, bl);
  85     ENCODE_FINISH(bl);
  86   }
  87
  88   void decode(bufferlist::const_iterator& bl) {
  89     DECODE_START_LEGACY_COMPAT_LEN_32(6, 2, 2, bl);
  90     decode(obj_size, bl);
  91     decode(objs, bl);
  92     DECODE_FINISH(bl);
  93   }
  94
  95   bool empty() {
  96     return objs.empty();
  97   }
  98 };
  99 WRITE_CLASS_ENCODER(OldObjManifest)
 100
 101 void append_head(list<rgw_obj> *objs, rgw_obj& head)
 102 {
 103   objs->push_back(head);
 104 }
 105
 106 void append_stripes(list<rgw_obj> *objs, RGWObjManifest& manifest, uint64_t obj_size, uint64_t stripe_size)
 107 {
 108   string prefix = manifest.get_prefix();
 109   rgw_bucket bucket = manifest.get_obj().bucket;
 110
 111   int i = 0;
 112   for (uint64_t ofs = manifest.get_max_head_size(); ofs < obj_size; ofs += stripe_size) {
 113     char buf[16];
 114     snprintf(buf, sizeof(buf), "%d", ++i);
 115     string oid = prefix + buf;
 116   cout << "oid=" << oid << std::endl;
 117     rgw_obj obj;
 118     obj.init_ns(bucket, oid, "shadow");
 119     objs->push_back(obj);
 120   }
 121 }
 122
 123 static void gen_obj(test_rgw_env& env, uint64_t obj_size, uint64_t head_max_size, uint64_t stripe_size,
 124                     RGWObjManifest *manifest, const rgw_placement_rule& placement_rule, rgw_bucket *bucket, rgw_obj *head, RGWObjManifest::generator *gen,
 125                     list<rgw_obj> *test_objs)
 126 {
 127   manifest->set_trivial_rule(head_max_size, stripe_size);
 128
 129   test_rgw_init_bucket(bucket, "buck");
 130
 131   *head = rgw_obj(*bucket, "oid");
 132   gen->create_begin(g_ceph_context, manifest, placement_rule, nullptr, *bucket, *head);
 133
 134   append_head(test_objs, *head);
 135   cout << "test_objs.size()=" << test_objs->size() << std::endl;
 136   append_stripes(test_objs, *manifest, obj_size, stripe_size);
 137
 138   cout << "test_objs.size()=" << test_objs->size() << std::endl;
 139
 140   ASSERT_EQ((int)manifest->get_obj_size(), 0);
 141   ASSERT_EQ((int)manifest->get_head_size(), 0);
 142   ASSERT_EQ(manifest->has_tail(), false);
 143
 144   uint64_t ofs = 0;
 145   list<rgw_obj>::iterator iter = test_objs->begin();
 146
 147   while (ofs < obj_size) {
 148     rgw_raw_obj obj = gen->get_cur_obj(env.zonegroup, env.zone_params);
 149     cout << "obj=" << obj << std::endl;
 150     rgw_raw_obj test_raw = rgw_obj_select(*iter).get_raw_obj(env.zonegroup, env.zone_params);
 151     ASSERT_TRUE(obj == test_raw);
 152
 153     ofs = std::min(ofs + gen->cur_stripe_max_size(), obj_size);
 154     gen->create_next(ofs);
 155
 156   cout << "obj=" << obj << " *iter=" << *iter << std::endl;
 157   cout << "test_objs.size()=" << test_objs->size() << std::endl;
 158     ++iter;
 159
 160   }
 161
 162   if (manifest->has_tail()) {
 163     rgw_raw_obj obj = gen->get_cur_obj(env.zonegroup, env.zone_params);
 164     rgw_raw_obj test_raw = rgw_obj_select(*iter).get_raw_obj(env.zonegroup, env.zone_params);
 165     ASSERT_TRUE(obj == test_raw);
 166     ++iter;
 167   }
 168   ASSERT_TRUE(iter == test_objs->end());
 169   ASSERT_EQ(manifest->get_obj_size(), obj_size);
 170   ASSERT_EQ(manifest->get_head_size(), std::min(obj_size, head_max_size));
 171   ASSERT_EQ(manifest->has_tail(), (obj_size > head_max_size));
 172 }
 173
 174 static void gen_old_obj(test_rgw_env& env, uint64_t obj_size, uint64_t head_max_size, uint64_t stripe_size,
 175                     OldObjManifest *manifest, old_rgw_bucket *bucket, old_rgw_obj *head,
 176                     list<old_rgw_obj> *test_objs)
 177 {
 178   test_rgw_init_old_bucket(bucket, "buck");
 179
 180   *head = old_rgw_obj(*bucket, "obj");
 181
 182   OldObjManifestPart part;
 183   part.loc = *head;
 184   part.size = head_max_size;
 185   part.loc_ofs = 0;
 186
 187   manifest->append(0, part);
 188   test_objs->push_back(part.loc);
 189
 190   string prefix;
 191   append_rand_alpha(g_ceph_context, prefix, prefix, 16);
 192
 193   int i = 0;
 194   for (uint64_t ofs = head_max_size; ofs < obj_size; ofs += stripe_size, i++) {
 195     char buf[32];
 196     snprintf(buf, sizeof(buf), "%s.%d", prefix.c_str(), i);
 197     old_rgw_obj loc(*bucket, buf);
 198     loc.set_ns("shadow");
 199     OldObjManifestPart part;
 200     part.loc = loc;
 201     part.size = min(stripe_size, obj_size - ofs);
 202     part.loc_ofs = 0;
 203
 204     manifest->append(ofs, part);
 205
 206     test_objs->push_back(loc);
 207   }
 208 }
 209
 210 TEST(TestRGWManifest, head_only_obj) {
 211   test_rgw_env env;
 212   RGWObjManifest manifest;
 213   rgw_bucket bucket;
 214   rgw_obj head;
 215   RGWObjManifest::generator gen;
 216
 217   int obj_size = 256 * 1024;
 218
 219   list<rgw_obj> objs;
 220
 221   gen_obj(env, obj_size, 512 * 1024, 4 * 1024 * 1024, &manifest, env.zonegroup.default_placement, &bucket, &head, &gen, &objs);
 222
 223   cout <<  " manifest.get_obj_size()=" << manifest.get_obj_size() << std::endl;
 224   cout <<  " manifest.get_head_size()=" << manifest.get_head_size() << std::endl;
 225   list<rgw_obj>::iterator liter;
 226
 227   RGWObjManifest::obj_iterator iter;
 228   for (iter = manifest.obj_begin(&dp), liter = objs.begin();
 229        iter != manifest.obj_end(&dp) && liter != objs.end();
 230        ++iter, ++liter) {
 231     ASSERT_TRUE(env.get_raw(*liter) == env.get_raw(iter.get_location()));
 232   }
 233
 234   ASSERT_TRUE(iter == manifest.obj_end(&dp));
 235   ASSERT_TRUE(liter == objs.end());
 236
 237   rgw_raw_obj raw_head;
 238
 239   iter = manifest.obj_find(&dp, 100 * 1024);
 240   ASSERT_TRUE(env.get_raw(iter.get_location()) == env.get_raw(head));
 241   ASSERT_EQ((int)iter.get_stripe_size(), obj_size);
 242 }
 243
 244 TEST(TestRGWManifest, obj_with_head_and_tail) {
 245   test_rgw_env env;
 246   RGWObjManifest manifest;
 247   rgw_bucket bucket;
 248   rgw_obj head;
 249   RGWObjManifest::generator gen;
 250
 251   list<rgw_obj> objs;
 252
 253   int obj_size = 21 * 1024 * 1024 + 1000;
 254   int stripe_size = 4 * 1024 * 1024;
 255   int head_size = 512 * 1024;
 256
 257   gen_obj(env, obj_size, head_size, stripe_size, &manifest, env.zonegroup.default_placement, &bucket, &head, &gen, &objs);
 258
 259   list<rgw_obj>::iterator liter;
 260
 261   rgw_obj_select last_obj;
 262
 263   RGWObjManifest::obj_iterator iter;
 264   for (iter = manifest.obj_begin(&dp), liter = objs.begin();
 265        iter != manifest.obj_end(&dp) && liter != objs.end();
 266        ++iter, ++liter) {
 267     cout << "*liter=" << *liter << " iter.get_location()=" << env.get_raw(iter.get_location()) << std::endl;
 268     ASSERT_TRUE(env.get_raw(*liter) == env.get_raw(iter.get_location()));
 269
 270     last_obj = iter.get_location();
 271   }
 272
 273   ASSERT_TRUE(iter == manifest.obj_end(&dp));
 274   ASSERT_TRUE(liter == objs.end());
 275
 276   iter = manifest.obj_find(&dp, 100 * 1024);
 277   ASSERT_TRUE(env.get_raw(iter.get_location()) == env.get_raw(head));
 278   ASSERT_EQ((int)iter.get_stripe_size(), head_size);
 279
 280   uint64_t ofs = 20 * 1024 * 1024 + head_size;
 281   iter = manifest.obj_find(&dp, ofs + 100);
 282
 283   ASSERT_TRUE(env.get_raw(iter.get_location()) == env.get_raw(last_obj));
 284   ASSERT_EQ(iter.get_stripe_ofs(), ofs);
 285   ASSERT_EQ(iter.get_stripe_size(), obj_size - ofs);
 286 }
 287
 288 TEST(TestRGWManifest, multipart) {
 289   test_rgw_env env;
 290   int num_parts = 16;
 291   vector <RGWObjManifest> pm(num_parts);
 292   rgw_bucket bucket;
 293   uint64_t part_size = 10 * 1024 * 1024;
 294   uint64_t stripe_size = 4 * 1024 * 1024;
 295
 296   string upload_id = "abc123";
 297
 298   for (int i = 0; i < num_parts; ++i) {
 299     RGWObjManifest& manifest = pm[i];
 300     RGWObjManifest::generator gen;
 301     manifest.set_prefix(upload_id);
 302
 303     manifest.set_multipart_part_rule(stripe_size, i + 1);
 304
 305     uint64_t ofs;
 306     rgw_obj head;
 307     for (ofs = 0; ofs < part_size; ofs += stripe_size) {
 308       if (ofs == 0) {
 309         rgw_placement_rule rule(env.zonegroup.default_placement.name, RGW_STORAGE_CLASS_STANDARD);
 310         int r = gen.create_begin(g_ceph_context, &manifest, rule, nullptr, bucket, head);
 311         ASSERT_EQ(r, 0);
 312         continue;
 313       }
 314       gen.create_next(ofs);
 315     }
 316
 317     if (ofs > part_size) {
 318       gen.create_next(part_size);
 319     }
 320   }
 321
 322   RGWObjManifest m;
 323
 324   for (int i = 0; i < num_parts; i++) {
 325     m.append(&dp, pm[i], env.zonegroup, env.zone_params);
 326   }
 327   RGWObjManifest::obj_iterator iter;
 328   for (iter = m.obj_begin(&dp); iter != m.obj_end(&dp); ++iter) {
 329     RGWObjManifest::obj_iterator fiter = m.obj_find(&dp, iter.get_ofs());
 330     ASSERT_TRUE(env.get_raw(fiter.get_location()) == env.get_raw(iter.get_location()));
 331   }
 332
 333   ASSERT_EQ(m.get_obj_size(), num_parts * part_size);
 334 }
 335
 336 TEST(TestRGWManifest, old_obj_manifest) {
 337   test_rgw_env env;
 338   OldObjManifest old_manifest;
 339   old_rgw_bucket old_bucket;
 340   old_rgw_obj old_head;
 341
 342   int obj_size = 40 * 1024 * 1024;
 343   uint64_t stripe_size = 4 * 1024 * 1024;
 344   uint64_t head_size = 512 * 1024;
 345
 346   list<old_rgw_obj> old_objs;
 347
 348   gen_old_obj(env, obj_size, head_size, stripe_size, &old_manifest, &old_bucket, &old_head, &old_objs);
 349
 350   ASSERT_EQ(old_objs.size(), 11u);
 351
 352
 353   bufferlist bl;
 354   encode(old_manifest , bl);
 355
 356   RGWObjManifest manifest;
 357
 358   try {
 359     auto iter = bl.cbegin();
 360     decode(manifest, iter);
 361   } catch (buffer::error& err) {
 362     ASSERT_TRUE(false);
 363   }
 364
 365   rgw_raw_obj last_obj;
 366
 367   RGWObjManifest::obj_iterator iter;
 368   auto liter = old_objs.begin();
 369   for (iter = manifest.obj_begin(&dp);
 370        iter != manifest.obj_end(&dp) && liter != old_objs.end();
 371        ++iter, ++liter) {
 372     rgw_pool old_pool(liter->bucket.data_pool);
 373     string old_oid;
 374     prepend_old_bucket_marker(old_bucket, liter->get_object(), old_oid);
 375     rgw_raw_obj raw_old(old_pool, old_oid);
 376     cout << "*liter=" << raw_old << " iter.get_location()=" << env.get_raw(iter.get_location()) << std::endl;
 377     ASSERT_EQ(raw_old, env.get_raw(iter.get_location()));
 378
 379     last_obj = env.get_raw(iter.get_location());
 380   }
 381
 382   ASSERT_TRUE(liter == old_objs.end());
 383   ASSERT_TRUE(iter == manifest.obj_end(&dp));
 384
 385 }
 386
 387
 388 int main(int argc, char **argv) {
 389   auto args = argv_to_vec(argc, argv);
 390   auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
 391                          CODE_ENVIRONMENT_UTILITY,
 392                          CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
 393   common_init_finish(g_ceph_context);
 394   ::testing::InitGoogleTest(&argc, argv);
 395   return RUN_ALL_TESTS();
 396 }
 397