ceph/src/test/libradosstriper/striping.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "include/types.h"
   5 #include "include/rados/librados.h"
   6 #include "include/rados/librados.hpp"
   7 #include "include/radosstriper/libradosstriper.h"
   8 #include "include/radosstriper/libradosstriper.hpp"
   9 #include "include/ceph_fs.h"
  10 #include "test/librados/test.h"
  11 #include "test/libradosstriper/TestCase.h"
  12
  13 #include <string>
  14 #include <errno.h>
  15 using namespace librados;
  16 using namespace libradosstriper;
  17
  18 class StriperTestRT : public StriperTestParam {
  19 public:
  20   StriperTestRT() : StriperTestParam() {}
  21 protected:
  22   char* getObjName(const std::string& soid, uint64_t nb)
  23   {
  24     char name[soid.size()+18];
  25     sprintf(name, "%s.%016llx", soid.c_str(), (long long unsigned int)nb);
  26     return strdup(name);
  27   }
  28
  29   void checkObjectFromRados(const std::string& soid, bufferlist &bl,
  30                             uint64_t exp_stripe_unit, uint64_t exp_stripe_count,
  31                             uint64_t exp_object_size, size_t size)
  32   {
  33     checkObjectFromRados(soid, bl, exp_stripe_unit, exp_stripe_count, exp_object_size, size, size);
  34   }
  35
  36   void checkObjectFromRados(const std::string& soid, bufferlist &bl,
  37                             uint64_t exp_stripe_unit, uint64_t exp_stripe_count,
  38                             uint64_t exp_object_size, size_t size,
  39                             size_t actual_size_if_sparse)
  40   {
  41     // checking first object's rados xattrs
  42     bufferlist xattrbl;
  43     char* firstOid = getObjName(soid, 0);
  44     ASSERT_LT(0, ioctx.getxattr(firstOid, "striper.layout.stripe_unit", xattrbl));
  45     std::string s_xattr(xattrbl.c_str(), xattrbl.length()); // adds 0 byte at the end
  46     uint64_t stripe_unit = strtoll(s_xattr.c_str(), NULL, 10);
  47     ASSERT_LT((unsigned)0, stripe_unit);
  48     ASSERT_EQ(stripe_unit, exp_stripe_unit);
  49     xattrbl.clear();
  50     ASSERT_LT(0, ioctx.getxattr(firstOid, "striper.layout.stripe_count", xattrbl));
  51     s_xattr = std::string(xattrbl.c_str(), xattrbl.length()); // adds 0 byte at the end
  52     uint64_t stripe_count = strtoll(s_xattr.c_str(), NULL, 10);
  53     ASSERT_LT(0U, stripe_count);
  54     ASSERT_EQ(stripe_count, exp_stripe_count);
  55     xattrbl.clear();
  56     ASSERT_LT(0, ioctx.getxattr(firstOid, "striper.layout.object_size", xattrbl));
  57     s_xattr = std::string(xattrbl.c_str(), xattrbl.length()); // adds 0 byte at the end
  58     uint64_t object_size = strtoll(s_xattr.c_str(), NULL, 10);
  59     ASSERT_EQ(object_size, exp_object_size);
  60     xattrbl.clear();
  61     ASSERT_LT(0, ioctx.getxattr(firstOid, "striper.size", xattrbl));
  62     s_xattr = std::string(xattrbl.c_str(), xattrbl.length()); // adds 0 byte at the end
  63     uint64_t xa_size = strtoll(s_xattr.c_str(), NULL, 10);
  64     ASSERT_EQ(xa_size, size);
  65     // checking object content from rados point of view
  66     // we will go stripe by stripe, read the content of each of them and
  67     // check with expectations
  68     uint64_t stripe_per_object = object_size / stripe_unit;
  69     uint64_t stripe_per_objectset = stripe_per_object * stripe_count;
  70     uint64_t nb_stripes_in_object = (size+stripe_unit-1)/stripe_unit;
  71     for (uint64_t stripe_nb = 0;
  72          stripe_nb < nb_stripes_in_object;
  73          stripe_nb++) {
  74       // find out where this stripe is stored
  75       uint64_t objectset = stripe_nb / stripe_per_objectset;
  76       uint64_t stripe_in_object_set = stripe_nb % stripe_per_objectset;
  77       uint64_t object_in_set = stripe_in_object_set % stripe_count;
  78       uint64_t stripe_in_object = stripe_in_object_set / stripe_count;
  79       uint64_t object_nb = objectset * stripe_count + object_in_set;
  80       uint64_t start = stripe_in_object * stripe_unit;
  81       uint64_t len = stripe_unit;
  82       if (stripe_nb == nb_stripes_in_object-1 and size % stripe_unit != 0) {
  83         len = size % stripe_unit;
  84       }
  85       // handle case of sparse object (can only be sparse at the end in our tests)
  86       if (actual_size_if_sparse < size and
  87           ((actual_size_if_sparse+stripe_unit-1)/stripe_unit)-1 == stripe_nb) {
  88         len = actual_size_if_sparse % stripe_unit;
  89         if (0 == len) len = stripe_unit;
  90       }
  91       bufferlist stripe_data;
  92       // check object content
  93       char* oid = getObjName(soid, object_nb);
  94       int rc = ioctx.read(oid, stripe_data, len, start);
  95       if (actual_size_if_sparse < size and
  96           (actual_size_if_sparse+stripe_unit-1)/stripe_unit <= stripe_nb) {
  97         // sparse object case : the stripe does not exist, but the rados object may
  98         uint64_t object_start = (object_in_set + objectset*stripe_per_objectset) * stripe_unit;
  99         if (actual_size_if_sparse <= object_start) {
 100           ASSERT_EQ(rc, -ENOENT);
 101         } else {
 102           ASSERT_EQ(rc, 0);
 103         }
 104       } else {
 105         ASSERT_EQ((uint64_t)rc, len);
 106         bufferlist original_data;
 107         original_data.substr_of(bl, stripe_nb*stripe_unit, len);
 108         ASSERT_EQ(0, memcmp(original_data.c_str(), stripe_data.c_str(), len));
 109       }
 110       free(oid);
 111     }
 112     // checking rados object sizes; we go object by object
 113     uint64_t nb_full_object_sets = nb_stripes_in_object / stripe_per_objectset;
 114     uint64_t nb_extra_objects = nb_stripes_in_object % stripe_per_objectset;
 115     if (nb_extra_objects > stripe_count) nb_extra_objects = stripe_count;
 116     uint64_t nb_objects = nb_full_object_sets * stripe_count + nb_extra_objects;
 117     for (uint64_t object_nb = 0; object_nb < nb_objects; object_nb++) {
 118       uint64_t rados_size;
 119       time_t mtime;
 120       char* oid = getObjName(soid, object_nb);
 121       uint64_t nb_full_object_set = object_nb / stripe_count;
 122       uint64_t object_index_in_set = object_nb % stripe_count;
 123       uint64_t object_start_stripe = nb_full_object_set * stripe_per_objectset + object_index_in_set;
 124       uint64_t object_start_off = object_start_stripe * stripe_unit;
 125       if (actual_size_if_sparse < size and actual_size_if_sparse <= object_start_off) {
 126         ASSERT_EQ(-ENOENT, ioctx.stat(oid, &rados_size, &mtime));
 127       } else {
 128         ASSERT_EQ(0, ioctx.stat(oid, &rados_size, &mtime));
 129         uint64_t offset;
 130         uint64_t stripe_size = stripe_count * stripe_unit;
 131         uint64_t set_size = stripe_count * object_size;
 132         uint64_t len = 0;
 133         for (offset = object_start_off;
 134              (offset < (object_start_off) + set_size) && (offset < actual_size_if_sparse);
 135              offset += stripe_size) {
 136           if (offset + stripe_unit > actual_size_if_sparse) {
 137             len += actual_size_if_sparse-offset;
 138           } else {
 139             len += stripe_unit;
 140           }
 141         }
 142         ASSERT_EQ(len, rados_size);
 143       }
 144       free(oid);
 145     }
 146     // check we do not have an extra object behind
 147     uint64_t rados_size;
 148     time_t mtime;
 149     char* oid = getObjName(soid, nb_objects);
 150     ASSERT_EQ(-ENOENT, ioctx.stat(oid, &rados_size, &mtime));
 151     free(oid);
 152     free(firstOid);
 153   }
 154 };
 155
 156 TEST_P(StriperTestRT, StripedRoundtrip) {
 157   // get striping parameters and apply them
 158   TestData testData = GetParam();
 159   ASSERT_EQ(0, striper.set_object_layout_stripe_unit(testData.stripe_unit));
 160   ASSERT_EQ(0, striper.set_object_layout_stripe_count(testData.stripe_count));
 161   ASSERT_EQ(0, striper.set_object_layout_object_size(testData.object_size));
 162   std::ostringstream oss;
 163   oss << "StripedRoundtrip_" << testData.stripe_unit << "_"
 164       << testData.stripe_count << "_" << testData.object_size
 165       << "_" << testData.size;
 166   std::string soid = oss.str();
 167   // writing striped data
 168   char* buf1;
 169   bufferlist bl1;
 170   {
 171     SCOPED_TRACE("Writing initial object");
 172     buf1 = (char*) calloc(1, testData.size);
 173     for (unsigned int i = 0; i < testData.size; i++) buf1[i] = 13*((unsigned char)i);
 174     bl1.append(buf1, testData.size);
 175     ASSERT_EQ(0, striper.write(soid, bl1, testData.size, 0));
 176     // checking object state from Rados point of view
 177     ASSERT_NO_FATAL_FAILURE(checkObjectFromRados(soid, bl1, testData.stripe_unit,
 178                                                  testData.stripe_count, testData.object_size,
 179                                                  testData.size));
 180   }
 181   // adding more data to object and checking again
 182   char* buf2;
 183   bufferlist bl2;
 184   {
 185     SCOPED_TRACE("Testing append");
 186     buf2 = (char*) calloc(1, testData.size);
 187     for (unsigned int i = 0; i < testData.size; i++) buf2[i] = 17*((unsigned char)i);
 188     bl2.append(buf2, testData.size);
 189     ASSERT_EQ(0, striper.append(soid, bl2, testData.size));
 190     bl1.append(buf2, testData.size);
 191     ASSERT_NO_FATAL_FAILURE(checkObjectFromRados(soid, bl1, testData.stripe_unit,
 192                                                  testData.stripe_count, testData.object_size,
 193                                                  testData.size*2));
 194   }
 195   // truncating to half original size and checking again
 196   {
 197     SCOPED_TRACE("Testing trunc to truncate object");
 198     ASSERT_EQ(0, striper.trunc(soid, testData.size/2));
 199     ASSERT_NO_FATAL_FAILURE(checkObjectFromRados(soid, bl1, testData.stripe_unit,
 200                                                  testData.stripe_count, testData.object_size,
 201                                                  testData.size/2));
 202   }
 203   // truncating back to original size and checking again (especially for 0s)
 204   {
 205     SCOPED_TRACE("Testing trunc to extend object with 0s");
 206     ASSERT_EQ(0, striper.trunc(soid, testData.size));
 207     bufferlist bl3;
 208     bl3.substr_of(bl1, 0, testData.size/2);
 209     bl3.append_zero(testData.size - testData.size/2);
 210     ASSERT_NO_FATAL_FAILURE(checkObjectFromRados(soid, bl3, testData.stripe_unit,
 211                                                  testData.stripe_count, testData.object_size,
 212                                                  testData.size, testData.size/2));
 213   }
 214   {
 215     SCOPED_TRACE("Testing write_full");
 216     // using write_full and checking again
 217     ASSERT_EQ(0, striper.write_full(soid, bl2));
 218     checkObjectFromRados(soid, bl2, testData.stripe_unit,
 219                          testData.stripe_count, testData.object_size,
 220                          testData.size);
 221   }
 222   {
 223     SCOPED_TRACE("Testing standard remove");
 224     // call remove
 225     ASSERT_EQ(0, striper.remove(soid));
 226     // check that the removal was successful
 227     uint64_t size;
 228     time_t mtime;
 229     for (uint64_t object_nb = 0;
 230          object_nb < testData.size*2/testData.object_size + testData.stripe_count;
 231          object_nb++) {
 232       char* oid = getObjName(soid, object_nb);
 233       ASSERT_EQ(-ENOENT, ioctx.stat(oid, &size, &mtime));
 234       free(oid);
 235     }
 236   }
 237   {
 238     SCOPED_TRACE("Testing remove when no object size");
 239     // recreate object
 240     ASSERT_EQ(0, striper.write(soid, bl1, testData.size*2, 0));
 241     // remove the object size attribute from the striped object
 242     char* firstOid = getObjName(soid, 0);
 243     ASSERT_EQ(0, ioctx.rmxattr(firstOid, "striper.size"));
 244     free(firstOid);
 245     // check that stat fails
 246     uint64_t size;
 247     time_t mtime;
 248     ASSERT_EQ(-ENODATA, striper.stat(soid, &size, &mtime));
 249     // call remove
 250     ASSERT_EQ(0, striper.remove(soid));
 251     // check that the removal was successful
 252     for (uint64_t object_nb = 0;
 253          object_nb < testData.size*2/testData.object_size + testData.stripe_count;
 254          object_nb++) {
 255       char* oid = getObjName(soid, object_nb);
 256       ASSERT_EQ(-ENOENT, ioctx.stat(oid, &size, &mtime));
 257       free(oid);
 258     }
 259   }
 260   free(buf1);
 261   free(buf2);
 262 }
 263
 264 const TestData simple_stripe_schemes[] = {
 265   // stripe_unit,        stripe_count, object_size,            size
 266   {CEPH_MIN_STRIPE_UNIT, 5,            CEPH_MIN_STRIPE_UNIT,   2},
 267   {CEPH_MIN_STRIPE_UNIT, 5,            CEPH_MIN_STRIPE_UNIT,   CEPH_MIN_STRIPE_UNIT},
 268   {CEPH_MIN_STRIPE_UNIT, 5,            CEPH_MIN_STRIPE_UNIT,   CEPH_MIN_STRIPE_UNIT-1},
 269   {CEPH_MIN_STRIPE_UNIT, 5,            CEPH_MIN_STRIPE_UNIT,   2*CEPH_MIN_STRIPE_UNIT},
 270   {CEPH_MIN_STRIPE_UNIT, 5,            CEPH_MIN_STRIPE_UNIT,   12*CEPH_MIN_STRIPE_UNIT},
 271   {CEPH_MIN_STRIPE_UNIT, 5,            CEPH_MIN_STRIPE_UNIT,   2*CEPH_MIN_STRIPE_UNIT-1},
 272   {CEPH_MIN_STRIPE_UNIT, 5,            CEPH_MIN_STRIPE_UNIT,   12*CEPH_MIN_STRIPE_UNIT-1},
 273   {CEPH_MIN_STRIPE_UNIT, 5,            CEPH_MIN_STRIPE_UNIT,   2*CEPH_MIN_STRIPE_UNIT+100},
 274   {CEPH_MIN_STRIPE_UNIT, 5,            CEPH_MIN_STRIPE_UNIT,   12*CEPH_MIN_STRIPE_UNIT+100},
 275   {CEPH_MIN_STRIPE_UNIT, 5,            3*CEPH_MIN_STRIPE_UNIT, 2*CEPH_MIN_STRIPE_UNIT+100},
 276   {CEPH_MIN_STRIPE_UNIT, 5,            3*CEPH_MIN_STRIPE_UNIT, 8*CEPH_MIN_STRIPE_UNIT+100},
 277   {CEPH_MIN_STRIPE_UNIT, 5,            3*CEPH_MIN_STRIPE_UNIT, 12*CEPH_MIN_STRIPE_UNIT+100},
 278   {CEPH_MIN_STRIPE_UNIT, 5,            3*CEPH_MIN_STRIPE_UNIT, 15*CEPH_MIN_STRIPE_UNIT+100},
 279   {CEPH_MIN_STRIPE_UNIT, 5,            3*CEPH_MIN_STRIPE_UNIT, 25*CEPH_MIN_STRIPE_UNIT+100},
 280   {CEPH_MIN_STRIPE_UNIT, 5,            3*CEPH_MIN_STRIPE_UNIT, 45*CEPH_MIN_STRIPE_UNIT+100},
 281   {262144,               5,            262144,                 2},
 282   {262144,               5,            262144,                 262144},
 283   {262144,               5,            262144,                 262144-1},
 284   {262144,               5,            262144,                 2*262144},
 285   {262144,               5,            262144,                 12*262144},
 286   {262144,               5,            262144,                 2*262144-1},
 287   {262144,               5,            262144,                 12*262144-1},
 288   {262144,               5,            262144,                 2*262144+100},
 289   {262144,               5,            262144,                 12*262144+100},
 290   {262144,               5,            3*262144,               2*262144+100},
 291   {262144,               5,            3*262144,               8*262144+100},
 292   {262144,               5,            3*262144,               12*262144+100},
 293   {262144,               5,            3*262144,               15*262144+100},
 294   {262144,               5,            3*262144,               25*262144+100},
 295   {262144,               5,            3*262144,               45*262144+100},
 296   {CEPH_MIN_STRIPE_UNIT, 1,            CEPH_MIN_STRIPE_UNIT,   2},
 297   {CEPH_MIN_STRIPE_UNIT, 1,            CEPH_MIN_STRIPE_UNIT,   CEPH_MIN_STRIPE_UNIT},
 298   {CEPH_MIN_STRIPE_UNIT, 1,            CEPH_MIN_STRIPE_UNIT,   CEPH_MIN_STRIPE_UNIT-1},
 299   {CEPH_MIN_STRIPE_UNIT, 1,            CEPH_MIN_STRIPE_UNIT,   2*CEPH_MIN_STRIPE_UNIT},
 300   {CEPH_MIN_STRIPE_UNIT, 1,            CEPH_MIN_STRIPE_UNIT,   12*CEPH_MIN_STRIPE_UNIT},
 301   {CEPH_MIN_STRIPE_UNIT, 1,            CEPH_MIN_STRIPE_UNIT,   2*CEPH_MIN_STRIPE_UNIT-1},
 302   {CEPH_MIN_STRIPE_UNIT, 1,            CEPH_MIN_STRIPE_UNIT,   12*CEPH_MIN_STRIPE_UNIT-1},
 303   {CEPH_MIN_STRIPE_UNIT, 1,            CEPH_MIN_STRIPE_UNIT,   2*CEPH_MIN_STRIPE_UNIT+100},
 304   {CEPH_MIN_STRIPE_UNIT, 1,            CEPH_MIN_STRIPE_UNIT,   12*CEPH_MIN_STRIPE_UNIT+100},
 305   {CEPH_MIN_STRIPE_UNIT, 1,            3*CEPH_MIN_STRIPE_UNIT, 2*CEPH_MIN_STRIPE_UNIT+100},
 306   {CEPH_MIN_STRIPE_UNIT, 1,            3*CEPH_MIN_STRIPE_UNIT, 8*CEPH_MIN_STRIPE_UNIT+100},
 307   {CEPH_MIN_STRIPE_UNIT, 1,            3*CEPH_MIN_STRIPE_UNIT, 12*CEPH_MIN_STRIPE_UNIT+100},
 308   {CEPH_MIN_STRIPE_UNIT, 1,            3*CEPH_MIN_STRIPE_UNIT, 15*CEPH_MIN_STRIPE_UNIT+100},
 309   {CEPH_MIN_STRIPE_UNIT, 1,            3*CEPH_MIN_STRIPE_UNIT, 25*CEPH_MIN_STRIPE_UNIT+100},
 310   {CEPH_MIN_STRIPE_UNIT, 1,            3*CEPH_MIN_STRIPE_UNIT, 45*CEPH_MIN_STRIPE_UNIT+100},
 311   {CEPH_MIN_STRIPE_UNIT, 50,           CEPH_MIN_STRIPE_UNIT,   2},
 312   {CEPH_MIN_STRIPE_UNIT, 50,           CEPH_MIN_STRIPE_UNIT,   CEPH_MIN_STRIPE_UNIT},
 313   {CEPH_MIN_STRIPE_UNIT, 50,           CEPH_MIN_STRIPE_UNIT,   CEPH_MIN_STRIPE_UNIT-1},
 314   {CEPH_MIN_STRIPE_UNIT, 50,           CEPH_MIN_STRIPE_UNIT,   2*CEPH_MIN_STRIPE_UNIT},
 315   {CEPH_MIN_STRIPE_UNIT, 50,           CEPH_MIN_STRIPE_UNIT,   12*CEPH_MIN_STRIPE_UNIT},
 316   {CEPH_MIN_STRIPE_UNIT, 50,           CEPH_MIN_STRIPE_UNIT,   2*CEPH_MIN_STRIPE_UNIT-1},
 317   {CEPH_MIN_STRIPE_UNIT, 50,           CEPH_MIN_STRIPE_UNIT,   12*CEPH_MIN_STRIPE_UNIT-1},
 318   {CEPH_MIN_STRIPE_UNIT, 50,           CEPH_MIN_STRIPE_UNIT,   2*CEPH_MIN_STRIPE_UNIT+100},
 319   {CEPH_MIN_STRIPE_UNIT, 50,           CEPH_MIN_STRIPE_UNIT,   12*CEPH_MIN_STRIPE_UNIT+100},
 320   {CEPH_MIN_STRIPE_UNIT, 50,           3*CEPH_MIN_STRIPE_UNIT, 2*CEPH_MIN_STRIPE_UNIT+100},
 321   {CEPH_MIN_STRIPE_UNIT, 50,           3*CEPH_MIN_STRIPE_UNIT, 8*CEPH_MIN_STRIPE_UNIT+100},
 322   {CEPH_MIN_STRIPE_UNIT, 50,           3*CEPH_MIN_STRIPE_UNIT, 12*CEPH_MIN_STRIPE_UNIT+100},
 323   {CEPH_MIN_STRIPE_UNIT, 50,           3*CEPH_MIN_STRIPE_UNIT, 15*CEPH_MIN_STRIPE_UNIT+100},
 324   {CEPH_MIN_STRIPE_UNIT, 50,           3*CEPH_MIN_STRIPE_UNIT, 25*CEPH_MIN_STRIPE_UNIT+100},
 325   {CEPH_MIN_STRIPE_UNIT, 50,           3*CEPH_MIN_STRIPE_UNIT, 45*CEPH_MIN_STRIPE_UNIT+100}
 326 };
 327
 328 INSTANTIATE_TEST_CASE_P(SimpleStriping,
 329                         StriperTestRT,
 330                         ::testing::ValuesIn(simple_stripe_schemes));