1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 #include "include/compat.h"
5 #include "include/types.h"
6 #include "include/rados/librados.h"
7 #include "include/rados/librados.hpp"
8 #include "include/radosstriper/libradosstriper.h"
9 #include "include/radosstriper/libradosstriper.hpp"
10 #include "include/ceph_fs.h"
11 #include "common/backport14.h"
12 #include "test/librados/test.h"
13 #include "test/libradosstriper/TestCase.h"
17 using namespace librados
;
18 using namespace libradosstriper
;
20 class StriperTestRT
: public StriperTestParam
{
22 StriperTestRT() : StriperTestParam() {}
24 char* getObjName(const std::string
& soid
, uint64_t nb
)
26 char name
[soid
.size()+18];
27 sprintf(name
, "%s.%016llx", soid
.c_str(), (long long unsigned int)nb
);
31 void checkObjectFromRados(const std::string
& soid
, bufferlist
&bl
,
32 uint64_t exp_stripe_unit
, uint64_t exp_stripe_count
,
33 uint64_t exp_object_size
, size_t size
)
35 checkObjectFromRados(soid
, bl
, exp_stripe_unit
, exp_stripe_count
, exp_object_size
, size
, size
);
38 void checkObjectFromRados(const std::string
& soid
, bufferlist
&bl
,
39 uint64_t exp_stripe_unit
, uint64_t exp_stripe_count
,
40 uint64_t exp_object_size
, size_t size
,
41 size_t actual_size_if_sparse
)
43 // checking first object's rados xattrs
45 char* firstOid
= getObjName(soid
, 0);
46 ASSERT_LT(0, ioctx
.getxattr(firstOid
, "striper.layout.stripe_unit", xattrbl
));
47 std::string
s_xattr(xattrbl
.c_str(), xattrbl
.length()); // adds 0 byte at the end
48 uint64_t stripe_unit
= strtoll(s_xattr
.c_str(), NULL
, 10);
49 ASSERT_LT((unsigned)0, stripe_unit
);
50 ASSERT_EQ(stripe_unit
, exp_stripe_unit
);
52 ASSERT_LT(0, ioctx
.getxattr(firstOid
, "striper.layout.stripe_count", xattrbl
));
53 s_xattr
= std::string(xattrbl
.c_str(), xattrbl
.length()); // adds 0 byte at the end
54 uint64_t stripe_count
= strtoll(s_xattr
.c_str(), NULL
, 10);
55 ASSERT_LT(0U, stripe_count
);
56 ASSERT_EQ(stripe_count
, exp_stripe_count
);
58 ASSERT_LT(0, ioctx
.getxattr(firstOid
, "striper.layout.object_size", xattrbl
));
59 s_xattr
= std::string(xattrbl
.c_str(), xattrbl
.length()); // adds 0 byte at the end
60 uint64_t object_size
= strtoll(s_xattr
.c_str(), NULL
, 10);
61 ASSERT_EQ(object_size
, exp_object_size
);
63 ASSERT_LT(0, ioctx
.getxattr(firstOid
, "striper.size", xattrbl
));
64 s_xattr
= std::string(xattrbl
.c_str(), xattrbl
.length()); // adds 0 byte at the end
65 uint64_t xa_size
= strtoll(s_xattr
.c_str(), NULL
, 10);
66 ASSERT_EQ(xa_size
, size
);
67 // checking object content from rados point of view
68 // we will go stripe by stripe, read the content of each of them and
69 // check with expectations
70 uint64_t stripe_per_object
= object_size
/ stripe_unit
;
71 uint64_t stripe_per_objectset
= stripe_per_object
* stripe_count
;
72 uint64_t nb_stripes_in_object
= (size
+stripe_unit
-1)/stripe_unit
;
73 for (uint64_t stripe_nb
= 0;
74 stripe_nb
< nb_stripes_in_object
;
76 // find out where this stripe is stored
77 uint64_t objectset
= stripe_nb
/ stripe_per_objectset
;
78 uint64_t stripe_in_object_set
= stripe_nb
% stripe_per_objectset
;
79 uint64_t object_in_set
= stripe_in_object_set
% stripe_count
;
80 uint64_t stripe_in_object
= stripe_in_object_set
/ stripe_count
;
81 uint64_t object_nb
= objectset
* stripe_count
+ object_in_set
;
82 uint64_t start
= stripe_in_object
* stripe_unit
;
83 uint64_t len
= stripe_unit
;
84 if (stripe_nb
== nb_stripes_in_object
-1 and size
% stripe_unit
!= 0) {
85 len
= size
% stripe_unit
;
87 // handle case of sparse object (can only be sparse at the end in our tests)
88 if (actual_size_if_sparse
< size
and
89 ((actual_size_if_sparse
+stripe_unit
-1)/stripe_unit
)-1 == stripe_nb
) {
90 len
= actual_size_if_sparse
% stripe_unit
;
91 if (0 == len
) len
= stripe_unit
;
93 bufferlist stripe_data
;
94 // check object content
95 char* oid
= getObjName(soid
, object_nb
);
96 int rc
= ioctx
.read(oid
, stripe_data
, len
, start
);
97 if (actual_size_if_sparse
< size
and
98 (actual_size_if_sparse
+stripe_unit
-1)/stripe_unit
<= stripe_nb
) {
99 // sparse object case : the stripe does not exist, but the rados object may
100 uint64_t object_start
= (object_in_set
+ objectset
*stripe_per_objectset
) * stripe_unit
;
101 if (actual_size_if_sparse
<= object_start
) {
102 ASSERT_EQ(rc
, -ENOENT
);
107 ASSERT_EQ((uint64_t)rc
, len
);
108 bufferlist original_data
;
109 original_data
.substr_of(bl
, stripe_nb
*stripe_unit
, len
);
110 ASSERT_EQ(0, memcmp(original_data
.c_str(), stripe_data
.c_str(), len
));
114 // checking rados object sizes; we go object by object
115 uint64_t nb_full_object_sets
= nb_stripes_in_object
/ stripe_per_objectset
;
116 uint64_t nb_extra_objects
= nb_stripes_in_object
% stripe_per_objectset
;
117 if (nb_extra_objects
> stripe_count
) nb_extra_objects
= stripe_count
;
118 uint64_t nb_objects
= nb_full_object_sets
* stripe_count
+ nb_extra_objects
;
119 for (uint64_t object_nb
= 0; object_nb
< nb_objects
; object_nb
++) {
122 char* oid
= getObjName(soid
, object_nb
);
123 uint64_t nb_full_object_set
= object_nb
/ stripe_count
;
124 uint64_t object_index_in_set
= object_nb
% stripe_count
;
125 uint64_t object_start_stripe
= nb_full_object_set
* stripe_per_objectset
+ object_index_in_set
;
126 uint64_t object_start_off
= object_start_stripe
* stripe_unit
;
127 if (actual_size_if_sparse
< size
and actual_size_if_sparse
<= object_start_off
) {
128 ASSERT_EQ(-ENOENT
, ioctx
.stat(oid
, &rados_size
, &mtime
));
130 ASSERT_EQ(0, ioctx
.stat(oid
, &rados_size
, &mtime
));
132 uint64_t stripe_size
= stripe_count
* stripe_unit
;
133 uint64_t set_size
= stripe_count
* object_size
;
135 for (offset
= object_start_off
;
136 (offset
< (object_start_off
) + set_size
) && (offset
< actual_size_if_sparse
);
137 offset
+= stripe_size
) {
138 if (offset
+ stripe_unit
> actual_size_if_sparse
) {
139 len
+= actual_size_if_sparse
-offset
;
144 ASSERT_EQ(len
, rados_size
);
148 // check we do not have an extra object behind
151 char* oid
= getObjName(soid
, nb_objects
);
152 ASSERT_EQ(-ENOENT
, ioctx
.stat(oid
, &rados_size
, &mtime
));
158 TEST_P(StriperTestRT
, StripedRoundtrip
) {
159 // get striping parameters and apply them
160 TestData testData
= GetParam();
161 ASSERT_EQ(0, striper
.set_object_layout_stripe_unit(testData
.stripe_unit
));
162 ASSERT_EQ(0, striper
.set_object_layout_stripe_count(testData
.stripe_count
));
163 ASSERT_EQ(0, striper
.set_object_layout_object_size(testData
.object_size
));
164 std::ostringstream oss
;
165 oss
<< "StripedRoundtrip_" << testData
.stripe_unit
<< "_"
166 << testData
.stripe_count
<< "_" << testData
.object_size
167 << "_" << testData
.size
;
168 std::string soid
= oss
.str();
169 // writing striped data
170 std::unique_ptr
<char[]> buf1
;
173 SCOPED_TRACE("Writing initial object");
174 buf1
= ceph::make_unique
<char[]>(testData
.size
);
175 for (unsigned int i
= 0; i
< testData
.size
; i
++) buf1
[i
] = 13*((unsigned char)i
);
176 bl1
.append(buf1
.get(), testData
.size
);
177 ASSERT_EQ(0, striper
.write(soid
, bl1
, testData
.size
, 0));
178 // checking object state from Rados point of view
179 ASSERT_NO_FATAL_FAILURE(checkObjectFromRados(soid
, bl1
, testData
.stripe_unit
,
180 testData
.stripe_count
, testData
.object_size
,
183 // adding more data to object and checking again
184 std::unique_ptr
<char[]> buf2
;
187 SCOPED_TRACE("Testing append");
188 buf2
= ceph::make_unique
<char[]>(testData
.size
);
189 for (unsigned int i
= 0; i
< testData
.size
; i
++) buf2
[i
] = 17*((unsigned char)i
);
190 bl2
.append(buf2
.get(), testData
.size
);
191 ASSERT_EQ(0, striper
.append(soid
, bl2
, testData
.size
));
192 bl1
.append(buf2
.get(), testData
.size
);
193 ASSERT_NO_FATAL_FAILURE(checkObjectFromRados(soid
, bl1
, testData
.stripe_unit
,
194 testData
.stripe_count
, testData
.object_size
,
197 // truncating to half original size and checking again
199 SCOPED_TRACE("Testing trunc to truncate object");
200 ASSERT_EQ(0, striper
.trunc(soid
, testData
.size
/2));
201 ASSERT_NO_FATAL_FAILURE(checkObjectFromRados(soid
, bl1
, testData
.stripe_unit
,
202 testData
.stripe_count
, testData
.object_size
,
205 // truncating back to original size and checking again (especially for 0s)
207 SCOPED_TRACE("Testing trunc to extend object with 0s");
208 ASSERT_EQ(0, striper
.trunc(soid
, testData
.size
));
210 bl3
.substr_of(bl1
, 0, testData
.size
/2);
211 bl3
.append_zero(testData
.size
- testData
.size
/2);
212 ASSERT_NO_FATAL_FAILURE(checkObjectFromRados(soid
, bl3
, testData
.stripe_unit
,
213 testData
.stripe_count
, testData
.object_size
,
214 testData
.size
, testData
.size
/2));
217 SCOPED_TRACE("Testing write_full");
218 // using write_full and checking again
219 ASSERT_EQ(0, striper
.write_full(soid
, bl2
));
220 checkObjectFromRados(soid
, bl2
, testData
.stripe_unit
,
221 testData
.stripe_count
, testData
.object_size
,
225 SCOPED_TRACE("Testing standard remove");
227 ASSERT_EQ(0, striper
.remove(soid
));
228 // check that the removal was successful
231 for (uint64_t object_nb
= 0;
232 object_nb
< testData
.size
*2/testData
.object_size
+ testData
.stripe_count
;
234 char* oid
= getObjName(soid
, object_nb
);
235 ASSERT_EQ(-ENOENT
, ioctx
.stat(oid
, &size
, &mtime
));
240 SCOPED_TRACE("Testing remove when no object size");
242 ASSERT_EQ(0, striper
.write(soid
, bl1
, testData
.size
*2, 0));
243 // remove the object size attribute from the striped object
244 char* firstOid
= getObjName(soid
, 0);
245 ASSERT_EQ(0, ioctx
.rmxattr(firstOid
, "striper.size"));
247 // check that stat fails
250 ASSERT_EQ(-ENODATA
, striper
.stat(soid
, &size
, &mtime
));
252 ASSERT_EQ(0, striper
.remove(soid
));
253 // check that the removal was successful
254 for (uint64_t object_nb
= 0;
255 object_nb
< testData
.size
*2/testData
.object_size
+ testData
.stripe_count
;
257 char* oid
= getObjName(soid
, object_nb
);
258 ASSERT_EQ(-ENOENT
, ioctx
.stat(oid
, &size
, &mtime
));
264 const TestData simple_stripe_schemes
[] = {
265 // stripe_unit, stripe_count, object_size, size
266 {CEPH_MIN_STRIPE_UNIT
, 5, CEPH_MIN_STRIPE_UNIT
, 2},
267 {CEPH_MIN_STRIPE_UNIT
, 5, CEPH_MIN_STRIPE_UNIT
, CEPH_MIN_STRIPE_UNIT
},
268 {CEPH_MIN_STRIPE_UNIT
, 5, CEPH_MIN_STRIPE_UNIT
, CEPH_MIN_STRIPE_UNIT
-1},
269 {CEPH_MIN_STRIPE_UNIT
, 5, CEPH_MIN_STRIPE_UNIT
, 2*CEPH_MIN_STRIPE_UNIT
},
270 {CEPH_MIN_STRIPE_UNIT
, 5, CEPH_MIN_STRIPE_UNIT
, 12*CEPH_MIN_STRIPE_UNIT
},
271 {CEPH_MIN_STRIPE_UNIT
, 5, CEPH_MIN_STRIPE_UNIT
, 2*CEPH_MIN_STRIPE_UNIT
-1},
272 {CEPH_MIN_STRIPE_UNIT
, 5, CEPH_MIN_STRIPE_UNIT
, 12*CEPH_MIN_STRIPE_UNIT
-1},
273 {CEPH_MIN_STRIPE_UNIT
, 5, CEPH_MIN_STRIPE_UNIT
, 2*CEPH_MIN_STRIPE_UNIT
+100},
274 {CEPH_MIN_STRIPE_UNIT
, 5, CEPH_MIN_STRIPE_UNIT
, 12*CEPH_MIN_STRIPE_UNIT
+100},
275 {CEPH_MIN_STRIPE_UNIT
, 5, 3*CEPH_MIN_STRIPE_UNIT
, 2*CEPH_MIN_STRIPE_UNIT
+100},
276 {CEPH_MIN_STRIPE_UNIT
, 5, 3*CEPH_MIN_STRIPE_UNIT
, 8*CEPH_MIN_STRIPE_UNIT
+100},
277 {CEPH_MIN_STRIPE_UNIT
, 5, 3*CEPH_MIN_STRIPE_UNIT
, 12*CEPH_MIN_STRIPE_UNIT
+100},
278 {CEPH_MIN_STRIPE_UNIT
, 5, 3*CEPH_MIN_STRIPE_UNIT
, 15*CEPH_MIN_STRIPE_UNIT
+100},
279 {CEPH_MIN_STRIPE_UNIT
, 5, 3*CEPH_MIN_STRIPE_UNIT
, 25*CEPH_MIN_STRIPE_UNIT
+100},
280 {CEPH_MIN_STRIPE_UNIT
, 5, 3*CEPH_MIN_STRIPE_UNIT
, 45*CEPH_MIN_STRIPE_UNIT
+100},
281 {262144, 5, 262144, 2},
282 {262144, 5, 262144, 262144},
283 {262144, 5, 262144, 262144-1},
284 {262144, 5, 262144, 2*262144},
285 {262144, 5, 262144, 12*262144},
286 {262144, 5, 262144, 2*262144-1},
287 {262144, 5, 262144, 12*262144-1},
288 {262144, 5, 262144, 2*262144+100},
289 {262144, 5, 262144, 12*262144+100},
290 {262144, 5, 3*262144, 2*262144+100},
291 {262144, 5, 3*262144, 8*262144+100},
292 {262144, 5, 3*262144, 12*262144+100},
293 {262144, 5, 3*262144, 15*262144+100},
294 {262144, 5, 3*262144, 25*262144+100},
295 {262144, 5, 3*262144, 45*262144+100},
296 {CEPH_MIN_STRIPE_UNIT
, 1, CEPH_MIN_STRIPE_UNIT
, 2},
297 {CEPH_MIN_STRIPE_UNIT
, 1, CEPH_MIN_STRIPE_UNIT
, CEPH_MIN_STRIPE_UNIT
},
298 {CEPH_MIN_STRIPE_UNIT
, 1, CEPH_MIN_STRIPE_UNIT
, CEPH_MIN_STRIPE_UNIT
-1},
299 {CEPH_MIN_STRIPE_UNIT
, 1, CEPH_MIN_STRIPE_UNIT
, 2*CEPH_MIN_STRIPE_UNIT
},
300 {CEPH_MIN_STRIPE_UNIT
, 1, CEPH_MIN_STRIPE_UNIT
, 12*CEPH_MIN_STRIPE_UNIT
},
301 {CEPH_MIN_STRIPE_UNIT
, 1, CEPH_MIN_STRIPE_UNIT
, 2*CEPH_MIN_STRIPE_UNIT
-1},
302 {CEPH_MIN_STRIPE_UNIT
, 1, CEPH_MIN_STRIPE_UNIT
, 12*CEPH_MIN_STRIPE_UNIT
-1},
303 {CEPH_MIN_STRIPE_UNIT
, 1, CEPH_MIN_STRIPE_UNIT
, 2*CEPH_MIN_STRIPE_UNIT
+100},
304 {CEPH_MIN_STRIPE_UNIT
, 1, CEPH_MIN_STRIPE_UNIT
, 12*CEPH_MIN_STRIPE_UNIT
+100},
305 {CEPH_MIN_STRIPE_UNIT
, 1, 3*CEPH_MIN_STRIPE_UNIT
, 2*CEPH_MIN_STRIPE_UNIT
+100},
306 {CEPH_MIN_STRIPE_UNIT
, 1, 3*CEPH_MIN_STRIPE_UNIT
, 8*CEPH_MIN_STRIPE_UNIT
+100},
307 {CEPH_MIN_STRIPE_UNIT
, 1, 3*CEPH_MIN_STRIPE_UNIT
, 12*CEPH_MIN_STRIPE_UNIT
+100},
308 {CEPH_MIN_STRIPE_UNIT
, 1, 3*CEPH_MIN_STRIPE_UNIT
, 15*CEPH_MIN_STRIPE_UNIT
+100},
309 {CEPH_MIN_STRIPE_UNIT
, 1, 3*CEPH_MIN_STRIPE_UNIT
, 25*CEPH_MIN_STRIPE_UNIT
+100},
310 {CEPH_MIN_STRIPE_UNIT
, 1, 3*CEPH_MIN_STRIPE_UNIT
, 45*CEPH_MIN_STRIPE_UNIT
+100},
311 {CEPH_MIN_STRIPE_UNIT
, 50, CEPH_MIN_STRIPE_UNIT
, 2},
312 {CEPH_MIN_STRIPE_UNIT
, 50, CEPH_MIN_STRIPE_UNIT
, CEPH_MIN_STRIPE_UNIT
},
313 {CEPH_MIN_STRIPE_UNIT
, 50, CEPH_MIN_STRIPE_UNIT
, CEPH_MIN_STRIPE_UNIT
-1},
314 {CEPH_MIN_STRIPE_UNIT
, 50, CEPH_MIN_STRIPE_UNIT
, 2*CEPH_MIN_STRIPE_UNIT
},
315 {CEPH_MIN_STRIPE_UNIT
, 50, CEPH_MIN_STRIPE_UNIT
, 12*CEPH_MIN_STRIPE_UNIT
},
316 {CEPH_MIN_STRIPE_UNIT
, 50, CEPH_MIN_STRIPE_UNIT
, 2*CEPH_MIN_STRIPE_UNIT
-1},
317 {CEPH_MIN_STRIPE_UNIT
, 50, CEPH_MIN_STRIPE_UNIT
, 12*CEPH_MIN_STRIPE_UNIT
-1},
318 {CEPH_MIN_STRIPE_UNIT
, 50, CEPH_MIN_STRIPE_UNIT
, 2*CEPH_MIN_STRIPE_UNIT
+100},
319 {CEPH_MIN_STRIPE_UNIT
, 50, CEPH_MIN_STRIPE_UNIT
, 12*CEPH_MIN_STRIPE_UNIT
+100},
320 {CEPH_MIN_STRIPE_UNIT
, 50, 3*CEPH_MIN_STRIPE_UNIT
, 2*CEPH_MIN_STRIPE_UNIT
+100},
321 {CEPH_MIN_STRIPE_UNIT
, 50, 3*CEPH_MIN_STRIPE_UNIT
, 8*CEPH_MIN_STRIPE_UNIT
+100},
322 {CEPH_MIN_STRIPE_UNIT
, 50, 3*CEPH_MIN_STRIPE_UNIT
, 12*CEPH_MIN_STRIPE_UNIT
+100},
323 {CEPH_MIN_STRIPE_UNIT
, 50, 3*CEPH_MIN_STRIPE_UNIT
, 15*CEPH_MIN_STRIPE_UNIT
+100},
324 {CEPH_MIN_STRIPE_UNIT
, 50, 3*CEPH_MIN_STRIPE_UNIT
, 25*CEPH_MIN_STRIPE_UNIT
+100},
325 {CEPH_MIN_STRIPE_UNIT
, 50, 3*CEPH_MIN_STRIPE_UNIT
, 45*CEPH_MIN_STRIPE_UNIT
+100}
328 INSTANTIATE_TEST_CASE_P(SimpleStriping
,
330 ::testing::ValuesIn(simple_stripe_schemes
));