1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 #include "include/compat.h"
5 #include "include/types.h"
6 #include "include/rados/librados.h"
7 #include "include/rados/librados.hpp"
8 #include "include/radosstriper/libradosstriper.h"
9 #include "include/radosstriper/libradosstriper.hpp"
10 #include "include/ceph_fs.h"
11 #include "test/librados/test.h"
12 #include "test/libradosstriper/TestCase.h"
16 using namespace librados
;
17 using namespace libradosstriper
;
19 class StriperTestRT
: public StriperTestParam
{
21 StriperTestRT() : StriperTestParam() {}
23 char* getObjName(const std::string
& soid
, uint64_t nb
)
25 char name
[soid
.size()+18];
26 sprintf(name
, "%s.%016llx", soid
.c_str(), (long long unsigned int)nb
);
30 void checkObjectFromRados(const std::string
& soid
, bufferlist
&bl
,
31 uint64_t exp_stripe_unit
, uint64_t exp_stripe_count
,
32 uint64_t exp_object_size
, size_t size
)
34 checkObjectFromRados(soid
, bl
, exp_stripe_unit
, exp_stripe_count
, exp_object_size
, size
, size
);
37 void checkObjectFromRados(const std::string
& soid
, bufferlist
&bl
,
38 uint64_t exp_stripe_unit
, uint64_t exp_stripe_count
,
39 uint64_t exp_object_size
, size_t size
,
40 size_t actual_size_if_sparse
)
42 // checking first object's rados xattrs
44 char* firstOid
= getObjName(soid
, 0);
45 ASSERT_LT(0, ioctx
.getxattr(firstOid
, "striper.layout.stripe_unit", xattrbl
));
46 std::string
s_xattr(xattrbl
.c_str(), xattrbl
.length()); // adds 0 byte at the end
47 uint64_t stripe_unit
= strtoll(s_xattr
.c_str(), NULL
, 10);
48 ASSERT_LT((unsigned)0, stripe_unit
);
49 ASSERT_EQ(stripe_unit
, exp_stripe_unit
);
51 ASSERT_LT(0, ioctx
.getxattr(firstOid
, "striper.layout.stripe_count", xattrbl
));
52 s_xattr
= std::string(xattrbl
.c_str(), xattrbl
.length()); // adds 0 byte at the end
53 uint64_t stripe_count
= strtoll(s_xattr
.c_str(), NULL
, 10);
54 ASSERT_LT(0U, stripe_count
);
55 ASSERT_EQ(stripe_count
, exp_stripe_count
);
57 ASSERT_LT(0, ioctx
.getxattr(firstOid
, "striper.layout.object_size", xattrbl
));
58 s_xattr
= std::string(xattrbl
.c_str(), xattrbl
.length()); // adds 0 byte at the end
59 uint64_t object_size
= strtoll(s_xattr
.c_str(), NULL
, 10);
60 ASSERT_EQ(object_size
, exp_object_size
);
62 ASSERT_LT(0, ioctx
.getxattr(firstOid
, "striper.size", xattrbl
));
63 s_xattr
= std::string(xattrbl
.c_str(), xattrbl
.length()); // adds 0 byte at the end
64 uint64_t xa_size
= strtoll(s_xattr
.c_str(), NULL
, 10);
65 ASSERT_EQ(xa_size
, size
);
66 // checking object content from rados point of view
67 // we will go stripe by stripe, read the content of each of them and
68 // check with expectations
69 uint64_t stripe_per_object
= object_size
/ stripe_unit
;
70 uint64_t stripe_per_objectset
= stripe_per_object
* stripe_count
;
71 uint64_t nb_stripes_in_object
= (size
+stripe_unit
-1)/stripe_unit
;
72 for (uint64_t stripe_nb
= 0;
73 stripe_nb
< nb_stripes_in_object
;
75 // find out where this stripe is stored
76 uint64_t objectset
= stripe_nb
/ stripe_per_objectset
;
77 uint64_t stripe_in_object_set
= stripe_nb
% stripe_per_objectset
;
78 uint64_t object_in_set
= stripe_in_object_set
% stripe_count
;
79 uint64_t stripe_in_object
= stripe_in_object_set
/ stripe_count
;
80 uint64_t object_nb
= objectset
* stripe_count
+ object_in_set
;
81 uint64_t start
= stripe_in_object
* stripe_unit
;
82 uint64_t len
= stripe_unit
;
83 if (stripe_nb
== nb_stripes_in_object
-1 and size
% stripe_unit
!= 0) {
84 len
= size
% stripe_unit
;
86 // handle case of sparse object (can only be sparse at the end in our tests)
87 if (actual_size_if_sparse
< size
and
88 ((actual_size_if_sparse
+stripe_unit
-1)/stripe_unit
)-1 == stripe_nb
) {
89 len
= actual_size_if_sparse
% stripe_unit
;
90 if (0 == len
) len
= stripe_unit
;
92 bufferlist stripe_data
;
93 // check object content
94 char* oid
= getObjName(soid
, object_nb
);
95 int rc
= ioctx
.read(oid
, stripe_data
, len
, start
);
96 if (actual_size_if_sparse
< size
and
97 (actual_size_if_sparse
+stripe_unit
-1)/stripe_unit
<= stripe_nb
) {
98 // sparse object case : the stripe does not exist, but the rados object may
99 uint64_t object_start
= (object_in_set
+ objectset
*stripe_per_objectset
) * stripe_unit
;
100 if (actual_size_if_sparse
<= object_start
) {
101 ASSERT_EQ(rc
, -ENOENT
);
106 ASSERT_EQ((uint64_t)rc
, len
);
107 bufferlist original_data
;
108 original_data
.substr_of(bl
, stripe_nb
*stripe_unit
, len
);
109 ASSERT_EQ(0, memcmp(original_data
.c_str(), stripe_data
.c_str(), len
));
113 // checking rados object sizes; we go object by object
114 uint64_t nb_full_object_sets
= nb_stripes_in_object
/ stripe_per_objectset
;
115 uint64_t nb_extra_objects
= nb_stripes_in_object
% stripe_per_objectset
;
116 if (nb_extra_objects
> stripe_count
) nb_extra_objects
= stripe_count
;
117 uint64_t nb_objects
= nb_full_object_sets
* stripe_count
+ nb_extra_objects
;
118 for (uint64_t object_nb
= 0; object_nb
< nb_objects
; object_nb
++) {
121 char* oid
= getObjName(soid
, object_nb
);
122 uint64_t nb_full_object_set
= object_nb
/ stripe_count
;
123 uint64_t object_index_in_set
= object_nb
% stripe_count
;
124 uint64_t object_start_stripe
= nb_full_object_set
* stripe_per_objectset
+ object_index_in_set
;
125 uint64_t object_start_off
= object_start_stripe
* stripe_unit
;
126 if (actual_size_if_sparse
< size
and actual_size_if_sparse
<= object_start_off
) {
127 ASSERT_EQ(-ENOENT
, ioctx
.stat(oid
, &rados_size
, &mtime
));
129 ASSERT_EQ(0, ioctx
.stat(oid
, &rados_size
, &mtime
));
131 uint64_t stripe_size
= stripe_count
* stripe_unit
;
132 uint64_t set_size
= stripe_count
* object_size
;
134 for (offset
= object_start_off
;
135 (offset
< (object_start_off
) + set_size
) && (offset
< actual_size_if_sparse
);
136 offset
+= stripe_size
) {
137 if (offset
+ stripe_unit
> actual_size_if_sparse
) {
138 len
+= actual_size_if_sparse
-offset
;
143 ASSERT_EQ(len
, rados_size
);
147 // check we do not have an extra object behind
150 char* oid
= getObjName(soid
, nb_objects
);
151 ASSERT_EQ(-ENOENT
, ioctx
.stat(oid
, &rados_size
, &mtime
));
157 TEST_P(StriperTestRT
, StripedRoundtrip
) {
158 // get striping parameters and apply them
159 TestData testData
= GetParam();
160 ASSERT_EQ(0, striper
.set_object_layout_stripe_unit(testData
.stripe_unit
));
161 ASSERT_EQ(0, striper
.set_object_layout_stripe_count(testData
.stripe_count
));
162 ASSERT_EQ(0, striper
.set_object_layout_object_size(testData
.object_size
));
163 std::ostringstream oss
;
164 oss
<< "StripedRoundtrip_" << testData
.stripe_unit
<< "_"
165 << testData
.stripe_count
<< "_" << testData
.object_size
166 << "_" << testData
.size
;
167 std::string soid
= oss
.str();
168 // writing striped data
169 std::unique_ptr
<char[]> buf1
;
172 SCOPED_TRACE("Writing initial object");
173 buf1
= std::make_unique
<char[]>(testData
.size
);
174 for (unsigned int i
= 0; i
< testData
.size
; i
++) buf1
[i
] = 13*((unsigned char)i
);
175 bl1
.append(buf1
.get(), testData
.size
);
176 ASSERT_EQ(0, striper
.write(soid
, bl1
, testData
.size
, 0));
177 // checking object state from Rados point of view
178 ASSERT_NO_FATAL_FAILURE(checkObjectFromRados(soid
, bl1
, testData
.stripe_unit
,
179 testData
.stripe_count
, testData
.object_size
,
182 // adding more data to object and checking again
183 std::unique_ptr
<char[]> buf2
;
186 SCOPED_TRACE("Testing append");
187 buf2
= std::make_unique
<char[]>(testData
.size
);
188 for (unsigned int i
= 0; i
< testData
.size
; i
++) buf2
[i
] = 17*((unsigned char)i
);
189 bl2
.append(buf2
.get(), testData
.size
);
190 ASSERT_EQ(0, striper
.append(soid
, bl2
, testData
.size
));
191 bl1
.append(buf2
.get(), testData
.size
);
192 ASSERT_NO_FATAL_FAILURE(checkObjectFromRados(soid
, bl1
, testData
.stripe_unit
,
193 testData
.stripe_count
, testData
.object_size
,
196 // truncating to half original size and checking again
198 SCOPED_TRACE("Testing trunc to truncate object");
199 ASSERT_EQ(0, striper
.trunc(soid
, testData
.size
/2));
200 ASSERT_NO_FATAL_FAILURE(checkObjectFromRados(soid
, bl1
, testData
.stripe_unit
,
201 testData
.stripe_count
, testData
.object_size
,
204 // truncating back to original size and checking again (especially for 0s)
206 SCOPED_TRACE("Testing trunc to extend object with 0s");
207 ASSERT_EQ(0, striper
.trunc(soid
, testData
.size
));
209 bl3
.substr_of(bl1
, 0, testData
.size
/2);
210 bl3
.append_zero(testData
.size
- testData
.size
/2);
211 ASSERT_NO_FATAL_FAILURE(checkObjectFromRados(soid
, bl3
, testData
.stripe_unit
,
212 testData
.stripe_count
, testData
.object_size
,
213 testData
.size
, testData
.size
/2));
216 SCOPED_TRACE("Testing write_full");
217 // using write_full and checking again
218 ASSERT_EQ(0, striper
.write_full(soid
, bl2
));
219 checkObjectFromRados(soid
, bl2
, testData
.stripe_unit
,
220 testData
.stripe_count
, testData
.object_size
,
224 SCOPED_TRACE("Testing standard remove");
226 ASSERT_EQ(0, striper
.remove(soid
));
227 // check that the removal was successful
230 for (uint64_t object_nb
= 0;
231 object_nb
< testData
.size
*2/testData
.object_size
+ testData
.stripe_count
;
233 char* oid
= getObjName(soid
, object_nb
);
234 ASSERT_EQ(-ENOENT
, ioctx
.stat(oid
, &size
, &mtime
));
239 SCOPED_TRACE("Testing remove when no object size");
241 ASSERT_EQ(0, striper
.write(soid
, bl1
, testData
.size
*2, 0));
242 // remove the object size attribute from the striped object
243 char* firstOid
= getObjName(soid
, 0);
244 ASSERT_EQ(0, ioctx
.rmxattr(firstOid
, "striper.size"));
246 // check that stat fails
249 ASSERT_EQ(-ENODATA
, striper
.stat(soid
, &size
, &mtime
));
251 ASSERT_EQ(0, striper
.remove(soid
));
252 // check that the removal was successful
253 for (uint64_t object_nb
= 0;
254 object_nb
< testData
.size
*2/testData
.object_size
+ testData
.stripe_count
;
256 char* oid
= getObjName(soid
, object_nb
);
257 ASSERT_EQ(-ENOENT
, ioctx
.stat(oid
, &size
, &mtime
));
263 const TestData simple_stripe_schemes
[] = {
264 // stripe_unit, stripe_count, object_size, size
265 {CEPH_MIN_STRIPE_UNIT
, 5, CEPH_MIN_STRIPE_UNIT
, 2},
266 {CEPH_MIN_STRIPE_UNIT
, 5, CEPH_MIN_STRIPE_UNIT
, CEPH_MIN_STRIPE_UNIT
},
267 {CEPH_MIN_STRIPE_UNIT
, 5, CEPH_MIN_STRIPE_UNIT
, CEPH_MIN_STRIPE_UNIT
-1},
268 {CEPH_MIN_STRIPE_UNIT
, 5, CEPH_MIN_STRIPE_UNIT
, 2*CEPH_MIN_STRIPE_UNIT
},
269 {CEPH_MIN_STRIPE_UNIT
, 5, CEPH_MIN_STRIPE_UNIT
, 12*CEPH_MIN_STRIPE_UNIT
},
270 {CEPH_MIN_STRIPE_UNIT
, 5, CEPH_MIN_STRIPE_UNIT
, 2*CEPH_MIN_STRIPE_UNIT
-1},
271 {CEPH_MIN_STRIPE_UNIT
, 5, CEPH_MIN_STRIPE_UNIT
, 12*CEPH_MIN_STRIPE_UNIT
-1},
272 {CEPH_MIN_STRIPE_UNIT
, 5, CEPH_MIN_STRIPE_UNIT
, 2*CEPH_MIN_STRIPE_UNIT
+100},
273 {CEPH_MIN_STRIPE_UNIT
, 5, CEPH_MIN_STRIPE_UNIT
, 12*CEPH_MIN_STRIPE_UNIT
+100},
274 {CEPH_MIN_STRIPE_UNIT
, 5, 3*CEPH_MIN_STRIPE_UNIT
, 2*CEPH_MIN_STRIPE_UNIT
+100},
275 {CEPH_MIN_STRIPE_UNIT
, 5, 3*CEPH_MIN_STRIPE_UNIT
, 8*CEPH_MIN_STRIPE_UNIT
+100},
276 {CEPH_MIN_STRIPE_UNIT
, 5, 3*CEPH_MIN_STRIPE_UNIT
, 12*CEPH_MIN_STRIPE_UNIT
+100},
277 {CEPH_MIN_STRIPE_UNIT
, 5, 3*CEPH_MIN_STRIPE_UNIT
, 15*CEPH_MIN_STRIPE_UNIT
+100},
278 {CEPH_MIN_STRIPE_UNIT
, 5, 3*CEPH_MIN_STRIPE_UNIT
, 25*CEPH_MIN_STRIPE_UNIT
+100},
279 {CEPH_MIN_STRIPE_UNIT
, 5, 3*CEPH_MIN_STRIPE_UNIT
, 45*CEPH_MIN_STRIPE_UNIT
+100},
280 {262144, 5, 262144, 2},
281 {262144, 5, 262144, 262144},
282 {262144, 5, 262144, 262144-1},
283 {262144, 5, 262144, 2*262144},
284 {262144, 5, 262144, 12*262144},
285 {262144, 5, 262144, 2*262144-1},
286 {262144, 5, 262144, 12*262144-1},
287 {262144, 5, 262144, 2*262144+100},
288 {262144, 5, 262144, 12*262144+100},
289 {262144, 5, 3*262144, 2*262144+100},
290 {262144, 5, 3*262144, 8*262144+100},
291 {262144, 5, 3*262144, 12*262144+100},
292 {262144, 5, 3*262144, 15*262144+100},
293 {262144, 5, 3*262144, 25*262144+100},
294 {262144, 5, 3*262144, 45*262144+100},
295 {CEPH_MIN_STRIPE_UNIT
, 1, CEPH_MIN_STRIPE_UNIT
, 2},
296 {CEPH_MIN_STRIPE_UNIT
, 1, CEPH_MIN_STRIPE_UNIT
, CEPH_MIN_STRIPE_UNIT
},
297 {CEPH_MIN_STRIPE_UNIT
, 1, CEPH_MIN_STRIPE_UNIT
, CEPH_MIN_STRIPE_UNIT
-1},
298 {CEPH_MIN_STRIPE_UNIT
, 1, CEPH_MIN_STRIPE_UNIT
, 2*CEPH_MIN_STRIPE_UNIT
},
299 {CEPH_MIN_STRIPE_UNIT
, 1, CEPH_MIN_STRIPE_UNIT
, 12*CEPH_MIN_STRIPE_UNIT
},
300 {CEPH_MIN_STRIPE_UNIT
, 1, CEPH_MIN_STRIPE_UNIT
, 2*CEPH_MIN_STRIPE_UNIT
-1},
301 {CEPH_MIN_STRIPE_UNIT
, 1, CEPH_MIN_STRIPE_UNIT
, 12*CEPH_MIN_STRIPE_UNIT
-1},
302 {CEPH_MIN_STRIPE_UNIT
, 1, CEPH_MIN_STRIPE_UNIT
, 2*CEPH_MIN_STRIPE_UNIT
+100},
303 {CEPH_MIN_STRIPE_UNIT
, 1, CEPH_MIN_STRIPE_UNIT
, 12*CEPH_MIN_STRIPE_UNIT
+100},
304 {CEPH_MIN_STRIPE_UNIT
, 1, 3*CEPH_MIN_STRIPE_UNIT
, 2*CEPH_MIN_STRIPE_UNIT
+100},
305 {CEPH_MIN_STRIPE_UNIT
, 1, 3*CEPH_MIN_STRIPE_UNIT
, 8*CEPH_MIN_STRIPE_UNIT
+100},
306 {CEPH_MIN_STRIPE_UNIT
, 1, 3*CEPH_MIN_STRIPE_UNIT
, 12*CEPH_MIN_STRIPE_UNIT
+100},
307 {CEPH_MIN_STRIPE_UNIT
, 1, 3*CEPH_MIN_STRIPE_UNIT
, 15*CEPH_MIN_STRIPE_UNIT
+100},
308 {CEPH_MIN_STRIPE_UNIT
, 1, 3*CEPH_MIN_STRIPE_UNIT
, 25*CEPH_MIN_STRIPE_UNIT
+100},
309 {CEPH_MIN_STRIPE_UNIT
, 1, 3*CEPH_MIN_STRIPE_UNIT
, 45*CEPH_MIN_STRIPE_UNIT
+100},
310 {CEPH_MIN_STRIPE_UNIT
, 50, CEPH_MIN_STRIPE_UNIT
, 2},
311 {CEPH_MIN_STRIPE_UNIT
, 50, CEPH_MIN_STRIPE_UNIT
, CEPH_MIN_STRIPE_UNIT
},
312 {CEPH_MIN_STRIPE_UNIT
, 50, CEPH_MIN_STRIPE_UNIT
, CEPH_MIN_STRIPE_UNIT
-1},
313 {CEPH_MIN_STRIPE_UNIT
, 50, CEPH_MIN_STRIPE_UNIT
, 2*CEPH_MIN_STRIPE_UNIT
},
314 {CEPH_MIN_STRIPE_UNIT
, 50, CEPH_MIN_STRIPE_UNIT
, 12*CEPH_MIN_STRIPE_UNIT
},
315 {CEPH_MIN_STRIPE_UNIT
, 50, CEPH_MIN_STRIPE_UNIT
, 2*CEPH_MIN_STRIPE_UNIT
-1},
316 {CEPH_MIN_STRIPE_UNIT
, 50, CEPH_MIN_STRIPE_UNIT
, 12*CEPH_MIN_STRIPE_UNIT
-1},
317 {CEPH_MIN_STRIPE_UNIT
, 50, CEPH_MIN_STRIPE_UNIT
, 2*CEPH_MIN_STRIPE_UNIT
+100},
318 {CEPH_MIN_STRIPE_UNIT
, 50, CEPH_MIN_STRIPE_UNIT
, 12*CEPH_MIN_STRIPE_UNIT
+100},
319 {CEPH_MIN_STRIPE_UNIT
, 50, 3*CEPH_MIN_STRIPE_UNIT
, 2*CEPH_MIN_STRIPE_UNIT
+100},
320 {CEPH_MIN_STRIPE_UNIT
, 50, 3*CEPH_MIN_STRIPE_UNIT
, 8*CEPH_MIN_STRIPE_UNIT
+100},
321 {CEPH_MIN_STRIPE_UNIT
, 50, 3*CEPH_MIN_STRIPE_UNIT
, 12*CEPH_MIN_STRIPE_UNIT
+100},
322 {CEPH_MIN_STRIPE_UNIT
, 50, 3*CEPH_MIN_STRIPE_UNIT
, 15*CEPH_MIN_STRIPE_UNIT
+100},
323 {CEPH_MIN_STRIPE_UNIT
, 50, 3*CEPH_MIN_STRIPE_UNIT
, 25*CEPH_MIN_STRIPE_UNIT
+100},
324 {CEPH_MIN_STRIPE_UNIT
, 50, 3*CEPH_MIN_STRIPE_UNIT
, 45*CEPH_MIN_STRIPE_UNIT
+100}
327 INSTANTIATE_TEST_SUITE_P(SimpleStriping
,
329 ::testing::ValuesIn(simple_stripe_schemes
));