1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph distributed storage system
6 * Copyright (C) 2013,2014 Cloudwatt <libre.licensing@cloudwatt.com>
7 * Copyright (C) 2014 Red Hat <contact@redhat.com>
9 * Author: Loic Dachary <loic@dachary.org>
11 * This library is free software; you can redistribute it and/or
12 * modify it under the terms of the GNU Lesser General Public
13 * License as published by the Free Software Foundation; either
14 * version 2.1 of the License, or (at your option) any later version.
21 #include "crush/CrushWrapper.h"
22 #include "include/stringify.h"
23 #include "erasure-code/jerasure/ErasureCodeJerasure.h"
24 #include "global/global_context.h"
25 #include "common/config.h"
26 #include "gtest/gtest.h"
30 class ErasureCodeTest
: public ::testing::Test
{
34 typedef ::testing::Types
<
35 ErasureCodeJerasureReedSolomonVandermonde
,
36 ErasureCodeJerasureReedSolomonRAID6
,
37 ErasureCodeJerasureCauchyOrig
,
38 ErasureCodeJerasureCauchyGood
,
39 ErasureCodeJerasureLiberation
,
40 ErasureCodeJerasureBlaumRoth
,
41 ErasureCodeJerasureLiber8tion
43 TYPED_TEST_CASE(ErasureCodeTest
, JerasureTypes
);
45 TYPED_TEST(ErasureCodeTest
, sanity_check_k
)
48 ErasureCodeProfile profile
;
51 profile
["packetsize"] = "8";
53 EXPECT_EQ(-EINVAL
, jerasure
.init(profile
, &errors
));
54 EXPECT_NE(std::string::npos
, errors
.str().find("must be >= 2"));
57 TYPED_TEST(ErasureCodeTest
, encode_decode
)
59 const char *per_chunk_alignments
[] = { "false", "true" };
60 for (int per_chunk_alignment
= 0 ;
61 per_chunk_alignment
< 2;
62 per_chunk_alignment
++) {
64 ErasureCodeProfile profile
;
67 profile
["packetsize"] = "8";
68 profile
["jerasure-per-chunk-alignment"] =
69 per_chunk_alignments
[per_chunk_alignment
];
70 jerasure
.init(profile
, &cerr
);
72 #define LARGE_ENOUGH 2048
73 bufferptr
in_ptr(buffer::create_page_aligned(LARGE_ENOUGH
));
77 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
78 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
79 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
80 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
81 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
82 in_ptr
.append(payload
, strlen(payload
));
84 in
.push_front(in_ptr
);
85 int want_to_encode
[] = { 0, 1, 2, 3 };
86 map
<int, bufferlist
> encoded
;
87 EXPECT_EQ(0, jerasure
.encode(set
<int>(want_to_encode
, want_to_encode
+4),
90 EXPECT_EQ(4u, encoded
.size());
91 unsigned length
= encoded
[0].length();
92 EXPECT_EQ(0, memcmp(encoded
[0].c_str(), in
.c_str(), length
));
93 EXPECT_EQ(0, memcmp(encoded
[1].c_str(), in
.c_str() + length
,
94 in
.length() - length
));
97 // all chunks are available
99 int want_to_decode
[] = { 0, 1 };
100 map
<int, bufferlist
> decoded
;
101 EXPECT_EQ(0, jerasure
.decode(set
<int>(want_to_decode
, want_to_decode
+2),
104 EXPECT_EQ(2u, decoded
.size());
105 EXPECT_EQ(length
, decoded
[0].length());
106 EXPECT_EQ(0, memcmp(decoded
[0].c_str(), in
.c_str(), length
));
107 EXPECT_EQ(0, memcmp(decoded
[1].c_str(), in
.c_str() + length
,
108 in
.length() - length
));
111 // two chunks are missing
113 map
<int, bufferlist
> degraded
= encoded
;
116 EXPECT_EQ(2u, degraded
.size());
117 int want_to_decode
[] = { 0, 1 };
118 map
<int, bufferlist
> decoded
;
119 EXPECT_EQ(0, jerasure
.decode(set
<int>(want_to_decode
, want_to_decode
+2),
122 // always decode all, regardless of want_to_decode
123 EXPECT_EQ(4u, decoded
.size());
124 EXPECT_EQ(length
, decoded
[0].length());
125 EXPECT_EQ(0, memcmp(decoded
[0].c_str(), in
.c_str(), length
));
126 EXPECT_EQ(0, memcmp(decoded
[1].c_str(), in
.c_str() + length
,
127 in
.length() - length
));
132 TYPED_TEST(ErasureCodeTest
, minimum_to_decode
)
135 ErasureCodeProfile profile
;
139 profile
["packetsize"] = "8";
140 jerasure
.init(profile
, &cerr
);
143 // If trying to read nothing, the minimum is empty.
146 set
<int> want_to_read
;
147 set
<int> available_chunks
;
150 EXPECT_EQ(0, jerasure
.minimum_to_decode(want_to_read
,
153 EXPECT_TRUE(minimum
.empty());
156 // There is no way to read a chunk if none are available.
159 set
<int> want_to_read
;
160 set
<int> available_chunks
;
163 want_to_read
.insert(0);
165 EXPECT_EQ(-EIO
, jerasure
.minimum_to_decode(want_to_read
,
170 // Reading a subset of the available chunks is always possible.
173 set
<int> want_to_read
;
174 set
<int> available_chunks
;
177 want_to_read
.insert(0);
178 available_chunks
.insert(0);
180 EXPECT_EQ(0, jerasure
.minimum_to_decode(want_to_read
,
183 EXPECT_EQ(want_to_read
, minimum
);
186 // There is no way to read a missing chunk if there is less than k
190 set
<int> want_to_read
;
191 set
<int> available_chunks
;
194 want_to_read
.insert(0);
195 want_to_read
.insert(1);
196 available_chunks
.insert(0);
198 EXPECT_EQ(-EIO
, jerasure
.minimum_to_decode(want_to_read
,
203 // When chunks are not available, the minimum can be made of any
204 // chunks. For instance, to read 1 and 3 below the minimum could be
205 // 2 and 3 which may seem better because it contains one of the
206 // chunks to be read. But it won't be more efficient than retrieving
207 // 0 and 2 instead because, in both cases, the decode function will
208 // need to run the same recovery operation and use the same amount
209 // of CPU and memory.
212 set
<int> want_to_read
;
213 set
<int> available_chunks
;
216 want_to_read
.insert(1);
217 want_to_read
.insert(3);
218 available_chunks
.insert(0);
219 available_chunks
.insert(2);
220 available_chunks
.insert(3);
222 EXPECT_EQ(0, jerasure
.minimum_to_decode(want_to_read
,
225 EXPECT_EQ(2u, minimum
.size());
226 EXPECT_EQ(0u, minimum
.count(3));
230 TEST(ErasureCodeTest
, encode
)
232 ErasureCodeJerasureReedSolomonVandermonde jerasure
;
233 ErasureCodeProfile profile
;
237 jerasure
.init(profile
, &cerr
);
239 unsigned aligned_object_size
= jerasure
.get_alignment() * 2;
242 // When the input bufferlist needs to be padded because
243 // it is not properly aligned, it is padded with zeros.
246 map
<int,bufferlist
> encoded
;
247 int want_to_encode
[] = { 0, 1, 2, 3 };
248 int trail_length
= 1;
249 in
.append(string(aligned_object_size
+ trail_length
, 'X'));
250 EXPECT_EQ(0, jerasure
.encode(set
<int>(want_to_encode
, want_to_encode
+4),
253 EXPECT_EQ(4u, encoded
.size());
254 char *last_chunk
= encoded
[1].c_str();
255 int length
=encoded
[1].length();
256 EXPECT_EQ('X', last_chunk
[0]);
257 EXPECT_EQ('\0', last_chunk
[length
- trail_length
]);
262 // When only the first chunk is required, the encoded map only
263 // contains the first chunk. Although the jerasure encode
264 // internally allocated a buffer because of padding requirements
265 // and also computes the coding chunks, they are released before
266 // the return of the method, as shown when running the tests thru
267 // valgrind (there is no leak).
270 map
<int,bufferlist
> encoded
;
271 set
<int> want_to_encode
;
272 want_to_encode
.insert(0);
273 int trail_length
= 1;
274 in
.append(string(aligned_object_size
+ trail_length
, 'X'));
275 EXPECT_EQ(0, jerasure
.encode(want_to_encode
, in
, &encoded
));
276 EXPECT_EQ(1u, encoded
.size());
280 TEST(ErasureCodeTest
, create_ruleset
)
282 CrushWrapper
*c
= new CrushWrapper
;
285 c
->set_type_name(root_type
, "root");
287 c
->set_type_name(host_type
, "host");
289 c
->set_type_name(osd_type
, "osd");
292 c
->add_bucket(0, CRUSH_BUCKET_STRAW
, CRUSH_HASH_RJENKINS1
,
293 root_type
, 0, NULL
, NULL
, &rootno
);
294 c
->set_item_name(rootno
, "default");
296 map
<string
,string
> loc
;
297 loc
["root"] = "default";
302 for (int h
=0; h
<num_host
; ++h
) {
303 loc
["host"] = string("host-") + stringify(h
);
304 for (int o
=0; o
<num_osd
; ++o
, ++osd
) {
305 c
->insert_item(g_ceph_context
, osd
, 1.0, string("osd.") + stringify(osd
), loc
);
313 ErasureCodeJerasureReedSolomonVandermonde jerasure
;
314 ErasureCodeProfile profile
;
318 jerasure
.init(profile
, &cerr
);
319 int ruleset
= jerasure
.create_ruleset("myrule", *c
, &ss
);
320 EXPECT_EQ(0, ruleset
);
321 EXPECT_EQ(-EEXIST
, jerasure
.create_ruleset("myrule", *c
, &ss
));
323 // the minimum that is expected from the created ruleset is to
324 // successfully map get_chunk_count() devices from the crushmap,
327 vector
<__u32
> weight(c
->get_max_devices(), 0x10000);
330 c
->do_rule(ruleset
, x
, out
, jerasure
.get_chunk_count(), weight
, 0);
331 ASSERT_EQ(out
.size(), jerasure
.get_chunk_count());
332 for (unsigned i
=0; i
<out
.size(); ++i
)
333 ASSERT_NE(CRUSH_ITEM_NONE
, out
[i
]);
337 ErasureCodeJerasureReedSolomonVandermonde jerasure
;
338 ErasureCodeProfile profile
;
342 profile
["ruleset-root"] = "BAD";
343 jerasure
.init(profile
, &cerr
);
344 EXPECT_EQ(-ENOENT
, jerasure
.create_ruleset("otherrule", *c
, &ss
));
345 EXPECT_EQ("root item BAD does not exist", ss
.str());
349 ErasureCodeJerasureReedSolomonVandermonde jerasure
;
350 ErasureCodeProfile profile
;
354 profile
["ruleset-failure-domain"] = "WORSE";
355 jerasure
.init(profile
, &cerr
);
356 EXPECT_EQ(-EINVAL
, jerasure
.create_ruleset("otherrule", *c
, &ss
));
357 EXPECT_EQ("unknown type WORSE", ss
.str());
363 * compile-command: "cd ../.. ;
364 * make -j4 unittest_erasure_code_jerasure &&
365 * valgrind --tool=memcheck \
366 * ./unittest_erasure_code_jerasure \
367 * --gtest_filter=*.* --log-to-stderr=true --debug-osd=20"