]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | /* | |
4 | * Ceph distributed storage system | |
5 | * | |
6 | * Copyright (C) 2015 Mirantis, Inc. | |
7 | * | |
8 | * Author: Alyona Kiseleva <akiselyova@mirantis.com> | |
9 | * | |
10 | * This library is free software; you can redistribute it and/or | |
11 | * modify it under the terms of the GNU Lesser General Public | |
12 | * License as published by the Free Software Foundation; either | |
13 | * version 2.1 of the License, or (at your option) any later version. | |
14 | * | |
15 | */ | |
16 | ||
17 | #include <errno.h> | |
18 | #include <signal.h> | |
19 | #include <stdlib.h> | |
20 | #include "gtest/gtest.h" | |
21 | #include "common/config.h" | |
22 | #include "compressor/Compressor.h" | |
23 | #include "compressor/CompressionPlugin.h" | |
24 | #include "global/global_context.h" | |
25 | ||
26 | class CompressorTest : public ::testing::Test, | |
27 | public ::testing::WithParamInterface<const char*> { | |
28 | public: | |
29 | std::string plugin; | |
30 | CompressorRef compressor; | |
31 | bool old_zlib_isal; | |
32 | ||
33 | CompressorTest() { | |
34 | // note for later | |
35 | old_zlib_isal = g_conf->compressor_zlib_isal; | |
36 | ||
37 | plugin = GetParam(); | |
38 | size_t pos = plugin.find('/'); | |
39 | if (pos != std::string::npos) { | |
40 | string isal = plugin.substr(pos + 1); | |
41 | plugin = plugin.substr(0, pos); | |
42 | if (isal == "isal") { | |
43 | g_conf->set_val("compressor_zlib_isal", "true"); | |
44 | g_ceph_context->_conf->apply_changes(NULL); | |
45 | } else if (isal == "noisal") { | |
46 | g_conf->set_val("compressor_zlib_isal", "false"); | |
47 | g_ceph_context->_conf->apply_changes(NULL); | |
48 | } else { | |
49 | assert(0 == "bad option"); | |
50 | } | |
51 | } | |
52 | cout << "[plugin " << plugin << " (" << GetParam() << ")]" << std::endl; | |
53 | } | |
54 | ~CompressorTest() override { | |
55 | g_conf->set_val("compressor_zlib_isal", old_zlib_isal ? "true" : "false"); | |
56 | g_ceph_context->_conf->apply_changes(NULL); | |
57 | } | |
58 | ||
59 | void SetUp() override { | |
60 | compressor = Compressor::create(g_ceph_context, plugin); | |
61 | ASSERT_TRUE(compressor); | |
62 | } | |
63 | void TearDown() override { | |
64 | compressor.reset(); | |
65 | } | |
66 | }; | |
67 | ||
68 | TEST_P(CompressorTest, load_plugin) | |
69 | { | |
70 | } | |
71 | ||
72 | TEST_P(CompressorTest, small_round_trip) | |
73 | { | |
74 | bufferlist orig; | |
75 | orig.append("This is a short string. There are many strings like it but this one is mine."); | |
76 | bufferlist compressed; | |
77 | int r = compressor->compress(orig, compressed); | |
78 | ASSERT_EQ(0, r); | |
79 | bufferlist decompressed; | |
80 | r = compressor->decompress(compressed, decompressed); | |
81 | ASSERT_EQ(0, r); | |
82 | ASSERT_EQ(decompressed.length(), orig.length()); | |
83 | ASSERT_TRUE(decompressed.contents_equal(orig)); | |
84 | cout << "orig " << orig.length() << " compressed " << compressed.length() | |
85 | << " with " << GetParam() << std::endl; | |
86 | } | |
87 | ||
88 | TEST_P(CompressorTest, big_round_trip_repeated) | |
89 | { | |
90 | unsigned len = 1048576 * 4; | |
91 | bufferlist orig; | |
92 | while (orig.length() < len) { | |
93 | orig.append("This is a short string. There are many strings like it but this one is mine."); | |
94 | } | |
95 | bufferlist compressed; | |
96 | int r = compressor->compress(orig, compressed); | |
97 | ASSERT_EQ(0, r); | |
98 | bufferlist decompressed; | |
99 | r = compressor->decompress(compressed, decompressed); | |
100 | ASSERT_EQ(0, r); | |
101 | ASSERT_EQ(decompressed.length(), orig.length()); | |
102 | ASSERT_TRUE(decompressed.contents_equal(orig)); | |
103 | cout << "orig " << orig.length() << " compressed " << compressed.length() | |
104 | << " with " << GetParam() << std::endl; | |
105 | } | |
106 | ||
107 | TEST_P(CompressorTest, big_round_trip_randomish) | |
108 | { | |
109 | unsigned len = 1048576 * 100;//269; | |
110 | bufferlist orig; | |
111 | const char *alphabet = "abcdefghijklmnopqrstuvwxyz"; | |
112 | if (false) { | |
113 | while (orig.length() < len) { | |
114 | orig.append(alphabet[rand() % 10]); | |
115 | } | |
116 | } else { | |
117 | bufferptr bp(len); | |
118 | char *p = bp.c_str(); | |
119 | for (unsigned i=0; i<len; ++i) { | |
120 | p[i] = alphabet[rand() % 10]; | |
121 | } | |
122 | orig.append(bp); | |
123 | } | |
124 | bufferlist compressed; | |
125 | int r = compressor->compress(orig, compressed); | |
126 | ASSERT_EQ(0, r); | |
127 | bufferlist decompressed; | |
128 | r = compressor->decompress(compressed, decompressed); | |
129 | ASSERT_EQ(0, r); | |
130 | ASSERT_EQ(decompressed.length(), orig.length()); | |
131 | ASSERT_TRUE(decompressed.contents_equal(orig)); | |
132 | cout << "orig " << orig.length() << " compressed " << compressed.length() | |
133 | << " with " << GetParam() << std::endl; | |
134 | } | |
135 | ||
136 | #if 0 | |
137 | TEST_P(CompressorTest, big_round_trip_file) | |
138 | { | |
139 | bufferlist orig; | |
140 | int fd = ::open("bin/ceph-osd", O_RDONLY); | |
141 | struct stat st; | |
142 | ::fstat(fd, &st); | |
143 | orig.read_fd(fd, st.st_size); | |
144 | ||
145 | bufferlist compressed; | |
146 | int r = compressor->compress(orig, compressed); | |
147 | ASSERT_EQ(0, r); | |
148 | bufferlist decompressed; | |
149 | r = compressor->decompress(compressed, decompressed); | |
150 | ASSERT_EQ(0, r); | |
151 | ASSERT_EQ(decompressed.length(), orig.length()); | |
152 | ASSERT_TRUE(decompressed.contents_equal(orig)); | |
153 | cout << "orig " << orig.length() << " compressed " << compressed.length() | |
154 | << " with " << GetParam() << std::endl; | |
155 | } | |
156 | #endif | |
157 | ||
158 | ||
159 | TEST_P(CompressorTest, compress_decompress) | |
160 | { | |
161 | const char* test = "This is test text"; | |
162 | int res; | |
163 | int len = strlen(test); | |
164 | bufferlist in, out; | |
165 | bufferlist after; | |
166 | bufferlist exp; | |
167 | in.append(test, len); | |
168 | res = compressor->compress(in, out); | |
169 | EXPECT_EQ(res, 0); | |
170 | res = compressor->decompress(out, after); | |
171 | EXPECT_EQ(res, 0); | |
172 | exp.append(test); | |
173 | EXPECT_TRUE(exp.contents_equal(after)); | |
174 | after.clear(); | |
175 | size_t compressed_len = out.length(); | |
176 | out.append_zero(12); | |
177 | auto it = out.begin(); | |
178 | res = compressor->decompress(it, compressed_len, after); | |
179 | EXPECT_EQ(res, 0); | |
180 | EXPECT_TRUE(exp.contents_equal(after)); | |
181 | ||
182 | //large block and non-begin iterator for continuous block | |
183 | std::string data; | |
184 | data.resize(0x10000 * 1); | |
185 | for(size_t i = 0; i < data.size(); i++) | |
186 | data[i] = i / 256; | |
187 | in.clear(); | |
188 | out.clear(); | |
189 | in.append(data); | |
190 | exp = in; | |
191 | res = compressor->compress(in, out); | |
192 | EXPECT_EQ(res, 0); | |
193 | compressed_len = out.length(); | |
194 | out.append_zero(0x10000 - out.length()); | |
195 | after.clear(); | |
196 | out.c_str(); | |
197 | bufferlist prefix; | |
198 | prefix.append(string("some prefix")); | |
199 | size_t prefix_len = prefix.length(); | |
200 | out.claim_prepend(prefix); | |
201 | it = out.begin(); | |
202 | it.advance(prefix_len); | |
203 | res = compressor->decompress(it, compressed_len, after); | |
204 | EXPECT_EQ(res, 0); | |
205 | EXPECT_TRUE(exp.contents_equal(after)); | |
206 | } | |
207 | ||
208 | TEST_P(CompressorTest, sharded_input_decompress) | |
209 | { | |
210 | const size_t small_prefix_size=3; | |
211 | ||
212 | string test(128*1024,0); | |
213 | int len = test.size(); | |
214 | bufferlist in, out; | |
215 | in.append(test.c_str(), len); | |
216 | int res = compressor->compress(in, out); | |
217 | EXPECT_EQ(res, 0); | |
218 | EXPECT_GT(out.length(), small_prefix_size); | |
219 | ||
220 | bufferlist out2, tmp; | |
221 | tmp.substr_of(out, 0, small_prefix_size ); | |
222 | out2.append( tmp ); | |
223 | size_t left = out.length()-small_prefix_size; | |
224 | size_t offs = small_prefix_size; | |
225 | while( left > 0 ){ | |
226 | size_t shard_size = MIN( 2048, left ); | |
227 | tmp.substr_of(out, offs, shard_size ); | |
228 | out2.append( tmp ); | |
229 | left -= shard_size; | |
230 | offs += shard_size; | |
231 | } | |
232 | ||
233 | bufferlist after; | |
234 | res = compressor->decompress(out2, after); | |
235 | EXPECT_EQ(res, 0); | |
236 | } | |
237 | ||
238 | void test_compress(CompressorRef compressor, size_t size) | |
239 | { | |
240 | char* data = (char*) malloc(size); | |
241 | for (size_t t = 0; t < size; t++) { | |
242 | data[t] = (t & 0xff) | (t >> 8); | |
243 | } | |
244 | bufferlist in; | |
245 | in.append(data, size); | |
246 | for (size_t t = 0; t < 100000; t++) { | |
247 | bufferlist out; | |
248 | int res = compressor->compress(in, out); | |
249 | EXPECT_EQ(res, 0); | |
250 | } | |
251 | } | |
252 | ||
253 | void test_decompress(CompressorRef compressor, size_t size) | |
254 | { | |
255 | char* data = (char*) malloc(size); | |
256 | for (size_t t = 0; t < size; t++) { | |
257 | data[t] = (t & 0xff) | (t >> 8); | |
258 | } | |
259 | bufferlist in, out; | |
260 | in.append(data, size); | |
261 | int res = compressor->compress(in, out); | |
262 | EXPECT_EQ(res, 0); | |
263 | for (size_t t = 0; t < 100000; t++) { | |
264 | bufferlist out_dec; | |
265 | int res = compressor->decompress(out, out_dec); | |
266 | EXPECT_EQ(res, 0); | |
267 | } | |
268 | } | |
269 | ||
270 | TEST_P(CompressorTest, compress_1024) | |
271 | { | |
272 | test_compress(compressor, 1024); | |
273 | } | |
274 | ||
275 | TEST_P(CompressorTest, compress_2048) | |
276 | { | |
277 | test_compress(compressor, 2048); | |
278 | } | |
279 | ||
280 | TEST_P(CompressorTest, compress_4096) | |
281 | { | |
282 | test_compress(compressor, 4096); | |
283 | } | |
284 | ||
285 | TEST_P(CompressorTest, compress_8192) | |
286 | { | |
287 | test_compress(compressor, 8192); | |
288 | } | |
289 | ||
290 | TEST_P(CompressorTest, compress_16384) | |
291 | { | |
292 | test_compress(compressor, 16384); | |
293 | } | |
294 | ||
295 | TEST_P(CompressorTest, decompress_1024) | |
296 | { | |
297 | test_decompress(compressor, 1024); | |
298 | } | |
299 | ||
300 | TEST_P(CompressorTest, decompress_2048) | |
301 | { | |
302 | test_decompress(compressor, 2048); | |
303 | } | |
304 | ||
305 | TEST_P(CompressorTest, decompress_4096) | |
306 | { | |
307 | test_decompress(compressor, 4096); | |
308 | } | |
309 | ||
310 | TEST_P(CompressorTest, decompress_8192) | |
311 | { | |
312 | test_decompress(compressor, 8192); | |
313 | } | |
314 | ||
315 | TEST_P(CompressorTest, decompress_16384) | |
316 | { | |
317 | test_decompress(compressor, 16384); | |
318 | } | |
319 | ||
320 | ||
321 | INSTANTIATE_TEST_CASE_P( | |
322 | Compressor, | |
323 | CompressorTest, | |
324 | ::testing::Values( | |
31f18b77 FG |
325 | #ifdef HAVE_LZ4 |
326 | "lz4", | |
327 | #endif | |
7c673cae FG |
328 | #ifdef __x86_64__ |
329 | "zlib/isal", | |
330 | #endif | |
331 | "zlib/noisal", | |
332 | "snappy", | |
333 | "zstd")); | |
334 | ||
335 | #ifdef __x86_64__ | |
336 | ||
337 | TEST(ZlibCompressor, zlib_isal_compatibility) | |
338 | { | |
339 | g_conf->set_val("compressor_zlib_isal", "true"); | |
340 | g_ceph_context->_conf->apply_changes(NULL); | |
341 | CompressorRef isal = Compressor::create(g_ceph_context, "zlib"); | |
342 | if (!isal) { | |
343 | // skip the test if the plugin is not ready | |
344 | return; | |
345 | } | |
346 | g_conf->set_val("compressor_zlib_isal", "false"); | |
347 | g_ceph_context->_conf->apply_changes(NULL); | |
348 | CompressorRef zlib = Compressor::create(g_ceph_context, "zlib"); | |
349 | char test[101]; | |
350 | srand(time(0)); | |
351 | for (int i=0; i<100; ++i) | |
352 | test[i] = 'a' + rand()%26; | |
353 | test[100] = '\0'; | |
354 | int len = strlen(test); | |
355 | bufferlist in, out; | |
356 | in.append(test, len); | |
357 | // isal -> zlib | |
358 | int res = isal->compress(in, out); | |
359 | EXPECT_EQ(res, 0); | |
360 | bufferlist after; | |
361 | res = zlib->decompress(out, after); | |
362 | EXPECT_EQ(res, 0); | |
363 | bufferlist exp; | |
364 | exp.append(test); | |
365 | EXPECT_TRUE(exp.contents_equal(after)); | |
366 | after.clear(); | |
367 | out.clear(); | |
368 | exp.clear(); | |
369 | // zlib -> isal | |
370 | res = zlib->compress(in, out); | |
371 | EXPECT_EQ(res, 0); | |
372 | res = isal->decompress(out, after); | |
373 | EXPECT_EQ(res, 0); | |
374 | exp.append(test); | |
375 | EXPECT_TRUE(exp.contents_equal(after)); | |
376 | } | |
377 | #endif | |
378 | ||
379 | TEST(CompressionPlugin, all) | |
380 | { | |
381 | const char* env = getenv("CEPH_LIB"); | |
382 | std::string directory(env ? env : ".libs"); | |
383 | CompressorRef compressor; | |
384 | PluginRegistry *reg = g_ceph_context->get_plugin_registry(); | |
385 | EXPECT_TRUE(reg); | |
386 | CompressionPlugin *factory = dynamic_cast<CompressionPlugin*>(reg->get_with_load("compressor", "invalid")); | |
387 | EXPECT_FALSE(factory); | |
388 | factory = dynamic_cast<CompressionPlugin*>(reg->get_with_load("compressor", "example")); | |
389 | EXPECT_TRUE(factory); | |
390 | stringstream ss; | |
391 | EXPECT_EQ(0, factory->factory(&compressor, &ss)); | |
392 | EXPECT_TRUE(compressor.get()); | |
393 | { | |
394 | Mutex::Locker l(reg->lock); | |
395 | EXPECT_EQ(-ENOENT, reg->remove("compressor", "does not exist")); | |
396 | EXPECT_EQ(0, reg->remove("compressor", "example")); | |
397 | EXPECT_EQ(0, reg->load("compressor", "example")); | |
398 | } | |
399 | } | |
400 | ||
401 | #ifdef __x86_64__ | |
402 | ||
403 | TEST(ZlibCompressor, isal_compress_zlib_decompress_random) | |
404 | { | |
405 | g_conf->set_val("compressor_zlib_isal", "true"); | |
406 | g_ceph_context->_conf->apply_changes(NULL); | |
407 | CompressorRef isal = Compressor::create(g_ceph_context, "zlib"); | |
408 | if (!isal) { | |
409 | // skip the test if the plugin is not ready | |
410 | return; | |
411 | } | |
412 | g_conf->set_val("compressor_zlib_isal", "false"); | |
413 | g_ceph_context->_conf->apply_changes(NULL); | |
414 | CompressorRef zlib = Compressor::create(g_ceph_context, "zlib"); | |
415 | ||
416 | for (int cnt=0; cnt<1000; cnt++) | |
417 | { | |
418 | srand(cnt + 1000); | |
419 | int log2 = (rand()%18) + 1; | |
420 | int size = (rand() % (1 << log2)) + 1; | |
421 | ||
422 | char test[size]; | |
423 | for (int i=0; i<size; ++i) | |
424 | test[i] = rand()%256; | |
425 | bufferlist in, out; | |
426 | in.append(test, size); | |
427 | ||
428 | int res = isal->compress(in, out); | |
429 | EXPECT_EQ(res, 0); | |
430 | bufferlist after; | |
431 | res = zlib->decompress(out, after); | |
432 | EXPECT_EQ(res, 0); | |
433 | bufferlist exp; | |
434 | exp.append(test, size); | |
435 | EXPECT_TRUE(exp.contents_equal(after)); | |
436 | } | |
437 | } | |
438 | ||
439 | TEST(ZlibCompressor, isal_compress_zlib_decompress_walk) | |
440 | { | |
441 | g_conf->set_val("compressor_zlib_isal", "true"); | |
442 | g_ceph_context->_conf->apply_changes(NULL); | |
443 | CompressorRef isal = Compressor::create(g_ceph_context, "zlib"); | |
444 | if (!isal) { | |
445 | // skip the test if the plugin is not ready | |
446 | return; | |
447 | } | |
448 | g_conf->set_val("compressor_zlib_isal", "false"); | |
449 | g_ceph_context->_conf->apply_changes(NULL); | |
450 | CompressorRef zlib = Compressor::create(g_ceph_context, "zlib"); | |
451 | ||
452 | for (int cnt=0; cnt<1000; cnt++) | |
453 | { | |
454 | srand(cnt + 1000); | |
455 | int log2 = (rand()%18) + 1; | |
456 | int size = (rand() % (1 << log2)) + 1; | |
457 | ||
458 | int range = 1; | |
459 | ||
460 | char test[size]; | |
461 | test[0] = rand()%256; | |
462 | for (int i=1; i<size; ++i) | |
463 | test[i] = test[i-1] + rand()%(range*2+1) - range; | |
464 | bufferlist in, out; | |
465 | in.append(test, size); | |
466 | ||
467 | int res = isal->compress(in, out); | |
468 | EXPECT_EQ(res, 0); | |
469 | bufferlist after; | |
470 | res = zlib->decompress(out, after); | |
471 | EXPECT_EQ(res, 0); | |
472 | bufferlist exp; | |
473 | exp.append(test, size); | |
474 | EXPECT_TRUE(exp.contents_equal(after)); | |
475 | } | |
476 | } | |
477 | ||
478 | #endif // __x86_64__ |