1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph distributed storage system
6 * Copyright (C) 2015 Mirantis, Inc.
8 * Author: Alyona Kiseleva <akiselyova@mirantis.com>
10 * This library is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU Lesser General Public
12 * License as published by the Free Software Foundation; either
13 * version 2.1 of the License, or (at your option) any later version.
20 #include "gtest/gtest.h"
21 #include "common/config.h"
22 #include "compressor/Compressor.h"
23 #include "compressor/CompressionPlugin.h"
24 #include "global/global_context.h"
26 class CompressorTest
: public ::testing::Test
,
27 public ::testing::WithParamInterface
<const char*> {
30 CompressorRef compressor
;
35 old_zlib_isal
= g_conf
->compressor_zlib_isal
;
38 size_t pos
= plugin
.find('/');
39 if (pos
!= std::string::npos
) {
40 string isal
= plugin
.substr(pos
+ 1);
41 plugin
= plugin
.substr(0, pos
);
43 g_conf
->set_val("compressor_zlib_isal", "true");
44 g_ceph_context
->_conf
->apply_changes(NULL
);
45 } else if (isal
== "noisal") {
46 g_conf
->set_val("compressor_zlib_isal", "false");
47 g_ceph_context
->_conf
->apply_changes(NULL
);
49 assert(0 == "bad option");
52 cout
<< "[plugin " << plugin
<< " (" << GetParam() << ")]" << std::endl
;
54 ~CompressorTest() override
{
55 g_conf
->set_val("compressor_zlib_isal", old_zlib_isal
? "true" : "false");
56 g_ceph_context
->_conf
->apply_changes(NULL
);
59 void SetUp() override
{
60 compressor
= Compressor::create(g_ceph_context
, plugin
);
61 ASSERT_TRUE(compressor
);
63 void TearDown() override
{
68 TEST_P(CompressorTest
, load_plugin
)
72 TEST_P(CompressorTest
, small_round_trip
)
75 orig
.append("This is a short string. There are many strings like it but this one is mine.");
76 bufferlist compressed
;
77 int r
= compressor
->compress(orig
, compressed
);
79 bufferlist decompressed
;
80 r
= compressor
->decompress(compressed
, decompressed
);
82 ASSERT_EQ(decompressed
.length(), orig
.length());
83 ASSERT_TRUE(decompressed
.contents_equal(orig
));
84 cout
<< "orig " << orig
.length() << " compressed " << compressed
.length()
85 << " with " << GetParam() << std::endl
;
88 TEST_P(CompressorTest
, big_round_trip_repeated
)
90 unsigned len
= 1048576 * 4;
92 while (orig
.length() < len
) {
93 orig
.append("This is a short string. There are many strings like it but this one is mine.");
95 bufferlist compressed
;
96 int r
= compressor
->compress(orig
, compressed
);
98 bufferlist decompressed
;
99 r
= compressor
->decompress(compressed
, decompressed
);
101 ASSERT_EQ(decompressed
.length(), orig
.length());
102 ASSERT_TRUE(decompressed
.contents_equal(orig
));
103 cout
<< "orig " << orig
.length() << " compressed " << compressed
.length()
104 << " with " << GetParam() << std::endl
;
107 TEST_P(CompressorTest
, big_round_trip_randomish
)
109 unsigned len
= 1048576 * 10;//269;
111 const char *alphabet
= "abcdefghijklmnopqrstuvwxyz";
113 while (orig
.length() < len
) {
114 orig
.append(alphabet
[rand() % 10]);
118 char *p
= bp
.c_str();
119 for (unsigned i
=0; i
<len
; ++i
) {
120 p
[i
] = alphabet
[rand() % 10];
124 bufferlist compressed
;
125 int r
= compressor
->compress(orig
, compressed
);
127 bufferlist decompressed
;
128 r
= compressor
->decompress(compressed
, decompressed
);
130 ASSERT_EQ(decompressed
.length(), orig
.length());
131 ASSERT_TRUE(decompressed
.contents_equal(orig
));
132 cout
<< "orig " << orig
.length() << " compressed " << compressed
.length()
133 << " with " << GetParam() << std::endl
;
137 TEST_P(CompressorTest
, big_round_trip_file
)
140 int fd
= ::open("bin/ceph-osd", O_RDONLY
);
143 orig
.read_fd(fd
, st
.st_size
);
145 bufferlist compressed
;
146 int r
= compressor
->compress(orig
, compressed
);
148 bufferlist decompressed
;
149 r
= compressor
->decompress(compressed
, decompressed
);
151 ASSERT_EQ(decompressed
.length(), orig
.length());
152 ASSERT_TRUE(decompressed
.contents_equal(orig
));
153 cout
<< "orig " << orig
.length() << " compressed " << compressed
.length()
154 << " with " << GetParam() << std::endl
;
159 TEST_P(CompressorTest
, compress_decompress
)
161 const char* test
= "This is test text";
163 int len
= strlen(test
);
167 in
.append(test
, len
);
168 res
= compressor
->compress(in
, out
);
170 res
= compressor
->decompress(out
, after
);
173 EXPECT_TRUE(exp
.contents_equal(after
));
175 size_t compressed_len
= out
.length();
177 auto it
= out
.begin();
178 res
= compressor
->decompress(it
, compressed_len
, after
);
180 EXPECT_TRUE(exp
.contents_equal(after
));
182 //large block and non-begin iterator for continuous block
184 data
.resize(0x10000 * 1);
185 for(size_t i
= 0; i
< data
.size(); i
++)
191 res
= compressor
->compress(in
, out
);
193 compressed_len
= out
.length();
194 out
.append_zero(0x10000 - out
.length());
198 prefix
.append(string("some prefix"));
199 size_t prefix_len
= prefix
.length();
200 out
.claim_prepend(prefix
);
202 it
.advance(prefix_len
);
203 res
= compressor
->decompress(it
, compressed_len
, after
);
205 EXPECT_TRUE(exp
.contents_equal(after
));
208 TEST_P(CompressorTest
, sharded_input_decompress
)
210 const size_t small_prefix_size
=3;
212 string
test(128*1024,0);
213 int len
= test
.size();
215 in
.append(test
.c_str(), len
);
216 int res
= compressor
->compress(in
, out
);
218 EXPECT_GT(out
.length(), small_prefix_size
);
220 bufferlist out2
, tmp
;
221 tmp
.substr_of(out
, 0, small_prefix_size
);
223 size_t left
= out
.length()-small_prefix_size
;
224 size_t offs
= small_prefix_size
;
226 size_t shard_size
= MIN( 2048, left
);
227 tmp
.substr_of(out
, offs
, shard_size
);
234 res
= compressor
->decompress(out2
, after
);
238 void test_compress(CompressorRef compressor
, size_t size
)
240 char* data
= (char*) malloc(size
);
241 for (size_t t
= 0; t
< size
; t
++) {
242 data
[t
] = (t
& 0xff) | (t
>> 8);
245 in
.append(data
, size
);
246 for (size_t t
= 0; t
< 10000; t
++) {
248 int res
= compressor
->compress(in
, out
);
253 void test_decompress(CompressorRef compressor
, size_t size
)
255 char* data
= (char*) malloc(size
);
256 for (size_t t
= 0; t
< size
; t
++) {
257 data
[t
] = (t
& 0xff) | (t
>> 8);
260 in
.append(data
, size
);
261 int res
= compressor
->compress(in
, out
);
263 for (size_t t
= 0; t
< 10000; t
++) {
265 int res
= compressor
->decompress(out
, out_dec
);
270 TEST_P(CompressorTest
, compress_1024
)
272 test_compress(compressor
, 1024);
275 TEST_P(CompressorTest
, compress_2048
)
277 test_compress(compressor
, 2048);
280 TEST_P(CompressorTest
, compress_4096
)
282 test_compress(compressor
, 4096);
285 TEST_P(CompressorTest
, compress_8192
)
287 test_compress(compressor
, 8192);
290 TEST_P(CompressorTest
, compress_16384
)
292 test_compress(compressor
, 16384);
295 TEST_P(CompressorTest
, decompress_1024
)
297 test_decompress(compressor
, 1024);
300 TEST_P(CompressorTest
, decompress_2048
)
302 test_decompress(compressor
, 2048);
305 TEST_P(CompressorTest
, decompress_4096
)
307 test_decompress(compressor
, 4096);
310 TEST_P(CompressorTest
, decompress_8192
)
312 test_decompress(compressor
, 8192);
315 TEST_P(CompressorTest
, decompress_16384
)
317 test_decompress(compressor
, 16384);
321 INSTANTIATE_TEST_CASE_P(
337 TEST(ZlibCompressor
, zlib_isal_compatibility
)
339 g_conf
->set_val("compressor_zlib_isal", "true");
340 g_ceph_context
->_conf
->apply_changes(NULL
);
341 CompressorRef isal
= Compressor::create(g_ceph_context
, "zlib");
343 // skip the test if the plugin is not ready
346 g_conf
->set_val("compressor_zlib_isal", "false");
347 g_ceph_context
->_conf
->apply_changes(NULL
);
348 CompressorRef zlib
= Compressor::create(g_ceph_context
, "zlib");
351 for (int i
=0; i
<100; ++i
)
352 test
[i
] = 'a' + rand()%26;
354 int len
= strlen(test
);
356 in
.append(test
, len
);
358 int res
= isal
->compress(in
, out
);
361 res
= zlib
->decompress(out
, after
);
365 EXPECT_TRUE(exp
.contents_equal(after
));
370 res
= zlib
->compress(in
, out
);
372 res
= isal
->decompress(out
, after
);
375 EXPECT_TRUE(exp
.contents_equal(after
));
379 TEST(CompressionPlugin
, all
)
381 const char* env
= getenv("CEPH_LIB");
382 std::string
directory(env
? env
: ".libs");
383 CompressorRef compressor
;
384 PluginRegistry
*reg
= g_ceph_context
->get_plugin_registry();
386 CompressionPlugin
*factory
= dynamic_cast<CompressionPlugin
*>(reg
->get_with_load("compressor", "invalid"));
387 EXPECT_FALSE(factory
);
388 factory
= dynamic_cast<CompressionPlugin
*>(reg
->get_with_load("compressor", "example"));
389 EXPECT_TRUE(factory
);
391 EXPECT_EQ(0, factory
->factory(&compressor
, &ss
));
392 EXPECT_TRUE(compressor
.get());
394 Mutex::Locker
l(reg
->lock
);
395 EXPECT_EQ(-ENOENT
, reg
->remove("compressor", "does not exist"));
396 EXPECT_EQ(0, reg
->remove("compressor", "example"));
397 EXPECT_EQ(0, reg
->load("compressor", "example"));
403 TEST(ZlibCompressor
, isal_compress_zlib_decompress_random
)
405 g_conf
->set_val("compressor_zlib_isal", "true");
406 g_ceph_context
->_conf
->apply_changes(NULL
);
407 CompressorRef isal
= Compressor::create(g_ceph_context
, "zlib");
409 // skip the test if the plugin is not ready
412 g_conf
->set_val("compressor_zlib_isal", "false");
413 g_ceph_context
->_conf
->apply_changes(NULL
);
414 CompressorRef zlib
= Compressor::create(g_ceph_context
, "zlib");
416 for (int cnt
=0; cnt
<100; cnt
++)
419 int log2
= (rand()%18) + 1;
420 int size
= (rand() % (1 << log2
)) + 1;
423 for (int i
=0; i
<size
; ++i
)
424 test
[i
] = rand()%256;
426 in
.append(test
, size
);
428 int res
= isal
->compress(in
, out
);
431 res
= zlib
->decompress(out
, after
);
434 exp
.append(test
, size
);
435 EXPECT_TRUE(exp
.contents_equal(after
));
439 TEST(ZlibCompressor
, isal_compress_zlib_decompress_walk
)
441 g_conf
->set_val("compressor_zlib_isal", "true");
442 g_ceph_context
->_conf
->apply_changes(NULL
);
443 CompressorRef isal
= Compressor::create(g_ceph_context
, "zlib");
445 // skip the test if the plugin is not ready
448 g_conf
->set_val("compressor_zlib_isal", "false");
449 g_ceph_context
->_conf
->apply_changes(NULL
);
450 CompressorRef zlib
= Compressor::create(g_ceph_context
, "zlib");
452 for (int cnt
=0; cnt
<100; cnt
++)
455 int log2
= (rand()%18) + 1;
456 int size
= (rand() % (1 << log2
)) + 1;
461 test
[0] = rand()%256;
462 for (int i
=1; i
<size
; ++i
)
463 test
[i
] = test
[i
-1] + rand()%(range
*2+1) - range
;
465 in
.append(test
, size
);
467 int res
= isal
->compress(in
, out
);
470 res
= zlib
->decompress(out
, after
);
473 exp
.append(test
, size
);
474 EXPECT_TRUE(exp
.contents_equal(after
));