1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph distributed storage system
6 * Copyright (C) 2015 Mirantis, Inc.
8 * Author: Alyona Kiseleva <akiselyova@mirantis.com>
10 * This library is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU Lesser General Public
12 * License as published by the Free Software Foundation; either
13 * version 2.1 of the License, or (at your option) any later version.
20 #include "gtest/gtest.h"
21 #include "common/config.h"
22 #include "compressor/Compressor.h"
23 #include "compressor/CompressionPlugin.h"
24 #include "global/global_context.h"
26 class CompressorTest
: public ::testing::Test
,
27 public ::testing::WithParamInterface
<const char*> {
30 CompressorRef compressor
;
35 old_zlib_isal
= g_conf
->compressor_zlib_isal
;
38 size_t pos
= plugin
.find('/');
39 if (pos
!= std::string::npos
) {
40 string isal
= plugin
.substr(pos
+ 1);
41 plugin
= plugin
.substr(0, pos
);
43 g_conf
->set_val("compressor_zlib_isal", "true");
44 g_ceph_context
->_conf
->apply_changes(NULL
);
45 } else if (isal
== "noisal") {
46 g_conf
->set_val("compressor_zlib_isal", "false");
47 g_ceph_context
->_conf
->apply_changes(NULL
);
49 assert(0 == "bad option");
52 cout
<< "[plugin " << plugin
<< " (" << GetParam() << ")]" << std::endl
;
54 ~CompressorTest() override
{
55 g_conf
->set_val("compressor_zlib_isal", old_zlib_isal
? "true" : "false");
56 g_ceph_context
->_conf
->apply_changes(NULL
);
59 void SetUp() override
{
60 compressor
= Compressor::create(g_ceph_context
, plugin
);
61 ASSERT_TRUE(compressor
);
63 void TearDown() override
{
68 TEST_P(CompressorTest
, load_plugin
)
72 TEST_P(CompressorTest
, small_round_trip
)
75 orig
.append("This is a short string. There are many strings like it but this one is mine.");
76 bufferlist compressed
;
77 int r
= compressor
->compress(orig
, compressed
);
79 bufferlist decompressed
;
80 r
= compressor
->decompress(compressed
, decompressed
);
82 ASSERT_EQ(decompressed
.length(), orig
.length());
83 ASSERT_TRUE(decompressed
.contents_equal(orig
));
84 cout
<< "orig " << orig
.length() << " compressed " << compressed
.length()
85 << " with " << GetParam() << std::endl
;
88 TEST_P(CompressorTest
, big_round_trip_repeated
)
90 unsigned len
= 1048576 * 4;
92 while (orig
.length() < len
) {
93 orig
.append("This is a short string. There are many strings like it but this one is mine.");
95 bufferlist compressed
;
96 int r
= compressor
->compress(orig
, compressed
);
98 bufferlist decompressed
;
99 r
= compressor
->decompress(compressed
, decompressed
);
101 ASSERT_EQ(decompressed
.length(), orig
.length());
102 ASSERT_TRUE(decompressed
.contents_equal(orig
));
103 cout
<< "orig " << orig
.length() << " compressed " << compressed
.length()
104 << " with " << GetParam() << std::endl
;
107 TEST_P(CompressorTest
, big_round_trip_randomish
)
109 unsigned len
= 1048576 * 100;//269;
111 const char *alphabet
= "abcdefghijklmnopqrstuvwxyz";
113 while (orig
.length() < len
) {
114 orig
.append(alphabet
[rand() % 10]);
118 char *p
= bp
.c_str();
119 for (unsigned i
=0; i
<len
; ++i
) {
120 p
[i
] = alphabet
[rand() % 10];
124 bufferlist compressed
;
125 int r
= compressor
->compress(orig
, compressed
);
127 bufferlist decompressed
;
128 r
= compressor
->decompress(compressed
, decompressed
);
130 ASSERT_EQ(decompressed
.length(), orig
.length());
131 ASSERT_TRUE(decompressed
.contents_equal(orig
));
132 cout
<< "orig " << orig
.length() << " compressed " << compressed
.length()
133 << " with " << GetParam() << std::endl
;
137 TEST_P(CompressorTest
, big_round_trip_file
)
140 int fd
= ::open("bin/ceph-osd", O_RDONLY
);
143 orig
.read_fd(fd
, st
.st_size
);
145 bufferlist compressed
;
146 int r
= compressor
->compress(orig
, compressed
);
148 bufferlist decompressed
;
149 r
= compressor
->decompress(compressed
, decompressed
);
151 ASSERT_EQ(decompressed
.length(), orig
.length());
152 ASSERT_TRUE(decompressed
.contents_equal(orig
));
153 cout
<< "orig " << orig
.length() << " compressed " << compressed
.length()
154 << " with " << GetParam() << std::endl
;
159 TEST_P(CompressorTest
, compress_decompress
)
161 const char* test
= "This is test text";
163 int len
= strlen(test
);
167 in
.append(test
, len
);
168 res
= compressor
->compress(in
, out
);
170 res
= compressor
->decompress(out
, after
);
173 EXPECT_TRUE(exp
.contents_equal(after
));
175 size_t compressed_len
= out
.length();
177 auto it
= out
.begin();
178 res
= compressor
->decompress(it
, compressed_len
, after
);
180 EXPECT_TRUE(exp
.contents_equal(after
));
182 //large block and non-begin iterator for continuous block
184 data
.resize(0x10000 * 1);
185 for(size_t i
= 0; i
< data
.size(); i
++)
191 res
= compressor
->compress(in
, out
);
193 compressed_len
= out
.length();
194 out
.append_zero(0x10000 - out
.length());
198 prefix
.append(string("some prefix"));
199 size_t prefix_len
= prefix
.length();
200 out
.claim_prepend(prefix
);
202 it
.advance(prefix_len
);
203 res
= compressor
->decompress(it
, compressed_len
, after
);
205 EXPECT_TRUE(exp
.contents_equal(after
));
208 TEST_P(CompressorTest
, sharded_input_decompress
)
210 const size_t small_prefix_size
=3;
212 string
test(128*1024,0);
213 int len
= test
.size();
215 in
.append(test
.c_str(), len
);
216 int res
= compressor
->compress(in
, out
);
218 EXPECT_GT(out
.length(), small_prefix_size
);
220 bufferlist out2
, tmp
;
221 tmp
.substr_of(out
, 0, small_prefix_size
);
223 size_t left
= out
.length()-small_prefix_size
;
224 size_t offs
= small_prefix_size
;
226 size_t shard_size
= MIN( 2048, left
);
227 tmp
.substr_of(out
, offs
, shard_size
);
234 res
= compressor
->decompress(out2
, after
);
238 void test_compress(CompressorRef compressor
, size_t size
)
240 char* data
= (char*) malloc(size
);
241 for (size_t t
= 0; t
< size
; t
++) {
242 data
[t
] = (t
& 0xff) | (t
>> 8);
245 in
.append(data
, size
);
246 for (size_t t
= 0; t
< 100000; t
++) {
248 int res
= compressor
->compress(in
, out
);
253 void test_decompress(CompressorRef compressor
, size_t size
)
255 char* data
= (char*) malloc(size
);
256 for (size_t t
= 0; t
< size
; t
++) {
257 data
[t
] = (t
& 0xff) | (t
>> 8);
260 in
.append(data
, size
);
261 int res
= compressor
->compress(in
, out
);
263 for (size_t t
= 0; t
< 100000; t
++) {
265 int res
= compressor
->decompress(out
, out_dec
);
270 TEST_P(CompressorTest
, compress_1024
)
272 test_compress(compressor
, 1024);
275 TEST_P(CompressorTest
, compress_2048
)
277 test_compress(compressor
, 2048);
280 TEST_P(CompressorTest
, compress_4096
)
282 test_compress(compressor
, 4096);
285 TEST_P(CompressorTest
, compress_8192
)
287 test_compress(compressor
, 8192);
290 TEST_P(CompressorTest
, compress_16384
)
292 test_compress(compressor
, 16384);
295 TEST_P(CompressorTest
, decompress_1024
)
297 test_decompress(compressor
, 1024);
300 TEST_P(CompressorTest
, decompress_2048
)
302 test_decompress(compressor
, 2048);
305 TEST_P(CompressorTest
, decompress_4096
)
307 test_decompress(compressor
, 4096);
310 TEST_P(CompressorTest
, decompress_8192
)
312 test_decompress(compressor
, 8192);
315 TEST_P(CompressorTest
, decompress_16384
)
317 test_decompress(compressor
, 16384);
321 INSTANTIATE_TEST_CASE_P(
334 TEST(ZlibCompressor
, zlib_isal_compatibility
)
336 g_conf
->set_val("compressor_zlib_isal", "true");
337 g_ceph_context
->_conf
->apply_changes(NULL
);
338 CompressorRef isal
= Compressor::create(g_ceph_context
, "zlib");
340 // skip the test if the plugin is not ready
343 g_conf
->set_val("compressor_zlib_isal", "false");
344 g_ceph_context
->_conf
->apply_changes(NULL
);
345 CompressorRef zlib
= Compressor::create(g_ceph_context
, "zlib");
348 for (int i
=0; i
<100; ++i
)
349 test
[i
] = 'a' + rand()%26;
351 int len
= strlen(test
);
353 in
.append(test
, len
);
355 int res
= isal
->compress(in
, out
);
358 res
= zlib
->decompress(out
, after
);
362 EXPECT_TRUE(exp
.contents_equal(after
));
367 res
= zlib
->compress(in
, out
);
369 res
= isal
->decompress(out
, after
);
372 EXPECT_TRUE(exp
.contents_equal(after
));
376 TEST(CompressionPlugin
, all
)
378 const char* env
= getenv("CEPH_LIB");
379 std::string
directory(env
? env
: ".libs");
380 CompressorRef compressor
;
381 PluginRegistry
*reg
= g_ceph_context
->get_plugin_registry();
383 CompressionPlugin
*factory
= dynamic_cast<CompressionPlugin
*>(reg
->get_with_load("compressor", "invalid"));
384 EXPECT_FALSE(factory
);
385 factory
= dynamic_cast<CompressionPlugin
*>(reg
->get_with_load("compressor", "example"));
386 EXPECT_TRUE(factory
);
388 EXPECT_EQ(0, factory
->factory(&compressor
, &ss
));
389 EXPECT_TRUE(compressor
.get());
391 Mutex::Locker
l(reg
->lock
);
392 EXPECT_EQ(-ENOENT
, reg
->remove("compressor", "does not exist"));
393 EXPECT_EQ(0, reg
->remove("compressor", "example"));
394 EXPECT_EQ(0, reg
->load("compressor", "example"));
400 TEST(ZlibCompressor
, isal_compress_zlib_decompress_random
)
402 g_conf
->set_val("compressor_zlib_isal", "true");
403 g_ceph_context
->_conf
->apply_changes(NULL
);
404 CompressorRef isal
= Compressor::create(g_ceph_context
, "zlib");
406 // skip the test if the plugin is not ready
409 g_conf
->set_val("compressor_zlib_isal", "false");
410 g_ceph_context
->_conf
->apply_changes(NULL
);
411 CompressorRef zlib
= Compressor::create(g_ceph_context
, "zlib");
413 for (int cnt
=0; cnt
<1000; cnt
++)
416 int log2
= (rand()%18) + 1;
417 int size
= (rand() % (1 << log2
)) + 1;
420 for (int i
=0; i
<size
; ++i
)
421 test
[i
] = rand()%256;
423 in
.append(test
, size
);
425 int res
= isal
->compress(in
, out
);
428 res
= zlib
->decompress(out
, after
);
431 exp
.append(test
, size
);
432 EXPECT_TRUE(exp
.contents_equal(after
));
436 TEST(ZlibCompressor
, isal_compress_zlib_decompress_walk
)
438 g_conf
->set_val("compressor_zlib_isal", "true");
439 g_ceph_context
->_conf
->apply_changes(NULL
);
440 CompressorRef isal
= Compressor::create(g_ceph_context
, "zlib");
442 // skip the test if the plugin is not ready
445 g_conf
->set_val("compressor_zlib_isal", "false");
446 g_ceph_context
->_conf
->apply_changes(NULL
);
447 CompressorRef zlib
= Compressor::create(g_ceph_context
, "zlib");
449 for (int cnt
=0; cnt
<1000; cnt
++)
452 int log2
= (rand()%18) + 1;
453 int size
= (rand() % (1 << log2
)) + 1;
458 test
[0] = rand()%256;
459 for (int i
=1; i
<size
; ++i
)
460 test
[i
] = test
[i
-1] + rand()%(range
*2+1) - range
;
462 in
.append(test
, size
);
464 int res
= isal
->compress(in
, out
);
467 res
= zlib
->decompress(out
, after
);
470 exp
.append(test
, size
);
471 EXPECT_TRUE(exp
.contents_equal(after
));