1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph distributed storage system
6 * Copyright (C) 2015 Mirantis, Inc.
8 * Author: Alyona Kiseleva <akiselyova@mirantis.com>
10 * This library is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU Lesser General Public
12 * License as published by the Free Software Foundation; either
13 * version 2.1 of the License, or (at your option) any later version.
20 #include "gtest/gtest.h"
21 #include "common/ceph_context.h"
22 #include "common/config.h"
23 #include "compressor/Compressor.h"
24 #include "compressor/CompressionPlugin.h"
25 #include "global/global_context.h"
26 #include "osd/OSDMap.h"
30 class CompressorTest
: public ::testing::Test
,
31 public ::testing::WithParamInterface
<const char*> {
34 CompressorRef compressor
;
39 old_zlib_isal
= g_conf()->compressor_zlib_isal
;
42 size_t pos
= plugin
.find('/');
43 if (pos
!= std::string::npos
) {
44 string isal
= plugin
.substr(pos
+ 1);
45 plugin
= plugin
.substr(0, pos
);
47 g_conf().set_val("compressor_zlib_isal", "true");
48 g_ceph_context
->_conf
.apply_changes(nullptr);
49 } else if (isal
== "noisal") {
50 g_conf().set_val("compressor_zlib_isal", "false");
51 g_ceph_context
->_conf
.apply_changes(nullptr);
53 ceph_abort_msg("bad option");
56 cout
<< "[plugin " << plugin
<< " (" << GetParam() << ")]" << std::endl
;
58 ~CompressorTest() override
{
59 g_conf().set_val("compressor_zlib_isal", old_zlib_isal
? "true" : "false");
60 g_ceph_context
->_conf
.apply_changes(nullptr);
63 void SetUp() override
{
64 compressor
= Compressor::create(g_ceph_context
, plugin
);
65 ASSERT_TRUE(compressor
);
67 void TearDown() override
{
72 TEST_P(CompressorTest
, load_plugin
)
76 TEST_P(CompressorTest
, small_round_trip
)
79 orig
.append("This is a short string. There are many strings like it but this one is mine.");
80 bufferlist compressed
;
81 std::optional
<int32_t> compressor_message
;
82 int r
= compressor
->compress(orig
, compressed
, compressor_message
);
84 bufferlist decompressed
;
85 r
= compressor
->decompress(compressed
, decompressed
, compressor_message
);
87 ASSERT_EQ(decompressed
.length(), orig
.length());
88 ASSERT_TRUE(decompressed
.contents_equal(orig
));
89 cout
<< "orig " << orig
.length() << " compressed " << compressed
.length()
90 << " with " << GetParam() << std::endl
;
93 TEST_P(CompressorTest
, big_round_trip_repeated
)
95 unsigned len
= 1048576 * 4;
97 while (orig
.length() < len
) {
98 orig
.append("This is a short string. There are many strings like it but this one is mine.");
100 bufferlist compressed
;
101 std::optional
<int32_t> compressor_message
;
102 int r
= compressor
->compress(orig
, compressed
, compressor_message
);
104 bufferlist decompressed
;
105 r
= compressor
->decompress(compressed
, decompressed
, compressor_message
);
107 ASSERT_EQ(decompressed
.length(), orig
.length());
108 ASSERT_TRUE(decompressed
.contents_equal(orig
));
109 cout
<< "orig " << orig
.length() << " compressed " << compressed
.length()
110 << " with " << GetParam() << std::endl
;
113 TEST_P(CompressorTest
, big_round_trip_randomish
)
115 unsigned len
= 1048576 * 10;//269;
117 const char *alphabet
= "abcdefghijklmnopqrstuvwxyz";
119 while (orig
.length() < len
) {
120 orig
.append(alphabet
[rand() % 10]);
124 char *p
= bp
.c_str();
125 for (unsigned i
=0; i
<len
; ++i
) {
126 p
[i
] = alphabet
[rand() % 10];
130 bufferlist compressed
;
131 std::optional
<int32_t> compressor_message
;
132 int r
= compressor
->compress(orig
, compressed
, compressor_message
);
134 bufferlist decompressed
;
135 r
= compressor
->decompress(compressed
, decompressed
, compressor_message
);
137 ASSERT_EQ(decompressed
.length(), orig
.length());
138 ASSERT_TRUE(decompressed
.contents_equal(orig
));
139 cout
<< "orig " << orig
.length() << " compressed " << compressed
.length()
140 << " with " << GetParam() << std::endl
;
144 TEST_P(CompressorTest
, big_round_trip_file
)
147 int fd
= ::open("bin/ceph-osd", O_RDONLY
);
150 orig
.read_fd(fd
, st
.st_size
);
152 bufferlist compressed
;
153 int r
= compressor
->compress(orig
, compressed
);
155 bufferlist decompressed
;
156 r
= compressor
->decompress(compressed
, decompressed
);
158 ASSERT_EQ(decompressed
.length(), orig
.length());
159 ASSERT_TRUE(decompressed
.contents_equal(orig
));
160 cout
<< "orig " << orig
.length() << " compressed " << compressed
.length()
161 << " with " << GetParam() << std::endl
;
166 TEST_P(CompressorTest
, round_trip_osdmap
)
168 #include "osdmaps/osdmap.2982809.h"
170 auto compressor
= Compressor::create(g_ceph_context
, plugin
);
172 orig
.append((char*)osdmap_a
, sizeof(osdmap_a
));
173 cout
<< "orig length " << orig
.length() << std::endl
;
174 uint32_t size
= 128*1024;
175 OSDMap
*o
= new OSDMap
;
178 o
->encode(fbl
, o
->get_encoding_features() | CEPH_FEATURE_RESERVED
);
179 ASSERT_TRUE(fbl
.contents_equal(orig
));
180 for (int j
= 0; j
< 3; j
++) {
182 uint32_t l
= std::min(size
, fbl
.length() - j
*size
);
183 chunk
.substr_of(fbl
, j
*size
, l
);
185 bufferlist compressed
;
186 std::optional
<int32_t> compressor_message
;
187 int r
= compressor
->compress(chunk
, compressed
, compressor_message
);
189 bufferlist decompressed
;
190 r
= compressor
->decompress(compressed
, decompressed
, compressor_message
);
192 ASSERT_EQ(decompressed
.length(), chunk
.length());
193 if (!decompressed
.contents_equal(chunk
)) {
194 cout
<< "FAILED, orig bl was\n" << fbl
<< std::endl
;
195 ASSERT_TRUE(decompressed
.contents_equal(chunk
));
197 cout
<< "chunk " << chunk
.length()
198 << " compressed " << compressed
.length()
199 << " decompressed " << decompressed
.length()
200 << " with " << plugin
<< std::endl
;
205 TEST_P(CompressorTest
, compress_decompress
)
207 const char* test
= "This is test text";
209 int len
= strlen(test
);
213 in
.append(test
, len
);
214 std::optional
<int32_t> compressor_message
;
215 res
= compressor
->compress(in
, out
, compressor_message
);
217 res
= compressor
->decompress(out
, after
, compressor_message
);
220 EXPECT_TRUE(exp
.contents_equal(after
));
222 size_t compressed_len
= out
.length();
224 auto it
= out
.cbegin();
225 res
= compressor
->decompress(it
, compressed_len
, after
, compressor_message
);
227 EXPECT_TRUE(exp
.contents_equal(after
));
229 //large block and non-begin iterator for continuous block
231 data
.resize(0x10000 * 1);
232 for(size_t i
= 0; i
< data
.size(); i
++)
238 res
= compressor
->compress(in
, out
, compressor_message
);
240 compressed_len
= out
.length();
241 out
.append_zero(0x10000 - out
.length());
245 prefix
.append(string("some prefix"));
246 size_t prefix_len
= prefix
.length();
247 prefix
.claim_append(out
);
251 res
= compressor
->decompress(it
, compressed_len
, after
, compressor_message
);
253 EXPECT_TRUE(exp
.contents_equal(after
));
256 TEST_P(CompressorTest
, sharded_input_decompress
)
258 const size_t small_prefix_size
=3;
260 string
test(128*1024,0);
261 int len
= test
.size();
263 in
.append(test
.c_str(), len
);
264 std::optional
<int32_t> compressor_message
;
265 int res
= compressor
->compress(in
, out
, compressor_message
);
267 EXPECT_GT(out
.length(), small_prefix_size
);
269 bufferlist out2
, tmp
;
270 tmp
.substr_of(out
, 0, small_prefix_size
);
272 size_t left
= out
.length()-small_prefix_size
;
273 size_t offs
= small_prefix_size
;
275 size_t shard_size
= std::min
<size_t>(2048, left
);
276 tmp
.substr_of(out
, offs
, shard_size
);
283 res
= compressor
->decompress(out2
, after
, compressor_message
);
287 void test_compress(CompressorRef compressor
, size_t size
)
289 char* data
= (char*) malloc(size
);
290 for (size_t t
= 0; t
< size
; t
++) {
291 data
[t
] = (t
& 0xff) | (t
>> 8);
294 in
.append(data
, size
);
295 for (size_t t
= 0; t
< 10000; t
++) {
297 std::optional
<int32_t> compressor_message
;
298 int res
= compressor
->compress(in
, out
, compressor_message
);
304 void test_decompress(CompressorRef compressor
, size_t size
)
306 char* data
= (char*) malloc(size
);
307 for (size_t t
= 0; t
< size
; t
++) {
308 data
[t
] = (t
& 0xff) | (t
>> 8);
311 in
.append(data
, size
);
312 std::optional
<int32_t> compressor_message
;
313 int res
= compressor
->compress(in
, out
, compressor_message
);
315 for (size_t t
= 0; t
< 10000; t
++) {
317 int res
= compressor
->decompress(out
, out_dec
, compressor_message
);
323 TEST_P(CompressorTest
, compress_1024
)
325 test_compress(compressor
, 1024);
328 TEST_P(CompressorTest
, compress_2048
)
330 test_compress(compressor
, 2048);
333 TEST_P(CompressorTest
, compress_4096
)
335 test_compress(compressor
, 4096);
338 TEST_P(CompressorTest
, compress_8192
)
340 test_compress(compressor
, 8192);
343 TEST_P(CompressorTest
, compress_16384
)
345 test_compress(compressor
, 16384);
348 TEST_P(CompressorTest
, decompress_1024
)
350 test_decompress(compressor
, 1024);
353 TEST_P(CompressorTest
, decompress_2048
)
355 test_decompress(compressor
, 2048);
358 TEST_P(CompressorTest
, decompress_4096
)
360 test_decompress(compressor
, 4096);
363 TEST_P(CompressorTest
, decompress_8192
)
365 test_decompress(compressor
, 8192);
368 TEST_P(CompressorTest
, decompress_16384
)
370 test_decompress(compressor
, 16384);
374 INSTANTIATE_TEST_SUITE_P(
381 #if defined(__x86_64__) || defined(__aarch64__)
391 #if defined(__x86_64__) || defined(__aarch64__)
393 TEST(ZlibCompressor
, zlib_isal_compatibility
)
395 g_conf().set_val("compressor_zlib_isal", "true");
396 g_ceph_context
->_conf
.apply_changes(nullptr);
397 CompressorRef isal
= Compressor::create(g_ceph_context
, "zlib");
399 // skip the test if the plugin is not ready
402 g_conf().set_val("compressor_zlib_isal", "false");
403 g_ceph_context
->_conf
.apply_changes(nullptr);
404 CompressorRef zlib
= Compressor::create(g_ceph_context
, "zlib");
407 for (int i
=0; i
<100; ++i
)
408 test
[i
] = 'a' + rand()%26;
410 int len
= strlen(test
);
412 in
.append(test
, len
);
414 std::optional
<int32_t> compressor_message
;
415 int res
= isal
->compress(in
, out
, compressor_message
);
418 res
= zlib
->decompress(out
, after
, compressor_message
);
421 exp
.append(static_cast<char*>(test
));
422 EXPECT_TRUE(exp
.contents_equal(after
));
427 res
= zlib
->compress(in
, out
, compressor_message
);
429 res
= isal
->decompress(out
, after
, compressor_message
);
431 exp
.append(static_cast<char*>(test
));
432 EXPECT_TRUE(exp
.contents_equal(after
));
436 TEST(CompressionPlugin
, all
)
438 CompressorRef compressor
;
439 PluginRegistry
*reg
= g_ceph_context
->get_plugin_registry();
441 CompressionPlugin
*factory
= dynamic_cast<CompressionPlugin
*>(reg
->get_with_load("compressor", "invalid"));
442 EXPECT_FALSE(factory
);
443 factory
= dynamic_cast<CompressionPlugin
*>(reg
->get_with_load("compressor", "example"));
444 ASSERT_TRUE(factory
);
446 EXPECT_EQ(0, factory
->factory(&compressor
, &ss
));
447 EXPECT_TRUE(compressor
.get());
449 std::lock_guard
l(reg
->lock
);
450 EXPECT_EQ(-ENOENT
, reg
->remove("compressor", "does not exist"));
451 EXPECT_EQ(0, reg
->remove("compressor", "example"));
452 EXPECT_EQ(0, reg
->load("compressor", "example"));
456 #if defined(__x86_64__) || defined(__aarch64__)
458 TEST(ZlibCompressor
, isal_compress_zlib_decompress_random
)
460 g_conf().set_val("compressor_zlib_isal", "true");
461 g_ceph_context
->_conf
.apply_changes(nullptr);
462 CompressorRef isal
= Compressor::create(g_ceph_context
, "zlib");
464 // skip the test if the plugin is not ready
467 g_conf().set_val("compressor_zlib_isal", "false");
468 g_ceph_context
->_conf
.apply_changes(nullptr);
469 CompressorRef zlib
= Compressor::create(g_ceph_context
, "zlib");
471 for (int cnt
=0; cnt
<100; cnt
++)
474 int log2
= (rand()%18) + 1;
475 int size
= (rand() % (1 << log2
)) + 1;
478 for (int i
=0; i
<size
; ++i
)
479 test
[i
] = rand()%256;
481 in
.append(test
, size
);
483 std::optional
<int32_t> compressor_message
;
484 int res
= isal
->compress(in
, out
, compressor_message
);
487 res
= zlib
->decompress(out
, after
, compressor_message
);
490 exp
.append(test
, size
);
491 EXPECT_TRUE(exp
.contents_equal(after
));
495 TEST(ZlibCompressor
, isal_compress_zlib_decompress_walk
)
497 g_conf().set_val("compressor_zlib_isal", "true");
498 g_ceph_context
->_conf
.apply_changes(nullptr);
499 CompressorRef isal
= Compressor::create(g_ceph_context
, "zlib");
501 // skip the test if the plugin is not ready
504 g_conf().set_val("compressor_zlib_isal", "false");
505 g_ceph_context
->_conf
.apply_changes(nullptr);
506 CompressorRef zlib
= Compressor::create(g_ceph_context
, "zlib");
508 for (int cnt
=0; cnt
<100; cnt
++)
511 int log2
= (rand()%18) + 1;
512 int size
= (rand() % (1 << log2
)) + 1;
517 test
[0] = rand()%256;
518 for (int i
=1; i
<size
; ++i
)
519 test
[i
] = test
[i
-1] + rand()%(range
*2+1) - range
;
521 in
.append(test
, size
);
523 std::optional
<int32_t> compressor_message
;
524 int res
= isal
->compress(in
, out
, compressor_message
);
527 res
= zlib
->decompress(out
, after
, compressor_message
);
530 exp
.append(test
, size
);
531 EXPECT_TRUE(exp
.contents_equal(after
));
538 TEST(QAT
, enc_qat_dec_noqat
) {
540 const char* alg_collection
[] = {"zlib", "lz4", "snappy"};
542 const char* alg_collection
[] = {"zlib", "snappy"};
544 for (auto alg
: alg_collection
) {
545 g_conf().set_val("qat_compressor_enabled", "true");
546 CompressorRef q
= Compressor::create(g_ceph_context
, alg
);
547 g_conf().set_val("qat_compressor_enabled", "false");
548 CompressorRef noq
= Compressor::create(g_ceph_context
, alg
);
550 // generate random buffer
551 for (int cnt
=0; cnt
<100; cnt
++) {
553 int log2
= (rand()%18) + 1;
554 int size
= (rand() % (1 << log2
)) + 1;
557 for (int i
=0; i
<size
; ++i
)
558 test
[i
] = rand()%256;
560 in
.append(test
, size
);
562 std::optional
<int32_t> compressor_message
;
563 int res
= q
->compress(in
, out
, compressor_message
);
566 res
= noq
->decompress(out
, after
, compressor_message
);
569 exp
.append(test
, size
);
570 EXPECT_TRUE(exp
.contents_equal(after
));
575 TEST(QAT
, enc_noqat_dec_qat
) {
577 const char* alg_collection
[] = {"zlib", "lz4", "snappy"};
579 const char* alg_collection
[] = {"zlib", "snappy"};
581 for (auto alg
: alg_collection
) {
582 g_conf().set_val("qat_compressor_enabled", "true");
583 CompressorRef q
= Compressor::create(g_ceph_context
, alg
);
584 g_conf().set_val("qat_compressor_enabled", "false");
585 CompressorRef noq
= Compressor::create(g_ceph_context
, alg
);
587 // generate random buffer
588 for (int cnt
=0; cnt
<100; cnt
++) {
590 int log2
= (rand()%18) + 1;
591 int size
= (rand() % (1 << log2
)) + 1;
594 for (int i
=0; i
<size
; ++i
)
595 test
[i
] = rand()%256;
597 in
.append(test
, size
);
599 std::optional
<int32_t> compressor_message
;
600 int res
= noq
->compress(in
, out
, compressor_message
);
603 res
= q
->decompress(out
, after
, compressor_message
);
606 exp
.append(test
, size
);
607 EXPECT_TRUE(exp
.contents_equal(after
));
612 #endif // HAVE_QATZIP