]>
git.proxmox.com Git - ceph.git/blob - ceph/src/test/lazy-omap-stats/lazy_omap_stats_test.cc
1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2019 Red Hat
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
17 #include <boost/algorithm/string/trim.hpp>
18 #include <boost/process.hpp>
19 #include <boost/tokenizer.hpp>
20 #include <boost/uuid/uuid.hpp> // uuid class
21 #include <boost/uuid/uuid_generators.hpp> // generators
22 #include <boost/uuid/uuid_io.hpp> // streaming operators etc.
28 #include "lazy_omap_stats_test.h"
29 #include "include/compat.h"
32 namespace bp
= boost::process
;
34 void LazyOmapStatsTest::init(const int argc
, const char** argv
)
36 int ret
= rados
.init("admin");
39 cerr
<< "Failed to initialise rados! Error: " << ret
<< " " << strerror(ret
)
44 ret
= rados
.conf_parse_argv(argc
, argv
);
47 cerr
<< "Failed to parse command line config options! Error: " << ret
<< " "
48 << strerror(ret
) << endl
;
52 rados
.conf_parse_env(NULL
);
55 cerr
<< "Failed to parse environment! Error: " << ret
<< " "
56 << strerror(ret
) << endl
;
60 rados
.conf_read_file(NULL
);
63 cerr
<< "Failed to read config file! Error: " << ret
<< " " << strerror(ret
)
68 ret
= rados
.connect();
71 cerr
<< "Failed to connect to running cluster! Error: " << ret
<< " "
72 << strerror(ret
) << endl
;
78 "prefix
": "osd pool create
",
79 "pool
": ")" + conf.pool_name +
81 "pool_type
": "replicated
",
82 "size
": )" + to_string(conf
.replica_count
) +
85 librados::bufferlist inbl
;
87 ret
= rados
.mon_command(command
, inbl
, nullptr, &output
);
88 if (output
.length()) cout
<< output
<< endl
;
91 cerr
<< "Failed to create pool! Error: " << ret
<< " " << strerror(ret
)
96 ret
= rados
.ioctx_create(conf
.pool_name
.c_str(), io_ctx
);
99 cerr
<< "Failed to create ioctx! Error: " << ret
<< " " << strerror(ret
)
105 void LazyOmapStatsTest::shutdown()
107 rados
.pool_delete(conf
.pool_name
.c_str());
111 void LazyOmapStatsTest::write_omap(const string
& object_name
)
113 librados::bufferlist bl
;
114 int ret
= io_ctx
.write_full(object_name
, bl
);
117 cerr
<< "Failed to create object! Error: " << ret
<< " " << strerror(ret
)
121 ret
= io_ctx
.omap_set(object_name
, payload
);
124 cerr
<< "Failed to write omap payload! Error: " << ret
<< " "
125 << strerror(ret
) << endl
;
128 cout
<< "Wrote " << conf
.keys
<< " omap keys of " << conf
.payload_size
130 << "the " << object_name
<< " object" << endl
;
133 const string
LazyOmapStatsTest::get_name() const
135 boost::uuids::uuid uuid
= boost::uuids::random_generator()();
136 return boost::uuids::to_string(uuid
);
139 void LazyOmapStatsTest::write_many(uint how_many
)
141 for (uint i
= 0; i
< how_many
; i
++) {
142 write_omap(get_name());
146 void LazyOmapStatsTest::create_payload()
148 librados::bufferlist Lorem
;
150 "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do "
151 "eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut "
152 "enim ad minim veniam, quis nostrud exercitation ullamco laboris "
153 "nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in "
154 "reprehenderit in voluptate velit esse cillum dolore eu fugiat "
155 "nulla pariatur. Excepteur sint occaecat cupidatat non proident, "
156 "sunt in culpa qui officia deserunt mollit anim id est laborum.");
157 conf
.payload_size
= Lorem
.length();
158 conf
.total_bytes
= conf
.keys
* conf
.payload_size
* conf
.how_many
;
159 conf
.total_keys
= conf
.keys
* conf
.how_many
;
161 for (i
= 1; i
< conf
.keys
+ 1; ++i
) {
162 payload
[get_name()] = Lorem
;
164 cout
<< "Created payload with " << conf
.keys
<< " keys of "
166 << " bytes each. Total size in bytes = " << conf
.keys
* conf
.payload_size
170 void LazyOmapStatsTest::scrub() const
172 // Use CLI because we need to block
174 cout
<< "Scrubbing" << endl
;
177 bp::child
c("ceph osd deep-scrub all --block");
180 cout
<< "Deep scrub command failed! Error: " << ec
.value() << " "
181 << ec
.message() << endl
;
186 const int LazyOmapStatsTest::find_matches(string
& output
, regex
& reg
) const
188 sregex_iterator
cur(output
.begin(), output
.end(), reg
);
190 for (auto end
= std::sregex_iterator(); cur
!= end
; ++cur
) {
191 cout
<< (*cur
)[1].str() << endl
;
197 const string
LazyOmapStatsTest::get_output(const string command
,
200 librados::bufferlist inbl
, outbl
;
202 int ret
= rados
.mgr_command(command
, inbl
, &outbl
, &output
);
203 if (output
.length() && !silent
) {
204 cout
<< output
<< endl
;
208 cerr
<< "Failed to get " << command
<< "! Error: " << ret
<< " "
209 << strerror(ret
) << endl
;
212 return string(outbl
.c_str(), outbl
.length());
215 void LazyOmapStatsTest::check_one()
217 string full_output
= get_output();
218 cout
<< full_output
<< endl
;
222 "\n)OSD_STAT"); // Strip OSD_STAT table so we don't find matches there
224 regex_search(full_output
, match
, reg
);
225 auto truncated_output
= match
[1].str();
226 cout
<< truncated_output
<< endl
;
230 to_string(conf
.keys
) +
234 cout
<< "Checking number of keys " << conf
.keys
<< endl
;
235 cout
<< "Found the following lines" << endl
;
236 cout
<< "*************************" << endl
;
237 uint result
= find_matches(truncated_output
, reg
);
238 cout
<< "**********************" << endl
;
239 cout
<< "Found " << result
<< " matching line(s)" << endl
;
245 to_string(conf
.payload_size
* conf
.keys
) +
248 cout
<< "Checking number of bytes "
249 << conf
.payload_size
* conf
.keys
<< endl
;
250 cout
<< "Found the following lines" << endl
;
251 cout
<< "*************************" << endl
;
252 result
= find_matches(truncated_output
, reg
);
253 cout
<< "**********************" << endl
;
254 cout
<< "Found " << result
<< " matching line(s)" << endl
;
258 cout
<< "Error: Found " << total
<< " matches, expected 6! Exiting..."
262 cout
<< "check_one successful. Found " << total
<< " matches as expected"
266 const int LazyOmapStatsTest::find_index(string
& haystack
, regex
& needle
,
270 regex_search(haystack
, match
, needle
);
271 auto line
= match
[1].str();
272 boost::algorithm::trim(line
);
273 boost::char_separator
<char> sep
{" "};
274 boost::tokenizer
<boost::char_separator
<char>> tok(line
, sep
);
275 vector
<string
> tokens(tok
.begin(), tok
.end());
276 auto it
= find(tokens
.begin(), tokens
.end(), label
);
277 if (it
!= tokens
.end()) {
278 return distance(tokens
.begin(), it
);
281 cerr
<< "find_index failed to find index for " << label
<< endl
;
283 return -1; // Unreachable
286 const uint
LazyOmapStatsTest::tally_column(const uint omap_bytes_index
,
290 istringstream
buffer(table
);
293 while (std::getline(buffer
, line
)) {
298 boost::char_separator
<char> sep
{" "};
299 boost::tokenizer
<boost::char_separator
<char>> tok(line
, sep
);
300 vector
<string
> tokens(tok
.begin(), tok
.end());
301 total
+= stoi(tokens
.at(omap_bytes_index
));
307 void LazyOmapStatsTest::check_column(const int index
, const string
& table
,
308 const string
& type
, bool header
) const
312 if (type
.compare("bytes") == 0) {
313 expected
= conf
.total_bytes
;
314 errormsg
= "Error. Got unexpected byte count!";
316 expected
= conf
.total_keys
;
317 errormsg
= "Error. Got unexpected key count!";
319 uint sum
= tally_column(index
, table
, header
);
320 cout
<< "Got: " << sum
<< " Expected: " << expected
<< endl
;
321 if (sum
!= expected
) {
322 cout
<< errormsg
<< endl
;
327 index_t
LazyOmapStatsTest::get_indexes(regex
& reg
, string
& output
) const
330 indexes
.byte_index
= find_index(output
, reg
, "OMAP_BYTES*");
331 indexes
.key_index
= find_index(output
, reg
, "OMAP_KEYS*");
336 const string
LazyOmapStatsTest::get_pool_id(string
& pool
)
338 cout
<< R
"(Querying pool id)" << endl
;
340 string command
= R
"({"prefix
": "osd pool ls
", "detail
": "detail
"})";
341 librados::bufferlist inbl
, outbl
;
343 int ret
= rados
.mon_command(command
, inbl
, &outbl
, &output
);
344 if (output
.length()) cout
<< output
<< endl
;
347 cerr
<< "Failed to get pool id! Error: " << ret
<< " " << strerror(ret
)
351 string
dump_output(outbl
.c_str(), outbl
.length());
352 cout
<< dump_output
<< endl
;
354 string poolregstring
= R
"(pool\s(\d+)\s')" + pool
+ "'";
355 regex
reg(poolregstring
);
357 regex_search(dump_output
, match
, reg
);
358 auto pool_id
= match
[1].str();
359 cout
<< "Found pool ID: " << pool_id
<< endl
;
364 void LazyOmapStatsTest::check_pg_dump()
366 cout
<< R
"(Checking "pg dump
" output)" << endl
;
368 string dump_output
= get_output();
369 cout
<< dump_output
<< endl
;
375 index_t indexes
= get_indexes(reg
, dump_output
);
379 R
"((PG_STAT[\s\S]*))"
382 regex_search(dump_output
, match
, reg
);
383 auto table
= match
[1].str();
385 cout
<< "Checking bytes" << endl
;
386 check_column(indexes
.byte_index
, table
, string("bytes"));
388 cout
<< "Checking keys" << endl
;
389 check_column(indexes
.key_index
, table
, string("keys"));
394 void LazyOmapStatsTest::check_pg_dump_summary()
396 cout
<< R
"(Checking "pg dump summary
" output)" << endl
;
398 string command
= R
"({"prefix
": "pg dump
", "dumpcontents
": ["summary
"]})";
399 string dump_output
= get_output(command
);
400 cout
<< dump_output
<< endl
;
406 index_t indexes
= get_indexes(reg
, dump_output
);
413 regex_search(dump_output
, match
, reg
);
414 auto table
= match
[1].str();
416 cout
<< "Checking bytes" << endl
;
417 check_column(indexes
.byte_index
, table
, string("bytes"), false);
419 cout
<< "Checking keys" << endl
;
420 check_column(indexes
.key_index
, table
, string("keys"), false);
424 void LazyOmapStatsTest::check_pg_dump_pgs()
426 cout
<< R
"(Checking "pg dump pgs
" output)" << endl
;
428 string command
= R
"({"prefix
": "pg dump
", "dumpcontents
": ["pgs
"]})";
429 string dump_output
= get_output(command
);
430 cout
<< dump_output
<< endl
;
432 regex
reg(R
"(^(PG_STAT\s.*))"
434 index_t indexes
= get_indexes(reg
, dump_output
);
436 reg
= R
"(^(PG_STAT[\s\S]*))"
439 regex_search(dump_output
, match
, reg
);
440 auto table
= match
[1].str();
442 cout
<< "Checking bytes" << endl
;
443 check_column(indexes
.byte_index
, table
, string("bytes"));
445 cout
<< "Checking keys" << endl
;
446 check_column(indexes
.key_index
, table
, string("keys"));
450 void LazyOmapStatsTest::check_pg_dump_pools()
452 cout
<< R
"(Checking "pg dump pools
" output)" << endl
;
454 string command
= R
"({"prefix
": "pg dump
", "dumpcontents
": ["pools
"]})";
455 string dump_output
= get_output(command
);
456 cout
<< dump_output
<< endl
;
458 regex
reg(R
"(^(POOLID\s.*))"
460 index_t indexes
= get_indexes(reg
, dump_output
);
462 auto pool_id
= get_pool_id(conf
.pool_name
);
471 regex_search(dump_output
, match
, reg
);
472 auto line
= match
[1].str();
474 cout
<< "Checking bytes" << endl
;
475 check_column(indexes
.byte_index
, line
, string("bytes"), false);
477 cout
<< "Checking keys" << endl
;
478 check_column(indexes
.key_index
, line
, string("keys"), false);
482 void LazyOmapStatsTest::check_pg_ls()
484 cout
<< R
"(Checking "pg ls
" output)" << endl
;
486 string command
= R
"({"prefix
": "pg ls
"})";
487 string dump_output
= get_output(command
);
488 cout
<< dump_output
<< endl
;
490 regex
reg(R
"(^(PG\s.*))"
492 index_t indexes
= get_indexes(reg
, dump_output
);
494 reg
= R
"(^(PG[\s\S]*))"
497 regex_search(dump_output
, match
, reg
);
498 auto table
= match
[1].str();
500 cout
<< "Checking bytes" << endl
;
501 check_column(indexes
.byte_index
, table
, string("bytes"));
503 cout
<< "Checking keys" << endl
;
504 check_column(indexes
.key_index
, table
, string("keys"));
508 void LazyOmapStatsTest::wait_for_active_clean()
510 cout
<< "Waiting for active+clean" << endl
;
515 R
"((PG_STAT[\s\S]*))"
517 string command
= R
"({"prefix
": "pg dump
"})";
520 string dump_output
= get_output(command
, true);
526 index
= find_index(dump_output
, ireg
, "STATE");
529 regex_search(dump_output
, match
, reg
);
530 istringstream
buffer(match
[1].str());
533 while (std::getline(buffer
, line
)) {
534 if (line
.compare(0, 1, "P") == 0) continue;
535 boost::char_separator
<char> sep
{" "};
536 boost::tokenizer
<boost::char_separator
<char>> tok(line
, sep
);
537 vector
<string
> tokens(tok
.begin(), tok
.end());
538 num_not_clean
+= tokens
.at(index
).compare("active+clean");
540 cout
<< "." << flush
;
541 this_thread::sleep_for(chrono::milliseconds(250));
542 } while (num_not_clean
);
547 const int LazyOmapStatsTest::run(const int argc
, const char** argv
)
551 wait_for_active_clean();
552 write_omap(get_name());
556 write_many(conf
.how_many
- 1); // Since we already wrote one
559 check_pg_dump_summary();
561 check_pg_dump_pools();
563 cout
<< "All tests passed. Success!" << endl
;