1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 #include "tools/rbd/ArgumentTypes.h"
5 #include "tools/rbd/Shell.h"
6 #include "tools/rbd/Utils.h"
7 #include "common/errno.h"
8 #include "common/strtol.h"
9 #include "common/Cond.h"
10 #include "common/Mutex.h"
12 #include <boost/accumulators/accumulators.hpp>
13 #include <boost/accumulators/statistics/stats.hpp>
14 #include <boost/accumulators/statistics/rolling_sum.hpp>
15 #include <boost/program_options.hpp>
21 namespace at
= argument_types
;
22 namespace po
= boost::program_options
;
37 void validate(boost::any
& v
, const std::vector
<std::string
>& values
,
38 Size
*target_type
, int) {
39 po::validators::check_first_occurrence(v
);
40 const std::string
&s
= po::validators::get_single_string(values
);
42 std::string parse_error
;
43 uint64_t size
= strict_iecstrtoll(s
.c_str(), &parse_error
);
44 if (!parse_error
.empty()) {
45 throw po::validation_error(po::validation_error::invalid_option_value
);
50 void validate(boost::any
& v
, const std::vector
<std::string
>& values
,
51 IOPattern
*target_type
, int) {
52 po::validators::check_first_occurrence(v
);
53 const std::string
&s
= po::validators::get_single_string(values
);
56 } else if (s
== "seq") {
57 v
= boost::any(false);
59 throw po::validation_error(po::validation_error::invalid_option_value
);
63 io_type_t
get_io_type(string io_type_string
) {
64 if (io_type_string
== "read")
66 else if (io_type_string
== "write")
72 void validate(boost::any
& v
, const std::vector
<std::string
>& values
,
73 IOType
*target_type
, int) {
74 po::validators::check_first_occurrence(v
);
75 const std::string
&s
= po::validators::get_single_string(values
);
76 io_type_t io_type
= get_io_type(s
);
77 if (io_type
>= IO_TYPE_NUM
)
78 throw po::validation_error(po::validation_error::invalid_option_value
);
80 v
= boost::any(io_type
);
83 } // anonymous namespace
85 static void rbd_bencher_completion(void *c
, void *pc
);
88 struct bencher_completer
{
93 bencher_completer(rbd_bencher
*bencher
, bufferlist
*bl
)
94 : bencher(bencher
), bl(bl
)
105 librbd::Image
*image
;
113 explicit rbd_bencher(librbd::Image
*i
, io_type_t io_type
, uint64_t io_size
)
115 lock("rbd_bencher::lock"),
120 if (io_type
== IO_TYPE_WRITE
) {
121 bufferptr
bp(io_size
);
122 memset(bp
.c_str(), rand() & 0xff, io_size
);
123 write_bl
.push_back(bp
);
128 bool start_io(int max
, uint64_t off
, uint64_t len
, int op_flags
)
131 Mutex::Locker
l(lock
);
132 if (in_flight
>= max
)
137 librbd::RBD::AioCompletion
*c
;
138 if (io_type
== IO_TYPE_READ
) {
139 bufferlist
*read_bl
= new bufferlist();
140 c
= new librbd::RBD::AioCompletion((void *)(new bencher_completer(this, read_bl
)),
141 rbd_bencher_completion
);
142 image
->aio_read2(off
, len
, *read_bl
, c
, op_flags
);
143 } else if (io_type
== IO_TYPE_WRITE
) {
144 c
= new librbd::RBD::AioCompletion((void *)(new bencher_completer(this, NULL
)),
145 rbd_bencher_completion
);
146 image
->aio_write2(off
, len
, write_bl
, c
, op_flags
);
148 assert(0 == "Invalid io_type");
150 //cout << "start " << c << " at " << off << "~" << len << std::endl;
154 void wait_for(int max
) {
155 Mutex::Locker
l(lock
);
156 while (in_flight
> max
) {
158 dur
.set_from_double(.2);
159 cond
.WaitInterval(lock
, dur
);
165 void rbd_bencher_completion(void *vc
, void *pc
)
167 librbd::RBD::AioCompletion
*c
= (librbd::RBD::AioCompletion
*)vc
;
168 bencher_completer
*bc
= static_cast<bencher_completer
*>(pc
);
169 rbd_bencher
*b
= bc
->bencher
;
170 //cout << "complete " << c << std::endl;
171 int ret
= c
->get_return_value();
172 if (b
->io_type
== IO_TYPE_WRITE
&& ret
!= 0) {
173 cout
<< "write error: " << cpp_strerror(ret
) << std::endl
;
174 exit(ret
< 0 ? -ret
: ret
);
175 } else if (b
->io_type
== IO_TYPE_READ
&& (unsigned int)ret
!= b
->io_size
) {
176 cout
<< "read error: " << cpp_strerror(ret
) << std::endl
;
177 exit(ret
< 0 ? -ret
: ret
);
187 int do_bench(librbd::Image
& image
, io_type_t io_type
,
188 uint64_t io_size
, uint64_t io_threads
,
189 uint64_t io_bytes
, bool random
)
193 if (io_size
> size
) {
194 std::cerr
<< "rbd: io-size " << byte_u_t(io_size
) << " "
195 << "larger than image size " << byte_u_t(size
) << std::endl
;
199 if (io_size
> std::numeric_limits
<uint32_t>::max()) {
200 std::cerr
<< "rbd: io-size should be less than 4G" << std::endl
;
204 rbd_bencher
b(&image
, io_type
, io_size
);
206 std::cout
<< "bench "
207 << " type " << (io_type
== IO_TYPE_READ
? "read" : "write")
208 << " io_size " << io_size
209 << " io_threads " << io_threads
210 << " bytes " << io_bytes
211 << " pattern " << (random
? "random" : "sequential")
214 srand(time(NULL
) % (unsigned long) -1);
216 utime_t start
= ceph_clock_now();
220 vector
<uint64_t> thread_offset
;
224 // disturb all thread's offset, used by seq IO
225 for (i
= 0; i
< io_threads
; i
++) {
226 start_pos
= (rand() % (size
/ io_size
)) * io_size
;
227 thread_offset
.push_back(start_pos
);
230 const int WINDOW_SIZE
= 5;
231 typedef boost::accumulators::accumulator_set
<
232 double, boost::accumulators::stats
<
233 boost::accumulators::tag::rolling_sum
> > RollingSum
;
236 boost::accumulators::tag::rolling_window::window_size
= WINDOW_SIZE
);
238 boost::accumulators::tag::rolling_window::window_size
= WINDOW_SIZE
);
240 boost::accumulators::tag::rolling_window::window_size
= WINDOW_SIZE
);
241 uint64_t cur_ios
= 0;
242 uint64_t cur_off
= 0;
246 op_flags
= LIBRADOS_OP_FLAG_FADVISE_RANDOM
;
248 op_flags
= LIBRADOS_OP_FLAG_FADVISE_SEQUENTIAL
;
251 printf(" SEC OPS OPS/SEC BYTES/SEC\n");
253 for (off
= 0; off
< io_bytes
; ) {
254 b
.wait_for(io_threads
- 1);
256 while (i
< io_threads
&& off
< io_bytes
) {
257 if (!b
.start_io(io_threads
, thread_offset
[i
], io_size
, op_flags
)) {
261 thread_offset
[i
] = (rand() % (size
/ io_size
)) * io_size
;
263 thread_offset
[i
] += io_size
;
264 if (thread_offset
[i
] + io_size
> size
)
265 thread_offset
[i
] = 0;
275 utime_t now
= ceph_clock_now();
276 utime_t elapsed
= now
- start
;
277 if (last
.is_zero()) {
279 } else if (elapsed
.sec() != last
.sec()) {
280 time_acc(elapsed
- last
);
281 ios_acc(static_cast<double>(cur_ios
));
282 off_acc(static_cast<double>(cur_off
));
286 double time_sum
= boost::accumulators::rolling_sum(time_acc
);
287 printf("%5d %8d %8.2lf %8.2lf\n",
289 (int)(ios
- io_threads
),
290 boost::accumulators::rolling_sum(ios_acc
) / time_sum
,
291 boost::accumulators::rolling_sum(off_acc
) / time_sum
);
296 int r
= image
.flush();
298 std::cerr
<< "Error flushing data at the end: " << cpp_strerror(r
)
302 utime_t now
= ceph_clock_now();
303 double elapsed
= now
- start
;
305 printf("elapsed: %5d ops: %8d ops/sec: %8.2lf bytes/sec: %8.2lf\n",
306 (int)elapsed
, ios
, (double)ios
/ elapsed
, (double)off
/ elapsed
);
311 void add_bench_common_options(po::options_description
*positional
,
312 po::options_description
*options
) {
313 at::add_image_spec_options(positional
, options
, at::ARGUMENT_MODIFIER_NONE
);
315 options
->add_options()
316 ("io-size", po::value
<Size
>(), "IO size (in B/K/M/G/T) [default: 4K]")
317 ("io-threads", po::value
<uint32_t>(), "ios in flight [default: 16]")
318 ("io-total", po::value
<Size
>(), "total size for IO (in B/K/M/G/T) [default: 1G]")
319 ("io-pattern", po::value
<IOPattern
>(), "IO pattern (rand or seq) [default: seq]");
322 void get_arguments_for_write(po::options_description
*positional
,
323 po::options_description
*options
) {
324 add_bench_common_options(positional
, options
);
327 void get_arguments_for_bench(po::options_description
*positional
,
328 po::options_description
*options
) {
329 add_bench_common_options(positional
, options
);
331 options
->add_options()
332 ("io-type", po::value
<IOType
>()->required(), "IO type (read or write)");
335 int bench_execute(const po::variables_map
&vm
, io_type_t bench_io_type
) {
336 size_t arg_index
= 0;
337 std::string pool_name
;
338 std::string image_name
;
339 std::string snap_name
;
340 utils::SnapshotPresence snap_presence
= utils::SNAPSHOT_PRESENCE_NONE
;
341 if (bench_io_type
== IO_TYPE_READ
)
342 snap_presence
= utils::SNAPSHOT_PRESENCE_PERMITTED
;
344 int r
= utils::get_pool_image_snapshot_names(
345 vm
, at::ARGUMENT_MODIFIER_NONE
, &arg_index
, &pool_name
, &image_name
,
346 &snap_name
, snap_presence
, utils::SPEC_VALIDATION_NONE
);
351 uint64_t bench_io_size
;
352 if (vm
.count("io-size")) {
353 bench_io_size
= vm
["io-size"].as
<uint64_t>();
355 bench_io_size
= 4096;
357 if (bench_io_size
== 0) {
358 std::cerr
<< "rbd: --io-size should be greater than zero." << std::endl
;
362 uint32_t bench_io_threads
;
363 if (vm
.count("io-threads")) {
364 bench_io_threads
= vm
["io-threads"].as
<uint32_t>();
366 bench_io_threads
= 16;
368 if (bench_io_threads
== 0) {
369 std::cerr
<< "rbd: --io-threads should be greater than zero." << std::endl
;
373 uint64_t bench_bytes
;
374 if (vm
.count("io-total")) {
375 bench_bytes
= vm
["io-total"].as
<uint64_t>();
377 bench_bytes
= 1 << 30;
381 if (vm
.count("io-pattern")) {
382 bench_random
= vm
["io-pattern"].as
<bool>();
384 bench_random
= false;
387 librados::Rados rados
;
388 librados::IoCtx io_ctx
;
390 r
= utils::init_and_open_image(pool_name
, image_name
, "", "", false, &rados
,
396 r
= do_bench(image
, bench_io_type
, bench_io_size
, bench_io_threads
,
397 bench_bytes
, bench_random
);
399 std::cerr
<< "bench failed: " << cpp_strerror(r
) << std::endl
;
405 int execute_for_write(const po::variables_map
&vm
) {
406 std::cerr
<< "rbd: bench-write is deprecated, use rbd bench --io-type write ..." << std::endl
;
407 return bench_execute(vm
, IO_TYPE_WRITE
);
410 int execute_for_bench(const po::variables_map
&vm
) {
411 io_type_t bench_io_type
;
412 if (vm
.count("io-type")) {
413 bench_io_type
= vm
["io-type"].as
<io_type_t
>();
415 std::cerr
<< "rbd: --io-type must be specified." << std::endl
;
419 return bench_execute(vm
, bench_io_type
);
422 Shell::Action
action_write(
423 {"bench-write"}, {}, "Simple write benchmark. (Deprecated, please use `rbd bench --io-type write` instead.)",
424 "", &get_arguments_for_write
, &execute_for_write
, false);
426 Shell::Action
action_bench(
427 {"bench"}, {}, "Simple benchmark.", "", &get_arguments_for_bench
, &execute_for_bench
);
430 } // namespace action