]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | /* |
2 | * KvStoreBench.cc | |
3 | * | |
4 | * Created on: Aug 23, 2012 | |
5 | * Author: eleanor | |
6 | */ | |
7 | ||
8 | #include "test/kv_store_bench.h" | |
9 | #include "key_value_store/key_value_structure.h" | |
10 | #include "key_value_store/kv_flat_btree_async.h" | |
11 | #include "include/rados/librados.hpp" | |
12 | #include "test/omap_bench.h" | |
13 | #include "common/ceph_argparse.h" | |
14 | ||
15 | ||
16 | #include <string> | |
17 | #include <climits> | |
18 | #include <iostream> | |
19 | #include <sstream> | |
20 | #include <cmath> | |
21 | ||
22 | KvStoreBench::KvStoreBench() | |
23 | : entries(30), | |
24 | ops(100), | |
25 | clients(5), | |
26 | key_size(5), | |
27 | val_size(7), | |
28 | max_ops_in_flight(8), | |
29 | clear_first(false), | |
30 | k(2), | |
31 | cache_size(10), | |
32 | cache_refresh(1), | |
33 | client_name("admin"), | |
34 | verbose(false), | |
35 | kvs(NULL), | |
7c673cae | 36 | ops_in_flight(0), |
7c673cae FG |
37 | rados_id("admin"), |
38 | pool_name("rbd"), | |
39 | io_ctx_ready(false) | |
40 | { | |
41 | probs[25] = 'i'; | |
42 | probs[50] = 'u'; | |
43 | probs[75] = 'd'; | |
44 | probs[100] = 'r'; | |
45 | } | |
46 | ||
47 | KvStoreBench::~KvStoreBench() | |
48 | { | |
49 | if (io_ctx_ready) { | |
50 | librados::ObjectWriteOperation owo; | |
51 | owo.remove(); | |
52 | io_ctx.operate(client_name + ".done-setting", &owo); | |
53 | } | |
54 | delete kvs; | |
55 | } | |
56 | ||
57 | int KvStoreBench::setup(int argc, const char** argv) { | |
58 | vector<const char*> args; | |
59 | argv_to_vec(argc,argv,args); | |
60 | srand(time(NULL)); | |
61 | ||
62 | stringstream help; | |
63 | help | |
64 | << "Usage: KvStoreBench [options]\n" | |
65 | << "Generate latency and throughput statistics for the key value store\n" | |
66 | << "\n" | |
67 | << "There are two sets of options - workload options affect the kind of\n" | |
68 | << "test to run, while algorithm options affect how the key value\n" | |
69 | << "store handles the workload.\n" | |
70 | << "\n" | |
71 | << "There are about entries / k objects in the store to begin with.\n" | |
72 | << "Higher k values reduce the likelihood of splits and the likelihood\n" | |
73 | << "multiple writers simultaneously faling to write because an object \n" | |
74 | << "is full, but having a high k also means there will be more object\n" | |
75 | << "contention.\n" | |
76 | << "\n" | |
77 | << "WORKLOAD OPTIONS\n" | |
78 | << " --name <client name> client name (default admin)\n" | |
79 | << " --entries <number> number of key/value pairs to store initially\n" | |
80 | << " (default " << entries << ")\n" | |
81 | << " --ops <number> number of operations to run\n" | |
82 | << " --keysize <number> number of characters per key (default " << key_size << ")\n" | |
83 | << " --valsize <number> number of characters per value (default " << val_size << ")\n" | |
84 | << " -t <number> number of operations in flight concurrently\n" | |
85 | << " (default " << max_ops_in_flight << ")\n" | |
86 | << " --clients <number> tells this instance how many total clients are. Note that\n" | |
87 | << " changing this does not change the number of clients." | |
88 | << " -d <insert> <update> <delete> <read> percent (1-100) of operations that should be of each type\n" | |
89 | << " (default 25 25 25 25)\n" | |
90 | << " -r <number> random seed to use (default time(0))\n" | |
91 | << "ALGORITHM OPTIONS\n" | |
92 | << " --kval k, where each object has a number of entries\n" | |
93 | << " >= k and <= 2k.\n" | |
94 | << " --cache-size number of index entries to keep in cache\n" | |
95 | << " (default " << cache_size << ")\n" | |
96 | << " --cache-refresh percent (1-100) of cache-size to read each \n" | |
97 | << " time the index is read\n" | |
98 | << "OTHER OPTIONS\n" | |
99 | << " --verbosity-on display debug output\n" | |
100 | << " --clear-first delete all existing objects in the pool before running tests\n"; | |
101 | for (unsigned i = 0; i < args.size(); i++) { | |
102 | if(i < args.size() - 1) { | |
103 | if (strcmp(args[i], "--ops") == 0) { | |
104 | ops = atoi(args[i+1]); | |
105 | } else if (strcmp(args[i], "--entries") == 0) { | |
106 | entries = atoi(args[i+1]); | |
107 | } else if (strcmp(args[i], "--kval") == 0) { | |
108 | k = atoi(args[i+1]); | |
109 | } else if (strcmp(args[i], "--keysize") == 0) { | |
110 | key_size = atoi(args[i+1]); | |
111 | } else if (strcmp(args[i], "--valsize") == 0) { | |
112 | val_size = atoi(args[i+1]); | |
113 | } else if (strcmp(args[i], "--cache-size") == 0) { | |
114 | cache_size = atoi(args[i+1]); | |
115 | } else if (strcmp(args[i], "--cache-refresh") == 0) { | |
11fdf7f2 TL |
116 | auto temp = atoi(args[i+1]); |
117 | assert (temp != 0); | |
118 | cache_refresh = 100 / (double)temp; | |
7c673cae FG |
119 | } else if (strcmp(args[i], "-t") == 0) { |
120 | max_ops_in_flight = atoi(args[i+1]); | |
121 | } else if (strcmp(args[i], "--clients") == 0) { | |
122 | clients = atoi(args[i+1]); | |
123 | } else if (strcmp(args[i], "-d") == 0) { | |
124 | if (i + 4 >= args.size()) { | |
125 | cout << "Invalid arguments after -d: there must be 4 of them." | |
126 | << std::endl; | |
127 | continue; | |
128 | } else { | |
129 | probs.clear(); | |
130 | int sum = atoi(args[i + 1]); | |
131 | probs[sum] = 'i'; | |
132 | sum += atoi(args[i + 2]); | |
133 | probs[sum] = 'u'; | |
134 | sum += atoi(args[i + 3]); | |
135 | probs[sum] = 'd'; | |
136 | sum += atoi(args[i + 4]); | |
137 | probs[sum] = 'r'; | |
138 | if (sum != 100) { | |
139 | cout << "Invalid arguments after -d: they must add to 100." | |
140 | << std::endl; | |
141 | } | |
142 | } | |
143 | } else if (strcmp(args[i], "--name") == 0) { | |
144 | client_name = args[i+1]; | |
145 | } else if (strcmp(args[i], "-r") == 0) { | |
146 | srand(atoi(args[i+1])); | |
147 | } | |
148 | } else if (strcmp(args[i], "--verbosity-on") == 0) { | |
149 | verbose = true; | |
150 | } else if (strcmp(args[i], "--clear-first") == 0) { | |
151 | clear_first = true; | |
152 | } else if (strcmp(args[i], "--help") == 0) { | |
153 | cout << help.str() << std::endl; | |
154 | exit(1); | |
155 | } | |
156 | } | |
157 | ||
158 | KvFlatBtreeAsync * kvba = new KvFlatBtreeAsync(k, client_name, cache_size, | |
159 | cache_refresh, verbose); | |
160 | kvs = kvba; | |
161 | ||
162 | int r = rados.init(rados_id.c_str()); | |
163 | if (r < 0) { | |
164 | cout << "error during init" << std::endl; | |
165 | return r; | |
166 | } | |
167 | r = rados.conf_parse_argv(argc, argv); | |
168 | if (r < 0) { | |
169 | cout << "error during parsing args" << std::endl; | |
170 | return r; | |
171 | } | |
172 | r = rados.conf_parse_env(NULL); | |
173 | if (r < 0) { | |
174 | cout << "error during parsing env" << std::endl; | |
175 | return r; | |
176 | } | |
177 | r = rados.conf_read_file(NULL); | |
178 | if (r < 0) { | |
179 | cout << "error during read file" << std::endl; | |
180 | return r; | |
181 | } | |
182 | r = rados.connect(); | |
183 | if (r < 0) { | |
184 | cout << "error during connect: " << r << std::endl; | |
185 | return r; | |
186 | } | |
187 | r = rados.ioctx_create(pool_name.c_str(), io_ctx); | |
188 | if (r < 0) { | |
189 | cout << "error creating io ctx" << std::endl; | |
190 | rados.shutdown(); | |
191 | return r; | |
192 | } | |
193 | io_ctx_ready = true; | |
194 | ||
195 | if (clear_first) { | |
196 | librados::NObjectIterator it; | |
197 | for (it = io_ctx.nobjects_begin(); it != io_ctx.nobjects_end(); ++it) { | |
198 | librados::ObjectWriteOperation rm; | |
199 | rm.remove(); | |
200 | io_ctx.operate(it->get_oid(), &rm); | |
201 | } | |
202 | } | |
203 | ||
204 | int err = kvs->setup(argc, argv); | |
205 | if (err < 0 && err != -17) { | |
206 | cout << "error during setup of kvs: " << err << std::endl; | |
207 | return err; | |
208 | } | |
209 | ||
210 | return 0; | |
211 | } | |
212 | ||
213 | string KvStoreBench::random_string(int len) { | |
214 | string ret; | |
215 | string alphanum = "0123456789" | |
216 | "ABCDEFGHIJKLMNOPQRSTUVWXYZ" | |
217 | "abcdefghijklmnopqrstuvwxyz"; | |
218 | for (int i = 0; i < len; ++i) { | |
219 | ret.push_back(alphanum[rand() % (alphanum.size() - 1)]); | |
220 | } | |
221 | ||
222 | return ret; | |
223 | } | |
224 | ||
225 | pair<string, bufferlist> KvStoreBench::rand_distr(bool new_elem) { | |
226 | pair<string, bufferlist> ret; | |
227 | if (new_elem) { | |
228 | ret = make_pair(random_string(key_size), | |
229 | KvFlatBtreeAsync::to_bl(random_string(val_size))); | |
230 | key_set.insert(ret.first); | |
231 | } else { | |
232 | if (key_set.size() == 0) { | |
233 | return make_pair("",KvFlatBtreeAsync::to_bl("")); | |
234 | } | |
235 | string get_string = random_string(key_size); | |
236 | std::set<string>::iterator it = key_set.lower_bound(get_string); | |
237 | if (it == key_set.end()) { | |
238 | ret.first = *(key_set.rbegin()); | |
239 | } else { | |
240 | ret.first = *it; | |
241 | } | |
242 | ret.second = KvFlatBtreeAsync::to_bl(random_string(val_size)); | |
243 | } | |
244 | return ret; | |
245 | } | |
246 | ||
247 | int KvStoreBench::test_random_insertions() { | |
248 | int err; | |
249 | if (entries == 0) { | |
250 | return 0; | |
251 | } | |
252 | stringstream prev_ss; | |
253 | prev_ss << (atoi(client_name.c_str()) - 1); | |
254 | string prev_rid = prev_ss.str(); | |
255 | stringstream last_ss; | |
256 | if (client_name.size() > 1) { | |
257 | last_ss << client_name.substr(0,client_name.size() - 2); | |
258 | } | |
259 | last_ss << clients - 1; | |
260 | string last_rid = client_name == "admin" ? "admin" : last_ss.str(); | |
261 | ||
262 | map<string, bufferlist> big_map; | |
263 | for (int i = 0; i < entries; i++) { | |
264 | bufferlist bfr; | |
265 | bfr.append(random_string(7)); | |
266 | big_map[random_string(5)] = bfr; | |
267 | } | |
268 | ||
269 | uint64_t uint; | |
270 | time_t t; | |
271 | if (client_name[client_name.size() - 1] != '0' && client_name != "admin") { | |
272 | do { | |
273 | librados::ObjectReadOperation oro; | |
274 | oro.stat(&uint, &t, &err); | |
275 | err = io_ctx.operate(prev_rid + ".done-setting", &oro, NULL); | |
276 | if (verbose) cout << "reading " << prev_rid << ": err = " << err | |
277 | << std::endl; | |
278 | } while (err != 0); | |
279 | cout << "detected " << prev_rid << ".done-setting" << std::endl; | |
280 | } | |
281 | ||
282 | cout << "testing random insertions"; | |
283 | err = kvs->set_many(big_map); | |
284 | if (err < 0) { | |
285 | cout << "error setting things" << std::endl; | |
286 | return err; | |
287 | } | |
288 | ||
289 | librados::ObjectWriteOperation owo; | |
290 | owo.create(true); | |
291 | io_ctx.operate(client_name + ".done-setting", &owo); | |
292 | cout << "created " << client_name + ".done-setting. waiting for " | |
293 | << last_rid << ".done-setting" << std::endl; | |
294 | ||
295 | do { | |
296 | librados::ObjectReadOperation oro; | |
297 | oro.stat(&uint, &t, &err); | |
298 | err = io_ctx.operate(last_rid + ".done-setting", &oro, NULL); | |
299 | } while (err != 0); | |
300 | cout << "detected " << last_rid << ".done-setting" << std::endl; | |
301 | ||
302 | return err; | |
303 | } | |
304 | ||
305 | void KvStoreBench::aio_callback_timed(int * err, void *arg) { | |
306 | timed_args *args = reinterpret_cast<timed_args *>(arg); | |
9f95a23c TL |
307 | ceph::mutex * ops_in_flight_lock = &args->kvsb->ops_in_flight_lock; |
308 | ceph::mutex * data_lock = &args->kvsb->data_lock; | |
309 | ceph::condition_variable * op_avail = &args->kvsb->op_avail; | |
7c673cae FG |
310 | int *ops_in_flight = &args->kvsb->ops_in_flight; |
311 | if (*err < 0 && *err != -61) { | |
312 | cerr << "Error during " << args->op << " operation: " << *err << std::endl; | |
313 | } | |
314 | ||
315 | args->sw.stop_time(); | |
316 | double time = args->sw.get_time(); | |
317 | args->sw.clear(); | |
318 | ||
9f95a23c | 319 | data_lock->lock(); |
7c673cae FG |
320 | //latency |
321 | args->kvsb->data.latency_jf.open_object_section("latency"); | |
322 | args->kvsb->data.latency_jf.dump_float(string(1, args->op).c_str(), | |
323 | time); | |
324 | args->kvsb->data.latency_jf.close_section(); | |
325 | ||
326 | //throughput | |
327 | args->kvsb->data.throughput_jf.open_object_section("throughput"); | |
328 | args->kvsb->data.throughput_jf.dump_unsigned(string(1, args->op).c_str(), | |
329 | ceph_clock_now()); | |
330 | args->kvsb->data.throughput_jf.close_section(); | |
331 | ||
9f95a23c | 332 | data_lock->unlock(); |
7c673cae | 333 | |
9f95a23c | 334 | ops_in_flight_lock->lock(); |
7c673cae | 335 | (*ops_in_flight)--; |
9f95a23c TL |
336 | op_avail->notify_all(); |
337 | ops_in_flight_lock->unlock(); | |
7c673cae FG |
338 | |
339 | delete args; | |
340 | } | |
341 | ||
342 | int KvStoreBench::test_teuthology_aio(next_gen_t distr, | |
343 | const map<int, char> &probs) | |
344 | { | |
345 | int err = 0; | |
346 | cout << "inserting initial entries..." << std::endl; | |
347 | err = test_random_insertions(); | |
348 | if (err < 0) { | |
349 | return err; | |
350 | } | |
351 | cout << "finished inserting initial entries. Waiting 10 seconds for everyone" | |
352 | << " to catch up..." << std::endl; | |
353 | ||
354 | sleep(10); | |
355 | ||
356 | cout << "done waiting. Starting random operations..." << std::endl; | |
357 | ||
9f95a23c | 358 | std::unique_lock l{ops_in_flight_lock}; |
7c673cae | 359 | for (int i = 0; i < ops; i++) { |
11fdf7f2 | 360 | ceph_assert(ops_in_flight <= max_ops_in_flight); |
7c673cae | 361 | if (ops_in_flight == max_ops_in_flight) { |
9f95a23c | 362 | op_avail.wait(l); |
11fdf7f2 | 363 | ceph_assert(ops_in_flight < max_ops_in_flight); |
7c673cae FG |
364 | } |
365 | cout << "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t" << i + 1 << " / " | |
366 | << ops << std::endl; | |
367 | timed_args * cb_args = new timed_args(this); | |
368 | pair<string, bufferlist> kv; | |
369 | int random = (rand() % 100); | |
370 | cb_args->op = probs.lower_bound(random)->second; | |
371 | switch (cb_args->op) { | |
372 | case 'i': | |
373 | kv = (((KvStoreBench *)this)->*distr)(true); | |
374 | if (kv.first == "") { | |
375 | i--; | |
376 | delete cb_args; | |
377 | continue; | |
378 | } | |
379 | ops_in_flight++; | |
380 | cb_args->sw.start_time(); | |
381 | kvs->aio_set(kv.first, kv.second, false, aio_callback_timed, | |
382 | cb_args, &cb_args->err); | |
383 | break; | |
384 | case 'u': | |
385 | kv = (((KvStoreBench *)this)->*distr)(false); | |
386 | if (kv.first == "") { | |
387 | i--; | |
388 | delete cb_args; | |
389 | continue; | |
390 | } | |
391 | ops_in_flight++; | |
392 | cb_args->sw.start_time(); | |
393 | kvs->aio_set(kv.first, kv.second, true, aio_callback_timed, | |
394 | cb_args, &cb_args->err); | |
395 | break; | |
396 | case 'd': | |
397 | kv = (((KvStoreBench *)this)->*distr)(false); | |
398 | if (kv.first == "") { | |
399 | i--; | |
400 | delete cb_args; | |
401 | continue; | |
402 | } | |
403 | key_set.erase(kv.first); | |
404 | ops_in_flight++; | |
405 | cb_args->sw.start_time(); | |
406 | kvs->aio_remove(kv.first, aio_callback_timed, cb_args, &cb_args->err); | |
407 | break; | |
408 | case 'r': | |
409 | kv = (((KvStoreBench *)this)->*distr)(false); | |
410 | if (kv.first == "") { | |
411 | i--; | |
412 | delete cb_args; | |
413 | continue; | |
414 | } | |
7c673cae FG |
415 | ops_in_flight++; |
416 | cb_args->sw.start_time(); | |
417 | kvs->aio_get(kv.first, &cb_args->val, aio_callback_timed, | |
418 | cb_args, &cb_args->err); | |
419 | break; | |
9f95a23c TL |
420 | default: |
421 | // shouldn't happen here | |
422 | assert(false); | |
7c673cae FG |
423 | } |
424 | ||
7c673cae FG |
425 | } |
426 | ||
9f95a23c | 427 | op_avail.wait(l, [this] { return ops_in_flight <= 0; }); |
7c673cae FG |
428 | |
429 | print_time_data(); | |
430 | return err; | |
431 | } | |
432 | ||
433 | int KvStoreBench::test_teuthology_sync(next_gen_t distr, | |
434 | const map<int, char> &probs) | |
435 | { | |
436 | int err = 0; | |
437 | err = test_random_insertions(); | |
438 | if (err < 0) { | |
439 | return err; | |
440 | } | |
441 | sleep(10); | |
442 | for (int i = 0; i < ops; i++) { | |
443 | StopWatch sw; | |
444 | pair<char, double> d; | |
445 | cout << "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t" << i + 1 << " / " | |
446 | << ops << std::endl; | |
447 | pair<string, bufferlist> kv; | |
448 | int random = (rand() % 100); | |
449 | d.first = probs.lower_bound(random)->second; | |
450 | switch (d.first) { | |
451 | case 'i': | |
452 | kv = (((KvStoreBench *)this)->*distr)(true); | |
453 | if (kv.first == "") { | |
454 | i--; | |
455 | continue; | |
456 | } | |
457 | sw.start_time(); | |
458 | err = kvs->set(kv.first, kv.second, true); | |
459 | sw.stop_time(); | |
460 | if (err < 0) { | |
461 | cout << "Error setting " << kv << ": " << err << std::endl; | |
462 | return err; | |
463 | } | |
464 | break; | |
465 | case 'u': | |
466 | kv = (((KvStoreBench *)this)->*distr)(false); | |
467 | if (kv.first == "") { | |
468 | i--; | |
469 | continue; | |
470 | } | |
471 | sw.start_time(); | |
472 | err = kvs->set(kv.first, kv.second, true); | |
473 | sw.stop_time(); | |
474 | if (err < 0 && err != -61) { | |
475 | cout << "Error updating " << kv << ": " << err << std::endl; | |
476 | return err; | |
477 | } | |
478 | break; | |
479 | case 'd': | |
480 | kv = (((KvStoreBench *)this)->*distr)(false); | |
481 | if (kv.first == "") { | |
482 | i--; | |
483 | continue; | |
484 | } | |
485 | key_set.erase(kv.first); | |
486 | sw.start_time(); | |
487 | err = kvs->remove(kv.first); | |
488 | sw.stop_time(); | |
489 | if (err < 0 && err != -61) { | |
490 | cout << "Error removing " << kv << ": " << err << std::endl; | |
491 | return err; | |
492 | } | |
493 | break; | |
494 | case 'r': | |
495 | kv = (((KvStoreBench *)this)->*distr)(false); | |
496 | if (kv.first == "") { | |
497 | i--; | |
498 | continue; | |
499 | } | |
500 | bufferlist val; | |
501 | sw.start_time(); | |
502 | err = kvs->get(kv.first, &kv.second); | |
503 | sw.stop_time(); | |
504 | if (err < 0 && err != -61) { | |
505 | cout << "Error getting " << kv << ": " << err << std::endl; | |
506 | return err; | |
507 | } | |
508 | break; | |
509 | } | |
510 | ||
511 | double time = sw.get_time(); | |
512 | d.second = time; | |
513 | sw.clear(); | |
514 | //latency | |
515 | data.latency_jf.open_object_section("latency"); | |
516 | data.latency_jf.dump_float(string(1, d.first).c_str(), | |
517 | time); | |
518 | data.latency_jf.close_section(); | |
519 | } | |
520 | ||
521 | print_time_data(); | |
522 | return err; | |
523 | } | |
524 | ||
525 | void KvStoreBench::print_time_data() { | |
526 | cout << "========================================================\n"; | |
527 | cout << "latency:" << std::endl; | |
528 | data.latency_jf.flush(cout); | |
529 | cout << std::endl; | |
530 | cout << "throughput:" << std::endl; | |
531 | data.throughput_jf.flush(cout); | |
532 | cout << "\n========================================================" | |
533 | << std::endl; | |
534 | } | |
535 | ||
536 | int KvStoreBench::teuthology_tests() { | |
537 | int err = 0; | |
538 | if (max_ops_in_flight > 1) { | |
9f95a23c | 539 | err = test_teuthology_aio(&KvStoreBench::rand_distr, probs); |
7c673cae FG |
540 | } else { |
541 | err = test_teuthology_sync(&KvStoreBench::rand_distr, probs); | |
542 | } | |
543 | return err; | |
544 | } | |
545 | ||
546 | int main(int argc, const char** argv) { | |
547 | KvStoreBench kvsb; | |
548 | int err = kvsb.setup(argc, argv); | |
549 | if (err == 0) cout << "setup successful" << std::endl; | |
550 | else{ | |
551 | cout << "error " << err << std::endl; | |
552 | return err; | |
553 | } | |
554 | err = kvsb.teuthology_tests(); | |
555 | if (err < 0) return err; | |
556 | return 0; | |
557 | }; |