]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | /* | |
4 | * Ceph - scalable distributed file system | |
5 | * | |
6 | * Copyright (C) 2012 New Dream Network | |
7 | * | |
8 | * This is free software; you can redistribute it and/or | |
9 | * modify it under the terms of the GNU Lesser General Public | |
10 | * License version 2.1, as published by the Free Software | |
11 | * Foundation. See file COPYING. | |
12 | */ | |
13 | #include <stdio.h> | |
14 | #include <string.h> | |
15 | #include <iostream> | |
16 | #include <fstream> | |
17 | #include <time.h> | |
18 | #include <stdlib.h> | |
19 | #include <signal.h> | |
20 | #include <sstream> | |
21 | #include "os/ObjectStore.h" | |
22 | #include "common/ceph_argparse.h" | |
23 | #include "global/global_init.h" | |
24 | #include "common/debug.h" | |
25 | #include <boost/scoped_ptr.hpp> | |
26 | #include <boost/lexical_cast.hpp> | |
27 | ||
28 | #include "DeterministicOpSequence.h" | |
29 | #include "common/config.h" | |
30 | #include "include/assert.h" | |
31 | ||
32 | #define dout_context g_ceph_context | |
33 | #define dout_subsys ceph_subsys_filestore | |
34 | #undef dout_prefix | |
35 | #define dout_prefix *_dout << "deterministic_seq " | |
36 | ||
37 | DeterministicOpSequence::DeterministicOpSequence(ObjectStore *store, | |
38 | std::string status) | |
39 | : TestObjectStoreState(store), | |
40 | txn(0), | |
41 | m_osr("OSR") | |
42 | { | |
43 | txn_object = hobject_t(sobject_t("txn", CEPH_NOSNAP)); | |
44 | ||
45 | if (!status.empty()) | |
46 | m_status.open(status.c_str()); | |
47 | } | |
48 | ||
49 | DeterministicOpSequence::~DeterministicOpSequence() | |
50 | { | |
51 | // TODO Auto-generated destructor stub | |
52 | } | |
53 | ||
54 | bool DeterministicOpSequence::run_one_op(int op, rngen_t& gen) | |
55 | { | |
56 | bool ok = false; | |
57 | switch (op) { | |
58 | case DSOP_TOUCH: | |
59 | ok = do_touch(gen); | |
60 | break; | |
61 | case DSOP_WRITE: | |
62 | ok = do_write(gen); | |
63 | break; | |
64 | case DSOP_CLONE: | |
65 | ok = do_clone(gen); | |
66 | break; | |
67 | case DSOP_CLONE_RANGE: | |
68 | ok = do_clone_range(gen); | |
69 | break; | |
70 | case DSOP_OBJ_REMOVE: | |
71 | ok = do_remove(gen); | |
72 | break; | |
73 | case DSOP_COLL_MOVE: | |
74 | ok = do_coll_move(gen); | |
75 | break; | |
76 | case DSOP_SET_ATTRS: | |
77 | ok = do_set_attrs(gen); | |
78 | break; | |
79 | case DSOP_COLL_CREATE: | |
80 | ok = do_coll_create(gen); | |
81 | break; | |
82 | ||
83 | default: | |
28e407b8 | 84 | cout << "bad op " << op << std::endl; |
7c673cae FG |
85 | assert(0 == "bad op"); |
86 | } | |
87 | return ok; | |
88 | } | |
89 | ||
90 | void DeterministicOpSequence::generate(int seed, int num_txs) | |
91 | { | |
92 | std::ostringstream ss; | |
93 | ss << "generate run " << num_txs << " --seed " << seed; | |
94 | ||
95 | if (m_status.is_open()) { | |
96 | m_status << ss.str() << std::endl; | |
97 | m_status.flush(); | |
98 | } | |
99 | ||
100 | dout(0) << ss.str() << dendl; | |
101 | ||
102 | rngen_t gen(seed); | |
103 | boost::uniform_int<> op_rng(DSOP_FIRST, DSOP_LAST); | |
104 | ||
105 | for (txn = 1; txn <= num_txs; ) { | |
106 | int op = op_rng(gen); | |
107 | _print_status(txn, op); | |
108 | dout(0) << "generate seq " << txn << " op " << op << dendl; | |
109 | if (run_one_op(op, gen)) | |
110 | txn++; | |
111 | } | |
112 | } | |
113 | ||
114 | void DeterministicOpSequence::_print_status(int seq, int op) | |
115 | { | |
116 | if (!m_status.is_open()) | |
117 | return; | |
118 | m_status << seq << " " << op << std::endl; | |
119 | m_status.flush(); | |
120 | } | |
121 | ||
122 | int DeterministicOpSequence::_gen_coll_id(rngen_t& gen) | |
123 | { | |
124 | boost::uniform_int<> coll_rng(0, m_collections_ids.size()-1); | |
125 | return coll_rng(gen); | |
126 | } | |
127 | ||
128 | int DeterministicOpSequence::_gen_obj_id(rngen_t& gen) | |
129 | { | |
130 | boost::uniform_int<> obj_rng(0, m_num_objects - 1); | |
131 | return obj_rng(gen); | |
132 | } | |
133 | ||
134 | void DeterministicOpSequence::note_txn(ObjectStore::Transaction *t) | |
135 | { | |
136 | bufferlist bl; | |
137 | ::encode(txn, bl); | |
138 | t->truncate(txn_coll, ghobject_t(txn_object), 0); | |
139 | t->write(txn_coll, ghobject_t(txn_object), 0, bl.length(), bl); | |
140 | dout(10) << __func__ << " " << txn << dendl; | |
141 | } | |
142 | ||
143 | bool DeterministicOpSequence::do_touch(rngen_t& gen) | |
144 | { | |
145 | int coll_id = _gen_coll_id(gen); | |
146 | int obj_id = _gen_obj_id(gen); | |
147 | ||
148 | coll_entry_t *entry = get_coll_at(coll_id); | |
149 | ceph_assert(entry != NULL); | |
150 | ||
151 | // Don't care about other collections if already exists | |
152 | if (!entry->check_for_obj(obj_id)) { | |
153 | bool other_found = false; | |
154 | map<int, coll_entry_t*>::iterator it = m_collections.begin(); | |
155 | for (; it != m_collections.end(); ++it) { | |
156 | if (it->second->check_for_obj(obj_id)) { | |
157 | ceph_assert(it->first != coll_id); | |
158 | other_found = true; | |
159 | } | |
160 | } | |
161 | if (other_found) { | |
162 | dout(0) << "do_touch new object in collection and exists in another" << dendl; | |
163 | return false; | |
164 | } | |
165 | } | |
166 | hobject_t *obj = entry->touch_obj(obj_id); | |
167 | ||
28e407b8 | 168 | dout(0) << "do_touch " << entry->m_coll << "/" << obj << dendl; |
7c673cae | 169 | |
28e407b8 | 170 | _do_touch(entry, *obj); |
7c673cae FG |
171 | return true; |
172 | } | |
173 | ||
174 | bool DeterministicOpSequence::do_remove(rngen_t& gen) | |
175 | { | |
176 | int coll_id = _gen_coll_id(gen); | |
177 | ||
178 | coll_entry_t *entry = get_coll_at(coll_id); | |
179 | ceph_assert(entry != NULL); | |
180 | ||
181 | if (entry->m_objects.size() == 0) { | |
182 | dout(0) << "do_remove no objects in collection" << dendl; | |
183 | return false; | |
184 | } | |
185 | int obj_id = entry->get_random_obj_id(gen); | |
186 | hobject_t *obj = entry->touch_obj(obj_id); | |
187 | ceph_assert(obj); | |
188 | ||
28e407b8 | 189 | dout(0) << "do_remove " << entry->m_coll << "/" << obj << dendl; |
7c673cae | 190 | |
28e407b8 | 191 | _do_remove(entry, *obj); |
7c673cae FG |
192 | hobject_t *rmobj = entry->remove_obj(obj_id); |
193 | ceph_assert(rmobj); | |
194 | delete rmobj; | |
195 | return true; | |
196 | } | |
197 | ||
198 | static void _gen_random(rngen_t& gen, | |
199 | size_t size, bufferlist& bl) { | |
200 | ||
201 | static const char alphanum[] = "0123456789" | |
202 | "ABCDEFGHIJKLMNOPQRSTUVWXYZ" | |
203 | "abcdefghijklmnopqrstuvwxyz"; | |
204 | ||
205 | boost::uniform_int<> char_rng(0, sizeof(alphanum)); | |
206 | bufferptr bp(size); | |
207 | for (unsigned int i = 0; i < size - 1; i++) { | |
208 | bp[i] = alphanum[char_rng(gen)]; | |
209 | } | |
210 | bp[size - 1] = '\0'; | |
211 | bl.append(bp); | |
212 | } | |
213 | ||
214 | static void gen_attrs(rngen_t &gen, | |
215 | map<string, bufferlist> *out) { | |
216 | boost::uniform_int<> num_rng(10, 30); | |
217 | boost::uniform_int<> key_size_rng(5, 10); | |
218 | boost::uniform_int<> val_size_rng(100, 1000); | |
219 | size_t num_attrs = static_cast<size_t>(num_rng(gen)); | |
220 | for (size_t i = 0; i < num_attrs; ++i) { | |
221 | size_t key_size = static_cast<size_t>(num_rng(gen)); | |
222 | size_t val_size = static_cast<size_t>(num_rng(gen)); | |
223 | bufferlist keybl; | |
224 | _gen_random(gen, key_size, keybl); | |
225 | string key(keybl.c_str(), keybl.length()); | |
226 | _gen_random(gen, val_size, (*out)[key]); | |
227 | } | |
228 | } | |
229 | ||
230 | bool DeterministicOpSequence::do_set_attrs(rngen_t& gen) | |
231 | { | |
232 | int coll_id = _gen_coll_id(gen); | |
233 | ||
234 | coll_entry_t *entry = get_coll_at(coll_id); | |
235 | ceph_assert(entry != NULL); | |
236 | ||
237 | if (entry->m_objects.size() == 0) { | |
238 | dout(0) << "do_set_attrs no objects in collection" << dendl; | |
239 | return false; | |
240 | } | |
241 | int obj_id = entry->get_random_obj_id(gen); | |
242 | hobject_t *obj = entry->touch_obj(obj_id); | |
243 | ceph_assert(obj); | |
244 | ||
245 | map<string, bufferlist> out; | |
246 | gen_attrs(gen, &out); | |
247 | ||
248 | dout(0) << "do_set_attrs " << out.size() << " entries" << dendl; | |
28e407b8 | 249 | _do_set_attrs(entry, *obj, out); |
7c673cae FG |
250 | return true; |
251 | } | |
252 | ||
253 | bool DeterministicOpSequence::do_write(rngen_t& gen) | |
254 | { | |
255 | int coll_id = _gen_coll_id(gen); | |
256 | ||
257 | coll_entry_t *entry = get_coll_at(coll_id); | |
258 | ceph_assert(entry != NULL); | |
259 | ||
260 | if (entry->m_objects.size() == 0) { | |
261 | dout(0) << "do_write no objects in collection" << dendl; | |
262 | return false; | |
263 | } | |
264 | int obj_id = entry->get_random_obj_id(gen); | |
265 | hobject_t *obj = entry->touch_obj(obj_id); | |
266 | ceph_assert(obj); | |
267 | ||
268 | boost::uniform_int<> size_rng(100, (2 << 19)); | |
269 | size_t size = (size_t) size_rng(gen); | |
270 | bufferlist bl; | |
271 | _gen_random(gen, size, bl); | |
272 | ||
28e407b8 | 273 | dout(0) << "do_write " << entry->m_coll << "/" << obj |
7c673cae FG |
274 | << " 0~" << size << dendl; |
275 | ||
28e407b8 | 276 | _do_write(entry, *obj, 0, bl.length(), bl); |
7c673cae FG |
277 | return true; |
278 | } | |
279 | ||
28e407b8 AA |
280 | bool DeterministicOpSequence::_prepare_clone( |
281 | rngen_t& gen, | |
282 | coll_entry_t **entry_ret, | |
283 | int *orig_obj_id, | |
284 | hobject_t *orig_obj_ret, | |
285 | int *new_obj_id, | |
286 | hobject_t *new_obj_ret) | |
7c673cae FG |
287 | { |
288 | int coll_id = _gen_coll_id(gen); | |
289 | ||
290 | coll_entry_t *entry = get_coll_at(coll_id); | |
291 | ceph_assert(entry != NULL); | |
292 | ||
28e407b8 AA |
293 | if (entry->m_objects.size() < 2) { |
294 | dout(0) << "_prepare_clone coll " << entry->m_coll | |
7c673cae FG |
295 | << " doesn't have 2 or more objects" << dendl; |
296 | return false; | |
297 | } | |
298 | ||
28e407b8 AA |
299 | *orig_obj_id = entry->get_random_obj_id(gen); |
300 | hobject_t *orig_obj = entry->touch_obj(*orig_obj_id); | |
7c673cae FG |
301 | ceph_assert(orig_obj); |
302 | ||
7c673cae | 303 | do { |
28e407b8 AA |
304 | *new_obj_id = entry->get_random_obj_id(gen); |
305 | } while (*new_obj_id == *orig_obj_id); | |
306 | hobject_t *new_obj = entry->touch_obj(*new_obj_id); | |
7c673cae FG |
307 | ceph_assert(new_obj); |
308 | ||
28e407b8 AA |
309 | *entry_ret = entry; |
310 | *orig_obj_ret = *orig_obj; | |
311 | *new_obj_ret = *new_obj; | |
7c673cae FG |
312 | return true; |
313 | } | |
314 | ||
315 | bool DeterministicOpSequence::do_clone(rngen_t& gen) | |
316 | { | |
28e407b8 AA |
317 | coll_entry_t *entry; |
318 | int orig_id, new_id; | |
7c673cae | 319 | hobject_t orig_obj, new_obj; |
28e407b8 | 320 | if (!_prepare_clone(gen, &entry, &orig_id, &orig_obj, &new_id, &new_obj)) { |
7c673cae FG |
321 | return false; |
322 | } | |
323 | ||
28e407b8 AA |
324 | dout(0) << "do_clone " << entry->m_coll << "/" << orig_obj |
325 | << " => " << entry->m_coll << "/" << new_obj << dendl; | |
7c673cae | 326 | |
28e407b8 | 327 | _do_clone(entry, orig_obj, new_obj); |
7c673cae FG |
328 | return true; |
329 | } | |
330 | ||
331 | bool DeterministicOpSequence::do_clone_range(rngen_t& gen) | |
332 | { | |
28e407b8 AA |
333 | coll_entry_t *entry; |
334 | int orig_id, new_id; | |
7c673cae | 335 | hobject_t orig_obj, new_obj; |
28e407b8 | 336 | if (!_prepare_clone(gen, &entry, &orig_id, &orig_obj, &new_id, &new_obj)) { |
7c673cae FG |
337 | return false; |
338 | } | |
339 | ||
340 | /* Whenever we have to make a clone_range() operation, just write to the | |
341 | * object first, so we know we have something to clone in the said range. | |
342 | * This may not be the best solution ever, but currently we're not keeping | |
343 | * track of the written-to objects, and until we know for sure we really | |
344 | * need to, let's just focus on the task at hand. | |
345 | */ | |
346 | ||
347 | boost::uniform_int<> write_size_rng(100, (2 << 19)); | |
348 | size_t size = (size_t) write_size_rng(gen); | |
349 | bufferlist bl; | |
350 | _gen_random(gen, size, bl); | |
351 | ||
352 | boost::uniform_int<> clone_len(1, bl.length()); | |
353 | size = (size_t) clone_len(gen); | |
354 | ||
28e407b8 | 355 | dout(0) << "do_clone_range " << entry->m_coll << "/" << orig_obj |
7c673cae | 356 | << " (0~" << size << ")" |
28e407b8 | 357 | << " => " << entry->m_coll << "/" << new_obj |
7c673cae | 358 | << " (0)" << dendl; |
28e407b8 | 359 | _do_write_and_clone_range(entry, orig_obj, new_obj, 0, size, 0, bl); |
7c673cae FG |
360 | return true; |
361 | } | |
362 | ||
7c673cae FG |
363 | bool DeterministicOpSequence::do_coll_move(rngen_t& gen) |
364 | { | |
28e407b8 AA |
365 | coll_entry_t *entry; |
366 | int orig_id, new_id; | |
367 | hobject_t orig_obj, new_obj; | |
368 | if (!_prepare_clone(gen, &entry, &orig_id, &orig_obj, &new_id, &new_obj)) { | |
7c673cae FG |
369 | return false; |
370 | } | |
7c673cae | 371 | |
28e407b8 AA |
372 | dout(0) << "do_coll_move " << entry->m_coll << "/" << orig_obj |
373 | << " => " << entry->m_coll << "/" << new_obj << dendl; | |
374 | entry->remove_obj(orig_id); | |
7c673cae | 375 | |
28e407b8 | 376 | _do_coll_move(entry, orig_obj, new_obj); |
7c673cae FG |
377 | |
378 | return true; | |
379 | } | |
380 | ||
381 | bool DeterministicOpSequence::do_coll_create(rngen_t& gen) | |
382 | { | |
28e407b8 AA |
383 | int i = m_collections.size(); |
384 | coll_entry_t *entry = coll_create(i); | |
385 | m_collections.insert(make_pair(i, entry)); | |
386 | m_collections_ids.push_back(i); | |
387 | ||
388 | _do_coll_create(entry, 10, 10); | |
389 | ||
7c673cae FG |
390 | return true; |
391 | } | |
392 | ||
28e407b8 | 393 | void DeterministicOpSequence::_do_coll_create(coll_entry_t *entry, uint32_t pg_num, uint64_t num_objs) |
7c673cae FG |
394 | { |
395 | ObjectStore::Transaction t; | |
396 | note_txn(&t); | |
28e407b8 | 397 | t.create_collection(entry->m_coll, 32); |
7c673cae | 398 | bufferlist hint; |
28e407b8 AA |
399 | encode(pg_num, hint); |
400 | encode(num_objs, hint); | |
401 | t.collection_hint(entry->m_coll, ObjectStore::Transaction::COLL_HINT_EXPECTED_NUM_OBJECTS, hint); | |
402 | dout(0) << "Give collection: " << entry->m_coll | |
403 | << " a hint, pg_num is: " << pg_num << ", num_objs is: " | |
404 | << num_objs << dendl; | |
7c673cae FG |
405 | |
406 | m_store->apply_transaction(&m_osr, std::move(t)); | |
407 | } | |
408 | ||
28e407b8 | 409 | void DeterministicOpSequence::_do_touch(coll_entry_t *entry, hobject_t& obj) |
7c673cae FG |
410 | { |
411 | ObjectStore::Transaction t; | |
412 | note_txn(&t); | |
28e407b8 | 413 | t.touch(entry->m_coll, ghobject_t(obj)); |
7c673cae FG |
414 | m_store->apply_transaction(&m_osr, std::move(t)); |
415 | } | |
416 | ||
28e407b8 | 417 | void DeterministicOpSequence::_do_remove(coll_entry_t *entry, hobject_t& obj) |
7c673cae FG |
418 | { |
419 | ObjectStore::Transaction t; | |
420 | note_txn(&t); | |
28e407b8 | 421 | t.remove(entry->m_coll, ghobject_t(obj)); |
7c673cae FG |
422 | m_store->apply_transaction(&m_osr, std::move(t)); |
423 | } | |
424 | ||
28e407b8 | 425 | void DeterministicOpSequence::_do_set_attrs(coll_entry_t *entry, |
7c673cae FG |
426 | hobject_t &obj, |
427 | const map<string, bufferlist> &attrs) | |
428 | { | |
429 | ObjectStore::Transaction t; | |
430 | note_txn(&t); | |
28e407b8 | 431 | t.omap_setkeys(entry->m_coll, ghobject_t(obj), attrs); |
7c673cae FG |
432 | m_store->apply_transaction(&m_osr, std::move(t)); |
433 | } | |
434 | ||
28e407b8 | 435 | void DeterministicOpSequence::_do_write(coll_entry_t *entry, hobject_t& obj, |
7c673cae FG |
436 | uint64_t off, uint64_t len, const bufferlist& data) |
437 | { | |
438 | ObjectStore::Transaction t; | |
439 | note_txn(&t); | |
28e407b8 | 440 | t.write(entry->m_coll, ghobject_t(obj), off, len, data); |
7c673cae FG |
441 | m_store->apply_transaction(&m_osr, std::move(t)); |
442 | } | |
443 | ||
28e407b8 | 444 | void DeterministicOpSequence::_do_clone(coll_entry_t *entry, hobject_t& orig_obj, |
7c673cae FG |
445 | hobject_t& new_obj) |
446 | { | |
447 | ObjectStore::Transaction t; | |
448 | note_txn(&t); | |
28e407b8 | 449 | t.clone(entry->m_coll, ghobject_t(orig_obj), ghobject_t(new_obj)); |
7c673cae FG |
450 | m_store->apply_transaction(&m_osr, std::move(t)); |
451 | } | |
452 | ||
28e407b8 | 453 | void DeterministicOpSequence::_do_clone_range(coll_entry_t *entry, |
7c673cae FG |
454 | hobject_t& orig_obj, hobject_t& new_obj, uint64_t srcoff, |
455 | uint64_t srclen, uint64_t dstoff) | |
456 | { | |
457 | ObjectStore::Transaction t; | |
458 | note_txn(&t); | |
28e407b8 | 459 | t.clone_range(entry->m_coll, ghobject_t(orig_obj), ghobject_t(new_obj), |
7c673cae FG |
460 | srcoff, srclen, dstoff); |
461 | m_store->apply_transaction(&m_osr, std::move(t)); | |
462 | } | |
463 | ||
28e407b8 | 464 | void DeterministicOpSequence::_do_write_and_clone_range(coll_entry_t *entry, |
7c673cae FG |
465 | hobject_t& orig_obj, |
466 | hobject_t& new_obj, | |
467 | uint64_t srcoff, | |
468 | uint64_t srclen, | |
469 | uint64_t dstoff, | |
470 | bufferlist& bl) | |
471 | { | |
472 | ObjectStore::Transaction t; | |
473 | note_txn(&t); | |
28e407b8 AA |
474 | t.write(entry->m_coll, ghobject_t(orig_obj), srcoff, bl.length(), bl); |
475 | t.clone_range(entry->m_coll, ghobject_t(orig_obj), ghobject_t(new_obj), | |
7c673cae FG |
476 | srcoff, srclen, dstoff); |
477 | m_store->apply_transaction(&m_osr, std::move(t)); | |
478 | } | |
479 | ||
28e407b8 AA |
480 | void DeterministicOpSequence::_do_coll_move(coll_entry_t *entry, |
481 | hobject_t& orig_obj, | |
482 | hobject_t& new_obj) | |
7c673cae FG |
483 | { |
484 | ObjectStore::Transaction t; | |
485 | note_txn(&t); | |
28e407b8 AA |
486 | t.remove(entry->m_coll, ghobject_t(new_obj)); |
487 | t.collection_move_rename(entry->m_coll, ghobject_t(orig_obj), | |
488 | entry->m_coll, ghobject_t(new_obj)); | |
7c673cae FG |
489 | m_store->apply_transaction(&m_osr, std::move(t)); |
490 | } | |
491 |