1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2012 New Dream Network
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
21 #include "os/ObjectStore.h"
22 #include "common/ceph_argparse.h"
23 #include "global/global_init.h"
24 #include "common/debug.h"
25 #include <boost/scoped_ptr.hpp>
26 #include <boost/lexical_cast.hpp>
28 #include "DeterministicOpSequence.h"
29 #include "common/config.h"
30 #include "include/assert.h"
32 #define dout_context g_ceph_context
33 #define dout_subsys ceph_subsys_filestore
35 #define dout_prefix *_dout << "deterministic_seq "
37 DeterministicOpSequence::DeterministicOpSequence(ObjectStore
*store
,
39 : TestObjectStoreState(store
),
43 txn_object
= hobject_t(sobject_t("txn", CEPH_NOSNAP
));
46 m_status
.open(status
.c_str());
49 DeterministicOpSequence::~DeterministicOpSequence()
51 // TODO Auto-generated destructor stub
54 bool DeterministicOpSequence::run_one_op(int op
, rngen_t
& gen
)
67 case DSOP_CLONE_RANGE
:
68 ok
= do_clone_range(gen
);
74 ok
= do_coll_move(gen
);
77 ok
= do_set_attrs(gen
);
79 case DSOP_COLL_CREATE
:
80 ok
= do_coll_create(gen
);
84 cout
<< "bad op " << op
<< std::endl
;
85 assert(0 == "bad op");
90 void DeterministicOpSequence::generate(int seed
, int num_txs
)
92 std::ostringstream ss
;
93 ss
<< "generate run " << num_txs
<< " --seed " << seed
;
95 if (m_status
.is_open()) {
96 m_status
<< ss
.str() << std::endl
;
100 dout(0) << ss
.str() << dendl
;
103 boost::uniform_int
<> op_rng(DSOP_FIRST
, DSOP_LAST
);
105 for (txn
= 1; txn
<= num_txs
; ) {
106 int op
= op_rng(gen
);
107 _print_status(txn
, op
);
108 dout(0) << "generate seq " << txn
<< " op " << op
<< dendl
;
109 if (run_one_op(op
, gen
))
114 void DeterministicOpSequence::_print_status(int seq
, int op
)
116 if (!m_status
.is_open())
118 m_status
<< seq
<< " " << op
<< std::endl
;
122 int DeterministicOpSequence::_gen_coll_id(rngen_t
& gen
)
124 boost::uniform_int
<> coll_rng(0, m_collections_ids
.size()-1);
125 return coll_rng(gen
);
128 int DeterministicOpSequence::_gen_obj_id(rngen_t
& gen
)
130 boost::uniform_int
<> obj_rng(0, m_num_objects
- 1);
134 void DeterministicOpSequence::note_txn(ObjectStore::Transaction
*t
)
138 t
->truncate(txn_coll
, ghobject_t(txn_object
), 0);
139 t
->write(txn_coll
, ghobject_t(txn_object
), 0, bl
.length(), bl
);
140 dout(10) << __func__
<< " " << txn
<< dendl
;
143 bool DeterministicOpSequence::do_touch(rngen_t
& gen
)
145 int coll_id
= _gen_coll_id(gen
);
146 int obj_id
= _gen_obj_id(gen
);
148 coll_entry_t
*entry
= get_coll_at(coll_id
);
149 ceph_assert(entry
!= NULL
);
151 // Don't care about other collections if already exists
152 if (!entry
->check_for_obj(obj_id
)) {
153 bool other_found
= false;
154 map
<int, coll_entry_t
*>::iterator it
= m_collections
.begin();
155 for (; it
!= m_collections
.end(); ++it
) {
156 if (it
->second
->check_for_obj(obj_id
)) {
157 ceph_assert(it
->first
!= coll_id
);
162 dout(0) << "do_touch new object in collection and exists in another" << dendl
;
166 hobject_t
*obj
= entry
->touch_obj(obj_id
);
168 dout(0) << "do_touch " << entry
->m_coll
<< "/" << obj
<< dendl
;
170 _do_touch(entry
, *obj
);
174 bool DeterministicOpSequence::do_remove(rngen_t
& gen
)
176 int coll_id
= _gen_coll_id(gen
);
178 coll_entry_t
*entry
= get_coll_at(coll_id
);
179 ceph_assert(entry
!= NULL
);
181 if (entry
->m_objects
.size() == 0) {
182 dout(0) << "do_remove no objects in collection" << dendl
;
185 int obj_id
= entry
->get_random_obj_id(gen
);
186 hobject_t
*obj
= entry
->touch_obj(obj_id
);
189 dout(0) << "do_remove " << entry
->m_coll
<< "/" << obj
<< dendl
;
191 _do_remove(entry
, *obj
);
192 hobject_t
*rmobj
= entry
->remove_obj(obj_id
);
198 static void _gen_random(rngen_t
& gen
,
199 size_t size
, bufferlist
& bl
) {
201 static const char alphanum
[] = "0123456789"
202 "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
203 "abcdefghijklmnopqrstuvwxyz";
205 boost::uniform_int
<> char_rng(0, sizeof(alphanum
));
207 for (unsigned int i
= 0; i
< size
- 1; i
++) {
208 bp
[i
] = alphanum
[char_rng(gen
)];
214 static void gen_attrs(rngen_t
&gen
,
215 map
<string
, bufferlist
> *out
) {
216 boost::uniform_int
<> num_rng(10, 30);
217 boost::uniform_int
<> key_size_rng(5, 10);
218 boost::uniform_int
<> val_size_rng(100, 1000);
219 size_t num_attrs
= static_cast<size_t>(num_rng(gen
));
220 for (size_t i
= 0; i
< num_attrs
; ++i
) {
221 size_t key_size
= static_cast<size_t>(num_rng(gen
));
222 size_t val_size
= static_cast<size_t>(num_rng(gen
));
224 _gen_random(gen
, key_size
, keybl
);
225 string
key(keybl
.c_str(), keybl
.length());
226 _gen_random(gen
, val_size
, (*out
)[key
]);
230 bool DeterministicOpSequence::do_set_attrs(rngen_t
& gen
)
232 int coll_id
= _gen_coll_id(gen
);
234 coll_entry_t
*entry
= get_coll_at(coll_id
);
235 ceph_assert(entry
!= NULL
);
237 if (entry
->m_objects
.size() == 0) {
238 dout(0) << "do_set_attrs no objects in collection" << dendl
;
241 int obj_id
= entry
->get_random_obj_id(gen
);
242 hobject_t
*obj
= entry
->touch_obj(obj_id
);
245 map
<string
, bufferlist
> out
;
246 gen_attrs(gen
, &out
);
248 dout(0) << "do_set_attrs " << out
.size() << " entries" << dendl
;
249 _do_set_attrs(entry
, *obj
, out
);
253 bool DeterministicOpSequence::do_write(rngen_t
& gen
)
255 int coll_id
= _gen_coll_id(gen
);
257 coll_entry_t
*entry
= get_coll_at(coll_id
);
258 ceph_assert(entry
!= NULL
);
260 if (entry
->m_objects
.size() == 0) {
261 dout(0) << "do_write no objects in collection" << dendl
;
264 int obj_id
= entry
->get_random_obj_id(gen
);
265 hobject_t
*obj
= entry
->touch_obj(obj_id
);
268 boost::uniform_int
<> size_rng(100, (2 << 19));
269 size_t size
= (size_t) size_rng(gen
);
271 _gen_random(gen
, size
, bl
);
273 dout(0) << "do_write " << entry
->m_coll
<< "/" << obj
274 << " 0~" << size
<< dendl
;
276 _do_write(entry
, *obj
, 0, bl
.length(), bl
);
280 bool DeterministicOpSequence::_prepare_clone(
282 coll_entry_t
**entry_ret
,
284 hobject_t
*orig_obj_ret
,
286 hobject_t
*new_obj_ret
)
288 int coll_id
= _gen_coll_id(gen
);
290 coll_entry_t
*entry
= get_coll_at(coll_id
);
291 ceph_assert(entry
!= NULL
);
293 if (entry
->m_objects
.size() < 2) {
294 dout(0) << "_prepare_clone coll " << entry
->m_coll
295 << " doesn't have 2 or more objects" << dendl
;
299 *orig_obj_id
= entry
->get_random_obj_id(gen
);
300 hobject_t
*orig_obj
= entry
->touch_obj(*orig_obj_id
);
301 ceph_assert(orig_obj
);
304 *new_obj_id
= entry
->get_random_obj_id(gen
);
305 } while (*new_obj_id
== *orig_obj_id
);
306 hobject_t
*new_obj
= entry
->touch_obj(*new_obj_id
);
307 ceph_assert(new_obj
);
310 *orig_obj_ret
= *orig_obj
;
311 *new_obj_ret
= *new_obj
;
315 bool DeterministicOpSequence::do_clone(rngen_t
& gen
)
319 hobject_t orig_obj
, new_obj
;
320 if (!_prepare_clone(gen
, &entry
, &orig_id
, &orig_obj
, &new_id
, &new_obj
)) {
324 dout(0) << "do_clone " << entry
->m_coll
<< "/" << orig_obj
325 << " => " << entry
->m_coll
<< "/" << new_obj
<< dendl
;
327 _do_clone(entry
, orig_obj
, new_obj
);
331 bool DeterministicOpSequence::do_clone_range(rngen_t
& gen
)
335 hobject_t orig_obj
, new_obj
;
336 if (!_prepare_clone(gen
, &entry
, &orig_id
, &orig_obj
, &new_id
, &new_obj
)) {
340 /* Whenever we have to make a clone_range() operation, just write to the
341 * object first, so we know we have something to clone in the said range.
342 * This may not be the best solution ever, but currently we're not keeping
343 * track of the written-to objects, and until we know for sure we really
344 * need to, let's just focus on the task at hand.
347 boost::uniform_int
<> write_size_rng(100, (2 << 19));
348 size_t size
= (size_t) write_size_rng(gen
);
350 _gen_random(gen
, size
, bl
);
352 boost::uniform_int
<> clone_len(1, bl
.length());
353 size
= (size_t) clone_len(gen
);
355 dout(0) << "do_clone_range " << entry
->m_coll
<< "/" << orig_obj
356 << " (0~" << size
<< ")"
357 << " => " << entry
->m_coll
<< "/" << new_obj
359 _do_write_and_clone_range(entry
, orig_obj
, new_obj
, 0, size
, 0, bl
);
363 bool DeterministicOpSequence::do_coll_move(rngen_t
& gen
)
367 hobject_t orig_obj
, new_obj
;
368 if (!_prepare_clone(gen
, &entry
, &orig_id
, &orig_obj
, &new_id
, &new_obj
)) {
372 dout(0) << "do_coll_move " << entry
->m_coll
<< "/" << orig_obj
373 << " => " << entry
->m_coll
<< "/" << new_obj
<< dendl
;
374 entry
->remove_obj(orig_id
);
376 _do_coll_move(entry
, orig_obj
, new_obj
);
381 bool DeterministicOpSequence::do_coll_create(rngen_t
& gen
)
383 int i
= m_collections
.size();
384 coll_entry_t
*entry
= coll_create(i
);
385 m_collections
.insert(make_pair(i
, entry
));
386 m_collections_ids
.push_back(i
);
388 _do_coll_create(entry
, 10, 10);
393 void DeterministicOpSequence::_do_coll_create(coll_entry_t
*entry
, uint32_t pg_num
, uint64_t num_objs
)
395 ObjectStore::Transaction t
;
397 t
.create_collection(entry
->m_coll
, 32);
399 encode(pg_num
, hint
);
400 encode(num_objs
, hint
);
401 t
.collection_hint(entry
->m_coll
, ObjectStore::Transaction::COLL_HINT_EXPECTED_NUM_OBJECTS
, hint
);
402 dout(0) << "Give collection: " << entry
->m_coll
403 << " a hint, pg_num is: " << pg_num
<< ", num_objs is: "
404 << num_objs
<< dendl
;
406 m_store
->apply_transaction(&m_osr
, std::move(t
));
409 void DeterministicOpSequence::_do_touch(coll_entry_t
*entry
, hobject_t
& obj
)
411 ObjectStore::Transaction t
;
413 t
.touch(entry
->m_coll
, ghobject_t(obj
));
414 m_store
->apply_transaction(&m_osr
, std::move(t
));
417 void DeterministicOpSequence::_do_remove(coll_entry_t
*entry
, hobject_t
& obj
)
419 ObjectStore::Transaction t
;
421 t
.remove(entry
->m_coll
, ghobject_t(obj
));
422 m_store
->apply_transaction(&m_osr
, std::move(t
));
425 void DeterministicOpSequence::_do_set_attrs(coll_entry_t
*entry
,
427 const map
<string
, bufferlist
> &attrs
)
429 ObjectStore::Transaction t
;
431 t
.omap_setkeys(entry
->m_coll
, ghobject_t(obj
), attrs
);
432 m_store
->apply_transaction(&m_osr
, std::move(t
));
435 void DeterministicOpSequence::_do_write(coll_entry_t
*entry
, hobject_t
& obj
,
436 uint64_t off
, uint64_t len
, const bufferlist
& data
)
438 ObjectStore::Transaction t
;
440 t
.write(entry
->m_coll
, ghobject_t(obj
), off
, len
, data
);
441 m_store
->apply_transaction(&m_osr
, std::move(t
));
444 void DeterministicOpSequence::_do_clone(coll_entry_t
*entry
, hobject_t
& orig_obj
,
447 ObjectStore::Transaction t
;
449 t
.clone(entry
->m_coll
, ghobject_t(orig_obj
), ghobject_t(new_obj
));
450 m_store
->apply_transaction(&m_osr
, std::move(t
));
453 void DeterministicOpSequence::_do_clone_range(coll_entry_t
*entry
,
454 hobject_t
& orig_obj
, hobject_t
& new_obj
, uint64_t srcoff
,
455 uint64_t srclen
, uint64_t dstoff
)
457 ObjectStore::Transaction t
;
459 t
.clone_range(entry
->m_coll
, ghobject_t(orig_obj
), ghobject_t(new_obj
),
460 srcoff
, srclen
, dstoff
);
461 m_store
->apply_transaction(&m_osr
, std::move(t
));
464 void DeterministicOpSequence::_do_write_and_clone_range(coll_entry_t
*entry
,
472 ObjectStore::Transaction t
;
474 t
.write(entry
->m_coll
, ghobject_t(orig_obj
), srcoff
, bl
.length(), bl
);
475 t
.clone_range(entry
->m_coll
, ghobject_t(orig_obj
), ghobject_t(new_obj
),
476 srcoff
, srclen
, dstoff
);
477 m_store
->apply_transaction(&m_osr
, std::move(t
));
480 void DeterministicOpSequence::_do_coll_move(coll_entry_t
*entry
,
484 ObjectStore::Transaction t
;
486 t
.remove(entry
->m_coll
, ghobject_t(new_obj
));
487 t
.collection_move_rename(entry
->m_coll
, ghobject_t(orig_obj
),
488 entry
->m_coll
, ghobject_t(new_obj
));
489 m_store
->apply_transaction(&m_osr
, std::move(t
));