1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 #include "kvstore_tool.h"
8 #include "common/errno.h"
9 #include "common/url_escape.h"
10 #include "include/buffer.h"
11 #include "kv/KeyValueDB.h"
13 StoreTool::StoreTool(const string
& type
,
21 g_conf()->rocksdb_perf
= true;
22 g_conf()->rocksdb_collect_compaction_stats
= true;
25 if (type
== "bluestore-kv") {
27 if (load_bluestore(path
, need_open_db
) != 0)
30 cerr
<< "bluestore not compiled in" << std::endl
;
34 auto db_ptr
= KeyValueDB::create(g_ceph_context
, type
, path
);
36 if (int r
= db_ptr
->open(std::cerr
); r
< 0) {
37 cerr
<< "failed to open type " << type
<< " path " << path
<< ": "
38 << cpp_strerror(r
) << std::endl
;
46 int StoreTool::load_bluestore(const string
& path
, bool need_open_db
)
48 auto bluestore
= new BlueStore(g_ceph_context
, path
);
50 int r
= bluestore
->start_kv_only(&db_ptr
, need_open_db
);
54 db
= decltype(db
){db_ptr
, Deleter(bluestore
)};
58 uint32_t StoreTool::traverse(const string
& prefix
,
60 const bool do_value_dump
,
63 KeyValueDB::WholeSpaceIterator iter
= db
->get_wholespace_iterator();
66 iter
->seek_to_first();
68 iter
->seek_to_first(prefix
);
72 while (iter
->valid()) {
73 pair
<string
,string
> rk
= iter
->raw_key();
74 if (!prefix
.empty() && (rk
.first
!= prefix
))
78 *out
<< url_escape(rk
.first
) << "\t" << url_escape(rk
.second
);
83 bl
.append(iter
->value());
87 *out
<< "\t" << bl
.crc32c(0);
92 if (out
&& do_value_dump
) {
93 bufferptr bp
= iter
->value_as_ptr();
98 std::cout
<< os
.str() << std::endl
;
106 void StoreTool::list(const string
& prefix
, const bool do_crc
,
107 const bool do_value_dump
)
109 traverse(prefix
, do_crc
, do_value_dump
,& std::cout
);
112 bool StoreTool::exists(const string
& prefix
)
114 ceph_assert(!prefix
.empty());
115 KeyValueDB::WholeSpaceIterator iter
= db
->get_wholespace_iterator();
116 iter
->seek_to_first(prefix
);
117 return (iter
->valid() && (iter
->raw_key().first
== prefix
));
120 bool StoreTool::exists(const string
& prefix
, const string
& key
)
122 ceph_assert(!prefix
.empty());
125 return exists(prefix
);
128 get(prefix
, key
, exists
);
132 bufferlist
StoreTool::get(const string
& prefix
,
136 ceph_assert(!prefix
.empty() && !key
.empty());
138 map
<string
,bufferlist
> result
;
139 std::set
<std::string
> keys
;
141 db
->get(prefix
, keys
, &result
);
143 if (result
.count(key
) > 0) {
152 uint64_t StoreTool::get_size()
154 map
<string
,uint64_t> extras
;
155 uint64_t s
= db
->get_estimated_size(extras
);
156 for (auto& [name
, size
] : extras
) {
157 std::cout
<< name
<< " - " << size
<< std::endl
;
159 std::cout
<< "total: " << s
<< std::endl
;
163 bool StoreTool::set(const string
&prefix
, const string
&key
, bufferlist
&val
)
165 ceph_assert(!prefix
.empty());
166 ceph_assert(!key
.empty());
167 ceph_assert(val
.length() > 0);
169 KeyValueDB::Transaction tx
= db
->get_transaction();
170 tx
->set(prefix
, key
, val
);
171 int ret
= db
->submit_transaction_sync(tx
);
176 bool StoreTool::rm(const string
& prefix
, const string
& key
)
178 ceph_assert(!prefix
.empty());
179 ceph_assert(!key
.empty());
181 KeyValueDB::Transaction tx
= db
->get_transaction();
182 tx
->rmkey(prefix
, key
);
183 int ret
= db
->submit_transaction_sync(tx
);
188 bool StoreTool::rm_prefix(const string
& prefix
)
190 ceph_assert(!prefix
.empty());
192 KeyValueDB::Transaction tx
= db
->get_transaction();
193 tx
->rmkeys_by_prefix(prefix
);
194 int ret
= db
->submit_transaction_sync(tx
);
199 void StoreTool::print_summary(const uint64_t total_keys
, const uint64_t total_size
,
200 const uint64_t total_txs
, const string
& store_path
,
201 const string
& other_path
, const int duration
) const
203 std::cout
<< "summary:" << std::endl
;
204 std::cout
<< " copied " << total_keys
<< " keys" << std::endl
;
205 std::cout
<< " used " << total_txs
<< " transactions" << std::endl
;
206 std::cout
<< " total size " << byte_u_t(total_size
) << std::endl
;
207 std::cout
<< " from '" << store_path
<< "' to '" << other_path
<< "'"
209 std::cout
<< " duration " << duration
<< " seconds" << std::endl
;
212 int StoreTool::print_stats() const
215 Formatter
* f
= Formatter::create("json-pretty", "json-pretty", "json-pretty");
217 if (g_conf()->rocksdb_perf
) {
218 db
->get_statistics(f
);
219 ostr
<< "db_statistics ";
223 ostr
<< "db_statistics not enabled";
226 std::cout
<< ostr
.str() << std::endl
;
231 int StoreTool::copy_store_to(const string
& type
, const string
& other_path
,
232 const int num_keys_per_tx
,
233 const string
& other_type
)
235 if (num_keys_per_tx
<= 0) {
236 std::cerr
<< "must specify a number of keys/tx > 0" << std::endl
;
240 // open or create a leveldb store at @p other_path
241 boost::scoped_ptr
<KeyValueDB
> other
;
242 KeyValueDB
*other_ptr
= KeyValueDB::create(g_ceph_context
,
245 if (int err
= other_ptr
->create_and_open(std::cerr
); err
< 0) {
248 other
.reset(other_ptr
);
250 KeyValueDB::WholeSpaceIterator it
= db
->get_wholespace_iterator();
252 uint64_t total_keys
= 0;
253 uint64_t total_size
= 0;
254 uint64_t total_txs
= 0;
256 auto duration
= [start
=coarse_mono_clock::now()] {
257 const auto now
= coarse_mono_clock::now();
258 auto seconds
= std::chrono::duration
<double>(now
- start
);
259 return seconds
.count();
265 KeyValueDB::Transaction tx
= other
->get_transaction();
267 while (it
->valid() && num_keys
< num_keys_per_tx
) {
268 auto [prefix
, key
] = it
->raw_key();
269 bufferlist v
= it
->value();
270 tx
->set(prefix
, key
, v
);
273 total_size
+= v
.length();
279 total_keys
+= num_keys
;
282 other
->submit_transaction_sync(tx
);
284 std::cout
<< "ts = " << duration() << "s, copied " << total_keys
285 << " keys so far (" << byte_u_t(total_size
) << ")"
288 } while (it
->valid());
290 print_summary(total_keys
, total_size
, total_txs
, store_path
, other_path
,
296 void StoreTool::compact()
301 void StoreTool::compact_prefix(const string
& prefix
)
303 db
->compact_prefix(prefix
);
306 void StoreTool::compact_range(const string
& prefix
,
310 db
->compact_range(prefix
, start
, end
);
313 int StoreTool::destructive_repair()
315 return db
->repair(std::cout
);