]> git.proxmox.com Git - ceph.git/blame - ceph/src/rocksdb/tools/ldb_cmd.cc
projects / ceph.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
update sources to ceph Nautilus 14.2.1
[ceph.git] / ceph / src / rocksdb / tools / ldb_cmd.cc
CommitLineData
7c673cae 1// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
11fdf7f2
TL		 2// This source code is licensed under both the GPLv2 (found in the
3// COPYING file in the root directory) and Apache 2.0 License
4// (found in the LICENSE.Apache file in the root directory).
7c673cae
FG		 5//
6#ifndef ROCKSDB_LITE
7#include "rocksdb/utilities/ldb_cmd.h"
8
9#ifndef __STDC_FORMAT_MACROS
10#define __STDC_FORMAT_MACROS
11#endif
12
13#include <inttypes.h>
14
15#include "db/db_impl.h"
16#include "db/dbformat.h"
17#include "db/log_reader.h"
18#include "db/write_batch_internal.h"
19#include "port/dirent.h"
20#include "rocksdb/cache.h"
21#include "rocksdb/table_properties.h"
22#include "rocksdb/utilities/backupable_db.h"
23#include "rocksdb/utilities/checkpoint.h"
11fdf7f2 24#include "rocksdb/utilities/debug.h"
7c673cae
FG		 25#include "rocksdb/utilities/object_registry.h"
26#include "rocksdb/utilities/options_util.h"
27#include "rocksdb/write_batch.h"
28#include "rocksdb/write_buffer_manager.h"
29#include "table/scoped_arena_iterator.h"
30#include "tools/ldb_cmd_impl.h"
31#include "tools/sst_dump_tool_imp.h"
11fdf7f2 32#include "util/cast_util.h"
7c673cae
FG		 33#include "util/coding.h"
34#include "util/filename.h"
35#include "util/stderr_logger.h"
36#include "util/string_util.h"
37#include "utilities/ttl/db_ttl_impl.h"
38
39#include <cstdlib>
40#include <ctime>
41#include <fstream>
42#include <functional>
43#include <iostream>
44#include <limits>
45#include <sstream>
46#include <stdexcept>
47#include <string>
48
49namespace rocksdb {
50
51const std::string LDBCommand::ARG_DB = "db";
52const std::string LDBCommand::ARG_PATH = "path";
53const std::string LDBCommand::ARG_HEX = "hex";
54const std::string LDBCommand::ARG_KEY_HEX = "key_hex";
55const std::string LDBCommand::ARG_VALUE_HEX = "value_hex";
56const std::string LDBCommand::ARG_CF_NAME = "column_family";
57const std::string LDBCommand::ARG_TTL = "ttl";
58const std::string LDBCommand::ARG_TTL_START = "start_time";
59const std::string LDBCommand::ARG_TTL_END = "end_time";
60const std::string LDBCommand::ARG_TIMESTAMP = "timestamp";
61const std::string LDBCommand::ARG_TRY_LOAD_OPTIONS = "try_load_options";
11fdf7f2
TL		 62const std::string LDBCommand::ARG_IGNORE_UNKNOWN_OPTIONS =
63 "ignore_unknown_options";
7c673cae
FG		 64const std::string LDBCommand::ARG_FROM = "from";
65const std::string LDBCommand::ARG_TO = "to";
66const std::string LDBCommand::ARG_MAX_KEYS = "max_keys";
67const std::string LDBCommand::ARG_BLOOM_BITS = "bloom_bits";
68const std::string LDBCommand::ARG_FIX_PREFIX_LEN = "fix_prefix_len";
69const std::string LDBCommand::ARG_COMPRESSION_TYPE = "compression_type";
70const std::string LDBCommand::ARG_COMPRESSION_MAX_DICT_BYTES =
71 "compression_max_dict_bytes";
72const std::string LDBCommand::ARG_BLOCK_SIZE = "block_size";
73const std::string LDBCommand::ARG_AUTO_COMPACTION = "auto_compaction";
74const std::string LDBCommand::ARG_DB_WRITE_BUFFER_SIZE = "db_write_buffer_size";
75const std::string LDBCommand::ARG_WRITE_BUFFER_SIZE = "write_buffer_size";
76const std::string LDBCommand::ARG_FILE_SIZE = "file_size";
77const std::string LDBCommand::ARG_CREATE_IF_MISSING = "create_if_missing";
78const std::string LDBCommand::ARG_NO_VALUE = "no_value";
79
80const char* LDBCommand::DELIM = " ==> ";
81
82namespace {
83
84void DumpWalFile(std::string wal_file, bool print_header, bool print_values,
11fdf7f2 85 bool is_write_committed, LDBCommandExecuteResult* exec_state);
7c673cae
FG		 86
87void DumpSstFile(std::string filename, bool output_hex, bool show_properties);
88};
89
90LDBCommand* LDBCommand::InitFromCmdLineArgs(
91 int argc, char** argv, const Options& options,
92 const LDBOptions& ldb_options,
93 const std::vector<ColumnFamilyDescriptor>* column_families) {
94 std::vector<std::string> args;
95 for (int i = 1; i < argc; i++) {
96 args.push_back(argv[i]);
97 }
98 return InitFromCmdLineArgs(args, options, ldb_options, column_families,
99 SelectCommand);
100}
101
102/**
103 * Parse the command-line arguments and create the appropriate LDBCommand2
104 * instance.
105 * The command line arguments must be in the following format:
106 * ./ldb --db=PATH_TO_DB [--commonOpt1=commonOpt1Val] ..
107 * COMMAND <PARAM1> <PARAM2> ... [-cmdSpecificOpt1=cmdSpecificOpt1Val] ..
108 * This is similar to the command line format used by HBaseClientTool.
109 * Command name is not included in args.
110 * Returns nullptr if the command-line cannot be parsed.
111 */
112LDBCommand* LDBCommand::InitFromCmdLineArgs(
113 const std::vector<std::string>& args, const Options& options,
114 const LDBOptions& ldb_options,
11fdf7f2 115 const std::vector<ColumnFamilyDescriptor>* /*column_families*/,
7c673cae
FG		 116 const std::function<LDBCommand*(const ParsedParams&)>& selector) {
117 // --x=y command line arguments are added as x->y map entries in
118 // parsed_params.option_map.
119 //
120 // Command-line arguments of the form --hex end up in this array as hex to
121 // parsed_params.flags
122 ParsedParams parsed_params;
123
124 // Everything other than option_map and flags. Represents commands
125 // and their parameters. For eg: put key1 value1 go into this vector.
126 std::vector<std::string> cmdTokens;
127
128 const std::string OPTION_PREFIX = "--";
129
130 for (const auto& arg : args) {
131 if (arg[0] == '-' && arg[1] == '-'){
132 std::vector<std::string> splits = StringSplit(arg, '=');
133 if (splits.size() == 2) {
134 std::string optionKey = splits[0].substr(OPTION_PREFIX.size());
135 parsed_params.option_map[optionKey] = splits[1];
136 } else {
137 std::string optionKey = splits[0].substr(OPTION_PREFIX.size());
138 parsed_params.flags.push_back(optionKey);
139 }
140 } else {
141 cmdTokens.push_back(arg);
142 }
143 }
144
145 if (cmdTokens.size() < 1) {
146 fprintf(stderr, "Command not specified!");
147 return nullptr;
148 }
149
150 parsed_params.cmd = cmdTokens[0];
151 parsed_params.cmd_params.assign(cmdTokens.begin() + 1, cmdTokens.end());
152
153 LDBCommand* command = selector(parsed_params);
154
155 if (command) {
156 command->SetDBOptions(options);
157 command->SetLDBOptions(ldb_options);
158 }
159 return command;
160}
161
162LDBCommand* LDBCommand::SelectCommand(const ParsedParams& parsed_params) {
163 if (parsed_params.cmd == GetCommand::Name()) {
164 return new GetCommand(parsed_params.cmd_params, parsed_params.option_map,
165 parsed_params.flags);
166 } else if (parsed_params.cmd == PutCommand::Name()) {
167 return new PutCommand(parsed_params.cmd_params, parsed_params.option_map,
168 parsed_params.flags);
169 } else if (parsed_params.cmd == BatchPutCommand::Name()) {
170 return new BatchPutCommand(parsed_params.cmd_params,
171 parsed_params.option_map, parsed_params.flags);
172 } else if (parsed_params.cmd == ScanCommand::Name()) {
173 return new ScanCommand(parsed_params.cmd_params, parsed_params.option_map,
174 parsed_params.flags);
175 } else if (parsed_params.cmd == DeleteCommand::Name()) {
176 return new DeleteCommand(parsed_params.cmd_params, parsed_params.option_map,
177 parsed_params.flags);
178 } else if (parsed_params.cmd == DeleteRangeCommand::Name()) {
179 return new DeleteRangeCommand(parsed_params.cmd_params,
180 parsed_params.option_map,
181 parsed_params.flags);
182 } else if (parsed_params.cmd == ApproxSizeCommand::Name()) {
183 return new ApproxSizeCommand(parsed_params.cmd_params,
184 parsed_params.option_map, parsed_params.flags);
185 } else if (parsed_params.cmd == DBQuerierCommand::Name()) {
186 return new DBQuerierCommand(parsed_params.cmd_params,
187 parsed_params.option_map, parsed_params.flags);
188 } else if (parsed_params.cmd == CompactorCommand::Name()) {
189 return new CompactorCommand(parsed_params.cmd_params,
190 parsed_params.option_map, parsed_params.flags);
191 } else if (parsed_params.cmd == WALDumperCommand::Name()) {
192 return new WALDumperCommand(parsed_params.cmd_params,
193 parsed_params.option_map, parsed_params.flags);
194 } else if (parsed_params.cmd == ReduceDBLevelsCommand::Name()) {
195 return new ReduceDBLevelsCommand(parsed_params.cmd_params,
196 parsed_params.option_map,
197 parsed_params.flags);
198 } else if (parsed_params.cmd == ChangeCompactionStyleCommand::Name()) {
199 return new ChangeCompactionStyleCommand(parsed_params.cmd_params,
200 parsed_params.option_map,
201 parsed_params.flags);
202 } else if (parsed_params.cmd == DBDumperCommand::Name()) {
203 return new DBDumperCommand(parsed_params.cmd_params,
204 parsed_params.option_map, parsed_params.flags);
205 } else if (parsed_params.cmd == DBLoaderCommand::Name()) {
206 return new DBLoaderCommand(parsed_params.cmd_params,
207 parsed_params.option_map, parsed_params.flags);
208 } else if (parsed_params.cmd == ManifestDumpCommand::Name()) {
209 return new ManifestDumpCommand(parsed_params.cmd_params,
210 parsed_params.option_map,
211 parsed_params.flags);
212 } else if (parsed_params.cmd == ListColumnFamiliesCommand::Name()) {
213 return new ListColumnFamiliesCommand(parsed_params.cmd_params,
214 parsed_params.option_map,
215 parsed_params.flags);
216 } else if (parsed_params.cmd == CreateColumnFamilyCommand::Name()) {
217 return new CreateColumnFamilyCommand(parsed_params.cmd_params,
218 parsed_params.option_map,
219 parsed_params.flags);
220 } else if (parsed_params.cmd == DBFileDumperCommand::Name()) {
221 return new DBFileDumperCommand(parsed_params.cmd_params,
222 parsed_params.option_map,
223 parsed_params.flags);
224 } else if (parsed_params.cmd == InternalDumpCommand::Name()) {
225 return new InternalDumpCommand(parsed_params.cmd_params,
226 parsed_params.option_map,
227 parsed_params.flags);
228 } else if (parsed_params.cmd == CheckConsistencyCommand::Name()) {
229 return new CheckConsistencyCommand(parsed_params.cmd_params,
230 parsed_params.option_map,
231 parsed_params.flags);
232 } else if (parsed_params.cmd == CheckPointCommand::Name()) {
233 return new CheckPointCommand(parsed_params.cmd_params,
234 parsed_params.option_map,
235 parsed_params.flags);
236 } else if (parsed_params.cmd == RepairCommand::Name()) {
237 return new RepairCommand(parsed_params.cmd_params, parsed_params.option_map,
238 parsed_params.flags);
239 } else if (parsed_params.cmd == BackupCommand::Name()) {
240 return new BackupCommand(parsed_params.cmd_params, parsed_params.option_map,
241 parsed_params.flags);
242 } else if (parsed_params.cmd == RestoreCommand::Name()) {
243 return new RestoreCommand(parsed_params.cmd_params,
244 parsed_params.option_map, parsed_params.flags);
11fdf7f2
TL		 245 } else if (parsed_params.cmd == WriteExternalSstFilesCommand::Name()) {
246 return new WriteExternalSstFilesCommand(parsed_params.cmd_params,
247 parsed_params.option_map,
248 parsed_params.flags);
249 } else if (parsed_params.cmd == IngestExternalSstFilesCommand::Name()) {
250 return new IngestExternalSstFilesCommand(parsed_params.cmd_params,
251 parsed_params.option_map,
252 parsed_params.flags);
7c673cae
FG		 253 }
254 return nullptr;
255}
256
257/* Run the command, and return the execute result. */
258void LDBCommand::Run() {
259 if (!exec_state_.IsNotStarted()) {
260 return;
261 }
262
263 if (db_ == nullptr && !NoDBOpen()) {
264 OpenDB();
265 if (exec_state_.IsFailed() && try_load_options_) {
266 // We don't always return if there is a failure because a WAL file or
267 // manifest file can be given to "dump" command so we should continue.
268 // --try_load_options is not valid in those cases.
269 return;
270 }
271 }
272
273 // We'll intentionally proceed even if the DB can't be opened because users
274 // can also specify a filename, not just a directory.
275 DoCommand();
276
277 if (exec_state_.IsNotStarted()) {
278 exec_state_ = LDBCommandExecuteResult::Succeed("");
279 }
280
281 if (db_ != nullptr) {
282 CloseDB();
283 }
284}
285
286LDBCommand::LDBCommand(const std::map<std::string, std::string>& options,
287 const std::vector<std::string>& flags, bool is_read_only,
288 const std::vector<std::string>& valid_cmd_line_options)
289 : db_(nullptr),
11fdf7f2 290 db_ttl_(nullptr),
7c673cae
FG		 291 is_read_only_(is_read_only),
292 is_key_hex_(false),
293 is_value_hex_(false),
294 is_db_ttl_(false),
295 timestamp_(false),
296 try_load_options_(false),
11fdf7f2 297 ignore_unknown_options_(false),
7c673cae
FG		 298 create_if_missing_(false),
299 option_map_(options),
300 flags_(flags),
301 valid_cmd_line_options_(valid_cmd_line_options) {
302 std::map<std::string, std::string>::const_iterator itr = options.find(ARG_DB);
303 if (itr != options.end()) {
304 db_path_ = itr->second;
305 }
306
307 itr = options.find(ARG_CF_NAME);
308 if (itr != options.end()) {
309 column_family_name_ = itr->second;
310 } else {
311 column_family_name_ = kDefaultColumnFamilyName;
312 }
313
314 is_key_hex_ = IsKeyHex(options, flags);
315 is_value_hex_ = IsValueHex(options, flags);
316 is_db_ttl_ = IsFlagPresent(flags, ARG_TTL);
317 timestamp_ = IsFlagPresent(flags, ARG_TIMESTAMP);
318 try_load_options_ = IsFlagPresent(flags, ARG_TRY_LOAD_OPTIONS);
11fdf7f2 319 ignore_unknown_options_ = IsFlagPresent(flags, ARG_IGNORE_UNKNOWN_OPTIONS);
7c673cae
FG		 320}
321
322void LDBCommand::OpenDB() {
7c673cae 323 if (!create_if_missing_ && try_load_options_) {
11fdf7f2
TL		 324 Status s = LoadLatestOptions(db_path_, Env::Default(), &options_,
325 &column_families_, ignore_unknown_options_);
326 if (!s.ok() && !s.IsNotFound()) {
7c673cae
FG		 327 // Option file exists but load option file error.
328 std::string msg = s.ToString();
329 exec_state_ = LDBCommandExecuteResult::Failed(msg);
330 db_ = nullptr;
331 return;
332 }
333 }
11fdf7f2 334 options_ = PrepareOptionsForOpenDB();
7c673cae
FG		 335 if (!exec_state_.IsNotStarted()) {
336 return;
337 }
338 // Open the DB.
339 Status st;
340 std::vector<ColumnFamilyHandle*> handles_opened;
341 if (is_db_ttl_) {
342 // ldb doesn't yet support TTL DB with multiple column families
343 if (!column_family_name_.empty() || !column_families_.empty()) {
344 exec_state_ = LDBCommandExecuteResult::Failed(
345 "ldb doesn't support TTL DB with multiple column families");
346 }
347 if (is_read_only_) {
11fdf7f2 348 st = DBWithTTL::Open(options_, db_path_, &db_ttl_, 0, true);
7c673cae 349 } else {
11fdf7f2 350 st = DBWithTTL::Open(options_, db_path_, &db_ttl_);
7c673cae
FG		 351 }
352 db_ = db_ttl_;
353 } else {
11fdf7f2 354 if (column_families_.empty()) {
7c673cae
FG		 355 // Try to figure out column family lists
356 std::vector<std::string> cf_list;
357 st = DB::ListColumnFamilies(DBOptions(), db_path_, &cf_list);
358 // There is possible the DB doesn't exist yet, for "create if not
359 // "existing case". The failure is ignored here. We rely on DB::Open()
360 // to give us the correct error message for problem with opening
361 // existing DB.
362 if (st.ok() && cf_list.size() > 1) {
363 // Ignore single column family DB.
364 for (auto cf_name : cf_list) {
11fdf7f2 365 column_families_.emplace_back(cf_name, options_);
7c673cae
FG		 366 }
367 }
368 }
369 if (is_read_only_) {
370 if (column_families_.empty()) {
11fdf7f2 371 st = DB::OpenForReadOnly(options_, db_path_, &db_);
7c673cae 372 } else {
11fdf7f2 373 st = DB::OpenForReadOnly(options_, db_path_, column_families_,
7c673cae
FG		 374 &handles_opened, &db_);
375 }
376 } else {
377 if (column_families_.empty()) {
11fdf7f2 378 st = DB::Open(options_, db_path_, &db_);
7c673cae 379 } else {
11fdf7f2
TL		 380 st = DB::Open(options_, db_path_, column_families_, &handles_opened,
381 &db_);
7c673cae
FG		 382 }
383 }
384 }
385 if (!st.ok()) {
386 std::string msg = st.ToString();
387 exec_state_ = LDBCommandExecuteResult::Failed(msg);
388 } else if (!handles_opened.empty()) {
389 assert(handles_opened.size() == column_families_.size());
390 bool found_cf_name = false;
391 for (size_t i = 0; i < handles_opened.size(); i++) {
392 cf_handles_[column_families_[i].name] = handles_opened[i];
393 if (column_family_name_ == column_families_[i].name) {
394 found_cf_name = true;
395 }
396 }
397 if (!found_cf_name) {
398 exec_state_ = LDBCommandExecuteResult::Failed(
399 "Non-existing column family " + column_family_name_);
400 CloseDB();
401 }
402 } else {
403 // We successfully opened DB in single column family mode.
404 assert(column_families_.empty());
405 if (column_family_name_ != kDefaultColumnFamilyName) {
406 exec_state_ = LDBCommandExecuteResult::Failed(
407 "Non-existing column family " + column_family_name_);
408 CloseDB();
409 }
410 }
7c673cae
FG		 411}
412
413void LDBCommand::CloseDB() {
414 if (db_ != nullptr) {
415 for (auto& pair : cf_handles_) {
416 delete pair.second;
417 }
418 delete db_;
419 db_ = nullptr;
420 }
421}
422
423ColumnFamilyHandle* LDBCommand::GetCfHandle() {
424 if (!cf_handles_.empty()) {
425 auto it = cf_handles_.find(column_family_name_);
426 if (it == cf_handles_.end()) {
427 exec_state_ = LDBCommandExecuteResult::Failed(
428 "Cannot find column family " + column_family_name_);
429 } else {
430 return it->second;
431 }
432 }
433 return db_->DefaultColumnFamily();
434}
435
436std::vector<std::string> LDBCommand::BuildCmdLineOptions(
437 std::vector<std::string> options) {
438 std::vector<std::string> ret = {ARG_DB,
439 ARG_BLOOM_BITS,
440 ARG_BLOCK_SIZE,
441 ARG_AUTO_COMPACTION,
442 ARG_COMPRESSION_TYPE,
443 ARG_COMPRESSION_MAX_DICT_BYTES,
444 ARG_WRITE_BUFFER_SIZE,
445 ARG_FILE_SIZE,
446 ARG_FIX_PREFIX_LEN,
447 ARG_TRY_LOAD_OPTIONS,
11fdf7f2 448 ARG_IGNORE_UNKNOWN_OPTIONS,
7c673cae
FG		 449 ARG_CF_NAME};
450 ret.insert(ret.end(), options.begin(), options.end());
451 return ret;
452}
453
454/**
455 * Parses the specific integer option and fills in the value.
456 * Returns true if the option is found.
457 * Returns false if the option is not found or if there is an error parsing the
458 * value. If there is an error, the specified exec_state is also
459 * updated.
460 */
461bool LDBCommand::ParseIntOption(
11fdf7f2 462 const std::map<std::string, std::string>& /*options*/,
7c673cae
FG		 463 const std::string& option, int& value,
464 LDBCommandExecuteResult& exec_state) {
465 std::map<std::string, std::string>::const_iterator itr =
466 option_map_.find(option);
467 if (itr != option_map_.end()) {
468 try {
469#if defined(CYGWIN)
470 value = strtol(itr->second.c_str(), 0, 10);
471#else
472 value = std::stoi(itr->second);
473#endif
474 return true;
475 } catch (const std::invalid_argument&) {
476 exec_state =
477 LDBCommandExecuteResult::Failed(option + " has an invalid value.");
478 } catch (const std::out_of_range&) {
479 exec_state = LDBCommandExecuteResult::Failed(
480 option + " has a value out-of-range.");
481 }
482 }
483 return false;
484}
485
486/**
487 * Parses the specified option and fills in the value.
488 * Returns true if the option is found.
489 * Returns false otherwise.
490 */
491bool LDBCommand::ParseStringOption(
11fdf7f2 492 const std::map<std::string, std::string>& /*options*/,
7c673cae
FG		 493 const std::string& option, std::string* value) {
494 auto itr = option_map_.find(option);
495 if (itr != option_map_.end()) {
496 *value = itr->second;
497 return true;
498 }
499 return false;
500}
501
502Options LDBCommand::PrepareOptionsForOpenDB() {
11fdf7f2
TL		 503 ColumnFamilyOptions* cf_opts;
504 auto column_families_iter =
505 std::find_if(column_families_.begin(), column_families_.end(),
506 [this](const ColumnFamilyDescriptor& cf_desc) {
507 return cf_desc.name == column_family_name_;
508 });
509 if (column_families_iter != column_families_.end()) {
510 cf_opts = &column_families_iter->options;
511 } else {
512 cf_opts = static_cast<ColumnFamilyOptions*>(&options_);
513 }
514 DBOptions* db_opts = static_cast<DBOptions*>(&options_);
515 db_opts->create_if_missing = false;
7c673cae
FG		 516
517 std::map<std::string, std::string>::const_iterator itr;
518
519 BlockBasedTableOptions table_options;
520 bool use_table_options = false;
521 int bits;
522 if (ParseIntOption(option_map_, ARG_BLOOM_BITS, bits, exec_state_)) {
523 if (bits > 0) {
524 use_table_options = true;
525 table_options.filter_policy.reset(NewBloomFilterPolicy(bits));
526 } else {
527 exec_state_ =
528 LDBCommandExecuteResult::Failed(ARG_BLOOM_BITS + " must be > 0.");
529 }
530 }
531
532 int block_size;
533 if (ParseIntOption(option_map_, ARG_BLOCK_SIZE, block_size, exec_state_)) {
534 if (block_size > 0) {
535 use_table_options = true;
536 table_options.block_size = block_size;
537 } else {
538 exec_state_ =
539 LDBCommandExecuteResult::Failed(ARG_BLOCK_SIZE + " must be > 0.");
540 }
541 }
542
543 if (use_table_options) {
11fdf7f2 544 cf_opts->table_factory.reset(NewBlockBasedTableFactory(table_options));
7c673cae
FG		 545 }
546
547 itr = option_map_.find(ARG_AUTO_COMPACTION);
548 if (itr != option_map_.end()) {
11fdf7f2 549 cf_opts->disable_auto_compactions = !StringToBool(itr->second);
7c673cae
FG		 550 }
551
552 itr = option_map_.find(ARG_COMPRESSION_TYPE);
553 if (itr != option_map_.end()) {
554 std::string comp = itr->second;
555 if (comp == "no") {
11fdf7f2 556 cf_opts->compression = kNoCompression;
7c673cae 557 } else if (comp == "snappy") {
11fdf7f2 558 cf_opts->compression = kSnappyCompression;
7c673cae 559 } else if (comp == "zlib") {
11fdf7f2 560 cf_opts->compression = kZlibCompression;
7c673cae 561 } else if (comp == "bzip2") {
11fdf7f2 562 cf_opts->compression = kBZip2Compression;
7c673cae 563 } else if (comp == "lz4") {
11fdf7f2 564 cf_opts->compression = kLZ4Compression;
7c673cae 565 } else if (comp == "lz4hc") {
11fdf7f2 566 cf_opts->compression = kLZ4HCCompression;
7c673cae 567 } else if (comp == "xpress") {
11fdf7f2 568 cf_opts->compression = kXpressCompression;
7c673cae 569 } else if (comp == "zstd") {
11fdf7f2 570 cf_opts->compression = kZSTD;
7c673cae
FG		 571 } else {
572 // Unknown compression.
573 exec_state_ =
574 LDBCommandExecuteResult::Failed("Unknown compression level: " + comp);
575 }
576 }
577
578 int compression_max_dict_bytes;
579 if (ParseIntOption(option_map_, ARG_COMPRESSION_MAX_DICT_BYTES,
580 compression_max_dict_bytes, exec_state_)) {
581 if (compression_max_dict_bytes >= 0) {
11fdf7f2 582 cf_opts->compression_opts.max_dict_bytes = compression_max_dict_bytes;
7c673cae
FG		 583 } else {
584 exec_state_ = LDBCommandExecuteResult::Failed(
585 ARG_COMPRESSION_MAX_DICT_BYTES + " must be >= 0.");
586 }
587 }
588
589 int db_write_buffer_size;
590 if (ParseIntOption(option_map_, ARG_DB_WRITE_BUFFER_SIZE,
591 db_write_buffer_size, exec_state_)) {
592 if (db_write_buffer_size >= 0) {
11fdf7f2 593 db_opts->db_write_buffer_size = db_write_buffer_size;
7c673cae
FG		 594 } else {
595 exec_state_ = LDBCommandExecuteResult::Failed(ARG_DB_WRITE_BUFFER_SIZE +
596 " must be >= 0.");
597 }
598 }
599
600 int write_buffer_size;
601 if (ParseIntOption(option_map_, ARG_WRITE_BUFFER_SIZE, write_buffer_size,
602 exec_state_)) {
603 if (write_buffer_size > 0) {
11fdf7f2 604 cf_opts->write_buffer_size = write_buffer_size;
7c673cae
FG		 605 } else {
606 exec_state_ = LDBCommandExecuteResult::Failed(ARG_WRITE_BUFFER_SIZE +
607 " must be > 0.");
608 }
609 }
610
611 int file_size;
612 if (ParseIntOption(option_map_, ARG_FILE_SIZE, file_size, exec_state_)) {
613 if (file_size > 0) {
11fdf7f2 614 cf_opts->target_file_size_base = file_size;
7c673cae
FG		 615 } else {
616 exec_state_ =
617 LDBCommandExecuteResult::Failed(ARG_FILE_SIZE + " must be > 0.");
618 }
619 }
620
11fdf7f2
TL		 621 if (db_opts->db_paths.size() == 0) {
622 db_opts->db_paths.emplace_back(db_path_,
623 std::numeric_limits<uint64_t>::max());
7c673cae
FG		 624 }
625
626 int fix_prefix_len;
627 if (ParseIntOption(option_map_, ARG_FIX_PREFIX_LEN, fix_prefix_len,
628 exec_state_)) {
629 if (fix_prefix_len > 0) {
11fdf7f2 630 cf_opts->prefix_extractor.reset(
7c673cae
FG		 631 NewFixedPrefixTransform(static_cast<size_t>(fix_prefix_len)));
632 } else {
633 exec_state_ =
634 LDBCommandExecuteResult::Failed(ARG_FIX_PREFIX_LEN + " must be > 0.");
635 }
636 }
11fdf7f2
TL		 637 // TODO(ajkr): this return value doesn't reflect the CF options changed, so
638 // subcommands that rely on this won't see the effect of CF-related CLI args.
639 // Such subcommands need to be changed to properly support CFs.
640 return options_;
7c673cae
FG		 641}
642
643bool LDBCommand::ParseKeyValue(const std::string& line, std::string* key,
644 std::string* value, bool is_key_hex,
645 bool is_value_hex) {
646 size_t pos = line.find(DELIM);
647 if (pos != std::string::npos) {
648 *key = line.substr(0, pos);
649 *value = line.substr(pos + strlen(DELIM));
650 if (is_key_hex) {
651 *key = HexToString(*key);
652 }
653 if (is_value_hex) {
654 *value = HexToString(*value);
655 }
656 return true;
657 } else {
658 return false;
659 }
660}
661
662/**
663 * Make sure that ONLY the command-line options and flags expected by this
664 * command are specified on the command-line. Extraneous options are usually
665 * the result of user error.
666 * Returns true if all checks pass. Else returns false, and prints an
667 * appropriate error msg to stderr.
668 */
669bool LDBCommand::ValidateCmdLineOptions() {
670 for (std::map<std::string, std::string>::const_iterator itr =
671 option_map_.begin();
672 itr != option_map_.end(); ++itr) {
673 if (std::find(valid_cmd_line_options_.begin(),
674 valid_cmd_line_options_.end(),
675 itr->first) == valid_cmd_line_options_.end()) {
676 fprintf(stderr, "Invalid command-line option %s\n", itr->first.c_str());
677 return false;
678 }
679 }
680
681 for (std::vector<std::string>::const_iterator itr = flags_.begin();
682 itr != flags_.end(); ++itr) {
683 if (std::find(valid_cmd_line_options_.begin(),
684 valid_cmd_line_options_.end(),
685 *itr) == valid_cmd_line_options_.end()) {
686 fprintf(stderr, "Invalid command-line flag %s\n", itr->c_str());
687 return false;
688 }
689 }
690
691 if (!NoDBOpen() && option_map_.find(ARG_DB) == option_map_.end() &&
692 option_map_.find(ARG_PATH) == option_map_.end()) {
693 fprintf(stderr, "Either %s or %s must be specified.\n", ARG_DB.c_str(),
694 ARG_PATH.c_str());
695 return false;
696 }
697
698 return true;
699}
700
701std::string LDBCommand::HexToString(const std::string& str) {
702 std::string result;
703 std::string::size_type len = str.length();
704 if (len < 2 || str[0] != '0' || str[1] != 'x') {
705 fprintf(stderr, "Invalid hex input %s. Must start with 0x\n", str.c_str());
706 throw "Invalid hex input";
707 }
708 if (!Slice(str.data() + 2, len - 2).DecodeHex(&result)) {
709 throw "Invalid hex input";
710 }
711 return result;
712}
713
714std::string LDBCommand::StringToHex(const std::string& str) {
715 std::string result("0x");
716 result.append(Slice(str).ToString(true));
717 return result;
718}
719
720std::string LDBCommand::PrintKeyValue(const std::string& key,
721 const std::string& value, bool is_key_hex,
722 bool is_value_hex) {
723 std::string result;
724 result.append(is_key_hex ? StringToHex(key) : key);
725 result.append(DELIM);
726 result.append(is_value_hex ? StringToHex(value) : value);
727 return result;
728}
729
730std::string LDBCommand::PrintKeyValue(const std::string& key,
731 const std::string& value, bool is_hex) {
732 return PrintKeyValue(key, value, is_hex, is_hex);
733}
734
735std::string LDBCommand::HelpRangeCmdArgs() {
736 std::ostringstream str_stream;
737 str_stream << " ";
738 str_stream << "[--" << ARG_FROM << "] ";
739 str_stream << "[--" << ARG_TO << "] ";
740 return str_stream.str();
741}
742
743bool LDBCommand::IsKeyHex(const std::map<std::string, std::string>& options,
744 const std::vector<std::string>& flags) {
745 return (IsFlagPresent(flags, ARG_HEX) || IsFlagPresent(flags, ARG_KEY_HEX) ||
746 ParseBooleanOption(options, ARG_HEX, false) ||
747 ParseBooleanOption(options, ARG_KEY_HEX, false));
748}
749
750bool LDBCommand::IsValueHex(const std::map<std::string, std::string>& options,
751 const std::vector<std::string>& flags) {
752 return (IsFlagPresent(flags, ARG_HEX) ||
753 IsFlagPresent(flags, ARG_VALUE_HEX) ||
754 ParseBooleanOption(options, ARG_HEX, false) ||
755 ParseBooleanOption(options, ARG_VALUE_HEX, false));
756}
757
758bool LDBCommand::ParseBooleanOption(
759 const std::map<std::string, std::string>& options,
760 const std::string& option, bool default_val) {
761 std::map<std::string, std::string>::const_iterator itr = options.find(option);
762 if (itr != options.end()) {
763 std::string option_val = itr->second;
764 return StringToBool(itr->second);
765 }
766 return default_val;
767}
768
769bool LDBCommand::StringToBool(std::string val) {
770 std::transform(val.begin(), val.end(), val.begin(),
771 [](char ch) -> char { return (char)::tolower(ch); });
772
773 if (val == "true") {
774 return true;
775 } else if (val == "false") {
776 return false;
777 } else {
778 throw "Invalid value for boolean argument";
779 }
780}
781
782CompactorCommand::CompactorCommand(
11fdf7f2 783 const std::vector<std::string>& /*params*/,
7c673cae
FG		 784 const std::map<std::string, std::string>& options,
785 const std::vector<std::string>& flags)
786 : LDBCommand(options, flags, false,
787 BuildCmdLineOptions({ARG_FROM, ARG_TO, ARG_HEX, ARG_KEY_HEX,
788 ARG_VALUE_HEX, ARG_TTL})),
789 null_from_(true),
790 null_to_(true) {
791 std::map<std::string, std::string>::const_iterator itr =
792 options.find(ARG_FROM);
793 if (itr != options.end()) {
794 null_from_ = false;
795 from_ = itr->second;
796 }
797
798 itr = options.find(ARG_TO);
799 if (itr != options.end()) {
800 null_to_ = false;
801 to_ = itr->second;
802 }
803
804 if (is_key_hex_) {
805 if (!null_from_) {
806 from_ = HexToString(from_);
807 }
808 if (!null_to_) {
809 to_ = HexToString(to_);
810 }
811 }
812}
813
814void CompactorCommand::Help(std::string& ret) {
815 ret.append(" ");
816 ret.append(CompactorCommand::Name());
817 ret.append(HelpRangeCmdArgs());
818 ret.append("\n");
819}
820
821void CompactorCommand::DoCommand() {
822 if (!db_) {
823 assert(GetExecuteState().IsFailed());
824 return;
825 }
826
827 Slice* begin = nullptr;
828 Slice* end = nullptr;
829 if (!null_from_) {
830 begin = new Slice(from_);
831 }
832 if (!null_to_) {
833 end = new Slice(to_);
834 }
835
836 CompactRangeOptions cro;
837 cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
838
839 db_->CompactRange(cro, GetCfHandle(), begin, end);
840 exec_state_ = LDBCommandExecuteResult::Succeed("");
841
842 delete begin;
843 delete end;
844}
845
846// ----------------------------------------------------------------------------
847
848const std::string DBLoaderCommand::ARG_DISABLE_WAL = "disable_wal";
849const std::string DBLoaderCommand::ARG_BULK_LOAD = "bulk_load";
850const std::string DBLoaderCommand::ARG_COMPACT = "compact";
851
852DBLoaderCommand::DBLoaderCommand(
11fdf7f2 853 const std::vector<std::string>& /*params*/,
7c673cae
FG		 854 const std::map<std::string, std::string>& options,
855 const std::vector<std::string>& flags)
856 : LDBCommand(
857 options, flags, false,
858 BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM,
859 ARG_TO, ARG_CREATE_IF_MISSING, ARG_DISABLE_WAL,
860 ARG_BULK_LOAD, ARG_COMPACT})),
861 disable_wal_(false),
862 bulk_load_(false),
863 compact_(false) {
864 create_if_missing_ = IsFlagPresent(flags, ARG_CREATE_IF_MISSING);
865 disable_wal_ = IsFlagPresent(flags, ARG_DISABLE_WAL);
866 bulk_load_ = IsFlagPresent(flags, ARG_BULK_LOAD);
867 compact_ = IsFlagPresent(flags, ARG_COMPACT);
868}
869
870void DBLoaderCommand::Help(std::string& ret) {
871 ret.append(" ");
872 ret.append(DBLoaderCommand::Name());
873 ret.append(" [--" + ARG_CREATE_IF_MISSING + "]");
874 ret.append(" [--" + ARG_DISABLE_WAL + "]");
875 ret.append(" [--" + ARG_BULK_LOAD + "]");
876 ret.append(" [--" + ARG_COMPACT + "]");
877 ret.append("\n");
878}
879
880Options DBLoaderCommand::PrepareOptionsForOpenDB() {
881 Options opt = LDBCommand::PrepareOptionsForOpenDB();
882 opt.create_if_missing = create_if_missing_;
883 if (bulk_load_) {
884 opt.PrepareForBulkLoad();
885 }
886 return opt;
887}
888
889void DBLoaderCommand::DoCommand() {
890 if (!db_) {
891 assert(GetExecuteState().IsFailed());
892 return;
893 }
894
895 WriteOptions write_options;
896 if (disable_wal_) {
897 write_options.disableWAL = true;
898 }
899
900 int bad_lines = 0;
901 std::string line;
902 // prefer ifstream getline performance vs that from std::cin istream
903 std::ifstream ifs_stdin("/dev/stdin");
904 std::istream* istream_p = ifs_stdin.is_open() ? &ifs_stdin : &std::cin;
905 while (getline(*istream_p, line, '\n')) {
906 std::string key;
907 std::string value;
908 if (ParseKeyValue(line, &key, &value, is_key_hex_, is_value_hex_)) {
909 db_->Put(write_options, GetCfHandle(), Slice(key), Slice(value));
910 } else if (0 == line.find("Keys in range:")) {
911 // ignore this line
912 } else if (0 == line.find("Created bg thread 0x")) {
913 // ignore this line
914 } else {
915 bad_lines ++;
916 }
917 }
918
919 if (bad_lines > 0) {
920 std::cout << "Warning: " << bad_lines << " bad lines ignored." << std::endl;
921 }
922 if (compact_) {
923 db_->CompactRange(CompactRangeOptions(), GetCfHandle(), nullptr, nullptr);
924 }
925}
926
927// ----------------------------------------------------------------------------
928
929namespace {
930
931void DumpManifestFile(std::string file, bool verbose, bool hex, bool json) {
932 Options options;
933 EnvOptions sopt;
934 std::string dbname("dummy");
935 std::shared_ptr<Cache> tc(NewLRUCache(options.max_open_files - 10,
936 options.table_cache_numshardbits));
937 // Notice we are using the default options not through SanitizeOptions(),
938 // if VersionSet::DumpManifest() depends on any option done by
939 // SanitizeOptions(), we need to initialize it manually.
940 options.db_paths.emplace_back("dummy", 0);
941 options.num_levels = 64;
942 WriteController wc(options.delayed_write_rate);
943 WriteBufferManager wb(options.db_write_buffer_size);
944 ImmutableDBOptions immutable_db_options(options);
945 VersionSet versions(dbname, &immutable_db_options, sopt, tc.get(), &wb, &wc);
946 Status s = versions.DumpManifest(options, file, verbose, hex, json);
947 if (!s.ok()) {
948 printf("Error in processing file %s %s\n", file.c_str(),
949 s.ToString().c_str());
950 }
951}
952
953} // namespace
954
955const std::string ManifestDumpCommand::ARG_VERBOSE = "verbose";
956const std::string ManifestDumpCommand::ARG_JSON = "json";
957const std::string ManifestDumpCommand::ARG_PATH = "path";
958
959void ManifestDumpCommand::Help(std::string& ret) {
960 ret.append(" ");
961 ret.append(ManifestDumpCommand::Name());
962 ret.append(" [--" + ARG_VERBOSE + "]");
963 ret.append(" [--" + ARG_JSON + "]");
964 ret.append(" [--" + ARG_PATH + "=<path_to_manifest_file>]");
965 ret.append("\n");
966}
967
968ManifestDumpCommand::ManifestDumpCommand(
11fdf7f2 969 const std::vector<std::string>& /*params*/,
7c673cae
FG		 970 const std::map<std::string, std::string>& options,
971 const std::vector<std::string>& flags)
972 : LDBCommand(
973 options, flags, false,
974 BuildCmdLineOptions({ARG_VERBOSE, ARG_PATH, ARG_HEX, ARG_JSON})),
975 verbose_(false),
976 json_(false),
977 path_("") {
978 verbose_ = IsFlagPresent(flags, ARG_VERBOSE);
979 json_ = IsFlagPresent(flags, ARG_JSON);
980
981 std::map<std::string, std::string>::const_iterator itr =
982 options.find(ARG_PATH);
983 if (itr != options.end()) {
984 path_ = itr->second;
985 if (path_.empty()) {
986 exec_state_ = LDBCommandExecuteResult::Failed("--path: missing pathname");
987 }
988 }
989}
990
991void ManifestDumpCommand::DoCommand() {
992
993 std::string manifestfile;
994
995 if (!path_.empty()) {
996 manifestfile = path_;
997 } else {
998 bool found = false;
999 // We need to find the manifest file by searching the directory
1000 // containing the db for files of the form MANIFEST_[0-9]+
1001
1002 auto CloseDir = [](DIR* p) { closedir(p); };
1003 std::unique_ptr<DIR, decltype(CloseDir)> d(opendir(db_path_.c_str()),
1004 CloseDir);
1005
1006 if (d == nullptr) {
1007 exec_state_ =
1008 LDBCommandExecuteResult::Failed(db_path_ + " is not a directory");
1009 return;
1010 }
1011 struct dirent* entry;
1012 while ((entry = readdir(d.get())) != nullptr) {
1013 unsigned int match;
1014 uint64_t num;
1015 if (sscanf(entry->d_name, "MANIFEST-%" PRIu64 "%n", &num, &match) &&
1016 match == strlen(entry->d_name)) {
1017 if (!found) {
1018 manifestfile = db_path_ + "/" + std::string(entry->d_name);
1019 found = true;
1020 } else {
1021 exec_state_ = LDBCommandExecuteResult::Failed(
1022 "Multiple MANIFEST files found; use --path to select one");
1023 return;
1024 }
1025 }
1026 }
1027 }
1028
1029 if (verbose_) {
1030 printf("Processing Manifest file %s\n", manifestfile.c_str());
1031 }
1032
1033 DumpManifestFile(manifestfile, verbose_, is_key_hex_, json_);
1034
1035 if (verbose_) {
1036 printf("Processing Manifest file %s done\n", manifestfile.c_str());
1037 }
1038}
1039
1040// ----------------------------------------------------------------------------
1041
1042void ListColumnFamiliesCommand::Help(std::string& ret) {
1043 ret.append(" ");
1044 ret.append(ListColumnFamiliesCommand::Name());
1045 ret.append(" full_path_to_db_directory ");
1046 ret.append("\n");
1047}
1048
1049ListColumnFamiliesCommand::ListColumnFamiliesCommand(
1050 const std::vector<std::string>& params,
1051 const std::map<std::string, std::string>& options,
1052 const std::vector<std::string>& flags)
1053 : LDBCommand(options, flags, false, {}) {
1054 if (params.size() != 1) {
1055 exec_state_ = LDBCommandExecuteResult::Failed(
1056 "dbname must be specified for the list_column_families command");
1057 } else {
1058 dbname_ = params[0];
1059 }
1060}
1061
1062void ListColumnFamiliesCommand::DoCommand() {
1063 std::vector<std::string> column_families;
1064 Status s = DB::ListColumnFamilies(DBOptions(), dbname_, &column_families);
1065 if (!s.ok()) {
1066 printf("Error in processing db %s %s\n", dbname_.c_str(),
1067 s.ToString().c_str());
1068 } else {
1069 printf("Column families in %s: \n{", dbname_.c_str());
1070 bool first = true;
1071 for (auto cf : column_families) {
1072 if (!first) {
1073 printf(", ");
1074 }
1075 first = false;
1076 printf("%s", cf.c_str());
1077 }
1078 printf("}\n");
1079 }
1080}
1081
1082void CreateColumnFamilyCommand::Help(std::string& ret) {
1083 ret.append(" ");
1084 ret.append(CreateColumnFamilyCommand::Name());
1085 ret.append(" --db=<db_path> <new_column_family_name>");
1086 ret.append("\n");
1087}
1088
1089CreateColumnFamilyCommand::CreateColumnFamilyCommand(
1090 const std::vector<std::string>& params,
1091 const std::map<std::string, std::string>& options,
1092 const std::vector<std::string>& flags)
1093 : LDBCommand(options, flags, true, {ARG_DB}) {
1094 if (params.size() != 1) {
1095 exec_state_ = LDBCommandExecuteResult::Failed(
1096 "new column family name must be specified");
1097 } else {
1098 new_cf_name_ = params[0];
1099 }
1100}
1101
1102void CreateColumnFamilyCommand::DoCommand() {
11fdf7f2 1103 ColumnFamilyHandle* new_cf_handle = nullptr;
7c673cae
FG		 1104 Status st = db_->CreateColumnFamily(options_, new_cf_name_, &new_cf_handle);
1105 if (st.ok()) {
1106 fprintf(stdout, "OK\n");
1107 } else {
1108 exec_state_ = LDBCommandExecuteResult::Failed(
1109 "Fail to create new column family: " + st.ToString());
1110 }
1111 delete new_cf_handle;
1112 CloseDB();
1113}
1114
1115// ----------------------------------------------------------------------------
1116
1117namespace {
1118
1119std::string ReadableTime(int unixtime) {
1120 char time_buffer [80];
1121 time_t rawtime = unixtime;
1122 struct tm tInfo;
1123 struct tm* timeinfo = localtime_r(&rawtime, &tInfo);
1124 assert(timeinfo == &tInfo);
1125 strftime(time_buffer, 80, "%c", timeinfo);
1126 return std::string(time_buffer);
1127}
1128
1129// This function only called when it's the sane case of >1 buckets in time-range
1130// Also called only when timekv falls between ttl_start and ttl_end provided
1131void IncBucketCounts(std::vector<uint64_t>& bucket_counts, int ttl_start,
1132 int time_range, int bucket_size, int timekv,
1133 int num_buckets) {
11fdf7f2
TL		 1134#ifdef NDEBUG
1135 (void)time_range;
1136 (void)num_buckets;
1137#endif
7c673cae
FG		 1138 assert(time_range > 0 && timekv >= ttl_start && bucket_size > 0 &&
1139 timekv < (ttl_start + time_range) && num_buckets > 1);
1140 int bucket = (timekv - ttl_start) / bucket_size;
1141 bucket_counts[bucket]++;
1142}
1143
1144void PrintBucketCounts(const std::vector<uint64_t>& bucket_counts,
1145 int ttl_start, int ttl_end, int bucket_size,
1146 int num_buckets) {
1147 int time_point = ttl_start;
1148 for(int i = 0; i < num_buckets - 1; i++, time_point += bucket_size) {
1149 fprintf(stdout, "Keys in range %s to %s : %lu\n",
1150 ReadableTime(time_point).c_str(),
1151 ReadableTime(time_point + bucket_size).c_str(),
1152 (unsigned long)bucket_counts[i]);
1153 }
1154 fprintf(stdout, "Keys in range %s to %s : %lu\n",
1155 ReadableTime(time_point).c_str(),
1156 ReadableTime(ttl_end).c_str(),
1157 (unsigned long)bucket_counts[num_buckets - 1]);
1158}
1159
1160} // namespace
1161
1162const std::string InternalDumpCommand::ARG_COUNT_ONLY = "count_only";
1163const std::string InternalDumpCommand::ARG_COUNT_DELIM = "count_delim";
1164const std::string InternalDumpCommand::ARG_STATS = "stats";
1165const std::string InternalDumpCommand::ARG_INPUT_KEY_HEX = "input_key_hex";
1166
1167InternalDumpCommand::InternalDumpCommand(
11fdf7f2 1168 const std::vector<std::string>& /*params*/,
7c673cae
FG		 1169 const std::map<std::string, std::string>& options,
1170 const std::vector<std::string>& flags)
1171 : LDBCommand(
1172 options, flags, true,
1173 BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM,
1174 ARG_TO, ARG_MAX_KEYS, ARG_COUNT_ONLY,
1175 ARG_COUNT_DELIM, ARG_STATS, ARG_INPUT_KEY_HEX})),
1176 has_from_(false),
1177 has_to_(false),
1178 max_keys_(-1),
1179 delim_("."),
1180 count_only_(false),
1181 count_delim_(false),
1182 print_stats_(false),
1183 is_input_key_hex_(false) {
1184 has_from_ = ParseStringOption(options, ARG_FROM, &from_);
1185 has_to_ = ParseStringOption(options, ARG_TO, &to_);
1186
1187 ParseIntOption(options, ARG_MAX_KEYS, max_keys_, exec_state_);
1188 std::map<std::string, std::string>::const_iterator itr =
1189 options.find(ARG_COUNT_DELIM);
1190 if (itr != options.end()) {
1191 delim_ = itr->second;
1192 count_delim_ = true;
1193 // fprintf(stdout,"delim = %c\n",delim_[0]);
1194 } else {
1195 count_delim_ = IsFlagPresent(flags, ARG_COUNT_DELIM);
1196 delim_=".";
1197 }
1198
1199 print_stats_ = IsFlagPresent(flags, ARG_STATS);
1200 count_only_ = IsFlagPresent(flags, ARG_COUNT_ONLY);
1201 is_input_key_hex_ = IsFlagPresent(flags, ARG_INPUT_KEY_HEX);
1202
1203 if (is_input_key_hex_) {
1204 if (has_from_) {
1205 from_ = HexToString(from_);
1206 }
1207 if (has_to_) {
1208 to_ = HexToString(to_);
1209 }
1210 }
1211}
1212
1213void InternalDumpCommand::Help(std::string& ret) {
1214 ret.append(" ");
1215 ret.append(InternalDumpCommand::Name());
1216 ret.append(HelpRangeCmdArgs());
1217 ret.append(" [--" + ARG_INPUT_KEY_HEX + "]");
1218 ret.append(" [--" + ARG_MAX_KEYS + "=<N>]");
1219 ret.append(" [--" + ARG_COUNT_ONLY + "]");
1220 ret.append(" [--" + ARG_COUNT_DELIM + "=<char>]");
1221 ret.append(" [--" + ARG_STATS + "]");
1222 ret.append("\n");
1223}
1224
1225void InternalDumpCommand::DoCommand() {
1226 if (!db_) {
1227 assert(GetExecuteState().IsFailed());
1228 return;
1229 }
1230
1231 if (print_stats_) {
1232 std::string stats;
1233 if (db_->GetProperty(GetCfHandle(), "rocksdb.stats", &stats)) {
1234 fprintf(stdout, "%s\n", stats.c_str());
1235 }
1236 }
1237
1238 // Cast as DBImpl to get internal iterator
11fdf7f2
TL		 1239 std::vector<KeyVersion> key_versions;
1240 Status st = GetAllKeyVersions(db_, from_, to_, max_keys_, &key_versions);
1241 if (!st.ok()) {
1242 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
7c673cae
FG		 1243 return;
1244 }
1245 std::string rtype1, rtype2, row, val;
1246 rtype2 = "";
1247 uint64_t c=0;
1248 uint64_t s1=0,s2=0;
7c673cae
FG		 1249
1250 long long count = 0;
11fdf7f2
TL		 1251 for (auto& key_version : key_versions) {
1252 InternalKey ikey(key_version.user_key, key_version.sequence,
1253 static_cast<ValueType>(key_version.type));
1254 if (has_to_ && ikey.user_key() == to_) {
1255 // GetAllKeyVersions() includes keys with user key `to_`, but idump has
1256 // traditionally excluded such keys.
7c673cae
FG		 1257 break;
1258 }
7c673cae
FG		 1259 ++count;
1260 int k;
1261 if (count_delim_) {
1262 rtype1 = "";
1263 s1=0;
11fdf7f2
TL		 1264 row = ikey.Encode().ToString();
1265 val = key_version.value;
7c673cae
FG		 1266 for(k=0;row[k]!='\x01' && row[k]!='\0';k++)
1267 s1++;
1268 for(k=0;val[k]!='\x01' && val[k]!='\0';k++)
1269 s1++;
1270 for(int j=0;row[j]!=delim_[0] && row[j]!='\0' && row[j]!='\x01';j++)
1271 rtype1+=row[j];
1272 if(rtype2.compare("") && rtype2.compare(rtype1)!=0) {
11fdf7f2
TL		 1273 fprintf(stdout, "%s => count:%" PRIu64 "\tsize:%" PRIu64 "\n",
1274 rtype2.c_str(), c, s2);
7c673cae
FG		 1275 c=1;
1276 s2=s1;
1277 rtype2 = rtype1;
1278 } else {
1279 c++;
1280 s2+=s1;
1281 rtype2=rtype1;
11fdf7f2 1282 }
7c673cae 1283 }
7c673cae
FG		 1284
1285 if (!count_only_ && !count_delim_) {
1286 std::string key = ikey.DebugString(is_key_hex_);
11fdf7f2 1287 std::string value = Slice(key_version.value).ToString(is_value_hex_);
7c673cae
FG		 1288 std::cout << key << " => " << value << "\n";
1289 }
1290
1291 // Terminate if maximum number of keys have been dumped
1292 if (max_keys_ > 0 && count >= max_keys_) break;
1293 }
1294 if(count_delim_) {
11fdf7f2
TL		 1295 fprintf(stdout, "%s => count:%" PRIu64 "\tsize:%" PRIu64 "\n",
1296 rtype2.c_str(), c, s2);
1297 } else {
1298 fprintf(stdout, "Internal keys in range: %lld\n", count);
1299 }
7c673cae
FG		 1300}
1301
1302const std::string DBDumperCommand::ARG_COUNT_ONLY = "count_only";
1303const std::string DBDumperCommand::ARG_COUNT_DELIM = "count_delim";
1304const std::string DBDumperCommand::ARG_STATS = "stats";
1305const std::string DBDumperCommand::ARG_TTL_BUCKET = "bucket";
1306
1307DBDumperCommand::DBDumperCommand(
11fdf7f2 1308 const std::vector<std::string>& /*params*/,
7c673cae
FG		 1309 const std::map<std::string, std::string>& options,
1310 const std::vector<std::string>& flags)
1311 : LDBCommand(options, flags, true,
1312 BuildCmdLineOptions(
1313 {ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM,
1314 ARG_TO, ARG_MAX_KEYS, ARG_COUNT_ONLY, ARG_COUNT_DELIM,
1315 ARG_STATS, ARG_TTL_START, ARG_TTL_END, ARG_TTL_BUCKET,
1316 ARG_TIMESTAMP, ARG_PATH})),
1317 null_from_(true),
1318 null_to_(true),
1319 max_keys_(-1),
1320 count_only_(false),
1321 count_delim_(false),
1322 print_stats_(false) {
1323 std::map<std::string, std::string>::const_iterator itr =
1324 options.find(ARG_FROM);
1325 if (itr != options.end()) {
1326 null_from_ = false;
1327 from_ = itr->second;
1328 }
1329
1330 itr = options.find(ARG_TO);
1331 if (itr != options.end()) {
1332 null_to_ = false;
1333 to_ = itr->second;
1334 }
1335
1336 itr = options.find(ARG_MAX_KEYS);
1337 if (itr != options.end()) {
1338 try {
1339#if defined(CYGWIN)
1340 max_keys_ = strtol(itr->second.c_str(), 0, 10);
1341#else
1342 max_keys_ = std::stoi(itr->second);
1343#endif
1344 } catch (const std::invalid_argument&) {
1345 exec_state_ = LDBCommandExecuteResult::Failed(ARG_MAX_KEYS +
1346 " has an invalid value");
1347 } catch (const std::out_of_range&) {
1348 exec_state_ = LDBCommandExecuteResult::Failed(
1349 ARG_MAX_KEYS + " has a value out-of-range");
1350 }
1351 }
1352 itr = options.find(ARG_COUNT_DELIM);
1353 if (itr != options.end()) {
1354 delim_ = itr->second;
1355 count_delim_ = true;
1356 } else {
1357 count_delim_ = IsFlagPresent(flags, ARG_COUNT_DELIM);
1358 delim_=".";
1359 }
1360
1361 print_stats_ = IsFlagPresent(flags, ARG_STATS);
1362 count_only_ = IsFlagPresent(flags, ARG_COUNT_ONLY);
1363
1364 if (is_key_hex_) {
1365 if (!null_from_) {
1366 from_ = HexToString(from_);
1367 }
1368 if (!null_to_) {
1369 to_ = HexToString(to_);
1370 }
1371 }
1372
1373 itr = options.find(ARG_PATH);
1374 if (itr != options.end()) {
1375 path_ = itr->second;
11fdf7f2
TL		 1376 if (db_path_.empty()) {
1377 db_path_ = path_;
1378 }
7c673cae
FG		 1379 }
1380}
1381
1382void DBDumperCommand::Help(std::string& ret) {
1383 ret.append(" ");
1384 ret.append(DBDumperCommand::Name());
1385 ret.append(HelpRangeCmdArgs());
1386 ret.append(" [--" + ARG_TTL + "]");
1387 ret.append(" [--" + ARG_MAX_KEYS + "=<N>]");
1388 ret.append(" [--" + ARG_TIMESTAMP + "]");
1389 ret.append(" [--" + ARG_COUNT_ONLY + "]");
1390 ret.append(" [--" + ARG_COUNT_DELIM + "=<char>]");
1391 ret.append(" [--" + ARG_STATS + "]");
1392 ret.append(" [--" + ARG_TTL_BUCKET + "=<N>]");
1393 ret.append(" [--" + ARG_TTL_START + "=<N>:- is inclusive]");
1394 ret.append(" [--" + ARG_TTL_END + "=<N>:- is exclusive]");
1395 ret.append(" [--" + ARG_PATH + "=<path_to_a_file>]");
1396 ret.append("\n");
1397}
1398
1399/**
1400 * Handles two separate cases:
1401 *
1402 * 1) --db is specified - just dump the database.
1403 *
1404 * 2) --path is specified - determine based on file extension what dumping
1405 * function to call. Please note that we intentionally use the extension
1406 * and avoid probing the file contents under the assumption that renaming
1407 * the files is not a supported scenario.
1408 *
1409 */
1410void DBDumperCommand::DoCommand() {
1411 if (!db_) {
1412 assert(!path_.empty());
1413 std::string fileName = GetFileNameFromPath(path_);
1414 uint64_t number;
1415 FileType type;
1416
1417 exec_state_ = LDBCommandExecuteResult::Succeed("");
1418
1419 if (!ParseFileName(fileName, &number, &type)) {
1420 exec_state_ =
1421 LDBCommandExecuteResult::Failed("Can't parse file type: " + path_);
1422 return;
1423 }
1424
1425 switch (type) {
1426 case kLogFile:
11fdf7f2 1427 // TODO(myabandeh): allow configuring is_write_commited
7c673cae 1428 DumpWalFile(path_, /* print_header_ */ true, /* print_values_ */ true,
11fdf7f2 1429 true /* is_write_commited */, &exec_state_);
7c673cae
FG		 1430 break;
1431 case kTableFile:
1432 DumpSstFile(path_, is_key_hex_, /* show_properties */ true);
1433 break;
1434 case kDescriptorFile:
1435 DumpManifestFile(path_, /* verbose_ */ false, is_key_hex_,
1436 /* json_ */ false);
1437 break;
1438 default:
1439 exec_state_ = LDBCommandExecuteResult::Failed(
1440 "File type not supported: " + path_);
1441 break;
1442 }
1443
1444 } else {
1445 DoDumpCommand();
1446 }
1447}
1448
1449void DBDumperCommand::DoDumpCommand() {
1450 assert(nullptr != db_);
1451 assert(path_.empty());
1452
1453 // Parse command line args
1454 uint64_t count = 0;
1455 if (print_stats_) {
1456 std::string stats;
1457 if (db_->GetProperty("rocksdb.stats", &stats)) {
1458 fprintf(stdout, "%s\n", stats.c_str());
1459 }
1460 }
1461
1462 // Setup key iterator
1463 Iterator* iter = db_->NewIterator(ReadOptions(), GetCfHandle());
1464 Status st = iter->status();
1465 if (!st.ok()) {
1466 exec_state_ =
1467 LDBCommandExecuteResult::Failed("Iterator error." + st.ToString());
1468 }
1469
1470 if (!null_from_) {
1471 iter->Seek(from_);
1472 } else {
1473 iter->SeekToFirst();
1474 }
1475
1476 int max_keys = max_keys_;
1477 int ttl_start;
1478 if (!ParseIntOption(option_map_, ARG_TTL_START, ttl_start, exec_state_)) {
1479 ttl_start = DBWithTTLImpl::kMinTimestamp; // TTL introduction time
1480 }
1481 int ttl_end;
1482 if (!ParseIntOption(option_map_, ARG_TTL_END, ttl_end, exec_state_)) {
1483 ttl_end = DBWithTTLImpl::kMaxTimestamp; // Max time allowed by TTL feature
1484 }
1485 if (ttl_end < ttl_start) {
1486 fprintf(stderr, "Error: End time can't be less than start time\n");
1487 delete iter;
1488 return;
1489 }
1490 int time_range = ttl_end - ttl_start;
1491 int bucket_size;
1492 if (!ParseIntOption(option_map_, ARG_TTL_BUCKET, bucket_size, exec_state_) ||
1493 bucket_size <= 0) {
1494 bucket_size = time_range; // Will have just 1 bucket by default
1495 }
1496 //cretaing variables for row count of each type
1497 std::string rtype1, rtype2, row, val;
1498 rtype2 = "";
1499 uint64_t c=0;
1500 uint64_t s1=0,s2=0;
1501
1502 // At this point, bucket_size=0 => time_range=0
1503 int num_buckets = (bucket_size >= time_range)
1504 ? 1
1505 : ((time_range + bucket_size - 1) / bucket_size);
1506 std::vector<uint64_t> bucket_counts(num_buckets, 0);
1507 if (is_db_ttl_ && !count_only_ && timestamp_ && !count_delim_) {
1508 fprintf(stdout, "Dumping key-values from %s to %s\n",
1509 ReadableTime(ttl_start).c_str(), ReadableTime(ttl_end).c_str());
1510 }
1511
11fdf7f2
TL		 1512 HistogramImpl vsize_hist;
1513
7c673cae
FG		 1514 for (; iter->Valid(); iter->Next()) {
1515 int rawtime = 0;
1516 // If end marker was specified, we stop before it
1517 if (!null_to_ && (iter->key().ToString() >= to_))
1518 break;
1519 // Terminate if maximum number of keys have been dumped
1520 if (max_keys == 0)
1521 break;
1522 if (is_db_ttl_) {
11fdf7f2 1523 TtlIterator* it_ttl = static_cast_with_check<TtlIterator, Iterator>(iter);
7c673cae
FG		 1524 rawtime = it_ttl->timestamp();
1525 if (rawtime < ttl_start || rawtime >= ttl_end) {
1526 continue;
1527 }
1528 }
1529 if (max_keys > 0) {
1530 --max_keys;
1531 }
1532 if (is_db_ttl_ && num_buckets > 1) {
1533 IncBucketCounts(bucket_counts, ttl_start, time_range, bucket_size,
1534 rawtime, num_buckets);
1535 }
1536 ++count;
1537 if (count_delim_) {
1538 rtype1 = "";
1539 row = iter->key().ToString();
1540 val = iter->value().ToString();
1541 s1 = row.size()+val.size();
1542 for(int j=0;row[j]!=delim_[0] && row[j]!='\0';j++)
1543 rtype1+=row[j];
1544 if(rtype2.compare("") && rtype2.compare(rtype1)!=0) {
11fdf7f2
TL		 1545 fprintf(stdout, "%s => count:%" PRIu64 "\tsize:%" PRIu64 "\n",
1546 rtype2.c_str(), c, s2);
7c673cae
FG		 1547 c=1;
1548 s2=s1;
1549 rtype2 = rtype1;
1550 } else {
1551 c++;
1552 s2+=s1;
1553 rtype2=rtype1;
1554 }
1555
1556 }
1557
11fdf7f2
TL		 1558 if (count_only_) {
1559 vsize_hist.Add(iter->value().size());
1560 }
7c673cae
FG		 1561
1562 if (!count_only_ && !count_delim_) {
1563 if (is_db_ttl_ && timestamp_) {
1564 fprintf(stdout, "%s ", ReadableTime(rawtime).c_str());
1565 }
1566 std::string str =
1567 PrintKeyValue(iter->key().ToString(), iter->value().ToString(),
1568 is_key_hex_, is_value_hex_);
1569 fprintf(stdout, "%s\n", str.c_str());
1570 }
1571 }
1572
1573 if (num_buckets > 1 && is_db_ttl_) {
1574 PrintBucketCounts(bucket_counts, ttl_start, ttl_end, bucket_size,
1575 num_buckets);
1576 } else if(count_delim_) {
11fdf7f2
TL		 1577 fprintf(stdout, "%s => count:%" PRIu64 "\tsize:%" PRIu64 "\n",
1578 rtype2.c_str(), c, s2);
7c673cae 1579 } else {
11fdf7f2
TL		 1580 fprintf(stdout, "Keys in range: %" PRIu64 "\n", count);
1581 }
1582
1583 if (count_only_) {
1584 fprintf(stdout, "Value size distribution: \n");
1585 fprintf(stdout, "%s\n", vsize_hist.ToString().c_str());
7c673cae
FG		 1586 }
1587 // Clean up
1588 delete iter;
1589}
1590
1591const std::string ReduceDBLevelsCommand::ARG_NEW_LEVELS = "new_levels";
1592const std::string ReduceDBLevelsCommand::ARG_PRINT_OLD_LEVELS =
1593 "print_old_levels";
1594
1595ReduceDBLevelsCommand::ReduceDBLevelsCommand(
11fdf7f2 1596 const std::vector<std::string>& /*params*/,
7c673cae
FG		 1597 const std::map<std::string, std::string>& options,
1598 const std::vector<std::string>& flags)
1599 : LDBCommand(options, flags, false,
1600 BuildCmdLineOptions({ARG_NEW_LEVELS, ARG_PRINT_OLD_LEVELS})),
1601 old_levels_(1 << 7),
1602 new_levels_(-1),
1603 print_old_levels_(false) {
1604 ParseIntOption(option_map_, ARG_NEW_LEVELS, new_levels_, exec_state_);
1605 print_old_levels_ = IsFlagPresent(flags, ARG_PRINT_OLD_LEVELS);
1606
1607 if(new_levels_ <= 0) {
1608 exec_state_ = LDBCommandExecuteResult::Failed(
1609 " Use --" + ARG_NEW_LEVELS + " to specify a new level number\n");
1610 }
1611}
1612
1613std::vector<std::string> ReduceDBLevelsCommand::PrepareArgs(
1614 const std::string& db_path, int new_levels, bool print_old_level) {
1615 std::vector<std::string> ret;
1616 ret.push_back("reduce_levels");
1617 ret.push_back("--" + ARG_DB + "=" + db_path);
1618 ret.push_back("--" + ARG_NEW_LEVELS + "=" + rocksdb::ToString(new_levels));
1619 if(print_old_level) {
1620 ret.push_back("--" + ARG_PRINT_OLD_LEVELS);
1621 }
1622 return ret;
1623}
1624
1625void ReduceDBLevelsCommand::Help(std::string& ret) {
1626 ret.append(" ");
1627 ret.append(ReduceDBLevelsCommand::Name());
1628 ret.append(" --" + ARG_NEW_LEVELS + "=<New number of levels>");
1629 ret.append(" [--" + ARG_PRINT_OLD_LEVELS + "]");
1630 ret.append("\n");
1631}
1632
1633Options ReduceDBLevelsCommand::PrepareOptionsForOpenDB() {
1634 Options opt = LDBCommand::PrepareOptionsForOpenDB();
1635 opt.num_levels = old_levels_;
1636 opt.max_bytes_for_level_multiplier_additional.resize(opt.num_levels, 1);
1637 // Disable size compaction
1638 opt.max_bytes_for_level_base = 1ULL << 50;
1639 opt.max_bytes_for_level_multiplier = 1;
1640 return opt;
1641}
1642
1643Status ReduceDBLevelsCommand::GetOldNumOfLevels(Options& opt,
1644 int* levels) {
1645 ImmutableDBOptions db_options(opt);
1646 EnvOptions soptions;
1647 std::shared_ptr<Cache> tc(
1648 NewLRUCache(opt.max_open_files - 10, opt.table_cache_numshardbits));
1649 const InternalKeyComparator cmp(opt.comparator);
1650 WriteController wc(opt.delayed_write_rate);
1651 WriteBufferManager wb(opt.db_write_buffer_size);
1652 VersionSet versions(db_path_, &db_options, soptions, tc.get(), &wb, &wc);
1653 std::vector<ColumnFamilyDescriptor> dummy;
1654 ColumnFamilyDescriptor dummy_descriptor(kDefaultColumnFamilyName,
1655 ColumnFamilyOptions(opt));
1656 dummy.push_back(dummy_descriptor);
1657 // We rely the VersionSet::Recover to tell us the internal data structures
1658 // in the db. And the Recover() should never do any change
1659 // (like LogAndApply) to the manifest file.
1660 Status st = versions.Recover(dummy);
1661 if (!st.ok()) {
1662 return st;
1663 }
1664 int max = -1;
1665 auto default_cfd = versions.GetColumnFamilySet()->GetDefault();
1666 for (int i = 0; i < default_cfd->NumberLevels(); i++) {
1667 if (default_cfd->current()->storage_info()->NumLevelFiles(i)) {
1668 max = i;
1669 }
1670 }
1671
1672 *levels = max + 1;
1673 return st;
1674}
1675
1676void ReduceDBLevelsCommand::DoCommand() {
1677 if (new_levels_ <= 1) {
1678 exec_state_ =
1679 LDBCommandExecuteResult::Failed("Invalid number of levels.\n");
1680 return;
1681 }
1682
1683 Status st;
1684 Options opt = PrepareOptionsForOpenDB();
1685 int old_level_num = -1;
1686 st = GetOldNumOfLevels(opt, &old_level_num);
1687 if (!st.ok()) {
1688 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
1689 return;
1690 }
1691
1692 if (print_old_levels_) {
1693 fprintf(stdout, "The old number of levels in use is %d\n", old_level_num);
1694 }
1695
1696 if (old_level_num <= new_levels_) {
1697 return;
1698 }
1699
1700 old_levels_ = old_level_num;
1701
1702 OpenDB();
1703 if (exec_state_.IsFailed()) {
1704 return;
1705 }
11fdf7f2 1706 assert(db_ != nullptr);
7c673cae
FG		 1707 // Compact the whole DB to put all files to the highest level.
1708 fprintf(stdout, "Compacting the db...\n");
1709 db_->CompactRange(CompactRangeOptions(), GetCfHandle(), nullptr, nullptr);
1710 CloseDB();
1711
1712 EnvOptions soptions;
1713 st = VersionSet::ReduceNumberOfLevels(db_path_, &opt, soptions, new_levels_);
1714 if (!st.ok()) {
1715 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
1716 return;
1717 }
1718}
1719
1720const std::string ChangeCompactionStyleCommand::ARG_OLD_COMPACTION_STYLE =
1721 "old_compaction_style";
1722const std::string ChangeCompactionStyleCommand::ARG_NEW_COMPACTION_STYLE =
1723 "new_compaction_style";
1724
1725ChangeCompactionStyleCommand::ChangeCompactionStyleCommand(
11fdf7f2 1726 const std::vector<std::string>& /*params*/,
7c673cae
FG		 1727 const std::map<std::string, std::string>& options,
1728 const std::vector<std::string>& flags)
1729 : LDBCommand(options, flags, false,
1730 BuildCmdLineOptions(
1731 {ARG_OLD_COMPACTION_STYLE, ARG_NEW_COMPACTION_STYLE})),
1732 old_compaction_style_(-1),
1733 new_compaction_style_(-1) {
1734 ParseIntOption(option_map_, ARG_OLD_COMPACTION_STYLE, old_compaction_style_,
1735 exec_state_);
1736 if (old_compaction_style_ != kCompactionStyleLevel &&
1737 old_compaction_style_ != kCompactionStyleUniversal) {
1738 exec_state_ = LDBCommandExecuteResult::Failed(
1739 "Use --" + ARG_OLD_COMPACTION_STYLE + " to specify old compaction " +
1740 "style. Check ldb help for proper compaction style value.\n");
1741 return;
1742 }
1743
1744 ParseIntOption(option_map_, ARG_NEW_COMPACTION_STYLE, new_compaction_style_,
1745 exec_state_);
1746 if (new_compaction_style_ != kCompactionStyleLevel &&
1747 new_compaction_style_ != kCompactionStyleUniversal) {
1748 exec_state_ = LDBCommandExecuteResult::Failed(
1749 "Use --" + ARG_NEW_COMPACTION_STYLE + " to specify new compaction " +
1750 "style. Check ldb help for proper compaction style value.\n");
1751 return;
1752 }
1753
1754 if (new_compaction_style_ == old_compaction_style_) {
1755 exec_state_ = LDBCommandExecuteResult::Failed(
1756 "Old compaction style is the same as new compaction style. "
1757 "Nothing to do.\n");
1758 return;
1759 }
1760
1761 if (old_compaction_style_ == kCompactionStyleUniversal &&
1762 new_compaction_style_ == kCompactionStyleLevel) {
1763 exec_state_ = LDBCommandExecuteResult::Failed(
1764 "Convert from universal compaction to level compaction. "
1765 "Nothing to do.\n");
1766 return;
1767 }
1768}
1769
1770void ChangeCompactionStyleCommand::Help(std::string& ret) {
1771 ret.append(" ");
1772 ret.append(ChangeCompactionStyleCommand::Name());
1773 ret.append(" --" + ARG_OLD_COMPACTION_STYLE + "=<Old compaction style: 0 " +
1774 "for level compaction, 1 for universal compaction>");
1775 ret.append(" --" + ARG_NEW_COMPACTION_STYLE + "=<New compaction style: 0 " +
1776 "for level compaction, 1 for universal compaction>");
1777 ret.append("\n");
1778}
1779
1780Options ChangeCompactionStyleCommand::PrepareOptionsForOpenDB() {
1781 Options opt = LDBCommand::PrepareOptionsForOpenDB();
1782
1783 if (old_compaction_style_ == kCompactionStyleLevel &&
1784 new_compaction_style_ == kCompactionStyleUniversal) {
1785 // In order to convert from level compaction to universal compaction, we
1786 // need to compact all data into a single file and move it to level 0.
1787 opt.disable_auto_compactions = true;
1788 opt.target_file_size_base = INT_MAX;
1789 opt.target_file_size_multiplier = 1;
1790 opt.max_bytes_for_level_base = INT_MAX;
1791 opt.max_bytes_for_level_multiplier = 1;
1792 }
1793
1794 return opt;
1795}
1796
1797void ChangeCompactionStyleCommand::DoCommand() {
1798 // print db stats before we have made any change
1799 std::string property;
1800 std::string files_per_level;
1801 for (int i = 0; i < db_->NumberLevels(GetCfHandle()); i++) {
1802 db_->GetProperty(GetCfHandle(),
1803 "rocksdb.num-files-at-level" + NumberToString(i),
1804 &property);
1805
1806 // format print string
1807 char buf[100];
1808 snprintf(buf, sizeof(buf), "%s%s", (i ? "," : ""), property.c_str());
1809 files_per_level += buf;
1810 }
1811 fprintf(stdout, "files per level before compaction: %s\n",
1812 files_per_level.c_str());
1813
1814 // manual compact into a single file and move the file to level 0
1815 CompactRangeOptions compact_options;
1816 compact_options.change_level = true;
1817 compact_options.target_level = 0;
1818 db_->CompactRange(compact_options, GetCfHandle(), nullptr, nullptr);
1819
1820 // verify compaction result
1821 files_per_level = "";
1822 int num_files = 0;
1823 for (int i = 0; i < db_->NumberLevels(GetCfHandle()); i++) {
1824 db_->GetProperty(GetCfHandle(),
1825 "rocksdb.num-files-at-level" + NumberToString(i),
1826 &property);
1827
1828 // format print string
1829 char buf[100];
1830 snprintf(buf, sizeof(buf), "%s%s", (i ? "," : ""), property.c_str());
1831 files_per_level += buf;
1832
1833 num_files = atoi(property.c_str());
1834
1835 // level 0 should have only 1 file
1836 if (i == 0 && num_files != 1) {
1837 exec_state_ = LDBCommandExecuteResult::Failed(
1838 "Number of db files at "
1839 "level 0 after compaction is " +
1840 ToString(num_files) + ", not 1.\n");
1841 return;
1842 }
1843 // other levels should have no file
1844 if (i > 0 && num_files != 0) {
1845 exec_state_ = LDBCommandExecuteResult::Failed(
1846 "Number of db files at "
1847 "level " +
1848 ToString(i) + " after compaction is " + ToString(num_files) +
1849 ", not 0.\n");
1850 return;
1851 }
1852 }
1853
1854 fprintf(stdout, "files per level after compaction: %s\n",
1855 files_per_level.c_str());
1856}
1857
1858// ----------------------------------------------------------------------------
1859
1860namespace {
1861
1862struct StdErrReporter : public log::Reader::Reporter {
11fdf7f2 1863 virtual void Corruption(size_t /*bytes*/, const Status& s) override {
7c673cae
FG		 1864 std::cerr << "Corruption detected in log file " << s.ToString() << "\n";
1865 }
1866};
1867
1868class InMemoryHandler : public WriteBatch::Handler {
1869 public:
11fdf7f2
TL		 1870 InMemoryHandler(std::stringstream& row, bool print_values,
1871 bool write_after_commit = false)
1872 : Handler(),
1873 row_(row),
1874 print_values_(print_values),
1875 write_after_commit_(write_after_commit) {}
7c673cae
FG		 1876
1877 void commonPutMerge(const Slice& key, const Slice& value) {
1878 std::string k = LDBCommand::StringToHex(key.ToString());
1879 if (print_values_) {
1880 std::string v = LDBCommand::StringToHex(value.ToString());
1881 row_ << k << " : ";
1882 row_ << v << " ";
1883 } else {
1884 row_ << k << " ";
1885 }
1886 }
1887
1888 virtual Status PutCF(uint32_t cf, const Slice& key,
1889 const Slice& value) override {
1890 row_ << "PUT(" << cf << ") : ";
1891 commonPutMerge(key, value);
1892 return Status::OK();
1893 }
1894
1895 virtual Status MergeCF(uint32_t cf, const Slice& key,
1896 const Slice& value) override {
1897 row_ << "MERGE(" << cf << ") : ";
1898 commonPutMerge(key, value);
1899 return Status::OK();
1900 }
1901
11fdf7f2
TL		 1902 virtual Status MarkNoop(bool)
1903 override {
1904 row_ << "NOOP ";
1905 return Status::OK();
1906 }
1907
7c673cae
FG		 1908 virtual Status DeleteCF(uint32_t cf, const Slice& key) override {
1909 row_ << "DELETE(" << cf << ") : ";
1910 row_ << LDBCommand::StringToHex(key.ToString()) << " ";
1911 return Status::OK();
1912 }
1913
1914 virtual Status SingleDeleteCF(uint32_t cf, const Slice& key) override {
1915 row_ << "SINGLE_DELETE(" << cf << ") : ";
1916 row_ << LDBCommand::StringToHex(key.ToString()) << " ";
1917 return Status::OK();
1918 }
1919
1920 virtual Status DeleteRangeCF(uint32_t cf, const Slice& begin_key,
1921 const Slice& end_key) override {
1922 row_ << "DELETE_RANGE(" << cf << ") : ";
1923 row_ << LDBCommand::StringToHex(begin_key.ToString()) << " ";
1924 row_ << LDBCommand::StringToHex(end_key.ToString()) << " ";
1925 return Status::OK();
1926 }
1927
11fdf7f2
TL		 1928 virtual Status MarkBeginPrepare(bool unprepare) override {
1929 row_ << "BEGIN_PREPARE(";
1930 row_ << (unprepare ? "true" : "false") << ") ";
7c673cae
FG		 1931 return Status::OK();
1932 }
1933
1934 virtual Status MarkEndPrepare(const Slice& xid) override {
1935 row_ << "END_PREPARE(";
1936 row_ << LDBCommand::StringToHex(xid.ToString()) << ") ";
1937 return Status::OK();
1938 }
1939
1940 virtual Status MarkRollback(const Slice& xid) override {
1941 row_ << "ROLLBACK(";
1942 row_ << LDBCommand::StringToHex(xid.ToString()) << ") ";
1943 return Status::OK();
1944 }
1945
1946 virtual Status MarkCommit(const Slice& xid) override {
1947 row_ << "COMMIT(";
1948 row_ << LDBCommand::StringToHex(xid.ToString()) << ") ";
1949 return Status::OK();
1950 }
1951
1952 virtual ~InMemoryHandler() {}
1953
11fdf7f2
TL		 1954 protected:
1955 virtual bool WriteAfterCommit() const override { return write_after_commit_; }
1956
7c673cae
FG		 1957 private:
1958 std::stringstream& row_;
1959 bool print_values_;
11fdf7f2 1960 bool write_after_commit_;
7c673cae
FG		 1961};
1962
1963void DumpWalFile(std::string wal_file, bool print_header, bool print_values,
11fdf7f2 1964 bool is_write_committed, LDBCommandExecuteResult* exec_state) {
7c673cae
FG		 1965 Env* env_ = Env::Default();
1966 EnvOptions soptions;
1967 unique_ptr<SequentialFileReader> wal_file_reader;
1968
1969 Status status;
1970 {
1971 unique_ptr<SequentialFile> file;
1972 status = env_->NewSequentialFile(wal_file, &file, soptions);
1973 if (status.ok()) {
11fdf7f2
TL		 1974 wal_file_reader.reset(
1975 new SequentialFileReader(std::move(file), wal_file));
7c673cae
FG		 1976 }
1977 }
1978 if (!status.ok()) {
1979 if (exec_state) {
1980 *exec_state = LDBCommandExecuteResult::Failed("Failed to open WAL file " +
1981 status.ToString());
1982 } else {
1983 std::cerr << "Error: Failed to open WAL file " << status.ToString()
1984 << std::endl;
1985 }
1986 } else {
1987 StdErrReporter reporter;
1988 uint64_t log_number;
1989 FileType type;
1990
1991 // we need the log number, but ParseFilename expects dbname/NNN.log.
1992 std::string sanitized = wal_file;
1993 size_t lastslash = sanitized.rfind('/');
1994 if (lastslash != std::string::npos)
1995 sanitized = sanitized.substr(lastslash + 1);
1996 if (!ParseFileName(sanitized, &log_number, &type)) {
1997 // bogus input, carry on as best we can
1998 log_number = 0;
1999 }
2000 DBOptions db_options;
2001 log::Reader reader(db_options.info_log, std::move(wal_file_reader),
11fdf7f2 2002 &reporter, true /* checksum */, log_number);
7c673cae
FG		 2003 std::string scratch;
2004 WriteBatch batch;
2005 Slice record;
2006 std::stringstream row;
2007 if (print_header) {
2008 std::cout << "Sequence,Count,ByteSize,Physical Offset,Key(s)";
2009 if (print_values) {
2010 std::cout << " : value ";
2011 }
2012 std::cout << "\n";
2013 }
2014 while (reader.ReadRecord(&record, &scratch)) {
2015 row.str("");
2016 if (record.size() < WriteBatchInternal::kHeader) {
2017 reporter.Corruption(record.size(),
2018 Status::Corruption("log record too small"));
2019 } else {
2020 WriteBatchInternal::SetContents(&batch, record);
2021 row << WriteBatchInternal::Sequence(&batch) << ",";
2022 row << WriteBatchInternal::Count(&batch) << ",";
2023 row << WriteBatchInternal::ByteSize(&batch) << ",";
2024 row << reader.LastRecordOffset() << ",";
11fdf7f2 2025 InMemoryHandler handler(row, print_values, is_write_committed);
7c673cae
FG		 2026 batch.Iterate(&handler);
2027 row << "\n";
2028 }
2029 std::cout << row.str();
2030 }
2031 }
2032}
2033
2034} // namespace
2035
2036const std::string WALDumperCommand::ARG_WAL_FILE = "walfile";
11fdf7f2 2037const std::string WALDumperCommand::ARG_WRITE_COMMITTED = "write_committed";
7c673cae
FG		 2038const std::string WALDumperCommand::ARG_PRINT_VALUE = "print_value";
2039const std::string WALDumperCommand::ARG_PRINT_HEADER = "header";
2040
2041WALDumperCommand::WALDumperCommand(
11fdf7f2 2042 const std::vector<std::string>& /*params*/,
7c673cae
FG		 2043 const std::map<std::string, std::string>& options,
2044 const std::vector<std::string>& flags)
2045 : LDBCommand(options, flags, true,
11fdf7f2
TL		 2046 BuildCmdLineOptions({ARG_WAL_FILE, ARG_WRITE_COMMITTED,
2047 ARG_PRINT_HEADER, ARG_PRINT_VALUE})),
7c673cae 2048 print_header_(false),
11fdf7f2
TL		 2049 print_values_(false),
2050 is_write_committed_(false) {
7c673cae
FG		 2051 wal_file_.clear();
2052
2053 std::map<std::string, std::string>::const_iterator itr =
2054 options.find(ARG_WAL_FILE);
2055 if (itr != options.end()) {
2056 wal_file_ = itr->second;
2057 }
2058
2059
2060 print_header_ = IsFlagPresent(flags, ARG_PRINT_HEADER);
2061 print_values_ = IsFlagPresent(flags, ARG_PRINT_VALUE);
11fdf7f2
TL		 2062 is_write_committed_ = ParseBooleanOption(options, ARG_WRITE_COMMITTED, true);
2063
7c673cae
FG		 2064 if (wal_file_.empty()) {
2065 exec_state_ = LDBCommandExecuteResult::Failed("Argument " + ARG_WAL_FILE +
2066 " must be specified.");
2067 }
2068}
2069
2070void WALDumperCommand::Help(std::string& ret) {
2071 ret.append(" ");
2072 ret.append(WALDumperCommand::Name());
2073 ret.append(" --" + ARG_WAL_FILE + "=<write_ahead_log_file_path>");
2074 ret.append(" [--" + ARG_PRINT_HEADER + "] ");
2075 ret.append(" [--" + ARG_PRINT_VALUE + "] ");
11fdf7f2 2076 ret.append(" [--" + ARG_WRITE_COMMITTED + "=true|false] ");
7c673cae
FG		 2077 ret.append("\n");
2078}
2079
2080void WALDumperCommand::DoCommand() {
11fdf7f2
TL		 2081 DumpWalFile(wal_file_, print_header_, print_values_, is_write_committed_,
2082 &exec_state_);
7c673cae
FG		 2083}
2084
2085// ----------------------------------------------------------------------------
2086
2087GetCommand::GetCommand(const std::vector<std::string>& params,
2088 const std::map<std::string, std::string>& options,
2089 const std::vector<std::string>& flags)
2090 : LDBCommand(
2091 options, flags, true,
2092 BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
2093 if (params.size() != 1) {
2094 exec_state_ = LDBCommandExecuteResult::Failed(
2095 "<key> must be specified for the get command");
2096 } else {
2097 key_ = params.at(0);
2098 }
2099
2100 if (is_key_hex_) {
2101 key_ = HexToString(key_);
2102 }
2103}
2104
2105void GetCommand::Help(std::string& ret) {
2106 ret.append(" ");
2107 ret.append(GetCommand::Name());
2108 ret.append(" <key>");
2109 ret.append(" [--" + ARG_TTL + "]");
2110 ret.append("\n");
2111}
2112
2113void GetCommand::DoCommand() {
2114 if (!db_) {
2115 assert(GetExecuteState().IsFailed());
2116 return;
2117 }
2118 std::string value;
2119 Status st = db_->Get(ReadOptions(), GetCfHandle(), key_, &value);
2120 if (st.ok()) {
2121 fprintf(stdout, "%s\n",
2122 (is_value_hex_ ? StringToHex(value) : value).c_str());
2123 } else {
2124 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2125 }
2126}
2127
2128// ----------------------------------------------------------------------------
2129
2130ApproxSizeCommand::ApproxSizeCommand(
11fdf7f2 2131 const std::vector<std::string>& /*params*/,
7c673cae
FG		 2132 const std::map<std::string, std::string>& options,
2133 const std::vector<std::string>& flags)
2134 : LDBCommand(options, flags, true,
2135 BuildCmdLineOptions(
2136 {ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM, ARG_TO})) {
2137 if (options.find(ARG_FROM) != options.end()) {
2138 start_key_ = options.find(ARG_FROM)->second;
2139 } else {
2140 exec_state_ = LDBCommandExecuteResult::Failed(
2141 ARG_FROM + " must be specified for approxsize command");
2142 return;
2143 }
2144
2145 if (options.find(ARG_TO) != options.end()) {
2146 end_key_ = options.find(ARG_TO)->second;
2147 } else {
2148 exec_state_ = LDBCommandExecuteResult::Failed(
2149 ARG_TO + " must be specified for approxsize command");
2150 return;
2151 }
2152
2153 if (is_key_hex_) {
2154 start_key_ = HexToString(start_key_);
2155 end_key_ = HexToString(end_key_);
2156 }
2157}
2158
2159void ApproxSizeCommand::Help(std::string& ret) {
2160 ret.append(" ");
2161 ret.append(ApproxSizeCommand::Name());
2162 ret.append(HelpRangeCmdArgs());
2163 ret.append("\n");
2164}
2165
2166void ApproxSizeCommand::DoCommand() {
2167 if (!db_) {
2168 assert(GetExecuteState().IsFailed());
2169 return;
2170 }
2171 Range ranges[1];
2172 ranges[0] = Range(start_key_, end_key_);
2173 uint64_t sizes[1];
2174 db_->GetApproximateSizes(GetCfHandle(), ranges, 1, sizes);
2175 fprintf(stdout, "%lu\n", (unsigned long)sizes[0]);
2176 /* Weird that GetApproximateSizes() returns void, although documentation
2177 * says that it returns a Status object.
2178 if (!st.ok()) {
2179 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2180 }
2181 */
2182}
2183
2184// ----------------------------------------------------------------------------
2185
2186BatchPutCommand::BatchPutCommand(
2187 const std::vector<std::string>& params,
2188 const std::map<std::string, std::string>& options,
2189 const std::vector<std::string>& flags)
2190 : LDBCommand(options, flags, false,
2191 BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX,
2192 ARG_VALUE_HEX, ARG_CREATE_IF_MISSING})) {
2193 if (params.size() < 2) {
2194 exec_state_ = LDBCommandExecuteResult::Failed(
2195 "At least one <key> <value> pair must be specified batchput.");
2196 } else if (params.size() % 2 != 0) {
2197 exec_state_ = LDBCommandExecuteResult::Failed(
2198 "Equal number of <key>s and <value>s must be specified for batchput.");
2199 } else {
2200 for (size_t i = 0; i < params.size(); i += 2) {
2201 std::string key = params.at(i);
2202 std::string value = params.at(i + 1);
2203 key_values_.push_back(std::pair<std::string, std::string>(
2204 is_key_hex_ ? HexToString(key) : key,
2205 is_value_hex_ ? HexToString(value) : value));
2206 }
2207 }
2208 create_if_missing_ = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING);
2209}
2210
2211void BatchPutCommand::Help(std::string& ret) {
2212 ret.append(" ");
2213 ret.append(BatchPutCommand::Name());
2214 ret.append(" <key> <value> [<key> <value>] [..]");
2215 ret.append(" [--" + ARG_TTL + "]");
2216 ret.append("\n");
2217}
2218
2219void BatchPutCommand::DoCommand() {
2220 if (!db_) {
2221 assert(GetExecuteState().IsFailed());
2222 return;
2223 }
2224 WriteBatch batch;
2225
2226 for (std::vector<std::pair<std::string, std::string>>::const_iterator itr =
2227 key_values_.begin();
2228 itr != key_values_.end(); ++itr) {
2229 batch.Put(GetCfHandle(), itr->first, itr->second);
2230 }
2231 Status st = db_->Write(WriteOptions(), &batch);
2232 if (st.ok()) {
2233 fprintf(stdout, "OK\n");
2234 } else {
2235 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2236 }
2237}
2238
2239Options BatchPutCommand::PrepareOptionsForOpenDB() {
2240 Options opt = LDBCommand::PrepareOptionsForOpenDB();
2241 opt.create_if_missing = create_if_missing_;
2242 return opt;
2243}
2244
2245// ----------------------------------------------------------------------------
2246
11fdf7f2 2247ScanCommand::ScanCommand(const std::vector<std::string>& /*params*/,
7c673cae
FG		 2248 const std::map<std::string, std::string>& options,
2249 const std::vector<std::string>& flags)
2250 : LDBCommand(
2251 options, flags, true,
2252 BuildCmdLineOptions({ARG_TTL, ARG_NO_VALUE, ARG_HEX, ARG_KEY_HEX,
2253 ARG_TO, ARG_VALUE_HEX, ARG_FROM, ARG_TIMESTAMP,
2254 ARG_MAX_KEYS, ARG_TTL_START, ARG_TTL_END})),
2255 start_key_specified_(false),
2256 end_key_specified_(false),
2257 max_keys_scanned_(-1),
2258 no_value_(false) {
2259 std::map<std::string, std::string>::const_iterator itr =
2260 options.find(ARG_FROM);
2261 if (itr != options.end()) {
2262 start_key_ = itr->second;
2263 if (is_key_hex_) {
2264 start_key_ = HexToString(start_key_);
2265 }
2266 start_key_specified_ = true;
2267 }
2268 itr = options.find(ARG_TO);
2269 if (itr != options.end()) {
2270 end_key_ = itr->second;
2271 if (is_key_hex_) {
2272 end_key_ = HexToString(end_key_);
2273 }
2274 end_key_specified_ = true;
2275 }
2276
2277 std::vector<std::string>::const_iterator vitr =
2278 std::find(flags.begin(), flags.end(), ARG_NO_VALUE);
2279 if (vitr != flags.end()) {
2280 no_value_ = true;
2281 }
2282
2283 itr = options.find(ARG_MAX_KEYS);
2284 if (itr != options.end()) {
2285 try {
2286#if defined(CYGWIN)
2287 max_keys_scanned_ = strtol(itr->second.c_str(), 0, 10);
2288#else
2289 max_keys_scanned_ = std::stoi(itr->second);
2290#endif
2291 } catch (const std::invalid_argument&) {
2292 exec_state_ = LDBCommandExecuteResult::Failed(ARG_MAX_KEYS +
2293 " has an invalid value");
2294 } catch (const std::out_of_range&) {
2295 exec_state_ = LDBCommandExecuteResult::Failed(
2296 ARG_MAX_KEYS + " has a value out-of-range");
2297 }
2298 }
2299}
2300
2301void ScanCommand::Help(std::string& ret) {
2302 ret.append(" ");
2303 ret.append(ScanCommand::Name());
2304 ret.append(HelpRangeCmdArgs());
2305 ret.append(" [--" + ARG_TTL + "]");
2306 ret.append(" [--" + ARG_TIMESTAMP + "]");
2307 ret.append(" [--" + ARG_MAX_KEYS + "=<N>q] ");
2308 ret.append(" [--" + ARG_TTL_START + "=<N>:- is inclusive]");
2309 ret.append(" [--" + ARG_TTL_END + "=<N>:- is exclusive]");
2310 ret.append(" [--" + ARG_NO_VALUE + "]");
2311 ret.append("\n");
2312}
2313
2314void ScanCommand::DoCommand() {
2315 if (!db_) {
2316 assert(GetExecuteState().IsFailed());
2317 return;
2318 }
2319
2320 int num_keys_scanned = 0;
2321 Iterator* it = db_->NewIterator(ReadOptions(), GetCfHandle());
2322 if (start_key_specified_) {
2323 it->Seek(start_key_);
2324 } else {
2325 it->SeekToFirst();
2326 }
2327 int ttl_start;
2328 if (!ParseIntOption(option_map_, ARG_TTL_START, ttl_start, exec_state_)) {
2329 ttl_start = DBWithTTLImpl::kMinTimestamp; // TTL introduction time
2330 }
2331 int ttl_end;
2332 if (!ParseIntOption(option_map_, ARG_TTL_END, ttl_end, exec_state_)) {
2333 ttl_end = DBWithTTLImpl::kMaxTimestamp; // Max time allowed by TTL feature
2334 }
2335 if (ttl_end < ttl_start) {
2336 fprintf(stderr, "Error: End time can't be less than start time\n");
2337 delete it;
2338 return;
2339 }
2340 if (is_db_ttl_ && timestamp_) {
2341 fprintf(stdout, "Scanning key-values from %s to %s\n",
2342 ReadableTime(ttl_start).c_str(), ReadableTime(ttl_end).c_str());
2343 }
2344 for ( ;
2345 it->Valid() && (!end_key_specified_ || it->key().ToString() < end_key_);
2346 it->Next()) {
2347 if (is_db_ttl_) {
11fdf7f2 2348 TtlIterator* it_ttl = static_cast_with_check<TtlIterator, Iterator>(it);
7c673cae
FG		 2349 int rawtime = it_ttl->timestamp();
2350 if (rawtime < ttl_start || rawtime >= ttl_end) {
2351 continue;
2352 }
2353 if (timestamp_) {
2354 fprintf(stdout, "%s ", ReadableTime(rawtime).c_str());
2355 }
2356 }
2357
2358 Slice key_slice = it->key();
2359
2360 std::string formatted_key;
2361 if (is_key_hex_) {
2362 formatted_key = "0x" + key_slice.ToString(true /* hex */);
2363 key_slice = formatted_key;
2364 } else if (ldb_options_.key_formatter) {
2365 formatted_key = ldb_options_.key_formatter->Format(key_slice);
2366 key_slice = formatted_key;
2367 }
2368
2369 if (no_value_) {
2370 fprintf(stdout, "%.*s\n", static_cast<int>(key_slice.size()),
2371 key_slice.data());
2372 } else {
2373 Slice val_slice = it->value();
2374 std::string formatted_value;
2375 if (is_value_hex_) {
2376 formatted_value = "0x" + val_slice.ToString(true /* hex */);
2377 val_slice = formatted_value;
2378 }
2379 fprintf(stdout, "%.*s : %.*s\n", static_cast<int>(key_slice.size()),
2380 key_slice.data(), static_cast<int>(val_slice.size()),
2381 val_slice.data());
2382 }
2383
2384 num_keys_scanned++;
2385 if (max_keys_scanned_ >= 0 && num_keys_scanned >= max_keys_scanned_) {
2386 break;
2387 }
2388 }
2389 if (!it->status().ok()) { // Check for any errors found during the scan
2390 exec_state_ = LDBCommandExecuteResult::Failed(it->status().ToString());
2391 }
2392 delete it;
2393}
2394
2395// ----------------------------------------------------------------------------
2396
2397DeleteCommand::DeleteCommand(const std::vector<std::string>& params,
2398 const std::map<std::string, std::string>& options,
2399 const std::vector<std::string>& flags)
2400 : LDBCommand(options, flags, false,
2401 BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
2402 if (params.size() != 1) {
2403 exec_state_ = LDBCommandExecuteResult::Failed(
2404 "KEY must be specified for the delete command");
2405 } else {
2406 key_ = params.at(0);
2407 if (is_key_hex_) {
2408 key_ = HexToString(key_);
2409 }
2410 }
2411}
2412
2413void DeleteCommand::Help(std::string& ret) {
2414 ret.append(" ");
2415 ret.append(DeleteCommand::Name() + " <key>");
2416 ret.append("\n");
2417}
2418
2419void DeleteCommand::DoCommand() {
2420 if (!db_) {
2421 assert(GetExecuteState().IsFailed());
2422 return;
2423 }
2424 Status st = db_->Delete(WriteOptions(), GetCfHandle(), key_);
2425 if (st.ok()) {
2426 fprintf(stdout, "OK\n");
2427 } else {
2428 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2429 }
2430}
2431
2432DeleteRangeCommand::DeleteRangeCommand(
2433 const std::vector<std::string>& params,
2434 const std::map<std::string, std::string>& options,
2435 const std::vector<std::string>& flags)
2436 : LDBCommand(options, flags, false,
2437 BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
2438 if (params.size() != 2) {
2439 exec_state_ = LDBCommandExecuteResult::Failed(
2440 "begin and end keys must be specified for the delete command");
2441 } else {
2442 begin_key_ = params.at(0);
2443 end_key_ = params.at(1);
2444 if (is_key_hex_) {
2445 begin_key_ = HexToString(begin_key_);
2446 end_key_ = HexToString(end_key_);
2447 }
2448 }
2449}
2450
2451void DeleteRangeCommand::Help(std::string& ret) {
2452 ret.append(" ");
2453 ret.append(DeleteRangeCommand::Name() + " <begin key> <end key>");
2454 ret.append("\n");
2455}
2456
2457void DeleteRangeCommand::DoCommand() {
2458 if (!db_) {
2459 assert(GetExecuteState().IsFailed());
2460 return;
2461 }
2462 Status st =
2463 db_->DeleteRange(WriteOptions(), GetCfHandle(), begin_key_, end_key_);
2464 if (st.ok()) {
2465 fprintf(stdout, "OK\n");
2466 } else {
2467 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2468 }
2469}
2470
2471PutCommand::PutCommand(const std::vector<std::string>& params,
2472 const std::map<std::string, std::string>& options,
2473 const std::vector<std::string>& flags)
2474 : LDBCommand(options, flags, false,
2475 BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX,
2476 ARG_VALUE_HEX, ARG_CREATE_IF_MISSING})) {
2477 if (params.size() != 2) {
2478 exec_state_ = LDBCommandExecuteResult::Failed(
2479 "<key> and <value> must be specified for the put command");
2480 } else {
2481 key_ = params.at(0);
2482 value_ = params.at(1);
2483 }
2484
2485 if (is_key_hex_) {
2486 key_ = HexToString(key_);
2487 }
2488
2489 if (is_value_hex_) {
2490 value_ = HexToString(value_);
2491 }
2492 create_if_missing_ = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING);
2493}
2494
2495void PutCommand::Help(std::string& ret) {
2496 ret.append(" ");
2497 ret.append(PutCommand::Name());
2498 ret.append(" <key> <value> ");
2499 ret.append(" [--" + ARG_TTL + "]");
2500 ret.append("\n");
2501}
2502
2503void PutCommand::DoCommand() {
2504 if (!db_) {
2505 assert(GetExecuteState().IsFailed());
2506 return;
2507 }
2508 Status st = db_->Put(WriteOptions(), GetCfHandle(), key_, value_);
2509 if (st.ok()) {
2510 fprintf(stdout, "OK\n");
2511 } else {
2512 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2513 }
2514}
2515
2516Options PutCommand::PrepareOptionsForOpenDB() {
2517 Options opt = LDBCommand::PrepareOptionsForOpenDB();
2518 opt.create_if_missing = create_if_missing_;
2519 return opt;
2520}
2521
2522// ----------------------------------------------------------------------------
2523
2524const char* DBQuerierCommand::HELP_CMD = "help";
2525const char* DBQuerierCommand::GET_CMD = "get";
2526const char* DBQuerierCommand::PUT_CMD = "put";
2527const char* DBQuerierCommand::DELETE_CMD = "delete";
2528
2529DBQuerierCommand::DBQuerierCommand(
11fdf7f2 2530 const std::vector<std::string>& /*params*/,
7c673cae
FG		 2531 const std::map<std::string, std::string>& options,
2532 const std::vector<std::string>& flags)
2533 : LDBCommand(
2534 options, flags, false,
2535 BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
2536
2537}
2538
2539void DBQuerierCommand::Help(std::string& ret) {
2540 ret.append(" ");
2541 ret.append(DBQuerierCommand::Name());
2542 ret.append(" [--" + ARG_TTL + "]");
2543 ret.append("\n");
2544 ret.append(" Starts a REPL shell. Type help for list of available "
2545 "commands.");
2546 ret.append("\n");
2547}
2548
2549void DBQuerierCommand::DoCommand() {
2550 if (!db_) {
2551 assert(GetExecuteState().IsFailed());
2552 return;
2553 }
2554
2555 ReadOptions read_options;
2556 WriteOptions write_options;
2557
2558 std::string line;
2559 std::string key;
2560 std::string value;
2561 while (getline(std::cin, line, '\n')) {
2562 // Parse line into std::vector<std::string>
2563 std::vector<std::string> tokens;
2564 size_t pos = 0;
2565 while (true) {
2566 size_t pos2 = line.find(' ', pos);
2567 if (pos2 == std::string::npos) {
2568 break;
2569 }
2570 tokens.push_back(line.substr(pos, pos2-pos));
2571 pos = pos2 + 1;
2572 }
2573 tokens.push_back(line.substr(pos));
2574
2575 const std::string& cmd = tokens[0];
2576
2577 if (cmd == HELP_CMD) {
2578 fprintf(stdout,
2579 "get <key>\n"
2580 "put <key> <value>\n"
2581 "delete <key>\n");
2582 } else if (cmd == DELETE_CMD && tokens.size() == 2) {
2583 key = (is_key_hex_ ? HexToString(tokens[1]) : tokens[1]);
2584 db_->Delete(write_options, GetCfHandle(), Slice(key));
2585 fprintf(stdout, "Successfully deleted %s\n", tokens[1].c_str());
2586 } else if (cmd == PUT_CMD && tokens.size() == 3) {
2587 key = (is_key_hex_ ? HexToString(tokens[1]) : tokens[1]);
2588 value = (is_value_hex_ ? HexToString(tokens[2]) : tokens[2]);
2589 db_->Put(write_options, GetCfHandle(), Slice(key), Slice(value));
2590 fprintf(stdout, "Successfully put %s %s\n",
2591 tokens[1].c_str(), tokens[2].c_str());
2592 } else if (cmd == GET_CMD && tokens.size() == 2) {
2593 key = (is_key_hex_ ? HexToString(tokens[1]) : tokens[1]);
2594 if (db_->Get(read_options, GetCfHandle(), Slice(key), &value).ok()) {
2595 fprintf(stdout, "%s\n", PrintKeyValue(key, value,
2596 is_key_hex_, is_value_hex_).c_str());
2597 } else {
2598 fprintf(stdout, "Not found %s\n", tokens[1].c_str());
2599 }
2600 } else {
2601 fprintf(stdout, "Unknown command %s\n", line.c_str());
2602 }
2603 }
2604}
2605
2606// ----------------------------------------------------------------------------
2607
2608CheckConsistencyCommand::CheckConsistencyCommand(
11fdf7f2 2609 const std::vector<std::string>& /*params*/,
7c673cae
FG		 2610 const std::map<std::string, std::string>& options,
2611 const std::vector<std::string>& flags)
2612 : LDBCommand(options, flags, false, BuildCmdLineOptions({})) {}
2613
2614void CheckConsistencyCommand::Help(std::string& ret) {
2615 ret.append(" ");
2616 ret.append(CheckConsistencyCommand::Name());
2617 ret.append("\n");
2618}
2619
2620void CheckConsistencyCommand::DoCommand() {
2621 Options opt = PrepareOptionsForOpenDB();
2622 opt.paranoid_checks = true;
2623 if (!exec_state_.IsNotStarted()) {
2624 return;
2625 }
2626 DB* db;
2627 Status st = DB::OpenForReadOnly(opt, db_path_, &db, false);
2628 delete db;
2629 if (st.ok()) {
2630 fprintf(stdout, "OK\n");
2631 } else {
2632 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2633 }
2634}
2635
2636// ----------------------------------------------------------------------------
2637
2638const std::string CheckPointCommand::ARG_CHECKPOINT_DIR = "checkpoint_dir";
2639
2640CheckPointCommand::CheckPointCommand(
11fdf7f2 2641 const std::vector<std::string>& /*params*/,
7c673cae
FG		 2642 const std::map<std::string, std::string>& options,
2643 const std::vector<std::string>& flags)
2644 : LDBCommand(options, flags, false /* is_read_only */,
2645 BuildCmdLineOptions({ARG_CHECKPOINT_DIR})) {
2646 auto itr = options.find(ARG_CHECKPOINT_DIR);
11fdf7f2 2647 if (itr != options.end()) {
7c673cae
FG		 2648 checkpoint_dir_ = itr->second;
2649 }
2650}
2651
2652void CheckPointCommand::Help(std::string& ret) {
2653 ret.append(" ");
2654 ret.append(CheckPointCommand::Name());
2655 ret.append(" [--" + ARG_CHECKPOINT_DIR + "] ");
2656 ret.append("\n");
2657}
2658
2659void CheckPointCommand::DoCommand() {
2660 if (!db_) {
2661 assert(GetExecuteState().IsFailed());
2662 return;
2663 }
2664 Checkpoint* checkpoint;
2665 Status status = Checkpoint::Create(db_, &checkpoint);
2666 status = checkpoint->CreateCheckpoint(checkpoint_dir_);
2667 if (status.ok()) {
2668 printf("OK\n");
2669 } else {
2670 exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
2671 }
2672}
2673
2674// ----------------------------------------------------------------------------
2675
11fdf7f2 2676RepairCommand::RepairCommand(const std::vector<std::string>& /*params*/,
7c673cae
FG		 2677 const std::map<std::string, std::string>& options,
2678 const std::vector<std::string>& flags)
2679 : LDBCommand(options, flags, false, BuildCmdLineOptions({})) {}
2680
2681void RepairCommand::Help(std::string& ret) {
2682 ret.append(" ");
2683 ret.append(RepairCommand::Name());
2684 ret.append("\n");
2685}
2686
2687void RepairCommand::DoCommand() {
2688 Options options = PrepareOptionsForOpenDB();
2689 options.info_log.reset(new StderrLogger(InfoLogLevel::WARN_LEVEL));
2690 Status status = RepairDB(db_path_, options);
2691 if (status.ok()) {
2692 printf("OK\n");
2693 } else {
2694 exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
2695 }
2696}
2697
2698// ----------------------------------------------------------------------------
2699
2700const std::string BackupableCommand::ARG_NUM_THREADS = "num_threads";
2701const std::string BackupableCommand::ARG_BACKUP_ENV_URI = "backup_env_uri";
2702const std::string BackupableCommand::ARG_BACKUP_DIR = "backup_dir";
2703const std::string BackupableCommand::ARG_STDERR_LOG_LEVEL = "stderr_log_level";
2704
2705BackupableCommand::BackupableCommand(
11fdf7f2 2706 const std::vector<std::string>& /*params*/,
7c673cae
FG		 2707 const std::map<std::string, std::string>& options,
2708 const std::vector<std::string>& flags)
2709 : LDBCommand(options, flags, false /* is_read_only */,
2710 BuildCmdLineOptions({ARG_BACKUP_ENV_URI, ARG_BACKUP_DIR,
2711 ARG_NUM_THREADS, ARG_STDERR_LOG_LEVEL})),
2712 num_threads_(1) {
2713 auto itr = options.find(ARG_NUM_THREADS);
2714 if (itr != options.end()) {
2715 num_threads_ = std::stoi(itr->second);
2716 }
2717 itr = options.find(ARG_BACKUP_ENV_URI);
2718 if (itr != options.end()) {
2719 backup_env_uri_ = itr->second;
2720 }
2721 itr = options.find(ARG_BACKUP_DIR);
2722 if (itr == options.end()) {
2723 exec_state_ = LDBCommandExecuteResult::Failed("--" + ARG_BACKUP_DIR +
2724 ": missing backup directory");
2725 } else {
2726 backup_dir_ = itr->second;
2727 }
2728
2729 itr = options.find(ARG_STDERR_LOG_LEVEL);
2730 if (itr != options.end()) {
2731 int stderr_log_level = std::stoi(itr->second);
2732 if (stderr_log_level < 0 ||
2733 stderr_log_level >= InfoLogLevel::NUM_INFO_LOG_LEVELS) {
2734 exec_state_ = LDBCommandExecuteResult::Failed(
2735 ARG_STDERR_LOG_LEVEL + " must be >= 0 and < " +
2736 std::to_string(InfoLogLevel::NUM_INFO_LOG_LEVELS) + ".");
2737 } else {
2738 logger_.reset(
2739 new StderrLogger(static_cast<InfoLogLevel>(stderr_log_level)));
2740 }
2741 }
2742}
2743
2744void BackupableCommand::Help(const std::string& name, std::string& ret) {
2745 ret.append(" ");
2746 ret.append(name);
2747 ret.append(" [--" + ARG_BACKUP_ENV_URI + "] ");
2748 ret.append(" [--" + ARG_BACKUP_DIR + "] ");
2749 ret.append(" [--" + ARG_NUM_THREADS + "] ");
2750 ret.append(" [--" + ARG_STDERR_LOG_LEVEL + "=<int (InfoLogLevel)>] ");
2751 ret.append("\n");
2752}
2753
2754// ----------------------------------------------------------------------------
2755
2756BackupCommand::BackupCommand(const std::vector<std::string>& params,
2757 const std::map<std::string, std::string>& options,
2758 const std::vector<std::string>& flags)
2759 : BackupableCommand(params, options, flags) {}
2760
2761void BackupCommand::Help(std::string& ret) {
2762 BackupableCommand::Help(Name(), ret);
2763}
2764
2765void BackupCommand::DoCommand() {
2766 BackupEngine* backup_engine;
2767 Status status;
2768 if (!db_) {
2769 assert(GetExecuteState().IsFailed());
2770 return;
2771 }
2772 printf("open db OK\n");
2773 std::unique_ptr<Env> custom_env_guard;
2774 Env* custom_env = NewCustomObject<Env>(backup_env_uri_, &custom_env_guard);
2775 BackupableDBOptions backup_options =
2776 BackupableDBOptions(backup_dir_, custom_env);
2777 backup_options.info_log = logger_.get();
2778 backup_options.max_background_operations = num_threads_;
2779 status = BackupEngine::Open(Env::Default(), backup_options, &backup_engine);
2780 if (status.ok()) {
2781 printf("open backup engine OK\n");
2782 } else {
2783 exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
2784 return;
2785 }
2786 status = backup_engine->CreateNewBackup(db_);
2787 if (status.ok()) {
2788 printf("create new backup OK\n");
2789 } else {
2790 exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
2791 return;
2792 }
2793}
2794
2795// ----------------------------------------------------------------------------
2796
2797RestoreCommand::RestoreCommand(
2798 const std::vector<std::string>& params,
2799 const std::map<std::string, std::string>& options,
2800 const std::vector<std::string>& flags)
2801 : BackupableCommand(params, options, flags) {}
2802
2803void RestoreCommand::Help(std::string& ret) {
2804 BackupableCommand::Help(Name(), ret);
2805}
2806
2807void RestoreCommand::DoCommand() {
2808 std::unique_ptr<Env> custom_env_guard;
2809 Env* custom_env = NewCustomObject<Env>(backup_env_uri_, &custom_env_guard);
2810 std::unique_ptr<BackupEngineReadOnly> restore_engine;
2811 Status status;
2812 {
2813 BackupableDBOptions opts(backup_dir_, custom_env);
2814 opts.info_log = logger_.get();
2815 opts.max_background_operations = num_threads_;
2816 BackupEngineReadOnly* raw_restore_engine_ptr;
2817 status = BackupEngineReadOnly::Open(Env::Default(), opts,
2818 &raw_restore_engine_ptr);
2819 if (status.ok()) {
2820 restore_engine.reset(raw_restore_engine_ptr);
2821 }
2822 }
2823 if (status.ok()) {
2824 printf("open restore engine OK\n");
2825 status = restore_engine->RestoreDBFromLatestBackup(db_path_, db_path_);
2826 }
2827 if (status.ok()) {
2828 printf("restore from backup OK\n");
2829 } else {
2830 exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
2831 }
2832}
2833
2834// ----------------------------------------------------------------------------
2835
2836namespace {
2837
2838void DumpSstFile(std::string filename, bool output_hex, bool show_properties) {
2839 std::string from_key;
2840 std::string to_key;
2841 if (filename.length() <= 4 ||
2842 filename.rfind(".sst") != filename.length() - 4) {
2843 std::cout << "Invalid sst file name." << std::endl;
2844 return;
2845 }
2846 // no verification
2847 rocksdb::SstFileReader reader(filename, false, output_hex);
11fdf7f2 2848 Status st = reader.ReadSequential(true, std::numeric_limits<uint64_t>::max(), false, // has_from
7c673cae
FG		 2849 from_key, false, // has_to
2850 to_key);
2851 if (!st.ok()) {
2852 std::cerr << "Error in reading SST file " << filename << st.ToString()
2853 << std::endl;
2854 return;
2855 }
2856
2857 if (show_properties) {
2858 const rocksdb::TableProperties* table_properties;
2859
2860 std::shared_ptr<const rocksdb::TableProperties>
2861 table_properties_from_reader;
2862 st = reader.ReadTableProperties(&table_properties_from_reader);
2863 if (!st.ok()) {
2864 std::cerr << filename << ": " << st.ToString()
2865 << ". Try to use initial table properties" << std::endl;
2866 table_properties = reader.GetInitTableProperties();
2867 } else {
2868 table_properties = table_properties_from_reader.get();
2869 }
2870 if (table_properties != nullptr) {
2871 std::cout << std::endl << "Table Properties:" << std::endl;
2872 std::cout << table_properties->ToString("\n") << std::endl;
2873 std::cout << "# deleted keys: "
2874 << rocksdb::GetDeletedKeys(
2875 table_properties->user_collected_properties)
2876 << std::endl;
2877 }
2878 }
2879}
2880
2881} // namespace
2882
2883DBFileDumperCommand::DBFileDumperCommand(
11fdf7f2 2884 const std::vector<std::string>& /*params*/,
7c673cae
FG		 2885 const std::map<std::string, std::string>& options,
2886 const std::vector<std::string>& flags)
2887 : LDBCommand(options, flags, true, BuildCmdLineOptions({})) {}
2888
2889void DBFileDumperCommand::Help(std::string& ret) {
2890 ret.append(" ");
2891 ret.append(DBFileDumperCommand::Name());
2892 ret.append("\n");
2893}
2894
2895void DBFileDumperCommand::DoCommand() {
2896 if (!db_) {
2897 assert(GetExecuteState().IsFailed());
2898 return;
2899 }
2900 Status s;
2901
2902 std::cout << "Manifest File" << std::endl;
2903 std::cout << "==============================" << std::endl;
2904 std::string manifest_filename;
2905 s = ReadFileToString(db_->GetEnv(), CurrentFileName(db_->GetName()),
2906 &manifest_filename);
2907 if (!s.ok() || manifest_filename.empty() ||
2908 manifest_filename.back() != '\n') {
2909 std::cerr << "Error when reading CURRENT file "
2910 << CurrentFileName(db_->GetName()) << std::endl;
2911 }
2912 // remove the trailing '\n'
2913 manifest_filename.resize(manifest_filename.size() - 1);
2914 std::string manifest_filepath = db_->GetName() + "/" + manifest_filename;
2915 std::cout << manifest_filepath << std::endl;
2916 DumpManifestFile(manifest_filepath, false, false, false);
2917 std::cout << std::endl;
2918
2919 std::cout << "SST Files" << std::endl;
2920 std::cout << "==============================" << std::endl;
2921 std::vector<LiveFileMetaData> metadata;
2922 db_->GetLiveFilesMetaData(&metadata);
2923 for (auto& fileMetadata : metadata) {
2924 std::string filename = fileMetadata.db_path + fileMetadata.name;
2925 std::cout << filename << " level:" << fileMetadata.level << std::endl;
2926 std::cout << "------------------------------" << std::endl;
2927 DumpSstFile(filename, false, true);
2928 std::cout << std::endl;
2929 }
2930 std::cout << std::endl;
2931
2932 std::cout << "Write Ahead Log Files" << std::endl;
2933 std::cout << "==============================" << std::endl;
2934 rocksdb::VectorLogPtr wal_files;
2935 s = db_->GetSortedWalFiles(wal_files);
2936 if (!s.ok()) {
2937 std::cerr << "Error when getting WAL files" << std::endl;
2938 } else {
2939 for (auto& wal : wal_files) {
2940 // TODO(qyang): option.wal_dir should be passed into ldb command
2941 std::string filename = db_->GetOptions().wal_dir + wal->PathName();
2942 std::cout << filename << std::endl;
11fdf7f2
TL		 2943 // TODO(myabandeh): allow configuring is_write_commited
2944 DumpWalFile(filename, true, true, true /* is_write_commited */,
2945 &exec_state_);
2946 }
2947 }
2948}
2949
2950void WriteExternalSstFilesCommand::Help(std::string& ret) {
2951 ret.append(" ");
2952 ret.append(WriteExternalSstFilesCommand::Name());
2953 ret.append(" <output_sst_path>");
2954 ret.append("\n");
2955}
2956
2957WriteExternalSstFilesCommand::WriteExternalSstFilesCommand(
2958 const std::vector<std::string>& params,
2959 const std::map<std::string, std::string>& options,
2960 const std::vector<std::string>& flags)
2961 : LDBCommand(
2962 options, flags, false /* is_read_only */,
2963 BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM,
2964 ARG_TO, ARG_CREATE_IF_MISSING})) {
2965 create_if_missing_ =
2966 IsFlagPresent(flags, ARG_CREATE_IF_MISSING) ||
2967 ParseBooleanOption(options, ARG_CREATE_IF_MISSING, false);
2968 if (params.size() != 1) {
2969 exec_state_ = LDBCommandExecuteResult::Failed(
2970 "output SST file path must be specified");
2971 } else {
2972 output_sst_path_ = params.at(0);
2973 }
2974}
2975
2976void WriteExternalSstFilesCommand::DoCommand() {
2977 if (!db_) {
2978 assert(GetExecuteState().IsFailed());
2979 return;
2980 }
2981 ColumnFamilyHandle* cfh = GetCfHandle();
2982 SstFileWriter sst_file_writer(EnvOptions(), db_->GetOptions(), cfh);
2983 Status status = sst_file_writer.Open(output_sst_path_);
2984 if (!status.ok()) {
2985 exec_state_ = LDBCommandExecuteResult::Failed("failed to open SST file: " +
2986 status.ToString());
2987 return;
2988 }
2989
2990 int bad_lines = 0;
2991 std::string line;
2992 std::ifstream ifs_stdin("/dev/stdin");
2993 std::istream* istream_p = ifs_stdin.is_open() ? &ifs_stdin : &std::cin;
2994 while (getline(*istream_p, line, '\n')) {
2995 std::string key;
2996 std::string value;
2997 if (ParseKeyValue(line, &key, &value, is_key_hex_, is_value_hex_)) {
2998 status = sst_file_writer.Put(key, value);
2999 if (!status.ok()) {
3000 exec_state_ = LDBCommandExecuteResult::Failed(
3001 "failed to write record to file: " + status.ToString());
3002 return;
3003 }
3004 } else if (0 == line.find("Keys in range:")) {
3005 // ignore this line
3006 } else if (0 == line.find("Created bg thread 0x")) {
3007 // ignore this line
3008 } else {
3009 bad_lines++;
7c673cae
FG		 3010 }
3011 }
11fdf7f2
TL		 3012
3013 status = sst_file_writer.Finish();
3014 if (!status.ok()) {
3015 exec_state_ = LDBCommandExecuteResult::Failed(
3016 "Failed to finish writing to file: " + status.ToString());
3017 return;
3018 }
3019
3020 if (bad_lines > 0) {
3021 fprintf(stderr, "Warning: %d bad lines ignored.\n", bad_lines);
3022 }
3023 exec_state_ = LDBCommandExecuteResult::Succeed(
3024 "external SST file written to " + output_sst_path_);
3025}
3026
3027Options WriteExternalSstFilesCommand::PrepareOptionsForOpenDB() {
3028 Options opt = LDBCommand::PrepareOptionsForOpenDB();
3029 opt.create_if_missing = create_if_missing_;
3030 return opt;
3031}
3032
3033const std::string IngestExternalSstFilesCommand::ARG_MOVE_FILES = "move_files";
3034const std::string IngestExternalSstFilesCommand::ARG_SNAPSHOT_CONSISTENCY =
3035 "snapshot_consistency";
3036const std::string IngestExternalSstFilesCommand::ARG_ALLOW_GLOBAL_SEQNO =
3037 "allow_global_seqno";
3038const std::string IngestExternalSstFilesCommand::ARG_ALLOW_BLOCKING_FLUSH =
3039 "allow_blocking_flush";
3040const std::string IngestExternalSstFilesCommand::ARG_INGEST_BEHIND =
3041 "ingest_behind";
3042const std::string IngestExternalSstFilesCommand::ARG_WRITE_GLOBAL_SEQNO =
3043 "write_global_seqno";
3044
3045void IngestExternalSstFilesCommand::Help(std::string& ret) {
3046 ret.append(" ");
3047 ret.append(IngestExternalSstFilesCommand::Name());
3048 ret.append(" <input_sst_path>");
3049 ret.append(" [--" + ARG_MOVE_FILES + "] ");
3050 ret.append(" [--" + ARG_SNAPSHOT_CONSISTENCY + "] ");
3051 ret.append(" [--" + ARG_ALLOW_GLOBAL_SEQNO + "] ");
3052 ret.append(" [--" + ARG_ALLOW_BLOCKING_FLUSH + "] ");
3053 ret.append(" [--" + ARG_INGEST_BEHIND + "] ");
3054 ret.append(" [--" + ARG_WRITE_GLOBAL_SEQNO + "] ");
3055 ret.append("\n");
3056}
3057
3058IngestExternalSstFilesCommand::IngestExternalSstFilesCommand(
3059 const std::vector<std::string>& params,
3060 const std::map<std::string, std::string>& options,
3061 const std::vector<std::string>& flags)
3062 : LDBCommand(
3063 options, flags, false /* is_read_only */,
3064 BuildCmdLineOptions({ARG_MOVE_FILES, ARG_SNAPSHOT_CONSISTENCY,
3065 ARG_ALLOW_GLOBAL_SEQNO, ARG_CREATE_IF_MISSING,
3066 ARG_ALLOW_BLOCKING_FLUSH, ARG_INGEST_BEHIND,
3067 ARG_WRITE_GLOBAL_SEQNO})),
3068 move_files_(false),
3069 snapshot_consistency_(true),
3070 allow_global_seqno_(true),
3071 allow_blocking_flush_(true),
3072 ingest_behind_(false),
3073 write_global_seqno_(true) {
3074 create_if_missing_ =
3075 IsFlagPresent(flags, ARG_CREATE_IF_MISSING) ||
3076 ParseBooleanOption(options, ARG_CREATE_IF_MISSING, false);
3077 move_files_ = IsFlagPresent(flags, ARG_MOVE_FILES) ||
3078 ParseBooleanOption(options, ARG_MOVE_FILES, false);
3079 snapshot_consistency_ =
3080 IsFlagPresent(flags, ARG_SNAPSHOT_CONSISTENCY) ||
3081 ParseBooleanOption(options, ARG_SNAPSHOT_CONSISTENCY, true);
3082 allow_global_seqno_ =
3083 IsFlagPresent(flags, ARG_ALLOW_GLOBAL_SEQNO) ||
3084 ParseBooleanOption(options, ARG_ALLOW_GLOBAL_SEQNO, true);
3085 allow_blocking_flush_ =
3086 IsFlagPresent(flags, ARG_ALLOW_BLOCKING_FLUSH) ||
3087 ParseBooleanOption(options, ARG_ALLOW_BLOCKING_FLUSH, true);
3088 ingest_behind_ = IsFlagPresent(flags, ARG_INGEST_BEHIND) ||
3089 ParseBooleanOption(options, ARG_INGEST_BEHIND, false);
3090 write_global_seqno_ =
3091 IsFlagPresent(flags, ARG_WRITE_GLOBAL_SEQNO) ||
3092 ParseBooleanOption(options, ARG_WRITE_GLOBAL_SEQNO, true);
3093
3094 if (allow_global_seqno_) {
3095 if (!write_global_seqno_) {
3096 fprintf(stderr,
3097 "Warning: not writing global_seqno to the ingested SST can\n"
3098 "prevent older versions of RocksDB from being able to open it\n");
3099 }
3100 } else {
3101 if (write_global_seqno_) {
3102 exec_state_ = LDBCommandExecuteResult::Failed(
3103 "ldb cannot write global_seqno to the ingested SST when global_seqno "
3104 "is not allowed");
3105 }
3106 }
3107
3108 if (params.size() != 1) {
3109 exec_state_ =
3110 LDBCommandExecuteResult::Failed("input SST path must be specified");
3111 } else {
3112 input_sst_path_ = params.at(0);
3113 }
3114}
3115
3116void IngestExternalSstFilesCommand::DoCommand() {
3117 if (!db_) {
3118 assert(GetExecuteState().IsFailed());
3119 return;
3120 }
3121 if (GetExecuteState().IsFailed()) {
3122 return;
3123 }
3124 ColumnFamilyHandle* cfh = GetCfHandle();
3125 IngestExternalFileOptions ifo;
3126 ifo.move_files = move_files_;
3127 ifo.snapshot_consistency = snapshot_consistency_;
3128 ifo.allow_global_seqno = allow_global_seqno_;
3129 ifo.allow_blocking_flush = allow_blocking_flush_;
3130 ifo.ingest_behind = ingest_behind_;
3131 ifo.write_global_seqno = write_global_seqno_;
3132 Status status = db_->IngestExternalFile(cfh, {input_sst_path_}, ifo);
3133 if (!status.ok()) {
3134 exec_state_ = LDBCommandExecuteResult::Failed(
3135 "failed to ingest external SST: " + status.ToString());
3136 } else {
3137 exec_state_ =
3138 LDBCommandExecuteResult::Succeed("external SST files ingested");
3139 }
3140}
3141
3142Options IngestExternalSstFilesCommand::PrepareOptionsForOpenDB() {
3143 Options opt = LDBCommand::PrepareOptionsForOpenDB();
3144 opt.create_if_missing = create_if_missing_;
3145 return opt;
7c673cae
FG		 3146}
3147
3148} // namespace rocksdb
3149#endif // ROCKSDB_LITE
Ceph