ceph/src/rocksdb/tools/ldb_cmd.cc

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under the BSD-style license found in the
   3 //  LICENSE file in the root directory of this source tree. An additional grant
   4 //  of patent rights can be found in the PATENTS file in the same directory.
   5 //
   6 #ifndef ROCKSDB_LITE
   7 #include "rocksdb/utilities/ldb_cmd.h"
   8
   9 #ifndef __STDC_FORMAT_MACROS
  10 #define __STDC_FORMAT_MACROS
  11 #endif
  12
  13 #include <inttypes.h>
  14
  15 #include "db/db_impl.h"
  16 #include "db/dbformat.h"
  17 #include "db/log_reader.h"
  18 #include "db/write_batch_internal.h"
  19 #include "port/dirent.h"
  20 #include "rocksdb/cache.h"
  21 #include "rocksdb/table_properties.h"
  22 #include "rocksdb/utilities/backupable_db.h"
  23 #include "rocksdb/utilities/checkpoint.h"
  24 #include "rocksdb/utilities/object_registry.h"
  25 #include "rocksdb/utilities/options_util.h"
  26 #include "rocksdb/write_batch.h"
  27 #include "rocksdb/write_buffer_manager.h"
  28 #include "table/scoped_arena_iterator.h"
  29 #include "tools/ldb_cmd_impl.h"
  30 #include "tools/sst_dump_tool_imp.h"
  31 #include "util/coding.h"
  32 #include "util/filename.h"
  33 #include "util/stderr_logger.h"
  34 #include "util/string_util.h"
  35 #include "utilities/ttl/db_ttl_impl.h"
  36
  37 #include <cstdlib>
  38 #include <ctime>
  39 #include <fstream>
  40 #include <functional>
  41 #include <iostream>
  42 #include <limits>
  43 #include <sstream>
  44 #include <stdexcept>
  45 #include <string>
  46
  47 namespace rocksdb {
  48
  49 const std::string LDBCommand::ARG_DB = "db";
  50 const std::string LDBCommand::ARG_PATH = "path";
  51 const std::string LDBCommand::ARG_HEX = "hex";
  52 const std::string LDBCommand::ARG_KEY_HEX = "key_hex";
  53 const std::string LDBCommand::ARG_VALUE_HEX = "value_hex";
  54 const std::string LDBCommand::ARG_CF_NAME = "column_family";
  55 const std::string LDBCommand::ARG_TTL = "ttl";
  56 const std::string LDBCommand::ARG_TTL_START = "start_time";
  57 const std::string LDBCommand::ARG_TTL_END = "end_time";
  58 const std::string LDBCommand::ARG_TIMESTAMP = "timestamp";
  59 const std::string LDBCommand::ARG_TRY_LOAD_OPTIONS = "try_load_options";
  60 const std::string LDBCommand::ARG_FROM = "from";
  61 const std::string LDBCommand::ARG_TO = "to";
  62 const std::string LDBCommand::ARG_MAX_KEYS = "max_keys";
  63 const std::string LDBCommand::ARG_BLOOM_BITS = "bloom_bits";
  64 const std::string LDBCommand::ARG_FIX_PREFIX_LEN = "fix_prefix_len";
  65 const std::string LDBCommand::ARG_COMPRESSION_TYPE = "compression_type";
  66 const std::string LDBCommand::ARG_COMPRESSION_MAX_DICT_BYTES =
  67     "compression_max_dict_bytes";
  68 const std::string LDBCommand::ARG_BLOCK_SIZE = "block_size";
  69 const std::string LDBCommand::ARG_AUTO_COMPACTION = "auto_compaction";
  70 const std::string LDBCommand::ARG_DB_WRITE_BUFFER_SIZE = "db_write_buffer_size";
  71 const std::string LDBCommand::ARG_WRITE_BUFFER_SIZE = "write_buffer_size";
  72 const std::string LDBCommand::ARG_FILE_SIZE = "file_size";
  73 const std::string LDBCommand::ARG_CREATE_IF_MISSING = "create_if_missing";
  74 const std::string LDBCommand::ARG_NO_VALUE = "no_value";
  75
  76 const char* LDBCommand::DELIM = " ==> ";
  77
  78 namespace {
  79
  80 void DumpWalFile(std::string wal_file, bool print_header, bool print_values,
  81                  LDBCommandExecuteResult* exec_state);
  82
  83 void DumpSstFile(std::string filename, bool output_hex, bool show_properties);
  84 };
  85
  86 LDBCommand* LDBCommand::InitFromCmdLineArgs(
  87     int argc, char** argv, const Options& options,
  88     const LDBOptions& ldb_options,
  89     const std::vector<ColumnFamilyDescriptor>* column_families) {
  90   std::vector<std::string> args;
  91   for (int i = 1; i < argc; i++) {
  92     args.push_back(argv[i]);
  93   }
  94   return InitFromCmdLineArgs(args, options, ldb_options, column_families,
  95                              SelectCommand);
  96 }
  97
  98 /**
  99  * Parse the command-line arguments and create the appropriate LDBCommand2
 100  * instance.
 101  * The command line arguments must be in the following format:
 102  * ./ldb --db=PATH_TO_DB [--commonOpt1=commonOpt1Val] ..
 103  *        COMMAND <PARAM1> <PARAM2> ... [-cmdSpecificOpt1=cmdSpecificOpt1Val] ..
 104  * This is similar to the command line format used by HBaseClientTool.
 105  * Command name is not included in args.
 106  * Returns nullptr if the command-line cannot be parsed.
 107  */
 108 LDBCommand* LDBCommand::InitFromCmdLineArgs(
 109     const std::vector<std::string>& args, const Options& options,
 110     const LDBOptions& ldb_options,
 111     const std::vector<ColumnFamilyDescriptor>* column_families,
 112     const std::function<LDBCommand*(const ParsedParams&)>& selector) {
 113   // --x=y command line arguments are added as x->y map entries in
 114   // parsed_params.option_map.
 115   //
 116   // Command-line arguments of the form --hex end up in this array as hex to
 117   // parsed_params.flags
 118   ParsedParams parsed_params;
 119
 120   // Everything other than option_map and flags. Represents commands
 121   // and their parameters.  For eg: put key1 value1 go into this vector.
 122   std::vector<std::string> cmdTokens;
 123
 124   const std::string OPTION_PREFIX = "--";
 125
 126   for (const auto& arg : args) {
 127     if (arg[0] == '-' && arg[1] == '-'){
 128       std::vector<std::string> splits = StringSplit(arg, '=');
 129       if (splits.size() == 2) {
 130         std::string optionKey = splits[0].substr(OPTION_PREFIX.size());
 131         parsed_params.option_map[optionKey] = splits[1];
 132       } else {
 133         std::string optionKey = splits[0].substr(OPTION_PREFIX.size());
 134         parsed_params.flags.push_back(optionKey);
 135       }
 136     } else {
 137       cmdTokens.push_back(arg);
 138     }
 139   }
 140
 141   if (cmdTokens.size() < 1) {
 142     fprintf(stderr, "Command not specified!");
 143     return nullptr;
 144   }
 145
 146   parsed_params.cmd = cmdTokens[0];
 147   parsed_params.cmd_params.assign(cmdTokens.begin() + 1, cmdTokens.end());
 148
 149   LDBCommand* command = selector(parsed_params);
 150
 151   if (command) {
 152     command->SetDBOptions(options);
 153     command->SetLDBOptions(ldb_options);
 154   }
 155   return command;
 156 }
 157
 158 LDBCommand* LDBCommand::SelectCommand(const ParsedParams& parsed_params) {
 159   if (parsed_params.cmd == GetCommand::Name()) {
 160     return new GetCommand(parsed_params.cmd_params, parsed_params.option_map,
 161                           parsed_params.flags);
 162   } else if (parsed_params.cmd == PutCommand::Name()) {
 163     return new PutCommand(parsed_params.cmd_params, parsed_params.option_map,
 164                           parsed_params.flags);
 165   } else if (parsed_params.cmd == BatchPutCommand::Name()) {
 166     return new BatchPutCommand(parsed_params.cmd_params,
 167                                parsed_params.option_map, parsed_params.flags);
 168   } else if (parsed_params.cmd == ScanCommand::Name()) {
 169     return new ScanCommand(parsed_params.cmd_params, parsed_params.option_map,
 170                            parsed_params.flags);
 171   } else if (parsed_params.cmd == DeleteCommand::Name()) {
 172     return new DeleteCommand(parsed_params.cmd_params, parsed_params.option_map,
 173                              parsed_params.flags);
 174   } else if (parsed_params.cmd == DeleteRangeCommand::Name()) {
 175     return new DeleteRangeCommand(parsed_params.cmd_params,
 176                                   parsed_params.option_map,
 177                                   parsed_params.flags);
 178   } else if (parsed_params.cmd == ApproxSizeCommand::Name()) {
 179     return new ApproxSizeCommand(parsed_params.cmd_params,
 180                                  parsed_params.option_map, parsed_params.flags);
 181   } else if (parsed_params.cmd == DBQuerierCommand::Name()) {
 182     return new DBQuerierCommand(parsed_params.cmd_params,
 183                                 parsed_params.option_map, parsed_params.flags);
 184   } else if (parsed_params.cmd == CompactorCommand::Name()) {
 185     return new CompactorCommand(parsed_params.cmd_params,
 186                                 parsed_params.option_map, parsed_params.flags);
 187   } else if (parsed_params.cmd == WALDumperCommand::Name()) {
 188     return new WALDumperCommand(parsed_params.cmd_params,
 189                                 parsed_params.option_map, parsed_params.flags);
 190   } else if (parsed_params.cmd == ReduceDBLevelsCommand::Name()) {
 191     return new ReduceDBLevelsCommand(parsed_params.cmd_params,
 192                                      parsed_params.option_map,
 193                                      parsed_params.flags);
 194   } else if (parsed_params.cmd == ChangeCompactionStyleCommand::Name()) {
 195     return new ChangeCompactionStyleCommand(parsed_params.cmd_params,
 196                                             parsed_params.option_map,
 197                                             parsed_params.flags);
 198   } else if (parsed_params.cmd == DBDumperCommand::Name()) {
 199     return new DBDumperCommand(parsed_params.cmd_params,
 200                                parsed_params.option_map, parsed_params.flags);
 201   } else if (parsed_params.cmd == DBLoaderCommand::Name()) {
 202     return new DBLoaderCommand(parsed_params.cmd_params,
 203                                parsed_params.option_map, parsed_params.flags);
 204   } else if (parsed_params.cmd == ManifestDumpCommand::Name()) {
 205     return new ManifestDumpCommand(parsed_params.cmd_params,
 206                                    parsed_params.option_map,
 207                                    parsed_params.flags);
 208   } else if (parsed_params.cmd == ListColumnFamiliesCommand::Name()) {
 209     return new ListColumnFamiliesCommand(parsed_params.cmd_params,
 210                                          parsed_params.option_map,
 211                                          parsed_params.flags);
 212   } else if (parsed_params.cmd == CreateColumnFamilyCommand::Name()) {
 213     return new CreateColumnFamilyCommand(parsed_params.cmd_params,
 214                                          parsed_params.option_map,
 215                                          parsed_params.flags);
 216   } else if (parsed_params.cmd == DBFileDumperCommand::Name()) {
 217     return new DBFileDumperCommand(parsed_params.cmd_params,
 218                                    parsed_params.option_map,
 219                                    parsed_params.flags);
 220   } else if (parsed_params.cmd == InternalDumpCommand::Name()) {
 221     return new InternalDumpCommand(parsed_params.cmd_params,
 222                                    parsed_params.option_map,
 223                                    parsed_params.flags);
 224   } else if (parsed_params.cmd == CheckConsistencyCommand::Name()) {
 225     return new CheckConsistencyCommand(parsed_params.cmd_params,
 226                                        parsed_params.option_map,
 227                                        parsed_params.flags);
 228   } else if (parsed_params.cmd == CheckPointCommand::Name()) {
 229     return new CheckPointCommand(parsed_params.cmd_params,
 230                                  parsed_params.option_map,
 231                                  parsed_params.flags);
 232   } else if (parsed_params.cmd == RepairCommand::Name()) {
 233     return new RepairCommand(parsed_params.cmd_params, parsed_params.option_map,
 234                              parsed_params.flags);
 235   } else if (parsed_params.cmd == BackupCommand::Name()) {
 236     return new BackupCommand(parsed_params.cmd_params, parsed_params.option_map,
 237                              parsed_params.flags);
 238   } else if (parsed_params.cmd == RestoreCommand::Name()) {
 239     return new RestoreCommand(parsed_params.cmd_params,
 240                               parsed_params.option_map, parsed_params.flags);
 241   }
 242   return nullptr;
 243 }
 244
 245 /* Run the command, and return the execute result. */
 246 void LDBCommand::Run() {
 247   if (!exec_state_.IsNotStarted()) {
 248     return;
 249   }
 250
 251   if (db_ == nullptr && !NoDBOpen()) {
 252     OpenDB();
 253     if (exec_state_.IsFailed() && try_load_options_) {
 254       // We don't always return if there is a failure because a WAL file or
 255       // manifest file can be given to "dump" command so we should continue.
 256       // --try_load_options is not valid in those cases.
 257       return;
 258     }
 259   }
 260
 261   // We'll intentionally proceed even if the DB can't be opened because users
 262   // can also specify a filename, not just a directory.
 263   DoCommand();
 264
 265   if (exec_state_.IsNotStarted()) {
 266     exec_state_ = LDBCommandExecuteResult::Succeed("");
 267   }
 268
 269   if (db_ != nullptr) {
 270     CloseDB();
 271   }
 272 }
 273
 274 LDBCommand::LDBCommand(const std::map<std::string, std::string>& options,
 275                        const std::vector<std::string>& flags, bool is_read_only,
 276                        const std::vector<std::string>& valid_cmd_line_options)
 277     : db_(nullptr),
 278       is_read_only_(is_read_only),
 279       is_key_hex_(false),
 280       is_value_hex_(false),
 281       is_db_ttl_(false),
 282       timestamp_(false),
 283       try_load_options_(false),
 284       create_if_missing_(false),
 285       option_map_(options),
 286       flags_(flags),
 287       valid_cmd_line_options_(valid_cmd_line_options) {
 288   std::map<std::string, std::string>::const_iterator itr = options.find(ARG_DB);
 289   if (itr != options.end()) {
 290     db_path_ = itr->second;
 291   }
 292
 293   itr = options.find(ARG_CF_NAME);
 294   if (itr != options.end()) {
 295     column_family_name_ = itr->second;
 296   } else {
 297     column_family_name_ = kDefaultColumnFamilyName;
 298   }
 299
 300   is_key_hex_ = IsKeyHex(options, flags);
 301   is_value_hex_ = IsValueHex(options, flags);
 302   is_db_ttl_ = IsFlagPresent(flags, ARG_TTL);
 303   timestamp_ = IsFlagPresent(flags, ARG_TIMESTAMP);
 304   try_load_options_ = IsFlagPresent(flags, ARG_TRY_LOAD_OPTIONS);
 305 }
 306
 307 void LDBCommand::OpenDB() {
 308   Options opt;
 309   bool opt_set = false;
 310   if (!create_if_missing_ && try_load_options_) {
 311     Status s =
 312         LoadLatestOptions(db_path_, Env::Default(), &opt, &column_families_);
 313     if (s.ok()) {
 314       opt_set = true;
 315     } else if (!s.IsNotFound()) {
 316       // Option file exists but load option file error.
 317       std::string msg = s.ToString();
 318       exec_state_ = LDBCommandExecuteResult::Failed(msg);
 319       db_ = nullptr;
 320       return;
 321     }
 322   }
 323   if (!opt_set) {
 324     opt = PrepareOptionsForOpenDB();
 325   }
 326   if (!exec_state_.IsNotStarted()) {
 327     return;
 328   }
 329   // Open the DB.
 330   Status st;
 331   std::vector<ColumnFamilyHandle*> handles_opened;
 332   if (is_db_ttl_) {
 333     // ldb doesn't yet support TTL DB with multiple column families
 334     if (!column_family_name_.empty() || !column_families_.empty()) {
 335       exec_state_ = LDBCommandExecuteResult::Failed(
 336           "ldb doesn't support TTL DB with multiple column families");
 337     }
 338     if (is_read_only_) {
 339       st = DBWithTTL::Open(opt, db_path_, &db_ttl_, 0, true);
 340     } else {
 341       st = DBWithTTL::Open(opt, db_path_, &db_ttl_);
 342     }
 343     db_ = db_ttl_;
 344   } else {
 345     if (!opt_set && column_families_.empty()) {
 346       // Try to figure out column family lists
 347       std::vector<std::string> cf_list;
 348       st = DB::ListColumnFamilies(DBOptions(), db_path_, &cf_list);
 349       // There is possible the DB doesn't exist yet, for "create if not
 350       // "existing case". The failure is ignored here. We rely on DB::Open()
 351       // to give us the correct error message for problem with opening
 352       // existing DB.
 353       if (st.ok() && cf_list.size() > 1) {
 354         // Ignore single column family DB.
 355         for (auto cf_name : cf_list) {
 356           column_families_.emplace_back(cf_name, opt);
 357         }
 358       }
 359     }
 360     if (is_read_only_) {
 361       if (column_families_.empty()) {
 362         st = DB::OpenForReadOnly(opt, db_path_, &db_);
 363       } else {
 364         st = DB::OpenForReadOnly(opt, db_path_, column_families_,
 365                                  &handles_opened, &db_);
 366       }
 367     } else {
 368       if (column_families_.empty()) {
 369         st = DB::Open(opt, db_path_, &db_);
 370       } else {
 371         st = DB::Open(opt, db_path_, column_families_, &handles_opened, &db_);
 372       }
 373     }
 374   }
 375   if (!st.ok()) {
 376     std::string msg = st.ToString();
 377     exec_state_ = LDBCommandExecuteResult::Failed(msg);
 378   } else if (!handles_opened.empty()) {
 379     assert(handles_opened.size() == column_families_.size());
 380     bool found_cf_name = false;
 381     for (size_t i = 0; i < handles_opened.size(); i++) {
 382       cf_handles_[column_families_[i].name] = handles_opened[i];
 383       if (column_family_name_ == column_families_[i].name) {
 384         found_cf_name = true;
 385       }
 386     }
 387     if (!found_cf_name) {
 388       exec_state_ = LDBCommandExecuteResult::Failed(
 389           "Non-existing column family " + column_family_name_);
 390       CloseDB();
 391     }
 392   } else {
 393     // We successfully opened DB in single column family mode.
 394     assert(column_families_.empty());
 395     if (column_family_name_ != kDefaultColumnFamilyName) {
 396       exec_state_ = LDBCommandExecuteResult::Failed(
 397           "Non-existing column family " + column_family_name_);
 398       CloseDB();
 399     }
 400   }
 401
 402   options_ = opt;
 403 }
 404
 405 void LDBCommand::CloseDB() {
 406   if (db_ != nullptr) {
 407     for (auto& pair : cf_handles_) {
 408       delete pair.second;
 409     }
 410     delete db_;
 411     db_ = nullptr;
 412   }
 413 }
 414
 415 ColumnFamilyHandle* LDBCommand::GetCfHandle() {
 416   if (!cf_handles_.empty()) {
 417     auto it = cf_handles_.find(column_family_name_);
 418     if (it == cf_handles_.end()) {
 419       exec_state_ = LDBCommandExecuteResult::Failed(
 420           "Cannot find column family " + column_family_name_);
 421     } else {
 422       return it->second;
 423     }
 424   }
 425   return db_->DefaultColumnFamily();
 426 }
 427
 428 std::vector<std::string> LDBCommand::BuildCmdLineOptions(
 429     std::vector<std::string> options) {
 430   std::vector<std::string> ret = {ARG_DB,
 431                                   ARG_BLOOM_BITS,
 432                                   ARG_BLOCK_SIZE,
 433                                   ARG_AUTO_COMPACTION,
 434                                   ARG_COMPRESSION_TYPE,
 435                                   ARG_COMPRESSION_MAX_DICT_BYTES,
 436                                   ARG_WRITE_BUFFER_SIZE,
 437                                   ARG_FILE_SIZE,
 438                                   ARG_FIX_PREFIX_LEN,
 439                                   ARG_TRY_LOAD_OPTIONS,
 440                                   ARG_CF_NAME};
 441   ret.insert(ret.end(), options.begin(), options.end());
 442   return ret;
 443 }
 444
 445 /**
 446  * Parses the specific integer option and fills in the value.
 447  * Returns true if the option is found.
 448  * Returns false if the option is not found or if there is an error parsing the
 449  * value.  If there is an error, the specified exec_state is also
 450  * updated.
 451  */
 452 bool LDBCommand::ParseIntOption(
 453     const std::map<std::string, std::string>& options,
 454     const std::string& option, int& value,
 455     LDBCommandExecuteResult& exec_state) {
 456   std::map<std::string, std::string>::const_iterator itr =
 457       option_map_.find(option);
 458   if (itr != option_map_.end()) {
 459     try {
 460 #if defined(CYGWIN)
 461       value = strtol(itr->second.c_str(), 0, 10);
 462 #else
 463       value = std::stoi(itr->second);
 464 #endif
 465       return true;
 466     } catch (const std::invalid_argument&) {
 467       exec_state =
 468           LDBCommandExecuteResult::Failed(option + " has an invalid value.");
 469     } catch (const std::out_of_range&) {
 470       exec_state = LDBCommandExecuteResult::Failed(
 471           option + " has a value out-of-range.");
 472     }
 473   }
 474   return false;
 475 }
 476
 477 /**
 478  * Parses the specified option and fills in the value.
 479  * Returns true if the option is found.
 480  * Returns false otherwise.
 481  */
 482 bool LDBCommand::ParseStringOption(
 483     const std::map<std::string, std::string>& options,
 484     const std::string& option, std::string* value) {
 485   auto itr = option_map_.find(option);
 486   if (itr != option_map_.end()) {
 487     *value = itr->second;
 488     return true;
 489   }
 490   return false;
 491 }
 492
 493 Options LDBCommand::PrepareOptionsForOpenDB() {
 494
 495   Options opt = options_;
 496   opt.create_if_missing = false;
 497
 498   std::map<std::string, std::string>::const_iterator itr;
 499
 500   BlockBasedTableOptions table_options;
 501   bool use_table_options = false;
 502   int bits;
 503   if (ParseIntOption(option_map_, ARG_BLOOM_BITS, bits, exec_state_)) {
 504     if (bits > 0) {
 505       use_table_options = true;
 506       table_options.filter_policy.reset(NewBloomFilterPolicy(bits));
 507     } else {
 508       exec_state_ =
 509           LDBCommandExecuteResult::Failed(ARG_BLOOM_BITS + " must be > 0.");
 510     }
 511   }
 512
 513   int block_size;
 514   if (ParseIntOption(option_map_, ARG_BLOCK_SIZE, block_size, exec_state_)) {
 515     if (block_size > 0) {
 516       use_table_options = true;
 517       table_options.block_size = block_size;
 518     } else {
 519       exec_state_ =
 520           LDBCommandExecuteResult::Failed(ARG_BLOCK_SIZE + " must be > 0.");
 521     }
 522   }
 523
 524   if (use_table_options) {
 525     opt.table_factory.reset(NewBlockBasedTableFactory(table_options));
 526   }
 527
 528   itr = option_map_.find(ARG_AUTO_COMPACTION);
 529   if (itr != option_map_.end()) {
 530     opt.disable_auto_compactions = ! StringToBool(itr->second);
 531   }
 532
 533   itr = option_map_.find(ARG_COMPRESSION_TYPE);
 534   if (itr != option_map_.end()) {
 535     std::string comp = itr->second;
 536     if (comp == "no") {
 537       opt.compression = kNoCompression;
 538     } else if (comp == "snappy") {
 539       opt.compression = kSnappyCompression;
 540     } else if (comp == "zlib") {
 541       opt.compression = kZlibCompression;
 542     } else if (comp == "bzip2") {
 543       opt.compression = kBZip2Compression;
 544     } else if (comp == "lz4") {
 545       opt.compression = kLZ4Compression;
 546     } else if (comp == "lz4hc") {
 547       opt.compression = kLZ4HCCompression;
 548     } else if (comp == "xpress") {
 549       opt.compression = kXpressCompression;
 550     } else if (comp == "zstd") {
 551       opt.compression = kZSTD;
 552     } else {
 553       // Unknown compression.
 554       exec_state_ =
 555           LDBCommandExecuteResult::Failed("Unknown compression level: " + comp);
 556     }
 557   }
 558
 559   int compression_max_dict_bytes;
 560   if (ParseIntOption(option_map_, ARG_COMPRESSION_MAX_DICT_BYTES,
 561                      compression_max_dict_bytes, exec_state_)) {
 562     if (compression_max_dict_bytes >= 0) {
 563       opt.compression_opts.max_dict_bytes = compression_max_dict_bytes;
 564     } else {
 565       exec_state_ = LDBCommandExecuteResult::Failed(
 566           ARG_COMPRESSION_MAX_DICT_BYTES + " must be >= 0.");
 567     }
 568   }
 569
 570   int db_write_buffer_size;
 571   if (ParseIntOption(option_map_, ARG_DB_WRITE_BUFFER_SIZE,
 572         db_write_buffer_size, exec_state_)) {
 573     if (db_write_buffer_size >= 0) {
 574       opt.db_write_buffer_size = db_write_buffer_size;
 575     } else {
 576       exec_state_ = LDBCommandExecuteResult::Failed(ARG_DB_WRITE_BUFFER_SIZE +
 577                                                     " must be >= 0.");
 578     }
 579   }
 580
 581   int write_buffer_size;
 582   if (ParseIntOption(option_map_, ARG_WRITE_BUFFER_SIZE, write_buffer_size,
 583         exec_state_)) {
 584     if (write_buffer_size > 0) {
 585       opt.write_buffer_size = write_buffer_size;
 586     } else {
 587       exec_state_ = LDBCommandExecuteResult::Failed(ARG_WRITE_BUFFER_SIZE +
 588                                                     " must be > 0.");
 589     }
 590   }
 591
 592   int file_size;
 593   if (ParseIntOption(option_map_, ARG_FILE_SIZE, file_size, exec_state_)) {
 594     if (file_size > 0) {
 595       opt.target_file_size_base = file_size;
 596     } else {
 597       exec_state_ =
 598           LDBCommandExecuteResult::Failed(ARG_FILE_SIZE + " must be > 0.");
 599     }
 600   }
 601
 602   if (opt.db_paths.size() == 0) {
 603     opt.db_paths.emplace_back(db_path_, std::numeric_limits<uint64_t>::max());
 604   }
 605
 606   int fix_prefix_len;
 607   if (ParseIntOption(option_map_, ARG_FIX_PREFIX_LEN, fix_prefix_len,
 608                      exec_state_)) {
 609     if (fix_prefix_len > 0) {
 610       opt.prefix_extractor.reset(
 611           NewFixedPrefixTransform(static_cast<size_t>(fix_prefix_len)));
 612     } else {
 613       exec_state_ =
 614           LDBCommandExecuteResult::Failed(ARG_FIX_PREFIX_LEN + " must be > 0.");
 615     }
 616   }
 617
 618   return opt;
 619 }
 620
 621 bool LDBCommand::ParseKeyValue(const std::string& line, std::string* key,
 622                                std::string* value, bool is_key_hex,
 623                                bool is_value_hex) {
 624   size_t pos = line.find(DELIM);
 625   if (pos != std::string::npos) {
 626     *key = line.substr(0, pos);
 627     *value = line.substr(pos + strlen(DELIM));
 628     if (is_key_hex) {
 629       *key = HexToString(*key);
 630     }
 631     if (is_value_hex) {
 632       *value = HexToString(*value);
 633     }
 634     return true;
 635   } else {
 636     return false;
 637   }
 638 }
 639
 640 /**
 641  * Make sure that ONLY the command-line options and flags expected by this
 642  * command are specified on the command-line.  Extraneous options are usually
 643  * the result of user error.
 644  * Returns true if all checks pass.  Else returns false, and prints an
 645  * appropriate error msg to stderr.
 646  */
 647 bool LDBCommand::ValidateCmdLineOptions() {
 648   for (std::map<std::string, std::string>::const_iterator itr =
 649            option_map_.begin();
 650        itr != option_map_.end(); ++itr) {
 651     if (std::find(valid_cmd_line_options_.begin(),
 652                   valid_cmd_line_options_.end(),
 653                   itr->first) == valid_cmd_line_options_.end()) {
 654       fprintf(stderr, "Invalid command-line option %s\n", itr->first.c_str());
 655       return false;
 656     }
 657   }
 658
 659   for (std::vector<std::string>::const_iterator itr = flags_.begin();
 660        itr != flags_.end(); ++itr) {
 661     if (std::find(valid_cmd_line_options_.begin(),
 662                   valid_cmd_line_options_.end(),
 663                   *itr) == valid_cmd_line_options_.end()) {
 664       fprintf(stderr, "Invalid command-line flag %s\n", itr->c_str());
 665       return false;
 666     }
 667   }
 668
 669   if (!NoDBOpen() && option_map_.find(ARG_DB) == option_map_.end() &&
 670       option_map_.find(ARG_PATH) == option_map_.end()) {
 671     fprintf(stderr, "Either %s or %s must be specified.\n", ARG_DB.c_str(),
 672             ARG_PATH.c_str());
 673     return false;
 674   }
 675
 676   return true;
 677 }
 678
 679 std::string LDBCommand::HexToString(const std::string& str) {
 680   std::string result;
 681   std::string::size_type len = str.length();
 682   if (len < 2 || str[0] != '0' || str[1] != 'x') {
 683     fprintf(stderr, "Invalid hex input %s.  Must start with 0x\n", str.c_str());
 684     throw "Invalid hex input";
 685   }
 686   if (!Slice(str.data() + 2, len - 2).DecodeHex(&result)) {
 687     throw "Invalid hex input";
 688   }
 689   return result;
 690 }
 691
 692 std::string LDBCommand::StringToHex(const std::string& str) {
 693   std::string result("0x");
 694   result.append(Slice(str).ToString(true));
 695   return result;
 696 }
 697
 698 std::string LDBCommand::PrintKeyValue(const std::string& key,
 699                                       const std::string& value, bool is_key_hex,
 700                                       bool is_value_hex) {
 701   std::string result;
 702   result.append(is_key_hex ? StringToHex(key) : key);
 703   result.append(DELIM);
 704   result.append(is_value_hex ? StringToHex(value) : value);
 705   return result;
 706 }
 707
 708 std::string LDBCommand::PrintKeyValue(const std::string& key,
 709                                       const std::string& value, bool is_hex) {
 710   return PrintKeyValue(key, value, is_hex, is_hex);
 711 }
 712
 713 std::string LDBCommand::HelpRangeCmdArgs() {
 714   std::ostringstream str_stream;
 715   str_stream << " ";
 716   str_stream << "[--" << ARG_FROM << "] ";
 717   str_stream << "[--" << ARG_TO << "] ";
 718   return str_stream.str();
 719 }
 720
 721 bool LDBCommand::IsKeyHex(const std::map<std::string, std::string>& options,
 722                           const std::vector<std::string>& flags) {
 723   return (IsFlagPresent(flags, ARG_HEX) || IsFlagPresent(flags, ARG_KEY_HEX) ||
 724           ParseBooleanOption(options, ARG_HEX, false) ||
 725           ParseBooleanOption(options, ARG_KEY_HEX, false));
 726 }
 727
 728 bool LDBCommand::IsValueHex(const std::map<std::string, std::string>& options,
 729                             const std::vector<std::string>& flags) {
 730   return (IsFlagPresent(flags, ARG_HEX) ||
 731           IsFlagPresent(flags, ARG_VALUE_HEX) ||
 732           ParseBooleanOption(options, ARG_HEX, false) ||
 733           ParseBooleanOption(options, ARG_VALUE_HEX, false));
 734 }
 735
 736 bool LDBCommand::ParseBooleanOption(
 737     const std::map<std::string, std::string>& options,
 738     const std::string& option, bool default_val) {
 739   std::map<std::string, std::string>::const_iterator itr = options.find(option);
 740   if (itr != options.end()) {
 741     std::string option_val = itr->second;
 742     return StringToBool(itr->second);
 743   }
 744   return default_val;
 745 }
 746
 747 bool LDBCommand::StringToBool(std::string val) {
 748   std::transform(val.begin(), val.end(), val.begin(),
 749                  [](char ch) -> char { return (char)::tolower(ch); });
 750
 751   if (val == "true") {
 752     return true;
 753   } else if (val == "false") {
 754     return false;
 755   } else {
 756     throw "Invalid value for boolean argument";
 757   }
 758 }
 759
 760 CompactorCommand::CompactorCommand(
 761     const std::vector<std::string>& params,
 762     const std::map<std::string, std::string>& options,
 763     const std::vector<std::string>& flags)
 764     : LDBCommand(options, flags, false,
 765                  BuildCmdLineOptions({ARG_FROM, ARG_TO, ARG_HEX, ARG_KEY_HEX,
 766                                       ARG_VALUE_HEX, ARG_TTL})),
 767       null_from_(true),
 768       null_to_(true) {
 769   std::map<std::string, std::string>::const_iterator itr =
 770       options.find(ARG_FROM);
 771   if (itr != options.end()) {
 772     null_from_ = false;
 773     from_ = itr->second;
 774   }
 775
 776   itr = options.find(ARG_TO);
 777   if (itr != options.end()) {
 778     null_to_ = false;
 779     to_ = itr->second;
 780   }
 781
 782   if (is_key_hex_) {
 783     if (!null_from_) {
 784       from_ = HexToString(from_);
 785     }
 786     if (!null_to_) {
 787       to_ = HexToString(to_);
 788     }
 789   }
 790 }
 791
 792 void CompactorCommand::Help(std::string& ret) {
 793   ret.append("  ");
 794   ret.append(CompactorCommand::Name());
 795   ret.append(HelpRangeCmdArgs());
 796   ret.append("\n");
 797 }
 798
 799 void CompactorCommand::DoCommand() {
 800   if (!db_) {
 801     assert(GetExecuteState().IsFailed());
 802     return;
 803   }
 804
 805   Slice* begin = nullptr;
 806   Slice* end = nullptr;
 807   if (!null_from_) {
 808     begin = new Slice(from_);
 809   }
 810   if (!null_to_) {
 811     end = new Slice(to_);
 812   }
 813
 814   CompactRangeOptions cro;
 815   cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
 816
 817   db_->CompactRange(cro, GetCfHandle(), begin, end);
 818   exec_state_ = LDBCommandExecuteResult::Succeed("");
 819
 820   delete begin;
 821   delete end;
 822 }
 823
 824 // ----------------------------------------------------------------------------
 825
 826 const std::string DBLoaderCommand::ARG_DISABLE_WAL = "disable_wal";
 827 const std::string DBLoaderCommand::ARG_BULK_LOAD = "bulk_load";
 828 const std::string DBLoaderCommand::ARG_COMPACT = "compact";
 829
 830 DBLoaderCommand::DBLoaderCommand(
 831     const std::vector<std::string>& params,
 832     const std::map<std::string, std::string>& options,
 833     const std::vector<std::string>& flags)
 834     : LDBCommand(
 835           options, flags, false,
 836           BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM,
 837                                ARG_TO, ARG_CREATE_IF_MISSING, ARG_DISABLE_WAL,
 838                                ARG_BULK_LOAD, ARG_COMPACT})),
 839       disable_wal_(false),
 840       bulk_load_(false),
 841       compact_(false) {
 842   create_if_missing_ = IsFlagPresent(flags, ARG_CREATE_IF_MISSING);
 843   disable_wal_ = IsFlagPresent(flags, ARG_DISABLE_WAL);
 844   bulk_load_ = IsFlagPresent(flags, ARG_BULK_LOAD);
 845   compact_ = IsFlagPresent(flags, ARG_COMPACT);
 846 }
 847
 848 void DBLoaderCommand::Help(std::string& ret) {
 849   ret.append("  ");
 850   ret.append(DBLoaderCommand::Name());
 851   ret.append(" [--" + ARG_CREATE_IF_MISSING + "]");
 852   ret.append(" [--" + ARG_DISABLE_WAL + "]");
 853   ret.append(" [--" + ARG_BULK_LOAD + "]");
 854   ret.append(" [--" + ARG_COMPACT + "]");
 855   ret.append("\n");
 856 }
 857
 858 Options DBLoaderCommand::PrepareOptionsForOpenDB() {
 859   Options opt = LDBCommand::PrepareOptionsForOpenDB();
 860   opt.create_if_missing = create_if_missing_;
 861   if (bulk_load_) {
 862     opt.PrepareForBulkLoad();
 863   }
 864   return opt;
 865 }
 866
 867 void DBLoaderCommand::DoCommand() {
 868   if (!db_) {
 869     assert(GetExecuteState().IsFailed());
 870     return;
 871   }
 872
 873   WriteOptions write_options;
 874   if (disable_wal_) {
 875     write_options.disableWAL = true;
 876   }
 877
 878   int bad_lines = 0;
 879   std::string line;
 880   // prefer ifstream getline performance vs that from std::cin istream
 881   std::ifstream ifs_stdin("/dev/stdin");
 882   std::istream* istream_p = ifs_stdin.is_open() ? &ifs_stdin : &std::cin;
 883   while (getline(*istream_p, line, '\n')) {
 884     std::string key;
 885     std::string value;
 886     if (ParseKeyValue(line, &key, &value, is_key_hex_, is_value_hex_)) {
 887       db_->Put(write_options, GetCfHandle(), Slice(key), Slice(value));
 888     } else if (0 == line.find("Keys in range:")) {
 889       // ignore this line
 890     } else if (0 == line.find("Created bg thread 0x")) {
 891       // ignore this line
 892     } else {
 893       bad_lines ++;
 894     }
 895   }
 896
 897   if (bad_lines > 0) {
 898     std::cout << "Warning: " << bad_lines << " bad lines ignored." << std::endl;
 899   }
 900   if (compact_) {
 901     db_->CompactRange(CompactRangeOptions(), GetCfHandle(), nullptr, nullptr);
 902   }
 903 }
 904
 905 // ----------------------------------------------------------------------------
 906
 907 namespace {
 908
 909 void DumpManifestFile(std::string file, bool verbose, bool hex, bool json) {
 910   Options options;
 911   EnvOptions sopt;
 912   std::string dbname("dummy");
 913   std::shared_ptr<Cache> tc(NewLRUCache(options.max_open_files - 10,
 914                                         options.table_cache_numshardbits));
 915   // Notice we are using the default options not through SanitizeOptions(),
 916   // if VersionSet::DumpManifest() depends on any option done by
 917   // SanitizeOptions(), we need to initialize it manually.
 918   options.db_paths.emplace_back("dummy", 0);
 919   options.num_levels = 64;
 920   WriteController wc(options.delayed_write_rate);
 921   WriteBufferManager wb(options.db_write_buffer_size);
 922   ImmutableDBOptions immutable_db_options(options);
 923   VersionSet versions(dbname, &immutable_db_options, sopt, tc.get(), &wb, &wc);
 924   Status s = versions.DumpManifest(options, file, verbose, hex, json);
 925   if (!s.ok()) {
 926     printf("Error in processing file %s %s\n", file.c_str(),
 927            s.ToString().c_str());
 928   }
 929 }
 930
 931 }  // namespace
 932
 933 const std::string ManifestDumpCommand::ARG_VERBOSE = "verbose";
 934 const std::string ManifestDumpCommand::ARG_JSON = "json";
 935 const std::string ManifestDumpCommand::ARG_PATH = "path";
 936
 937 void ManifestDumpCommand::Help(std::string& ret) {
 938   ret.append("  ");
 939   ret.append(ManifestDumpCommand::Name());
 940   ret.append(" [--" + ARG_VERBOSE + "]");
 941   ret.append(" [--" + ARG_JSON + "]");
 942   ret.append(" [--" + ARG_PATH + "=<path_to_manifest_file>]");
 943   ret.append("\n");
 944 }
 945
 946 ManifestDumpCommand::ManifestDumpCommand(
 947     const std::vector<std::string>& params,
 948     const std::map<std::string, std::string>& options,
 949     const std::vector<std::string>& flags)
 950     : LDBCommand(
 951           options, flags, false,
 952           BuildCmdLineOptions({ARG_VERBOSE, ARG_PATH, ARG_HEX, ARG_JSON})),
 953       verbose_(false),
 954       json_(false),
 955       path_("") {
 956   verbose_ = IsFlagPresent(flags, ARG_VERBOSE);
 957   json_ = IsFlagPresent(flags, ARG_JSON);
 958
 959   std::map<std::string, std::string>::const_iterator itr =
 960       options.find(ARG_PATH);
 961   if (itr != options.end()) {
 962     path_ = itr->second;
 963     if (path_.empty()) {
 964       exec_state_ = LDBCommandExecuteResult::Failed("--path: missing pathname");
 965     }
 966   }
 967 }
 968
 969 void ManifestDumpCommand::DoCommand() {
 970
 971   std::string manifestfile;
 972
 973   if (!path_.empty()) {
 974     manifestfile = path_;
 975   } else {
 976     bool found = false;
 977     // We need to find the manifest file by searching the directory
 978     // containing the db for files of the form MANIFEST_[0-9]+
 979
 980     auto CloseDir = [](DIR* p) { closedir(p); };
 981     std::unique_ptr<DIR, decltype(CloseDir)> d(opendir(db_path_.c_str()),
 982                                                CloseDir);
 983
 984     if (d == nullptr) {
 985       exec_state_ =
 986           LDBCommandExecuteResult::Failed(db_path_ + " is not a directory");
 987       return;
 988     }
 989     struct dirent* entry;
 990     while ((entry = readdir(d.get())) != nullptr) {
 991       unsigned int match;
 992       uint64_t num;
 993       if (sscanf(entry->d_name, "MANIFEST-%" PRIu64 "%n", &num, &match) &&
 994           match == strlen(entry->d_name)) {
 995         if (!found) {
 996           manifestfile = db_path_ + "/" + std::string(entry->d_name);
 997           found = true;
 998         } else {
 999           exec_state_ = LDBCommandExecuteResult::Failed(
1000               "Multiple MANIFEST files found; use --path to select one");
1001           return;
1002         }
1003       }
1004     }
1005   }
1006
1007   if (verbose_) {
1008     printf("Processing Manifest file %s\n", manifestfile.c_str());
1009   }
1010
1011   DumpManifestFile(manifestfile, verbose_, is_key_hex_, json_);
1012
1013   if (verbose_) {
1014     printf("Processing Manifest file %s done\n", manifestfile.c_str());
1015   }
1016 }
1017
1018 // ----------------------------------------------------------------------------
1019
1020 void ListColumnFamiliesCommand::Help(std::string& ret) {
1021   ret.append("  ");
1022   ret.append(ListColumnFamiliesCommand::Name());
1023   ret.append(" full_path_to_db_directory ");
1024   ret.append("\n");
1025 }
1026
1027 ListColumnFamiliesCommand::ListColumnFamiliesCommand(
1028     const std::vector<std::string>& params,
1029     const std::map<std::string, std::string>& options,
1030     const std::vector<std::string>& flags)
1031     : LDBCommand(options, flags, false, {}) {
1032   if (params.size() != 1) {
1033     exec_state_ = LDBCommandExecuteResult::Failed(
1034         "dbname must be specified for the list_column_families command");
1035   } else {
1036     dbname_ = params[0];
1037   }
1038 }
1039
1040 void ListColumnFamiliesCommand::DoCommand() {
1041   std::vector<std::string> column_families;
1042   Status s = DB::ListColumnFamilies(DBOptions(), dbname_, &column_families);
1043   if (!s.ok()) {
1044     printf("Error in processing db %s %s\n", dbname_.c_str(),
1045            s.ToString().c_str());
1046   } else {
1047     printf("Column families in %s: \n{", dbname_.c_str());
1048     bool first = true;
1049     for (auto cf : column_families) {
1050       if (!first) {
1051         printf(", ");
1052       }
1053       first = false;
1054       printf("%s", cf.c_str());
1055     }
1056     printf("}\n");
1057   }
1058 }
1059
1060 void CreateColumnFamilyCommand::Help(std::string& ret) {
1061   ret.append("  ");
1062   ret.append(CreateColumnFamilyCommand::Name());
1063   ret.append(" --db=<db_path> <new_column_family_name>");
1064   ret.append("\n");
1065 }
1066
1067 CreateColumnFamilyCommand::CreateColumnFamilyCommand(
1068     const std::vector<std::string>& params,
1069     const std::map<std::string, std::string>& options,
1070     const std::vector<std::string>& flags)
1071     : LDBCommand(options, flags, true, {ARG_DB}) {
1072   if (params.size() != 1) {
1073     exec_state_ = LDBCommandExecuteResult::Failed(
1074         "new column family name must be specified");
1075   } else {
1076     new_cf_name_ = params[0];
1077   }
1078 }
1079
1080 void CreateColumnFamilyCommand::DoCommand() {
1081   ColumnFamilyHandle* new_cf_handle;
1082   Status st = db_->CreateColumnFamily(options_, new_cf_name_, &new_cf_handle);
1083   if (st.ok()) {
1084     fprintf(stdout, "OK\n");
1085   } else {
1086     exec_state_ = LDBCommandExecuteResult::Failed(
1087         "Fail to create new column family: " + st.ToString());
1088   }
1089   delete new_cf_handle;
1090   CloseDB();
1091 }
1092
1093 // ----------------------------------------------------------------------------
1094
1095 namespace {
1096
1097 std::string ReadableTime(int unixtime) {
1098   char time_buffer [80];
1099   time_t rawtime = unixtime;
1100   struct tm tInfo;
1101   struct tm* timeinfo = localtime_r(&rawtime, &tInfo);
1102   assert(timeinfo == &tInfo);
1103   strftime(time_buffer, 80, "%c", timeinfo);
1104   return std::string(time_buffer);
1105 }
1106
1107 // This function only called when it's the sane case of >1 buckets in time-range
1108 // Also called only when timekv falls between ttl_start and ttl_end provided
1109 void IncBucketCounts(std::vector<uint64_t>& bucket_counts, int ttl_start,
1110                      int time_range, int bucket_size, int timekv,
1111                      int num_buckets) {
1112   assert(time_range > 0 && timekv >= ttl_start && bucket_size > 0 &&
1113     timekv < (ttl_start + time_range) && num_buckets > 1);
1114   int bucket = (timekv - ttl_start) / bucket_size;
1115   bucket_counts[bucket]++;
1116 }
1117
1118 void PrintBucketCounts(const std::vector<uint64_t>& bucket_counts,
1119                        int ttl_start, int ttl_end, int bucket_size,
1120                        int num_buckets) {
1121   int time_point = ttl_start;
1122   for(int i = 0; i < num_buckets - 1; i++, time_point += bucket_size) {
1123     fprintf(stdout, "Keys in range %s to %s : %lu\n",
1124             ReadableTime(time_point).c_str(),
1125             ReadableTime(time_point + bucket_size).c_str(),
1126             (unsigned long)bucket_counts[i]);
1127   }
1128   fprintf(stdout, "Keys in range %s to %s : %lu\n",
1129           ReadableTime(time_point).c_str(),
1130           ReadableTime(ttl_end).c_str(),
1131           (unsigned long)bucket_counts[num_buckets - 1]);
1132 }
1133
1134 }  // namespace
1135
1136 const std::string InternalDumpCommand::ARG_COUNT_ONLY = "count_only";
1137 const std::string InternalDumpCommand::ARG_COUNT_DELIM = "count_delim";
1138 const std::string InternalDumpCommand::ARG_STATS = "stats";
1139 const std::string InternalDumpCommand::ARG_INPUT_KEY_HEX = "input_key_hex";
1140
1141 InternalDumpCommand::InternalDumpCommand(
1142     const std::vector<std::string>& params,
1143     const std::map<std::string, std::string>& options,
1144     const std::vector<std::string>& flags)
1145     : LDBCommand(
1146           options, flags, true,
1147           BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM,
1148                                ARG_TO, ARG_MAX_KEYS, ARG_COUNT_ONLY,
1149                                ARG_COUNT_DELIM, ARG_STATS, ARG_INPUT_KEY_HEX})),
1150       has_from_(false),
1151       has_to_(false),
1152       max_keys_(-1),
1153       delim_("."),
1154       count_only_(false),
1155       count_delim_(false),
1156       print_stats_(false),
1157       is_input_key_hex_(false) {
1158   has_from_ = ParseStringOption(options, ARG_FROM, &from_);
1159   has_to_ = ParseStringOption(options, ARG_TO, &to_);
1160
1161   ParseIntOption(options, ARG_MAX_KEYS, max_keys_, exec_state_);
1162   std::map<std::string, std::string>::const_iterator itr =
1163       options.find(ARG_COUNT_DELIM);
1164   if (itr != options.end()) {
1165     delim_ = itr->second;
1166     count_delim_ = true;
1167    // fprintf(stdout,"delim = %c\n",delim_[0]);
1168   } else {
1169     count_delim_ = IsFlagPresent(flags, ARG_COUNT_DELIM);
1170     delim_=".";
1171   }
1172
1173   print_stats_ = IsFlagPresent(flags, ARG_STATS);
1174   count_only_ = IsFlagPresent(flags, ARG_COUNT_ONLY);
1175   is_input_key_hex_ = IsFlagPresent(flags, ARG_INPUT_KEY_HEX);
1176
1177   if (is_input_key_hex_) {
1178     if (has_from_) {
1179       from_ = HexToString(from_);
1180     }
1181     if (has_to_) {
1182       to_ = HexToString(to_);
1183     }
1184   }
1185 }
1186
1187 void InternalDumpCommand::Help(std::string& ret) {
1188   ret.append("  ");
1189   ret.append(InternalDumpCommand::Name());
1190   ret.append(HelpRangeCmdArgs());
1191   ret.append(" [--" + ARG_INPUT_KEY_HEX + "]");
1192   ret.append(" [--" + ARG_MAX_KEYS + "=<N>]");
1193   ret.append(" [--" + ARG_COUNT_ONLY + "]");
1194   ret.append(" [--" + ARG_COUNT_DELIM + "=<char>]");
1195   ret.append(" [--" + ARG_STATS + "]");
1196   ret.append("\n");
1197 }
1198
1199 void InternalDumpCommand::DoCommand() {
1200   if (!db_) {
1201     assert(GetExecuteState().IsFailed());
1202     return;
1203   }
1204
1205   if (print_stats_) {
1206     std::string stats;
1207     if (db_->GetProperty(GetCfHandle(), "rocksdb.stats", &stats)) {
1208       fprintf(stdout, "%s\n", stats.c_str());
1209     }
1210   }
1211
1212   // Cast as DBImpl to get internal iterator
1213   DBImpl* idb = dynamic_cast<DBImpl*>(db_);
1214   if (!idb) {
1215     exec_state_ = LDBCommandExecuteResult::Failed("DB is not DBImpl");
1216     return;
1217   }
1218   std::string rtype1, rtype2, row, val;
1219   rtype2 = "";
1220   uint64_t c=0;
1221   uint64_t s1=0,s2=0;
1222   // Setup internal key iterator
1223   Arena arena;
1224   auto icmp = InternalKeyComparator(options_.comparator);
1225   RangeDelAggregator range_del_agg(icmp, {} /* snapshots */);
1226   ScopedArenaIterator iter(idb->NewInternalIterator(&arena, &range_del_agg));
1227   Status st = iter->status();
1228   if (!st.ok()) {
1229     exec_state_ =
1230         LDBCommandExecuteResult::Failed("Iterator error:" + st.ToString());
1231   }
1232
1233   if (has_from_) {
1234     InternalKey ikey;
1235     ikey.SetMaxPossibleForUserKey(from_);
1236     iter->Seek(ikey.Encode());
1237   } else {
1238     iter->SeekToFirst();
1239   }
1240
1241   long long count = 0;
1242   for (; iter->Valid(); iter->Next()) {
1243     ParsedInternalKey ikey;
1244     if (!ParseInternalKey(iter->key(), &ikey)) {
1245       fprintf(stderr, "Internal Key [%s] parse error!\n",
1246               iter->key().ToString(true /* in hex*/).data());
1247       // TODO: add error counter
1248       continue;
1249     }
1250
1251     // If end marker was specified, we stop before it
1252     if (has_to_ && options_.comparator->Compare(ikey.user_key, to_) >= 0) {
1253       break;
1254     }
1255
1256     ++count;
1257     int k;
1258     if (count_delim_) {
1259       rtype1 = "";
1260       s1=0;
1261       row = iter->key().ToString();
1262       val = iter->value().ToString();
1263       for(k=0;row[k]!='\x01' && row[k]!='\0';k++)
1264         s1++;
1265       for(k=0;val[k]!='\x01' && val[k]!='\0';k++)
1266         s1++;
1267       for(int j=0;row[j]!=delim_[0] && row[j]!='\0' && row[j]!='\x01';j++)
1268         rtype1+=row[j];
1269       if(rtype2.compare("") && rtype2.compare(rtype1)!=0) {
1270         fprintf(stdout,"%s => count:%lld\tsize:%lld\n",rtype2.c_str(),
1271             (long long)c,(long long)s2);
1272         c=1;
1273         s2=s1;
1274         rtype2 = rtype1;
1275       } else {
1276         c++;
1277         s2+=s1;
1278         rtype2=rtype1;
1279     }
1280   }
1281
1282     if (!count_only_ && !count_delim_) {
1283       std::string key = ikey.DebugString(is_key_hex_);
1284       std::string value = iter->value().ToString(is_value_hex_);
1285       std::cout << key << " => " << value << "\n";
1286     }
1287
1288     // Terminate if maximum number of keys have been dumped
1289     if (max_keys_ > 0 && count >= max_keys_) break;
1290   }
1291   if(count_delim_) {
1292     fprintf(stdout,"%s => count:%lld\tsize:%lld\n", rtype2.c_str(),
1293         (long long)c,(long long)s2);
1294   } else
1295   fprintf(stdout, "Internal keys in range: %lld\n", (long long) count);
1296 }
1297
1298 const std::string DBDumperCommand::ARG_COUNT_ONLY = "count_only";
1299 const std::string DBDumperCommand::ARG_COUNT_DELIM = "count_delim";
1300 const std::string DBDumperCommand::ARG_STATS = "stats";
1301 const std::string DBDumperCommand::ARG_TTL_BUCKET = "bucket";
1302
1303 DBDumperCommand::DBDumperCommand(
1304     const std::vector<std::string>& params,
1305     const std::map<std::string, std::string>& options,
1306     const std::vector<std::string>& flags)
1307     : LDBCommand(options, flags, true,
1308                  BuildCmdLineOptions(
1309                      {ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM,
1310                       ARG_TO, ARG_MAX_KEYS, ARG_COUNT_ONLY, ARG_COUNT_DELIM,
1311                       ARG_STATS, ARG_TTL_START, ARG_TTL_END, ARG_TTL_BUCKET,
1312                       ARG_TIMESTAMP, ARG_PATH})),
1313       null_from_(true),
1314       null_to_(true),
1315       max_keys_(-1),
1316       count_only_(false),
1317       count_delim_(false),
1318       print_stats_(false) {
1319   std::map<std::string, std::string>::const_iterator itr =
1320       options.find(ARG_FROM);
1321   if (itr != options.end()) {
1322     null_from_ = false;
1323     from_ = itr->second;
1324   }
1325
1326   itr = options.find(ARG_TO);
1327   if (itr != options.end()) {
1328     null_to_ = false;
1329     to_ = itr->second;
1330   }
1331
1332   itr = options.find(ARG_MAX_KEYS);
1333   if (itr != options.end()) {
1334     try {
1335 #if defined(CYGWIN)
1336       max_keys_ = strtol(itr->second.c_str(), 0, 10);
1337 #else
1338       max_keys_ = std::stoi(itr->second);
1339 #endif
1340     } catch (const std::invalid_argument&) {
1341       exec_state_ = LDBCommandExecuteResult::Failed(ARG_MAX_KEYS +
1342                                                     " has an invalid value");
1343     } catch (const std::out_of_range&) {
1344       exec_state_ = LDBCommandExecuteResult::Failed(
1345           ARG_MAX_KEYS + " has a value out-of-range");
1346     }
1347   }
1348   itr = options.find(ARG_COUNT_DELIM);
1349   if (itr != options.end()) {
1350     delim_ = itr->second;
1351     count_delim_ = true;
1352   } else {
1353     count_delim_ = IsFlagPresent(flags, ARG_COUNT_DELIM);
1354     delim_=".";
1355   }
1356
1357   print_stats_ = IsFlagPresent(flags, ARG_STATS);
1358   count_only_ = IsFlagPresent(flags, ARG_COUNT_ONLY);
1359
1360   if (is_key_hex_) {
1361     if (!null_from_) {
1362       from_ = HexToString(from_);
1363     }
1364     if (!null_to_) {
1365       to_ = HexToString(to_);
1366     }
1367   }
1368
1369   itr = options.find(ARG_PATH);
1370   if (itr != options.end()) {
1371     path_ = itr->second;
1372   }
1373 }
1374
1375 void DBDumperCommand::Help(std::string& ret) {
1376   ret.append("  ");
1377   ret.append(DBDumperCommand::Name());
1378   ret.append(HelpRangeCmdArgs());
1379   ret.append(" [--" + ARG_TTL + "]");
1380   ret.append(" [--" + ARG_MAX_KEYS + "=<N>]");
1381   ret.append(" [--" + ARG_TIMESTAMP + "]");
1382   ret.append(" [--" + ARG_COUNT_ONLY + "]");
1383   ret.append(" [--" + ARG_COUNT_DELIM + "=<char>]");
1384   ret.append(" [--" + ARG_STATS + "]");
1385   ret.append(" [--" + ARG_TTL_BUCKET + "=<N>]");
1386   ret.append(" [--" + ARG_TTL_START + "=<N>:- is inclusive]");
1387   ret.append(" [--" + ARG_TTL_END + "=<N>:- is exclusive]");
1388   ret.append(" [--" + ARG_PATH + "=<path_to_a_file>]");
1389   ret.append("\n");
1390 }
1391
1392 /**
1393  * Handles two separate cases:
1394  *
1395  * 1) --db is specified - just dump the database.
1396  *
1397  * 2) --path is specified - determine based on file extension what dumping
1398  *    function to call. Please note that we intentionally use the extension
1399  *    and avoid probing the file contents under the assumption that renaming
1400  *    the files is not a supported scenario.
1401  *
1402  */
1403 void DBDumperCommand::DoCommand() {
1404   if (!db_) {
1405     assert(!path_.empty());
1406     std::string fileName = GetFileNameFromPath(path_);
1407     uint64_t number;
1408     FileType type;
1409
1410     exec_state_ = LDBCommandExecuteResult::Succeed("");
1411
1412     if (!ParseFileName(fileName, &number, &type)) {
1413       exec_state_ =
1414           LDBCommandExecuteResult::Failed("Can't parse file type: " + path_);
1415       return;
1416     }
1417
1418     switch (type) {
1419       case kLogFile:
1420         DumpWalFile(path_, /* print_header_ */ true, /* print_values_ */ true,
1421                     &exec_state_);
1422         break;
1423       case kTableFile:
1424         DumpSstFile(path_, is_key_hex_, /* show_properties */ true);
1425         break;
1426       case kDescriptorFile:
1427         DumpManifestFile(path_, /* verbose_ */ false, is_key_hex_,
1428                          /*  json_ */ false);
1429         break;
1430       default:
1431         exec_state_ = LDBCommandExecuteResult::Failed(
1432             "File type not supported: " + path_);
1433         break;
1434     }
1435
1436   } else {
1437     DoDumpCommand();
1438   }
1439 }
1440
1441 void DBDumperCommand::DoDumpCommand() {
1442   assert(nullptr != db_);
1443   assert(path_.empty());
1444
1445   // Parse command line args
1446   uint64_t count = 0;
1447   if (print_stats_) {
1448     std::string stats;
1449     if (db_->GetProperty("rocksdb.stats", &stats)) {
1450       fprintf(stdout, "%s\n", stats.c_str());
1451     }
1452   }
1453
1454   // Setup key iterator
1455   Iterator* iter = db_->NewIterator(ReadOptions(), GetCfHandle());
1456   Status st = iter->status();
1457   if (!st.ok()) {
1458     exec_state_ =
1459         LDBCommandExecuteResult::Failed("Iterator error." + st.ToString());
1460   }
1461
1462   if (!null_from_) {
1463     iter->Seek(from_);
1464   } else {
1465     iter->SeekToFirst();
1466   }
1467
1468   int max_keys = max_keys_;
1469   int ttl_start;
1470   if (!ParseIntOption(option_map_, ARG_TTL_START, ttl_start, exec_state_)) {
1471     ttl_start = DBWithTTLImpl::kMinTimestamp;  // TTL introduction time
1472   }
1473   int ttl_end;
1474   if (!ParseIntOption(option_map_, ARG_TTL_END, ttl_end, exec_state_)) {
1475     ttl_end = DBWithTTLImpl::kMaxTimestamp;  // Max time allowed by TTL feature
1476   }
1477   if (ttl_end < ttl_start) {
1478     fprintf(stderr, "Error: End time can't be less than start time\n");
1479     delete iter;
1480     return;
1481   }
1482   int time_range = ttl_end - ttl_start;
1483   int bucket_size;
1484   if (!ParseIntOption(option_map_, ARG_TTL_BUCKET, bucket_size, exec_state_) ||
1485       bucket_size <= 0) {
1486     bucket_size = time_range; // Will have just 1 bucket by default
1487   }
1488   //cretaing variables for row count of each type
1489   std::string rtype1, rtype2, row, val;
1490   rtype2 = "";
1491   uint64_t c=0;
1492   uint64_t s1=0,s2=0;
1493
1494   // At this point, bucket_size=0 => time_range=0
1495   int num_buckets = (bucket_size >= time_range)
1496                         ? 1
1497                         : ((time_range + bucket_size - 1) / bucket_size);
1498   std::vector<uint64_t> bucket_counts(num_buckets, 0);
1499   if (is_db_ttl_ && !count_only_ && timestamp_ && !count_delim_) {
1500     fprintf(stdout, "Dumping key-values from %s to %s\n",
1501             ReadableTime(ttl_start).c_str(), ReadableTime(ttl_end).c_str());
1502   }
1503
1504   for (; iter->Valid(); iter->Next()) {
1505     int rawtime = 0;
1506     // If end marker was specified, we stop before it
1507     if (!null_to_ && (iter->key().ToString() >= to_))
1508       break;
1509     // Terminate if maximum number of keys have been dumped
1510     if (max_keys == 0)
1511       break;
1512     if (is_db_ttl_) {
1513       TtlIterator* it_ttl = dynamic_cast<TtlIterator*>(iter);
1514       assert(it_ttl);
1515       rawtime = it_ttl->timestamp();
1516       if (rawtime < ttl_start || rawtime >= ttl_end) {
1517         continue;
1518       }
1519     }
1520     if (max_keys > 0) {
1521       --max_keys;
1522     }
1523     if (is_db_ttl_ && num_buckets > 1) {
1524       IncBucketCounts(bucket_counts, ttl_start, time_range, bucket_size,
1525                       rawtime, num_buckets);
1526     }
1527     ++count;
1528     if (count_delim_) {
1529       rtype1 = "";
1530       row = iter->key().ToString();
1531       val = iter->value().ToString();
1532       s1 = row.size()+val.size();
1533       for(int j=0;row[j]!=delim_[0] && row[j]!='\0';j++)
1534         rtype1+=row[j];
1535       if(rtype2.compare("") && rtype2.compare(rtype1)!=0) {
1536         fprintf(stdout,"%s => count:%lld\tsize:%lld\n",rtype2.c_str(),
1537             (long long )c,(long long)s2);
1538         c=1;
1539         s2=s1;
1540         rtype2 = rtype1;
1541       } else {
1542           c++;
1543           s2+=s1;
1544           rtype2=rtype1;
1545       }
1546
1547     }
1548
1549
1550
1551     if (!count_only_ && !count_delim_) {
1552       if (is_db_ttl_ && timestamp_) {
1553         fprintf(stdout, "%s ", ReadableTime(rawtime).c_str());
1554       }
1555       std::string str =
1556           PrintKeyValue(iter->key().ToString(), iter->value().ToString(),
1557                         is_key_hex_, is_value_hex_);
1558       fprintf(stdout, "%s\n", str.c_str());
1559     }
1560   }
1561
1562   if (num_buckets > 1 && is_db_ttl_) {
1563     PrintBucketCounts(bucket_counts, ttl_start, ttl_end, bucket_size,
1564                       num_buckets);
1565   } else if(count_delim_) {
1566     fprintf(stdout,"%s => count:%lld\tsize:%lld\n",rtype2.c_str(),
1567         (long long )c,(long long)s2);
1568   } else {
1569     fprintf(stdout, "Keys in range: %lld\n", (long long) count);
1570   }
1571   // Clean up
1572   delete iter;
1573 }
1574
1575 const std::string ReduceDBLevelsCommand::ARG_NEW_LEVELS = "new_levels";
1576 const std::string ReduceDBLevelsCommand::ARG_PRINT_OLD_LEVELS =
1577     "print_old_levels";
1578
1579 ReduceDBLevelsCommand::ReduceDBLevelsCommand(
1580     const std::vector<std::string>& params,
1581     const std::map<std::string, std::string>& options,
1582     const std::vector<std::string>& flags)
1583     : LDBCommand(options, flags, false,
1584                  BuildCmdLineOptions({ARG_NEW_LEVELS, ARG_PRINT_OLD_LEVELS})),
1585       old_levels_(1 << 7),
1586       new_levels_(-1),
1587       print_old_levels_(false) {
1588   ParseIntOption(option_map_, ARG_NEW_LEVELS, new_levels_, exec_state_);
1589   print_old_levels_ = IsFlagPresent(flags, ARG_PRINT_OLD_LEVELS);
1590
1591   if(new_levels_ <= 0) {
1592     exec_state_ = LDBCommandExecuteResult::Failed(
1593         " Use --" + ARG_NEW_LEVELS + " to specify a new level number\n");
1594   }
1595 }
1596
1597 std::vector<std::string> ReduceDBLevelsCommand::PrepareArgs(
1598     const std::string& db_path, int new_levels, bool print_old_level) {
1599   std::vector<std::string> ret;
1600   ret.push_back("reduce_levels");
1601   ret.push_back("--" + ARG_DB + "=" + db_path);
1602   ret.push_back("--" + ARG_NEW_LEVELS + "=" + rocksdb::ToString(new_levels));
1603   if(print_old_level) {
1604     ret.push_back("--" + ARG_PRINT_OLD_LEVELS);
1605   }
1606   return ret;
1607 }
1608
1609 void ReduceDBLevelsCommand::Help(std::string& ret) {
1610   ret.append("  ");
1611   ret.append(ReduceDBLevelsCommand::Name());
1612   ret.append(" --" + ARG_NEW_LEVELS + "=<New number of levels>");
1613   ret.append(" [--" + ARG_PRINT_OLD_LEVELS + "]");
1614   ret.append("\n");
1615 }
1616
1617 Options ReduceDBLevelsCommand::PrepareOptionsForOpenDB() {
1618   Options opt = LDBCommand::PrepareOptionsForOpenDB();
1619   opt.num_levels = old_levels_;
1620   opt.max_bytes_for_level_multiplier_additional.resize(opt.num_levels, 1);
1621   // Disable size compaction
1622   opt.max_bytes_for_level_base = 1ULL << 50;
1623   opt.max_bytes_for_level_multiplier = 1;
1624   return opt;
1625 }
1626
1627 Status ReduceDBLevelsCommand::GetOldNumOfLevels(Options& opt,
1628     int* levels) {
1629   ImmutableDBOptions db_options(opt);
1630   EnvOptions soptions;
1631   std::shared_ptr<Cache> tc(
1632       NewLRUCache(opt.max_open_files - 10, opt.table_cache_numshardbits));
1633   const InternalKeyComparator cmp(opt.comparator);
1634   WriteController wc(opt.delayed_write_rate);
1635   WriteBufferManager wb(opt.db_write_buffer_size);
1636   VersionSet versions(db_path_, &db_options, soptions, tc.get(), &wb, &wc);
1637   std::vector<ColumnFamilyDescriptor> dummy;
1638   ColumnFamilyDescriptor dummy_descriptor(kDefaultColumnFamilyName,
1639                                           ColumnFamilyOptions(opt));
1640   dummy.push_back(dummy_descriptor);
1641   // We rely the VersionSet::Recover to tell us the internal data structures
1642   // in the db. And the Recover() should never do any change
1643   // (like LogAndApply) to the manifest file.
1644   Status st = versions.Recover(dummy);
1645   if (!st.ok()) {
1646     return st;
1647   }
1648   int max = -1;
1649   auto default_cfd = versions.GetColumnFamilySet()->GetDefault();
1650   for (int i = 0; i < default_cfd->NumberLevels(); i++) {
1651     if (default_cfd->current()->storage_info()->NumLevelFiles(i)) {
1652       max = i;
1653     }
1654   }
1655
1656   *levels = max + 1;
1657   return st;
1658 }
1659
1660 void ReduceDBLevelsCommand::DoCommand() {
1661   if (new_levels_ <= 1) {
1662     exec_state_ =
1663         LDBCommandExecuteResult::Failed("Invalid number of levels.\n");
1664     return;
1665   }
1666
1667   Status st;
1668   Options opt = PrepareOptionsForOpenDB();
1669   int old_level_num = -1;
1670   st = GetOldNumOfLevels(opt, &old_level_num);
1671   if (!st.ok()) {
1672     exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
1673     return;
1674   }
1675
1676   if (print_old_levels_) {
1677     fprintf(stdout, "The old number of levels in use is %d\n", old_level_num);
1678   }
1679
1680   if (old_level_num <= new_levels_) {
1681     return;
1682   }
1683
1684   old_levels_ = old_level_num;
1685
1686   OpenDB();
1687   if (exec_state_.IsFailed()) {
1688     return;
1689   }
1690   // Compact the whole DB to put all files to the highest level.
1691   fprintf(stdout, "Compacting the db...\n");
1692   db_->CompactRange(CompactRangeOptions(), GetCfHandle(), nullptr, nullptr);
1693   CloseDB();
1694
1695   EnvOptions soptions;
1696   st = VersionSet::ReduceNumberOfLevels(db_path_, &opt, soptions, new_levels_);
1697   if (!st.ok()) {
1698     exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
1699     return;
1700   }
1701 }
1702
1703 const std::string ChangeCompactionStyleCommand::ARG_OLD_COMPACTION_STYLE =
1704     "old_compaction_style";
1705 const std::string ChangeCompactionStyleCommand::ARG_NEW_COMPACTION_STYLE =
1706     "new_compaction_style";
1707
1708 ChangeCompactionStyleCommand::ChangeCompactionStyleCommand(
1709     const std::vector<std::string>& params,
1710     const std::map<std::string, std::string>& options,
1711     const std::vector<std::string>& flags)
1712     : LDBCommand(options, flags, false,
1713                  BuildCmdLineOptions(
1714                      {ARG_OLD_COMPACTION_STYLE, ARG_NEW_COMPACTION_STYLE})),
1715       old_compaction_style_(-1),
1716       new_compaction_style_(-1) {
1717   ParseIntOption(option_map_, ARG_OLD_COMPACTION_STYLE, old_compaction_style_,
1718     exec_state_);
1719   if (old_compaction_style_ != kCompactionStyleLevel &&
1720      old_compaction_style_ != kCompactionStyleUniversal) {
1721     exec_state_ = LDBCommandExecuteResult::Failed(
1722         "Use --" + ARG_OLD_COMPACTION_STYLE + " to specify old compaction " +
1723         "style. Check ldb help for proper compaction style value.\n");
1724     return;
1725   }
1726
1727   ParseIntOption(option_map_, ARG_NEW_COMPACTION_STYLE, new_compaction_style_,
1728     exec_state_);
1729   if (new_compaction_style_ != kCompactionStyleLevel &&
1730      new_compaction_style_ != kCompactionStyleUniversal) {
1731     exec_state_ = LDBCommandExecuteResult::Failed(
1732         "Use --" + ARG_NEW_COMPACTION_STYLE + " to specify new compaction " +
1733         "style. Check ldb help for proper compaction style value.\n");
1734     return;
1735   }
1736
1737   if (new_compaction_style_ == old_compaction_style_) {
1738     exec_state_ = LDBCommandExecuteResult::Failed(
1739         "Old compaction style is the same as new compaction style. "
1740         "Nothing to do.\n");
1741     return;
1742   }
1743
1744   if (old_compaction_style_ == kCompactionStyleUniversal &&
1745       new_compaction_style_ == kCompactionStyleLevel) {
1746     exec_state_ = LDBCommandExecuteResult::Failed(
1747         "Convert from universal compaction to level compaction. "
1748         "Nothing to do.\n");
1749     return;
1750   }
1751 }
1752
1753 void ChangeCompactionStyleCommand::Help(std::string& ret) {
1754   ret.append("  ");
1755   ret.append(ChangeCompactionStyleCommand::Name());
1756   ret.append(" --" + ARG_OLD_COMPACTION_STYLE + "=<Old compaction style: 0 " +
1757              "for level compaction, 1 for universal compaction>");
1758   ret.append(" --" + ARG_NEW_COMPACTION_STYLE + "=<New compaction style: 0 " +
1759              "for level compaction, 1 for universal compaction>");
1760   ret.append("\n");
1761 }
1762
1763 Options ChangeCompactionStyleCommand::PrepareOptionsForOpenDB() {
1764   Options opt = LDBCommand::PrepareOptionsForOpenDB();
1765
1766   if (old_compaction_style_ == kCompactionStyleLevel &&
1767       new_compaction_style_ == kCompactionStyleUniversal) {
1768     // In order to convert from level compaction to universal compaction, we
1769     // need to compact all data into a single file and move it to level 0.
1770     opt.disable_auto_compactions = true;
1771     opt.target_file_size_base = INT_MAX;
1772     opt.target_file_size_multiplier = 1;
1773     opt.max_bytes_for_level_base = INT_MAX;
1774     opt.max_bytes_for_level_multiplier = 1;
1775   }
1776
1777   return opt;
1778 }
1779
1780 void ChangeCompactionStyleCommand::DoCommand() {
1781   // print db stats before we have made any change
1782   std::string property;
1783   std::string files_per_level;
1784   for (int i = 0; i < db_->NumberLevels(GetCfHandle()); i++) {
1785     db_->GetProperty(GetCfHandle(),
1786                      "rocksdb.num-files-at-level" + NumberToString(i),
1787                      &property);
1788
1789     // format print string
1790     char buf[100];
1791     snprintf(buf, sizeof(buf), "%s%s", (i ? "," : ""), property.c_str());
1792     files_per_level += buf;
1793   }
1794   fprintf(stdout, "files per level before compaction: %s\n",
1795           files_per_level.c_str());
1796
1797   // manual compact into a single file and move the file to level 0
1798   CompactRangeOptions compact_options;
1799   compact_options.change_level = true;
1800   compact_options.target_level = 0;
1801   db_->CompactRange(compact_options, GetCfHandle(), nullptr, nullptr);
1802
1803   // verify compaction result
1804   files_per_level = "";
1805   int num_files = 0;
1806   for (int i = 0; i < db_->NumberLevels(GetCfHandle()); i++) {
1807     db_->GetProperty(GetCfHandle(),
1808                      "rocksdb.num-files-at-level" + NumberToString(i),
1809                      &property);
1810
1811     // format print string
1812     char buf[100];
1813     snprintf(buf, sizeof(buf), "%s%s", (i ? "," : ""), property.c_str());
1814     files_per_level += buf;
1815
1816     num_files = atoi(property.c_str());
1817
1818     // level 0 should have only 1 file
1819     if (i == 0 && num_files != 1) {
1820       exec_state_ = LDBCommandExecuteResult::Failed(
1821           "Number of db files at "
1822           "level 0 after compaction is " +
1823           ToString(num_files) + ", not 1.\n");
1824       return;
1825     }
1826     // other levels should have no file
1827     if (i > 0 && num_files != 0) {
1828       exec_state_ = LDBCommandExecuteResult::Failed(
1829           "Number of db files at "
1830           "level " +
1831           ToString(i) + " after compaction is " + ToString(num_files) +
1832           ", not 0.\n");
1833       return;
1834     }
1835   }
1836
1837   fprintf(stdout, "files per level after compaction: %s\n",
1838           files_per_level.c_str());
1839 }
1840
1841 // ----------------------------------------------------------------------------
1842
1843 namespace {
1844
1845 struct StdErrReporter : public log::Reader::Reporter {
1846   virtual void Corruption(size_t bytes, const Status& s) override {
1847     std::cerr << "Corruption detected in log file " << s.ToString() << "\n";
1848   }
1849 };
1850
1851 class InMemoryHandler : public WriteBatch::Handler {
1852  public:
1853   InMemoryHandler(std::stringstream& row, bool print_values)
1854       : Handler(), row_(row) {
1855     print_values_ = print_values;
1856   }
1857
1858   void commonPutMerge(const Slice& key, const Slice& value) {
1859     std::string k = LDBCommand::StringToHex(key.ToString());
1860     if (print_values_) {
1861       std::string v = LDBCommand::StringToHex(value.ToString());
1862       row_ << k << " : ";
1863       row_ << v << " ";
1864     } else {
1865       row_ << k << " ";
1866     }
1867   }
1868
1869   virtual Status PutCF(uint32_t cf, const Slice& key,
1870                        const Slice& value) override {
1871     row_ << "PUT(" << cf << ") : ";
1872     commonPutMerge(key, value);
1873     return Status::OK();
1874   }
1875
1876   virtual Status MergeCF(uint32_t cf, const Slice& key,
1877                          const Slice& value) override {
1878     row_ << "MERGE(" << cf << ") : ";
1879     commonPutMerge(key, value);
1880     return Status::OK();
1881   }
1882
1883   virtual Status DeleteCF(uint32_t cf, const Slice& key) override {
1884     row_ << "DELETE(" << cf << ") : ";
1885     row_ << LDBCommand::StringToHex(key.ToString()) << " ";
1886     return Status::OK();
1887   }
1888
1889   virtual Status SingleDeleteCF(uint32_t cf, const Slice& key) override {
1890     row_ << "SINGLE_DELETE(" << cf << ") : ";
1891     row_ << LDBCommand::StringToHex(key.ToString()) << " ";
1892     return Status::OK();
1893   }
1894
1895   virtual Status DeleteRangeCF(uint32_t cf, const Slice& begin_key,
1896                                const Slice& end_key) override {
1897     row_ << "DELETE_RANGE(" << cf << ") : ";
1898     row_ << LDBCommand::StringToHex(begin_key.ToString()) << " ";
1899     row_ << LDBCommand::StringToHex(end_key.ToString()) << " ";
1900     return Status::OK();
1901   }
1902
1903   virtual Status MarkBeginPrepare() override {
1904     row_ << "BEGIN_PREARE ";
1905     return Status::OK();
1906   }
1907
1908   virtual Status MarkEndPrepare(const Slice& xid) override {
1909     row_ << "END_PREPARE(";
1910     row_ << LDBCommand::StringToHex(xid.ToString()) << ") ";
1911     return Status::OK();
1912   }
1913
1914   virtual Status MarkRollback(const Slice& xid) override {
1915     row_ << "ROLLBACK(";
1916     row_ << LDBCommand::StringToHex(xid.ToString()) << ") ";
1917     return Status::OK();
1918   }
1919
1920   virtual Status MarkCommit(const Slice& xid) override {
1921     row_ << "COMMIT(";
1922     row_ << LDBCommand::StringToHex(xid.ToString()) << ") ";
1923     return Status::OK();
1924   }
1925
1926   virtual ~InMemoryHandler() {}
1927
1928  private:
1929   std::stringstream& row_;
1930   bool print_values_;
1931 };
1932
1933 void DumpWalFile(std::string wal_file, bool print_header, bool print_values,
1934                  LDBCommandExecuteResult* exec_state) {
1935   Env* env_ = Env::Default();
1936   EnvOptions soptions;
1937   unique_ptr<SequentialFileReader> wal_file_reader;
1938
1939   Status status;
1940   {
1941     unique_ptr<SequentialFile> file;
1942     status = env_->NewSequentialFile(wal_file, &file, soptions);
1943     if (status.ok()) {
1944       wal_file_reader.reset(new SequentialFileReader(std::move(file)));
1945     }
1946   }
1947   if (!status.ok()) {
1948     if (exec_state) {
1949       *exec_state = LDBCommandExecuteResult::Failed("Failed to open WAL file " +
1950                                                     status.ToString());
1951     } else {
1952       std::cerr << "Error: Failed to open WAL file " << status.ToString()
1953                 << std::endl;
1954     }
1955   } else {
1956     StdErrReporter reporter;
1957     uint64_t log_number;
1958     FileType type;
1959
1960     // we need the log number, but ParseFilename expects dbname/NNN.log.
1961     std::string sanitized = wal_file;
1962     size_t lastslash = sanitized.rfind('/');
1963     if (lastslash != std::string::npos)
1964       sanitized = sanitized.substr(lastslash + 1);
1965     if (!ParseFileName(sanitized, &log_number, &type)) {
1966       // bogus input, carry on as best we can
1967       log_number = 0;
1968     }
1969     DBOptions db_options;
1970     log::Reader reader(db_options.info_log, std::move(wal_file_reader),
1971                        &reporter, true, 0, log_number);
1972     std::string scratch;
1973     WriteBatch batch;
1974     Slice record;
1975     std::stringstream row;
1976     if (print_header) {
1977       std::cout << "Sequence,Count,ByteSize,Physical Offset,Key(s)";
1978       if (print_values) {
1979         std::cout << " : value ";
1980       }
1981       std::cout << "\n";
1982     }
1983     while (reader.ReadRecord(&record, &scratch)) {
1984       row.str("");
1985       if (record.size() < WriteBatchInternal::kHeader) {
1986         reporter.Corruption(record.size(),
1987                             Status::Corruption("log record too small"));
1988       } else {
1989         WriteBatchInternal::SetContents(&batch, record);
1990         row << WriteBatchInternal::Sequence(&batch) << ",";
1991         row << WriteBatchInternal::Count(&batch) << ",";
1992         row << WriteBatchInternal::ByteSize(&batch) << ",";
1993         row << reader.LastRecordOffset() << ",";
1994         InMemoryHandler handler(row, print_values);
1995         batch.Iterate(&handler);
1996         row << "\n";
1997       }
1998       std::cout << row.str();
1999     }
2000   }
2001 }
2002
2003 }  // namespace
2004
2005 const std::string WALDumperCommand::ARG_WAL_FILE = "walfile";
2006 const std::string WALDumperCommand::ARG_PRINT_VALUE = "print_value";
2007 const std::string WALDumperCommand::ARG_PRINT_HEADER = "header";
2008
2009 WALDumperCommand::WALDumperCommand(
2010     const std::vector<std::string>& params,
2011     const std::map<std::string, std::string>& options,
2012     const std::vector<std::string>& flags)
2013     : LDBCommand(options, flags, true,
2014                  BuildCmdLineOptions(
2015                      {ARG_WAL_FILE, ARG_PRINT_HEADER, ARG_PRINT_VALUE})),
2016       print_header_(false),
2017       print_values_(false) {
2018   wal_file_.clear();
2019
2020   std::map<std::string, std::string>::const_iterator itr =
2021       options.find(ARG_WAL_FILE);
2022   if (itr != options.end()) {
2023     wal_file_ = itr->second;
2024   }
2025
2026
2027   print_header_ = IsFlagPresent(flags, ARG_PRINT_HEADER);
2028   print_values_ = IsFlagPresent(flags, ARG_PRINT_VALUE);
2029   if (wal_file_.empty()) {
2030     exec_state_ = LDBCommandExecuteResult::Failed("Argument " + ARG_WAL_FILE +
2031                                                   " must be specified.");
2032   }
2033 }
2034
2035 void WALDumperCommand::Help(std::string& ret) {
2036   ret.append("  ");
2037   ret.append(WALDumperCommand::Name());
2038   ret.append(" --" + ARG_WAL_FILE + "=<write_ahead_log_file_path>");
2039   ret.append(" [--" + ARG_PRINT_HEADER + "] ");
2040   ret.append(" [--" + ARG_PRINT_VALUE + "] ");
2041   ret.append("\n");
2042 }
2043
2044 void WALDumperCommand::DoCommand() {
2045   DumpWalFile(wal_file_, print_header_, print_values_, &exec_state_);
2046 }
2047
2048 // ----------------------------------------------------------------------------
2049
2050 GetCommand::GetCommand(const std::vector<std::string>& params,
2051                        const std::map<std::string, std::string>& options,
2052                        const std::vector<std::string>& flags)
2053     : LDBCommand(
2054           options, flags, true,
2055           BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
2056   if (params.size() != 1) {
2057     exec_state_ = LDBCommandExecuteResult::Failed(
2058         "<key> must be specified for the get command");
2059   } else {
2060     key_ = params.at(0);
2061   }
2062
2063   if (is_key_hex_) {
2064     key_ = HexToString(key_);
2065   }
2066 }
2067
2068 void GetCommand::Help(std::string& ret) {
2069   ret.append("  ");
2070   ret.append(GetCommand::Name());
2071   ret.append(" <key>");
2072   ret.append(" [--" + ARG_TTL + "]");
2073   ret.append("\n");
2074 }
2075
2076 void GetCommand::DoCommand() {
2077   if (!db_) {
2078     assert(GetExecuteState().IsFailed());
2079     return;
2080   }
2081   std::string value;
2082   Status st = db_->Get(ReadOptions(), GetCfHandle(), key_, &value);
2083   if (st.ok()) {
2084     fprintf(stdout, "%s\n",
2085               (is_value_hex_ ? StringToHex(value) : value).c_str());
2086   } else {
2087     exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2088   }
2089 }
2090
2091 // ----------------------------------------------------------------------------
2092
2093 ApproxSizeCommand::ApproxSizeCommand(
2094     const std::vector<std::string>& params,
2095     const std::map<std::string, std::string>& options,
2096     const std::vector<std::string>& flags)
2097     : LDBCommand(options, flags, true,
2098                  BuildCmdLineOptions(
2099                      {ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM, ARG_TO})) {
2100   if (options.find(ARG_FROM) != options.end()) {
2101     start_key_ = options.find(ARG_FROM)->second;
2102   } else {
2103     exec_state_ = LDBCommandExecuteResult::Failed(
2104         ARG_FROM + " must be specified for approxsize command");
2105     return;
2106   }
2107
2108   if (options.find(ARG_TO) != options.end()) {
2109     end_key_ = options.find(ARG_TO)->second;
2110   } else {
2111     exec_state_ = LDBCommandExecuteResult::Failed(
2112         ARG_TO + " must be specified for approxsize command");
2113     return;
2114   }
2115
2116   if (is_key_hex_) {
2117     start_key_ = HexToString(start_key_);
2118     end_key_ = HexToString(end_key_);
2119   }
2120 }
2121
2122 void ApproxSizeCommand::Help(std::string& ret) {
2123   ret.append("  ");
2124   ret.append(ApproxSizeCommand::Name());
2125   ret.append(HelpRangeCmdArgs());
2126   ret.append("\n");
2127 }
2128
2129 void ApproxSizeCommand::DoCommand() {
2130   if (!db_) {
2131     assert(GetExecuteState().IsFailed());
2132     return;
2133   }
2134   Range ranges[1];
2135   ranges[0] = Range(start_key_, end_key_);
2136   uint64_t sizes[1];
2137   db_->GetApproximateSizes(GetCfHandle(), ranges, 1, sizes);
2138   fprintf(stdout, "%lu\n", (unsigned long)sizes[0]);
2139   /* Weird that GetApproximateSizes() returns void, although documentation
2140    * says that it returns a Status object.
2141   if (!st.ok()) {
2142     exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2143   }
2144   */
2145 }
2146
2147 // ----------------------------------------------------------------------------
2148
2149 BatchPutCommand::BatchPutCommand(
2150     const std::vector<std::string>& params,
2151     const std::map<std::string, std::string>& options,
2152     const std::vector<std::string>& flags)
2153     : LDBCommand(options, flags, false,
2154                  BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX,
2155                                       ARG_VALUE_HEX, ARG_CREATE_IF_MISSING})) {
2156   if (params.size() < 2) {
2157     exec_state_ = LDBCommandExecuteResult::Failed(
2158         "At least one <key> <value> pair must be specified batchput.");
2159   } else if (params.size() % 2 != 0) {
2160     exec_state_ = LDBCommandExecuteResult::Failed(
2161         "Equal number of <key>s and <value>s must be specified for batchput.");
2162   } else {
2163     for (size_t i = 0; i < params.size(); i += 2) {
2164       std::string key = params.at(i);
2165       std::string value = params.at(i + 1);
2166       key_values_.push_back(std::pair<std::string, std::string>(
2167           is_key_hex_ ? HexToString(key) : key,
2168           is_value_hex_ ? HexToString(value) : value));
2169     }
2170   }
2171   create_if_missing_ = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING);
2172 }
2173
2174 void BatchPutCommand::Help(std::string& ret) {
2175   ret.append("  ");
2176   ret.append(BatchPutCommand::Name());
2177   ret.append(" <key> <value> [<key> <value>] [..]");
2178   ret.append(" [--" + ARG_TTL + "]");
2179   ret.append("\n");
2180 }
2181
2182 void BatchPutCommand::DoCommand() {
2183   if (!db_) {
2184     assert(GetExecuteState().IsFailed());
2185     return;
2186   }
2187   WriteBatch batch;
2188
2189   for (std::vector<std::pair<std::string, std::string>>::const_iterator itr =
2190            key_values_.begin();
2191        itr != key_values_.end(); ++itr) {
2192     batch.Put(GetCfHandle(), itr->first, itr->second);
2193   }
2194   Status st = db_->Write(WriteOptions(), &batch);
2195   if (st.ok()) {
2196     fprintf(stdout, "OK\n");
2197   } else {
2198     exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2199   }
2200 }
2201
2202 Options BatchPutCommand::PrepareOptionsForOpenDB() {
2203   Options opt = LDBCommand::PrepareOptionsForOpenDB();
2204   opt.create_if_missing = create_if_missing_;
2205   return opt;
2206 }
2207
2208 // ----------------------------------------------------------------------------
2209
2210 ScanCommand::ScanCommand(const std::vector<std::string>& params,
2211                          const std::map<std::string, std::string>& options,
2212                          const std::vector<std::string>& flags)
2213     : LDBCommand(
2214           options, flags, true,
2215           BuildCmdLineOptions({ARG_TTL, ARG_NO_VALUE, ARG_HEX, ARG_KEY_HEX,
2216                                ARG_TO, ARG_VALUE_HEX, ARG_FROM, ARG_TIMESTAMP,
2217                                ARG_MAX_KEYS, ARG_TTL_START, ARG_TTL_END})),
2218       start_key_specified_(false),
2219       end_key_specified_(false),
2220       max_keys_scanned_(-1),
2221       no_value_(false) {
2222   std::map<std::string, std::string>::const_iterator itr =
2223       options.find(ARG_FROM);
2224   if (itr != options.end()) {
2225     start_key_ = itr->second;
2226     if (is_key_hex_) {
2227       start_key_ = HexToString(start_key_);
2228     }
2229     start_key_specified_ = true;
2230   }
2231   itr = options.find(ARG_TO);
2232   if (itr != options.end()) {
2233     end_key_ = itr->second;
2234     if (is_key_hex_) {
2235       end_key_ = HexToString(end_key_);
2236     }
2237     end_key_specified_ = true;
2238   }
2239
2240   std::vector<std::string>::const_iterator vitr =
2241       std::find(flags.begin(), flags.end(), ARG_NO_VALUE);
2242   if (vitr != flags.end()) {
2243     no_value_ = true;
2244   }
2245
2246   itr = options.find(ARG_MAX_KEYS);
2247   if (itr != options.end()) {
2248     try {
2249 #if defined(CYGWIN)
2250       max_keys_scanned_ = strtol(itr->second.c_str(), 0, 10);
2251 #else
2252       max_keys_scanned_ = std::stoi(itr->second);
2253 #endif
2254     } catch (const std::invalid_argument&) {
2255       exec_state_ = LDBCommandExecuteResult::Failed(ARG_MAX_KEYS +
2256                                                     " has an invalid value");
2257     } catch (const std::out_of_range&) {
2258       exec_state_ = LDBCommandExecuteResult::Failed(
2259           ARG_MAX_KEYS + " has a value out-of-range");
2260     }
2261   }
2262 }
2263
2264 void ScanCommand::Help(std::string& ret) {
2265   ret.append("  ");
2266   ret.append(ScanCommand::Name());
2267   ret.append(HelpRangeCmdArgs());
2268   ret.append(" [--" + ARG_TTL + "]");
2269   ret.append(" [--" + ARG_TIMESTAMP + "]");
2270   ret.append(" [--" + ARG_MAX_KEYS + "=<N>q] ");
2271   ret.append(" [--" + ARG_TTL_START + "=<N>:- is inclusive]");
2272   ret.append(" [--" + ARG_TTL_END + "=<N>:- is exclusive]");
2273   ret.append(" [--" + ARG_NO_VALUE + "]");
2274   ret.append("\n");
2275 }
2276
2277 void ScanCommand::DoCommand() {
2278   if (!db_) {
2279     assert(GetExecuteState().IsFailed());
2280     return;
2281   }
2282
2283   int num_keys_scanned = 0;
2284   Iterator* it = db_->NewIterator(ReadOptions(), GetCfHandle());
2285   if (start_key_specified_) {
2286     it->Seek(start_key_);
2287   } else {
2288     it->SeekToFirst();
2289   }
2290   int ttl_start;
2291   if (!ParseIntOption(option_map_, ARG_TTL_START, ttl_start, exec_state_)) {
2292     ttl_start = DBWithTTLImpl::kMinTimestamp;  // TTL introduction time
2293   }
2294   int ttl_end;
2295   if (!ParseIntOption(option_map_, ARG_TTL_END, ttl_end, exec_state_)) {
2296     ttl_end = DBWithTTLImpl::kMaxTimestamp;  // Max time allowed by TTL feature
2297   }
2298   if (ttl_end < ttl_start) {
2299     fprintf(stderr, "Error: End time can't be less than start time\n");
2300     delete it;
2301     return;
2302   }
2303   if (is_db_ttl_ && timestamp_) {
2304     fprintf(stdout, "Scanning key-values from %s to %s\n",
2305             ReadableTime(ttl_start).c_str(), ReadableTime(ttl_end).c_str());
2306   }
2307   for ( ;
2308         it->Valid() && (!end_key_specified_ || it->key().ToString() < end_key_);
2309         it->Next()) {
2310     if (is_db_ttl_) {
2311       TtlIterator* it_ttl = dynamic_cast<TtlIterator*>(it);
2312       assert(it_ttl);
2313       int rawtime = it_ttl->timestamp();
2314       if (rawtime < ttl_start || rawtime >= ttl_end) {
2315         continue;
2316       }
2317       if (timestamp_) {
2318         fprintf(stdout, "%s ", ReadableTime(rawtime).c_str());
2319       }
2320     }
2321
2322     Slice key_slice = it->key();
2323
2324     std::string formatted_key;
2325     if (is_key_hex_) {
2326       formatted_key = "0x" + key_slice.ToString(true /* hex */);
2327       key_slice = formatted_key;
2328     } else if (ldb_options_.key_formatter) {
2329       formatted_key = ldb_options_.key_formatter->Format(key_slice);
2330       key_slice = formatted_key;
2331     }
2332
2333     if (no_value_) {
2334       fprintf(stdout, "%.*s\n", static_cast<int>(key_slice.size()),
2335               key_slice.data());
2336     } else {
2337       Slice val_slice = it->value();
2338       std::string formatted_value;
2339       if (is_value_hex_) {
2340         formatted_value = "0x" + val_slice.ToString(true /* hex */);
2341         val_slice = formatted_value;
2342       }
2343       fprintf(stdout, "%.*s : %.*s\n", static_cast<int>(key_slice.size()),
2344               key_slice.data(), static_cast<int>(val_slice.size()),
2345               val_slice.data());
2346     }
2347
2348     num_keys_scanned++;
2349     if (max_keys_scanned_ >= 0 && num_keys_scanned >= max_keys_scanned_) {
2350       break;
2351     }
2352   }
2353   if (!it->status().ok()) {  // Check for any errors found during the scan
2354     exec_state_ = LDBCommandExecuteResult::Failed(it->status().ToString());
2355   }
2356   delete it;
2357 }
2358
2359 // ----------------------------------------------------------------------------
2360
2361 DeleteCommand::DeleteCommand(const std::vector<std::string>& params,
2362                              const std::map<std::string, std::string>& options,
2363                              const std::vector<std::string>& flags)
2364     : LDBCommand(options, flags, false,
2365                  BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
2366   if (params.size() != 1) {
2367     exec_state_ = LDBCommandExecuteResult::Failed(
2368         "KEY must be specified for the delete command");
2369   } else {
2370     key_ = params.at(0);
2371     if (is_key_hex_) {
2372       key_ = HexToString(key_);
2373     }
2374   }
2375 }
2376
2377 void DeleteCommand::Help(std::string& ret) {
2378   ret.append("  ");
2379   ret.append(DeleteCommand::Name() + " <key>");
2380   ret.append("\n");
2381 }
2382
2383 void DeleteCommand::DoCommand() {
2384   if (!db_) {
2385     assert(GetExecuteState().IsFailed());
2386     return;
2387   }
2388   Status st = db_->Delete(WriteOptions(), GetCfHandle(), key_);
2389   if (st.ok()) {
2390     fprintf(stdout, "OK\n");
2391   } else {
2392     exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2393   }
2394 }
2395
2396 DeleteRangeCommand::DeleteRangeCommand(
2397     const std::vector<std::string>& params,
2398     const std::map<std::string, std::string>& options,
2399     const std::vector<std::string>& flags)
2400     : LDBCommand(options, flags, false,
2401                  BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
2402   if (params.size() != 2) {
2403     exec_state_ = LDBCommandExecuteResult::Failed(
2404         "begin and end keys must be specified for the delete command");
2405   } else {
2406     begin_key_ = params.at(0);
2407     end_key_ = params.at(1);
2408     if (is_key_hex_) {
2409       begin_key_ = HexToString(begin_key_);
2410       end_key_ = HexToString(end_key_);
2411     }
2412   }
2413 }
2414
2415 void DeleteRangeCommand::Help(std::string& ret) {
2416   ret.append("  ");
2417   ret.append(DeleteRangeCommand::Name() + " <begin key> <end key>");
2418   ret.append("\n");
2419 }
2420
2421 void DeleteRangeCommand::DoCommand() {
2422   if (!db_) {
2423     assert(GetExecuteState().IsFailed());
2424     return;
2425   }
2426   Status st =
2427       db_->DeleteRange(WriteOptions(), GetCfHandle(), begin_key_, end_key_);
2428   if (st.ok()) {
2429     fprintf(stdout, "OK\n");
2430   } else {
2431     exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2432   }
2433 }
2434
2435 PutCommand::PutCommand(const std::vector<std::string>& params,
2436                        const std::map<std::string, std::string>& options,
2437                        const std::vector<std::string>& flags)
2438     : LDBCommand(options, flags, false,
2439                  BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX,
2440                                       ARG_VALUE_HEX, ARG_CREATE_IF_MISSING})) {
2441   if (params.size() != 2) {
2442     exec_state_ = LDBCommandExecuteResult::Failed(
2443         "<key> and <value> must be specified for the put command");
2444   } else {
2445     key_ = params.at(0);
2446     value_ = params.at(1);
2447   }
2448
2449   if (is_key_hex_) {
2450     key_ = HexToString(key_);
2451   }
2452
2453   if (is_value_hex_) {
2454     value_ = HexToString(value_);
2455   }
2456   create_if_missing_ = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING);
2457 }
2458
2459 void PutCommand::Help(std::string& ret) {
2460   ret.append("  ");
2461   ret.append(PutCommand::Name());
2462   ret.append(" <key> <value> ");
2463   ret.append(" [--" + ARG_TTL + "]");
2464   ret.append("\n");
2465 }
2466
2467 void PutCommand::DoCommand() {
2468   if (!db_) {
2469     assert(GetExecuteState().IsFailed());
2470     return;
2471   }
2472   Status st = db_->Put(WriteOptions(), GetCfHandle(), key_, value_);
2473   if (st.ok()) {
2474     fprintf(stdout, "OK\n");
2475   } else {
2476     exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2477   }
2478 }
2479
2480 Options PutCommand::PrepareOptionsForOpenDB() {
2481   Options opt = LDBCommand::PrepareOptionsForOpenDB();
2482   opt.create_if_missing = create_if_missing_;
2483   return opt;
2484 }
2485
2486 // ----------------------------------------------------------------------------
2487
2488 const char* DBQuerierCommand::HELP_CMD = "help";
2489 const char* DBQuerierCommand::GET_CMD = "get";
2490 const char* DBQuerierCommand::PUT_CMD = "put";
2491 const char* DBQuerierCommand::DELETE_CMD = "delete";
2492
2493 DBQuerierCommand::DBQuerierCommand(
2494     const std::vector<std::string>& params,
2495     const std::map<std::string, std::string>& options,
2496     const std::vector<std::string>& flags)
2497     : LDBCommand(
2498           options, flags, false,
2499           BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
2500
2501 }
2502
2503 void DBQuerierCommand::Help(std::string& ret) {
2504   ret.append("  ");
2505   ret.append(DBQuerierCommand::Name());
2506   ret.append(" [--" + ARG_TTL + "]");
2507   ret.append("\n");
2508   ret.append("    Starts a REPL shell.  Type help for list of available "
2509              "commands.");
2510   ret.append("\n");
2511 }
2512
2513 void DBQuerierCommand::DoCommand() {
2514   if (!db_) {
2515     assert(GetExecuteState().IsFailed());
2516     return;
2517   }
2518
2519   ReadOptions read_options;
2520   WriteOptions write_options;
2521
2522   std::string line;
2523   std::string key;
2524   std::string value;
2525   while (getline(std::cin, line, '\n')) {
2526     // Parse line into std::vector<std::string>
2527     std::vector<std::string> tokens;
2528     size_t pos = 0;
2529     while (true) {
2530       size_t pos2 = line.find(' ', pos);
2531       if (pos2 == std::string::npos) {
2532         break;
2533       }
2534       tokens.push_back(line.substr(pos, pos2-pos));
2535       pos = pos2 + 1;
2536     }
2537     tokens.push_back(line.substr(pos));
2538
2539     const std::string& cmd = tokens[0];
2540
2541     if (cmd == HELP_CMD) {
2542       fprintf(stdout,
2543               "get <key>\n"
2544               "put <key> <value>\n"
2545               "delete <key>\n");
2546     } else if (cmd == DELETE_CMD && tokens.size() == 2) {
2547       key = (is_key_hex_ ? HexToString(tokens[1]) : tokens[1]);
2548       db_->Delete(write_options, GetCfHandle(), Slice(key));
2549       fprintf(stdout, "Successfully deleted %s\n", tokens[1].c_str());
2550     } else if (cmd == PUT_CMD && tokens.size() == 3) {
2551       key = (is_key_hex_ ? HexToString(tokens[1]) : tokens[1]);
2552       value = (is_value_hex_ ? HexToString(tokens[2]) : tokens[2]);
2553       db_->Put(write_options, GetCfHandle(), Slice(key), Slice(value));
2554       fprintf(stdout, "Successfully put %s %s\n",
2555               tokens[1].c_str(), tokens[2].c_str());
2556     } else if (cmd == GET_CMD && tokens.size() == 2) {
2557       key = (is_key_hex_ ? HexToString(tokens[1]) : tokens[1]);
2558       if (db_->Get(read_options, GetCfHandle(), Slice(key), &value).ok()) {
2559         fprintf(stdout, "%s\n", PrintKeyValue(key, value,
2560               is_key_hex_, is_value_hex_).c_str());
2561       } else {
2562         fprintf(stdout, "Not found %s\n", tokens[1].c_str());
2563       }
2564     } else {
2565       fprintf(stdout, "Unknown command %s\n", line.c_str());
2566     }
2567   }
2568 }
2569
2570 // ----------------------------------------------------------------------------
2571
2572 CheckConsistencyCommand::CheckConsistencyCommand(
2573     const std::vector<std::string>& params,
2574     const std::map<std::string, std::string>& options,
2575     const std::vector<std::string>& flags)
2576     : LDBCommand(options, flags, false, BuildCmdLineOptions({})) {}
2577
2578 void CheckConsistencyCommand::Help(std::string& ret) {
2579   ret.append("  ");
2580   ret.append(CheckConsistencyCommand::Name());
2581   ret.append("\n");
2582 }
2583
2584 void CheckConsistencyCommand::DoCommand() {
2585   Options opt = PrepareOptionsForOpenDB();
2586   opt.paranoid_checks = true;
2587   if (!exec_state_.IsNotStarted()) {
2588     return;
2589   }
2590   DB* db;
2591   Status st = DB::OpenForReadOnly(opt, db_path_, &db, false);
2592   delete db;
2593   if (st.ok()) {
2594     fprintf(stdout, "OK\n");
2595   } else {
2596     exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2597   }
2598 }
2599
2600 // ----------------------------------------------------------------------------
2601
2602 const std::string CheckPointCommand::ARG_CHECKPOINT_DIR = "checkpoint_dir";
2603
2604 CheckPointCommand::CheckPointCommand(
2605     const std::vector<std::string>& params,
2606     const std::map<std::string, std::string>& options,
2607     const std::vector<std::string>& flags)
2608     : LDBCommand(options, flags, false /* is_read_only */,
2609                  BuildCmdLineOptions({ARG_CHECKPOINT_DIR})) {
2610   auto itr = options.find(ARG_CHECKPOINT_DIR);
2611   if (itr == options.end()) {
2612     exec_state_ = LDBCommandExecuteResult::Failed(
2613         "--" + ARG_CHECKPOINT_DIR + ": missing checkpoint directory");
2614   } else {
2615     checkpoint_dir_ = itr->second;
2616   }
2617 }
2618
2619 void CheckPointCommand::Help(std::string& ret) {
2620   ret.append("  ");
2621   ret.append(CheckPointCommand::Name());
2622   ret.append(" [--" + ARG_CHECKPOINT_DIR + "] ");
2623   ret.append("\n");
2624 }
2625
2626 void CheckPointCommand::DoCommand() {
2627   if (!db_) {
2628     assert(GetExecuteState().IsFailed());
2629     return;
2630   }
2631   Checkpoint* checkpoint;
2632   Status status = Checkpoint::Create(db_, &checkpoint);
2633   status = checkpoint->CreateCheckpoint(checkpoint_dir_);
2634   if (status.ok()) {
2635     printf("OK\n");
2636   } else {
2637     exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
2638   }
2639 }
2640
2641 // ----------------------------------------------------------------------------
2642
2643 RepairCommand::RepairCommand(const std::vector<std::string>& params,
2644                              const std::map<std::string, std::string>& options,
2645                              const std::vector<std::string>& flags)
2646     : LDBCommand(options, flags, false, BuildCmdLineOptions({})) {}
2647
2648 void RepairCommand::Help(std::string& ret) {
2649   ret.append("  ");
2650   ret.append(RepairCommand::Name());
2651   ret.append("\n");
2652 }
2653
2654 void RepairCommand::DoCommand() {
2655   Options options = PrepareOptionsForOpenDB();
2656   options.info_log.reset(new StderrLogger(InfoLogLevel::WARN_LEVEL));
2657   Status status = RepairDB(db_path_, options);
2658   if (status.ok()) {
2659     printf("OK\n");
2660   } else {
2661     exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
2662   }
2663 }
2664
2665 // ----------------------------------------------------------------------------
2666
2667 const std::string BackupableCommand::ARG_NUM_THREADS = "num_threads";
2668 const std::string BackupableCommand::ARG_BACKUP_ENV_URI = "backup_env_uri";
2669 const std::string BackupableCommand::ARG_BACKUP_DIR = "backup_dir";
2670 const std::string BackupableCommand::ARG_STDERR_LOG_LEVEL = "stderr_log_level";
2671
2672 BackupableCommand::BackupableCommand(
2673     const std::vector<std::string>& params,
2674     const std::map<std::string, std::string>& options,
2675     const std::vector<std::string>& flags)
2676     : LDBCommand(options, flags, false /* is_read_only */,
2677                  BuildCmdLineOptions({ARG_BACKUP_ENV_URI, ARG_BACKUP_DIR,
2678                                       ARG_NUM_THREADS, ARG_STDERR_LOG_LEVEL})),
2679       num_threads_(1) {
2680   auto itr = options.find(ARG_NUM_THREADS);
2681   if (itr != options.end()) {
2682     num_threads_ = std::stoi(itr->second);
2683   }
2684   itr = options.find(ARG_BACKUP_ENV_URI);
2685   if (itr != options.end()) {
2686     backup_env_uri_ = itr->second;
2687   }
2688   itr = options.find(ARG_BACKUP_DIR);
2689   if (itr == options.end()) {
2690     exec_state_ = LDBCommandExecuteResult::Failed("--" + ARG_BACKUP_DIR +
2691                                                   ": missing backup directory");
2692   } else {
2693     backup_dir_ = itr->second;
2694   }
2695
2696   itr = options.find(ARG_STDERR_LOG_LEVEL);
2697   if (itr != options.end()) {
2698     int stderr_log_level = std::stoi(itr->second);
2699     if (stderr_log_level < 0 ||
2700         stderr_log_level >= InfoLogLevel::NUM_INFO_LOG_LEVELS) {
2701       exec_state_ = LDBCommandExecuteResult::Failed(
2702           ARG_STDERR_LOG_LEVEL + " must be >= 0 and < " +
2703           std::to_string(InfoLogLevel::NUM_INFO_LOG_LEVELS) + ".");
2704     } else {
2705       logger_.reset(
2706           new StderrLogger(static_cast<InfoLogLevel>(stderr_log_level)));
2707     }
2708   }
2709 }
2710
2711 void BackupableCommand::Help(const std::string& name, std::string& ret) {
2712   ret.append("  ");
2713   ret.append(name);
2714   ret.append(" [--" + ARG_BACKUP_ENV_URI + "] ");
2715   ret.append(" [--" + ARG_BACKUP_DIR + "] ");
2716   ret.append(" [--" + ARG_NUM_THREADS + "] ");
2717   ret.append(" [--" + ARG_STDERR_LOG_LEVEL + "=<int (InfoLogLevel)>] ");
2718   ret.append("\n");
2719 }
2720
2721 // ----------------------------------------------------------------------------
2722
2723 BackupCommand::BackupCommand(const std::vector<std::string>& params,
2724                              const std::map<std::string, std::string>& options,
2725                              const std::vector<std::string>& flags)
2726     : BackupableCommand(params, options, flags) {}
2727
2728 void BackupCommand::Help(std::string& ret) {
2729   BackupableCommand::Help(Name(), ret);
2730 }
2731
2732 void BackupCommand::DoCommand() {
2733   BackupEngine* backup_engine;
2734   Status status;
2735   if (!db_) {
2736     assert(GetExecuteState().IsFailed());
2737     return;
2738   }
2739   printf("open db OK\n");
2740   std::unique_ptr<Env> custom_env_guard;
2741   Env* custom_env = NewCustomObject<Env>(backup_env_uri_, &custom_env_guard);
2742   BackupableDBOptions backup_options =
2743       BackupableDBOptions(backup_dir_, custom_env);
2744   backup_options.info_log = logger_.get();
2745   backup_options.max_background_operations = num_threads_;
2746   status = BackupEngine::Open(Env::Default(), backup_options, &backup_engine);
2747   if (status.ok()) {
2748     printf("open backup engine OK\n");
2749   } else {
2750     exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
2751     return;
2752   }
2753   status = backup_engine->CreateNewBackup(db_);
2754   if (status.ok()) {
2755     printf("create new backup OK\n");
2756   } else {
2757     exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
2758     return;
2759   }
2760 }
2761
2762 // ----------------------------------------------------------------------------
2763
2764 RestoreCommand::RestoreCommand(
2765     const std::vector<std::string>& params,
2766     const std::map<std::string, std::string>& options,
2767     const std::vector<std::string>& flags)
2768     : BackupableCommand(params, options, flags) {}
2769
2770 void RestoreCommand::Help(std::string& ret) {
2771   BackupableCommand::Help(Name(), ret);
2772 }
2773
2774 void RestoreCommand::DoCommand() {
2775   std::unique_ptr<Env> custom_env_guard;
2776   Env* custom_env = NewCustomObject<Env>(backup_env_uri_, &custom_env_guard);
2777   std::unique_ptr<BackupEngineReadOnly> restore_engine;
2778   Status status;
2779   {
2780     BackupableDBOptions opts(backup_dir_, custom_env);
2781     opts.info_log = logger_.get();
2782     opts.max_background_operations = num_threads_;
2783     BackupEngineReadOnly* raw_restore_engine_ptr;
2784     status = BackupEngineReadOnly::Open(Env::Default(), opts,
2785                                         &raw_restore_engine_ptr);
2786     if (status.ok()) {
2787       restore_engine.reset(raw_restore_engine_ptr);
2788     }
2789   }
2790   if (status.ok()) {
2791     printf("open restore engine OK\n");
2792     status = restore_engine->RestoreDBFromLatestBackup(db_path_, db_path_);
2793   }
2794   if (status.ok()) {
2795     printf("restore from backup OK\n");
2796   } else {
2797     exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
2798   }
2799 }
2800
2801 // ----------------------------------------------------------------------------
2802
2803 namespace {
2804
2805 void DumpSstFile(std::string filename, bool output_hex, bool show_properties) {
2806   std::string from_key;
2807   std::string to_key;
2808   if (filename.length() <= 4 ||
2809       filename.rfind(".sst") != filename.length() - 4) {
2810     std::cout << "Invalid sst file name." << std::endl;
2811     return;
2812   }
2813   // no verification
2814   rocksdb::SstFileReader reader(filename, false, output_hex);
2815   Status st = reader.ReadSequential(true, -1, false,  // has_from
2816                                     from_key, false,  // has_to
2817                                     to_key);
2818   if (!st.ok()) {
2819     std::cerr << "Error in reading SST file " << filename << st.ToString()
2820               << std::endl;
2821     return;
2822   }
2823
2824   if (show_properties) {
2825     const rocksdb::TableProperties* table_properties;
2826
2827     std::shared_ptr<const rocksdb::TableProperties>
2828         table_properties_from_reader;
2829     st = reader.ReadTableProperties(&table_properties_from_reader);
2830     if (!st.ok()) {
2831       std::cerr << filename << ": " << st.ToString()
2832                 << ". Try to use initial table properties" << std::endl;
2833       table_properties = reader.GetInitTableProperties();
2834     } else {
2835       table_properties = table_properties_from_reader.get();
2836     }
2837     if (table_properties != nullptr) {
2838       std::cout << std::endl << "Table Properties:" << std::endl;
2839       std::cout << table_properties->ToString("\n") << std::endl;
2840       std::cout << "# deleted keys: "
2841                 << rocksdb::GetDeletedKeys(
2842                        table_properties->user_collected_properties)
2843                 << std::endl;
2844     }
2845   }
2846 }
2847
2848 }  // namespace
2849
2850 DBFileDumperCommand::DBFileDumperCommand(
2851     const std::vector<std::string>& params,
2852     const std::map<std::string, std::string>& options,
2853     const std::vector<std::string>& flags)
2854     : LDBCommand(options, flags, true, BuildCmdLineOptions({})) {}
2855
2856 void DBFileDumperCommand::Help(std::string& ret) {
2857   ret.append("  ");
2858   ret.append(DBFileDumperCommand::Name());
2859   ret.append("\n");
2860 }
2861
2862 void DBFileDumperCommand::DoCommand() {
2863   if (!db_) {
2864     assert(GetExecuteState().IsFailed());
2865     return;
2866   }
2867   Status s;
2868
2869   std::cout << "Manifest File" << std::endl;
2870   std::cout << "==============================" << std::endl;
2871   std::string manifest_filename;
2872   s = ReadFileToString(db_->GetEnv(), CurrentFileName(db_->GetName()),
2873                        &manifest_filename);
2874   if (!s.ok() || manifest_filename.empty() ||
2875       manifest_filename.back() != '\n') {
2876     std::cerr << "Error when reading CURRENT file "
2877               << CurrentFileName(db_->GetName()) << std::endl;
2878   }
2879   // remove the trailing '\n'
2880   manifest_filename.resize(manifest_filename.size() - 1);
2881   std::string manifest_filepath = db_->GetName() + "/" + manifest_filename;
2882   std::cout << manifest_filepath << std::endl;
2883   DumpManifestFile(manifest_filepath, false, false, false);
2884   std::cout << std::endl;
2885
2886   std::cout << "SST Files" << std::endl;
2887   std::cout << "==============================" << std::endl;
2888   std::vector<LiveFileMetaData> metadata;
2889   db_->GetLiveFilesMetaData(&metadata);
2890   for (auto& fileMetadata : metadata) {
2891     std::string filename = fileMetadata.db_path + fileMetadata.name;
2892     std::cout << filename << " level:" << fileMetadata.level << std::endl;
2893     std::cout << "------------------------------" << std::endl;
2894     DumpSstFile(filename, false, true);
2895     std::cout << std::endl;
2896   }
2897   std::cout << std::endl;
2898
2899   std::cout << "Write Ahead Log Files" << std::endl;
2900   std::cout << "==============================" << std::endl;
2901   rocksdb::VectorLogPtr wal_files;
2902   s = db_->GetSortedWalFiles(wal_files);
2903   if (!s.ok()) {
2904     std::cerr << "Error when getting WAL files" << std::endl;
2905   } else {
2906     for (auto& wal : wal_files) {
2907       // TODO(qyang): option.wal_dir should be passed into ldb command
2908       std::string filename = db_->GetOptions().wal_dir + wal->PathName();
2909       std::cout << filename << std::endl;
2910       DumpWalFile(filename, true, true, &exec_state_);
2911     }
2912   }
2913 }
2914
2915 }   // namespace rocksdb
2916 #endif  // ROCKSDB_LITE