]> git.proxmox.com Git - ceph.git/blame - ceph/src/rocksdb/tools/ldb_cmd.cc
projects / ceph.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
update ceph source to reef 18.1.2
[ceph.git] / ceph / src / rocksdb / tools / ldb_cmd.cc
CommitLineData
f67539c2 1
7c673cae 2// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
11fdf7f2
TL		 3// This source code is licensed under both the GPLv2 (found in the
4// COPYING file in the root directory) and Apache 2.0 License
5// (found in the LICENSE.Apache file in the root directory).
7c673cae
FG		 6//
7#ifndef ROCKSDB_LITE
8#include "rocksdb/utilities/ldb_cmd.h"
9
f67539c2 10#include <cinttypes>
20effc67
TL		 11#include <cstdlib>
12#include <ctime>
13#include <fstream>
14#include <functional>
15#include <iostream>
16#include <limits>
17#include <sstream>
18#include <stdexcept>
19#include <string>
7c673cae 20
1e59de90 21#include "db/blob/blob_index.h"
f67539c2 22#include "db/db_impl/db_impl.h"
7c673cae
FG		 23#include "db/dbformat.h"
24#include "db/log_reader.h"
1e59de90 25#include "db/version_util.h"
7c673cae 26#include "db/write_batch_internal.h"
f67539c2 27#include "file/filename.h"
7c673cae 28#include "rocksdb/cache.h"
1e59de90 29#include "rocksdb/experimental.h"
f67539c2 30#include "rocksdb/file_checksum.h"
1e59de90
TL		 31#include "rocksdb/filter_policy.h"
32#include "rocksdb/options.h"
7c673cae 33#include "rocksdb/table_properties.h"
1e59de90 34#include "rocksdb/utilities/backup_engine.h"
7c673cae 35#include "rocksdb/utilities/checkpoint.h"
11fdf7f2 36#include "rocksdb/utilities/debug.h"
7c673cae
FG		 37#include "rocksdb/utilities/options_util.h"
38#include "rocksdb/write_batch.h"
39#include "rocksdb/write_buffer_manager.h"
40#include "table/scoped_arena_iterator.h"
20effc67 41#include "table/sst_file_dumper.h"
7c673cae 42#include "tools/ldb_cmd_impl.h"
11fdf7f2 43#include "util/cast_util.h"
7c673cae 44#include "util/coding.h"
f67539c2 45#include "util/file_checksum_helper.h"
7c673cae
FG		 46#include "util/stderr_logger.h"
47#include "util/string_util.h"
1e59de90 48#include "utilities/blob_db/blob_dump_tool.h"
f67539c2 49#include "utilities/merge_operators.h"
7c673cae
FG		 50#include "utilities/ttl/db_ttl_impl.h"
51
f67539c2 52namespace ROCKSDB_NAMESPACE {
7c673cae 53
20effc67
TL		 54class FileChecksumGenCrc32c;
55class FileChecksumGenCrc32cFactory;
f67539c2
TL		 56
57const std::string LDBCommand::ARG_ENV_URI = "env_uri";
1e59de90 58const std::string LDBCommand::ARG_FS_URI = "fs_uri";
7c673cae
FG		 59const std::string LDBCommand::ARG_DB = "db";
60const std::string LDBCommand::ARG_PATH = "path";
f67539c2 61const std::string LDBCommand::ARG_SECONDARY_PATH = "secondary_path";
7c673cae
FG		 62const std::string LDBCommand::ARG_HEX = "hex";
63const std::string LDBCommand::ARG_KEY_HEX = "key_hex";
64const std::string LDBCommand::ARG_VALUE_HEX = "value_hex";
65const std::string LDBCommand::ARG_CF_NAME = "column_family";
66const std::string LDBCommand::ARG_TTL = "ttl";
67const std::string LDBCommand::ARG_TTL_START = "start_time";
68const std::string LDBCommand::ARG_TTL_END = "end_time";
69const std::string LDBCommand::ARG_TIMESTAMP = "timestamp";
70const std::string LDBCommand::ARG_TRY_LOAD_OPTIONS = "try_load_options";
20effc67
TL		 71const std::string LDBCommand::ARG_DISABLE_CONSISTENCY_CHECKS =
72 "disable_consistency_checks";
11fdf7f2
TL		 73const std::string LDBCommand::ARG_IGNORE_UNKNOWN_OPTIONS =
74 "ignore_unknown_options";
7c673cae
FG		 75const std::string LDBCommand::ARG_FROM = "from";
76const std::string LDBCommand::ARG_TO = "to";
77const std::string LDBCommand::ARG_MAX_KEYS = "max_keys";
78const std::string LDBCommand::ARG_BLOOM_BITS = "bloom_bits";
79const std::string LDBCommand::ARG_FIX_PREFIX_LEN = "fix_prefix_len";
80const std::string LDBCommand::ARG_COMPRESSION_TYPE = "compression_type";
81const std::string LDBCommand::ARG_COMPRESSION_MAX_DICT_BYTES =
82 "compression_max_dict_bytes";
83const std::string LDBCommand::ARG_BLOCK_SIZE = "block_size";
84const std::string LDBCommand::ARG_AUTO_COMPACTION = "auto_compaction";
85const std::string LDBCommand::ARG_DB_WRITE_BUFFER_SIZE = "db_write_buffer_size";
86const std::string LDBCommand::ARG_WRITE_BUFFER_SIZE = "write_buffer_size";
87const std::string LDBCommand::ARG_FILE_SIZE = "file_size";
88const std::string LDBCommand::ARG_CREATE_IF_MISSING = "create_if_missing";
89const std::string LDBCommand::ARG_NO_VALUE = "no_value";
1e59de90
TL		 90const std::string LDBCommand::ARG_ENABLE_BLOB_FILES = "enable_blob_files";
91const std::string LDBCommand::ARG_MIN_BLOB_SIZE = "min_blob_size";
92const std::string LDBCommand::ARG_BLOB_FILE_SIZE = "blob_file_size";
93const std::string LDBCommand::ARG_BLOB_COMPRESSION_TYPE =
94 "blob_compression_type";
95const std::string LDBCommand::ARG_ENABLE_BLOB_GARBAGE_COLLECTION =
96 "enable_blob_garbage_collection";
97const std::string LDBCommand::ARG_BLOB_GARBAGE_COLLECTION_AGE_CUTOFF =
98 "blob_garbage_collection_age_cutoff";
99const std::string LDBCommand::ARG_BLOB_GARBAGE_COLLECTION_FORCE_THRESHOLD =
100 "blob_garbage_collection_force_threshold";
101const std::string LDBCommand::ARG_BLOB_COMPACTION_READAHEAD_SIZE =
102 "blob_compaction_readahead_size";
103const std::string LDBCommand::ARG_BLOB_FILE_STARTING_LEVEL =
104 "blob_file_starting_level";
105const std::string LDBCommand::ARG_PREPOPULATE_BLOB_CACHE =
106 "prepopulate_blob_cache";
107const std::string LDBCommand::ARG_DECODE_BLOB_INDEX = "decode_blob_index";
108const std::string LDBCommand::ARG_DUMP_UNCOMPRESSED_BLOBS =
109 "dump_uncompressed_blobs";
7c673cae
FG		 110
111const char* LDBCommand::DELIM = " ==> ";
112
113namespace {
114
494da23a
TL		 115void DumpWalFile(Options options, std::string wal_file, bool print_header,
116 bool print_values, bool is_write_committed,
117 LDBCommandExecuteResult* exec_state);
7c673cae 118
494da23a 119void DumpSstFile(Options options, std::string filename, bool output_hex,
1e59de90
TL		 120 bool show_properties, bool decode_blob_index,
121 std::string from_key = "", std::string to_key = "");
122
123void DumpBlobFile(const std::string& filename, bool is_key_hex,
124 bool is_value_hex, bool dump_uncompressed_blobs);
125}; // namespace
7c673cae
FG		 126
127LDBCommand* LDBCommand::InitFromCmdLineArgs(
20effc67 128 int argc, char const* const* argv, const Options& options,
7c673cae
FG		 129 const LDBOptions& ldb_options,
130 const std::vector<ColumnFamilyDescriptor>* column_families) {
131 std::vector<std::string> args;
132 for (int i = 1; i < argc; i++) {
133 args.push_back(argv[i]);
134 }
135 return InitFromCmdLineArgs(args, options, ldb_options, column_families,
136 SelectCommand);
137}
138
139/**
140 * Parse the command-line arguments and create the appropriate LDBCommand2
141 * instance.
142 * The command line arguments must be in the following format:
143 * ./ldb --db=PATH_TO_DB [--commonOpt1=commonOpt1Val] ..
144 * COMMAND <PARAM1> <PARAM2> ... [-cmdSpecificOpt1=cmdSpecificOpt1Val] ..
145 * This is similar to the command line format used by HBaseClientTool.
146 * Command name is not included in args.
147 * Returns nullptr if the command-line cannot be parsed.
148 */
149LDBCommand* LDBCommand::InitFromCmdLineArgs(
150 const std::vector<std::string>& args, const Options& options,
151 const LDBOptions& ldb_options,
11fdf7f2 152 const std::vector<ColumnFamilyDescriptor>* /*column_families*/,
7c673cae
FG		 153 const std::function<LDBCommand*(const ParsedParams&)>& selector) {
154 // --x=y command line arguments are added as x->y map entries in
155 // parsed_params.option_map.
156 //
157 // Command-line arguments of the form --hex end up in this array as hex to
158 // parsed_params.flags
159 ParsedParams parsed_params;
160
161 // Everything other than option_map and flags. Represents commands
162 // and their parameters. For eg: put key1 value1 go into this vector.
163 std::vector<std::string> cmdTokens;
164
165 const std::string OPTION_PREFIX = "--";
166
167 for (const auto& arg : args) {
1e59de90 168 if (arg[0] == '-' && arg[1] == '-') {
7c673cae 169 std::vector<std::string> splits = StringSplit(arg, '=');
494da23a 170 // --option_name=option_value
7c673cae
FG		 171 if (splits.size() == 2) {
172 std::string optionKey = splits[0].substr(OPTION_PREFIX.size());
173 parsed_params.option_map[optionKey] = splits[1];
494da23a
TL		 174 } else if (splits.size() == 1) {
175 // --flag_name
7c673cae
FG		 176 std::string optionKey = splits[0].substr(OPTION_PREFIX.size());
177 parsed_params.flags.push_back(optionKey);
494da23a
TL		 178 } else {
179 // --option_name=option_value, option_value contains '='
180 std::string optionKey = splits[0].substr(OPTION_PREFIX.size());
181 parsed_params.option_map[optionKey] =
182 arg.substr(splits[0].length() + 1);
7c673cae
FG		 183 }
184 } else {
185 cmdTokens.push_back(arg);
186 }
187 }
188
189 if (cmdTokens.size() < 1) {
190 fprintf(stderr, "Command not specified!");
191 return nullptr;
192 }
193
194 parsed_params.cmd = cmdTokens[0];
195 parsed_params.cmd_params.assign(cmdTokens.begin() + 1, cmdTokens.end());
196
197 LDBCommand* command = selector(parsed_params);
198
199 if (command) {
200 command->SetDBOptions(options);
201 command->SetLDBOptions(ldb_options);
202 }
203 return command;
204}
205
206LDBCommand* LDBCommand::SelectCommand(const ParsedParams& parsed_params) {
207 if (parsed_params.cmd == GetCommand::Name()) {
208 return new GetCommand(parsed_params.cmd_params, parsed_params.option_map,
209 parsed_params.flags);
210 } else if (parsed_params.cmd == PutCommand::Name()) {
211 return new PutCommand(parsed_params.cmd_params, parsed_params.option_map,
212 parsed_params.flags);
213 } else if (parsed_params.cmd == BatchPutCommand::Name()) {
214 return new BatchPutCommand(parsed_params.cmd_params,
215 parsed_params.option_map, parsed_params.flags);
216 } else if (parsed_params.cmd == ScanCommand::Name()) {
217 return new ScanCommand(parsed_params.cmd_params, parsed_params.option_map,
218 parsed_params.flags);
219 } else if (parsed_params.cmd == DeleteCommand::Name()) {
220 return new DeleteCommand(parsed_params.cmd_params, parsed_params.option_map,
221 parsed_params.flags);
1e59de90
TL		 222 } else if (parsed_params.cmd == SingleDeleteCommand::Name()) {
223 return new SingleDeleteCommand(parsed_params.cmd_params,
224 parsed_params.option_map,
225 parsed_params.flags);
7c673cae
FG		 226 } else if (parsed_params.cmd == DeleteRangeCommand::Name()) {
227 return new DeleteRangeCommand(parsed_params.cmd_params,
228 parsed_params.option_map,
229 parsed_params.flags);
230 } else if (parsed_params.cmd == ApproxSizeCommand::Name()) {
231 return new ApproxSizeCommand(parsed_params.cmd_params,
232 parsed_params.option_map, parsed_params.flags);
233 } else if (parsed_params.cmd == DBQuerierCommand::Name()) {
234 return new DBQuerierCommand(parsed_params.cmd_params,
235 parsed_params.option_map, parsed_params.flags);
236 } else if (parsed_params.cmd == CompactorCommand::Name()) {
237 return new CompactorCommand(parsed_params.cmd_params,
238 parsed_params.option_map, parsed_params.flags);
239 } else if (parsed_params.cmd == WALDumperCommand::Name()) {
240 return new WALDumperCommand(parsed_params.cmd_params,
241 parsed_params.option_map, parsed_params.flags);
242 } else if (parsed_params.cmd == ReduceDBLevelsCommand::Name()) {
243 return new ReduceDBLevelsCommand(parsed_params.cmd_params,
244 parsed_params.option_map,
245 parsed_params.flags);
246 } else if (parsed_params.cmd == ChangeCompactionStyleCommand::Name()) {
247 return new ChangeCompactionStyleCommand(parsed_params.cmd_params,
248 parsed_params.option_map,
249 parsed_params.flags);
250 } else if (parsed_params.cmd == DBDumperCommand::Name()) {
251 return new DBDumperCommand(parsed_params.cmd_params,
252 parsed_params.option_map, parsed_params.flags);
253 } else if (parsed_params.cmd == DBLoaderCommand::Name()) {
254 return new DBLoaderCommand(parsed_params.cmd_params,
255 parsed_params.option_map, parsed_params.flags);
256 } else if (parsed_params.cmd == ManifestDumpCommand::Name()) {
257 return new ManifestDumpCommand(parsed_params.cmd_params,
258 parsed_params.option_map,
259 parsed_params.flags);
f67539c2
TL		 260 } else if (parsed_params.cmd == FileChecksumDumpCommand::Name()) {
261 return new FileChecksumDumpCommand(parsed_params.cmd_params,
262 parsed_params.option_map,
263 parsed_params.flags);
1e59de90
TL		 264 } else if (parsed_params.cmd == GetPropertyCommand::Name()) {
265 return new GetPropertyCommand(parsed_params.cmd_params,
266 parsed_params.option_map,
267 parsed_params.flags);
7c673cae
FG		 268 } else if (parsed_params.cmd == ListColumnFamiliesCommand::Name()) {
269 return new ListColumnFamiliesCommand(parsed_params.cmd_params,
270 parsed_params.option_map,
271 parsed_params.flags);
272 } else if (parsed_params.cmd == CreateColumnFamilyCommand::Name()) {
273 return new CreateColumnFamilyCommand(parsed_params.cmd_params,
274 parsed_params.option_map,
275 parsed_params.flags);
f67539c2
TL		 276 } else if (parsed_params.cmd == DropColumnFamilyCommand::Name()) {
277 return new DropColumnFamilyCommand(parsed_params.cmd_params,
278 parsed_params.option_map,
279 parsed_params.flags);
7c673cae
FG		 280 } else if (parsed_params.cmd == DBFileDumperCommand::Name()) {
281 return new DBFileDumperCommand(parsed_params.cmd_params,
282 parsed_params.option_map,
283 parsed_params.flags);
1e59de90
TL		 284 } else if (parsed_params.cmd == DBLiveFilesMetadataDumperCommand::Name()) {
285 return new DBLiveFilesMetadataDumperCommand(parsed_params.cmd_params,
286 parsed_params.option_map,
287 parsed_params.flags);
7c673cae
FG		 288 } else if (parsed_params.cmd == InternalDumpCommand::Name()) {
289 return new InternalDumpCommand(parsed_params.cmd_params,
290 parsed_params.option_map,
291 parsed_params.flags);
292 } else if (parsed_params.cmd == CheckConsistencyCommand::Name()) {
293 return new CheckConsistencyCommand(parsed_params.cmd_params,
294 parsed_params.option_map,
295 parsed_params.flags);
296 } else if (parsed_params.cmd == CheckPointCommand::Name()) {
297 return new CheckPointCommand(parsed_params.cmd_params,
1e59de90 298 parsed_params.option_map, parsed_params.flags);
7c673cae
FG		 299 } else if (parsed_params.cmd == RepairCommand::Name()) {
300 return new RepairCommand(parsed_params.cmd_params, parsed_params.option_map,
301 parsed_params.flags);
302 } else if (parsed_params.cmd == BackupCommand::Name()) {
303 return new BackupCommand(parsed_params.cmd_params, parsed_params.option_map,
304 parsed_params.flags);
305 } else if (parsed_params.cmd == RestoreCommand::Name()) {
306 return new RestoreCommand(parsed_params.cmd_params,
307 parsed_params.option_map, parsed_params.flags);
11fdf7f2
TL		 308 } else if (parsed_params.cmd == WriteExternalSstFilesCommand::Name()) {
309 return new WriteExternalSstFilesCommand(parsed_params.cmd_params,
310 parsed_params.option_map,
311 parsed_params.flags);
312 } else if (parsed_params.cmd == IngestExternalSstFilesCommand::Name()) {
313 return new IngestExternalSstFilesCommand(parsed_params.cmd_params,
314 parsed_params.option_map,
315 parsed_params.flags);
f67539c2
TL		 316 } else if (parsed_params.cmd == ListFileRangeDeletesCommand::Name()) {
317 return new ListFileRangeDeletesCommand(parsed_params.option_map,
318 parsed_params.flags);
20effc67
TL		 319 } else if (parsed_params.cmd == UnsafeRemoveSstFileCommand::Name()) {
320 return new UnsafeRemoveSstFileCommand(parsed_params.cmd_params,
321 parsed_params.option_map,
322 parsed_params.flags);
1e59de90
TL		 323 } else if (parsed_params.cmd == UpdateManifestCommand::Name()) {
324 return new UpdateManifestCommand(parsed_params.cmd_params,
325 parsed_params.option_map,
326 parsed_params.flags);
7c673cae
FG		 327 }
328 return nullptr;
329}
330
331/* Run the command, and return the execute result. */
332void LDBCommand::Run() {
333 if (!exec_state_.IsNotStarted()) {
334 return;
335 }
336
f67539c2
TL		 337 if (!options_.env || options_.env == Env::Default()) {
338 Env* env = Env::Default();
1e59de90
TL		 339 Status s = Env::CreateFromUri(config_options_, env_uri_, fs_uri_, &env,
340 &env_guard_);
341 if (!s.ok()) {
342 fprintf(stderr, "%s\n", s.ToString().c_str());
f67539c2
TL		 343 exec_state_ = LDBCommandExecuteResult::Failed(s.ToString());
344 return;
345 }
346 options_.env = env;
347 }
348
7c673cae
FG		 349 if (db_ == nullptr && !NoDBOpen()) {
350 OpenDB();
351 if (exec_state_.IsFailed() && try_load_options_) {
352 // We don't always return if there is a failure because a WAL file or
353 // manifest file can be given to "dump" command so we should continue.
354 // --try_load_options is not valid in those cases.
355 return;
356 }
357 }
358
359 // We'll intentionally proceed even if the DB can't be opened because users
360 // can also specify a filename, not just a directory.
361 DoCommand();
362
363 if (exec_state_.IsNotStarted()) {
364 exec_state_ = LDBCommandExecuteResult::Succeed("");
365 }
366
367 if (db_ != nullptr) {
368 CloseDB();
369 }
370}
371
372LDBCommand::LDBCommand(const std::map<std::string, std::string>& options,
373 const std::vector<std::string>& flags, bool is_read_only,
374 const std::vector<std::string>& valid_cmd_line_options)
375 : db_(nullptr),
11fdf7f2 376 db_ttl_(nullptr),
7c673cae
FG		 377 is_read_only_(is_read_only),
378 is_key_hex_(false),
379 is_value_hex_(false),
380 is_db_ttl_(false),
381 timestamp_(false),
382 try_load_options_(false),
383 create_if_missing_(false),
384 option_map_(options),
385 flags_(flags),
386 valid_cmd_line_options_(valid_cmd_line_options) {
1e59de90 387 auto itr = options.find(ARG_DB);
7c673cae
FG		 388 if (itr != options.end()) {
389 db_path_ = itr->second;
390 }
391
f67539c2
TL		 392 itr = options.find(ARG_ENV_URI);
393 if (itr != options.end()) {
394 env_uri_ = itr->second;
395 }
396
1e59de90
TL		 397 itr = options.find(ARG_FS_URI);
398 if (itr != options.end()) {
399 fs_uri_ = itr->second;
400 }
401
7c673cae
FG		 402 itr = options.find(ARG_CF_NAME);
403 if (itr != options.end()) {
404 column_family_name_ = itr->second;
405 } else {
406 column_family_name_ = kDefaultColumnFamilyName;
407 }
408
f67539c2
TL		 409 itr = options.find(ARG_SECONDARY_PATH);
410 secondary_path_ = "";
411 if (itr != options.end()) {
412 secondary_path_ = itr->second;
413 }
414
7c673cae
FG		 415 is_key_hex_ = IsKeyHex(options, flags);
416 is_value_hex_ = IsValueHex(options, flags);
417 is_db_ttl_ = IsFlagPresent(flags, ARG_TTL);
418 timestamp_ = IsFlagPresent(flags, ARG_TIMESTAMP);
1e59de90 419 try_load_options_ = IsTryLoadOptions(options, flags);
20effc67
TL		 420 force_consistency_checks_ =
421 !IsFlagPresent(flags, ARG_DISABLE_CONSISTENCY_CHECKS);
1e59de90
TL		 422 enable_blob_files_ = IsFlagPresent(flags, ARG_ENABLE_BLOB_FILES);
423 enable_blob_garbage_collection_ =
424 IsFlagPresent(flags, ARG_ENABLE_BLOB_GARBAGE_COLLECTION);
20effc67
TL		 425 config_options_.ignore_unknown_options =
426 IsFlagPresent(flags, ARG_IGNORE_UNKNOWN_OPTIONS);
7c673cae
FG		 427}
428
429void LDBCommand::OpenDB() {
20effc67 430 PrepareOptions();
7c673cae
FG		 431 if (!exec_state_.IsNotStarted()) {
432 return;
433 }
f67539c2
TL		 434 if (column_families_.empty() && !options_.merge_operator) {
435 // No harm to add a general merge operator if it is not specified.
436 options_.merge_operator = MergeOperators::CreateStringAppendOperator(':');
437 }
7c673cae
FG		 438 // Open the DB.
439 Status st;
440 std::vector<ColumnFamilyHandle*> handles_opened;
441 if (is_db_ttl_) {
442 // ldb doesn't yet support TTL DB with multiple column families
443 if (!column_family_name_.empty() || !column_families_.empty()) {
444 exec_state_ = LDBCommandExecuteResult::Failed(
445 "ldb doesn't support TTL DB with multiple column families");
446 }
f67539c2
TL		 447 if (!secondary_path_.empty()) {
448 exec_state_ = LDBCommandExecuteResult::Failed(
449 "Open as secondary is not supported for TTL DB yet.");
450 }
7c673cae 451 if (is_read_only_) {
11fdf7f2 452 st = DBWithTTL::Open(options_, db_path_, &db_ttl_, 0, true);
7c673cae 453 } else {
11fdf7f2 454 st = DBWithTTL::Open(options_, db_path_, &db_ttl_);
7c673cae
FG		 455 }
456 db_ = db_ttl_;
457 } else {
f67539c2 458 if (is_read_only_ && secondary_path_.empty()) {
7c673cae 459 if (column_families_.empty()) {
11fdf7f2 460 st = DB::OpenForReadOnly(options_, db_path_, &db_);
7c673cae 461 } else {
11fdf7f2 462 st = DB::OpenForReadOnly(options_, db_path_, column_families_,
7c673cae
FG		 463 &handles_opened, &db_);
464 }
465 } else {
466 if (column_families_.empty()) {
f67539c2
TL		 467 if (secondary_path_.empty()) {
468 st = DB::Open(options_, db_path_, &db_);
469 } else {
470 st = DB::OpenAsSecondary(options_, db_path_, secondary_path_, &db_);
471 }
7c673cae 472 } else {
f67539c2
TL		 473 if (secondary_path_.empty()) {
474 st = DB::Open(options_, db_path_, column_families_, &handles_opened,
475 &db_);
476 } else {
477 st = DB::OpenAsSecondary(options_, db_path_, secondary_path_,
478 column_families_, &handles_opened, &db_);
479 }
7c673cae
FG		 480 }
481 }
482 }
483 if (!st.ok()) {
484 std::string msg = st.ToString();
485 exec_state_ = LDBCommandExecuteResult::Failed(msg);
486 } else if (!handles_opened.empty()) {
487 assert(handles_opened.size() == column_families_.size());
488 bool found_cf_name = false;
489 for (size_t i = 0; i < handles_opened.size(); i++) {
490 cf_handles_[column_families_[i].name] = handles_opened[i];
491 if (column_family_name_ == column_families_[i].name) {
492 found_cf_name = true;
493 }
494 }
495 if (!found_cf_name) {
496 exec_state_ = LDBCommandExecuteResult::Failed(
497 "Non-existing column family " + column_family_name_);
498 CloseDB();
499 }
500 } else {
501 // We successfully opened DB in single column family mode.
502 assert(column_families_.empty());
503 if (column_family_name_ != kDefaultColumnFamilyName) {
504 exec_state_ = LDBCommandExecuteResult::Failed(
505 "Non-existing column family " + column_family_name_);
506 CloseDB();
507 }
508 }
7c673cae
FG		 509}
510
511void LDBCommand::CloseDB() {
512 if (db_ != nullptr) {
513 for (auto& pair : cf_handles_) {
514 delete pair.second;
515 }
1e59de90
TL		 516 Status s = db_->Close();
517 s.PermitUncheckedError();
7c673cae
FG		 518 delete db_;
519 db_ = nullptr;
520 }
521}
522
523ColumnFamilyHandle* LDBCommand::GetCfHandle() {
524 if (!cf_handles_.empty()) {
525 auto it = cf_handles_.find(column_family_name_);
526 if (it == cf_handles_.end()) {
527 exec_state_ = LDBCommandExecuteResult::Failed(
528 "Cannot find column family " + column_family_name_);
529 } else {
530 return it->second;
531 }
532 }
533 return db_->DefaultColumnFamily();
534}
535
536std::vector<std::string> LDBCommand::BuildCmdLineOptions(
537 std::vector<std::string> options) {
f67539c2 538 std::vector<std::string> ret = {ARG_ENV_URI,
1e59de90 539 ARG_FS_URI,
f67539c2
TL		 540 ARG_DB,
541 ARG_SECONDARY_PATH,
7c673cae
FG		 542 ARG_BLOOM_BITS,
543 ARG_BLOCK_SIZE,
544 ARG_AUTO_COMPACTION,
545 ARG_COMPRESSION_TYPE,
546 ARG_COMPRESSION_MAX_DICT_BYTES,
547 ARG_WRITE_BUFFER_SIZE,
548 ARG_FILE_SIZE,
549 ARG_FIX_PREFIX_LEN,
550 ARG_TRY_LOAD_OPTIONS,
20effc67 551 ARG_DISABLE_CONSISTENCY_CHECKS,
1e59de90
TL		 552 ARG_ENABLE_BLOB_FILES,
553 ARG_MIN_BLOB_SIZE,
554 ARG_BLOB_FILE_SIZE,
555 ARG_BLOB_COMPRESSION_TYPE,
556 ARG_ENABLE_BLOB_GARBAGE_COLLECTION,
557 ARG_BLOB_GARBAGE_COLLECTION_AGE_CUTOFF,
558 ARG_BLOB_GARBAGE_COLLECTION_FORCE_THRESHOLD,
559 ARG_BLOB_COMPACTION_READAHEAD_SIZE,
560 ARG_BLOB_FILE_STARTING_LEVEL,
561 ARG_PREPOPULATE_BLOB_CACHE,
11fdf7f2 562 ARG_IGNORE_UNKNOWN_OPTIONS,
7c673cae
FG		 563 ARG_CF_NAME};
564 ret.insert(ret.end(), options.begin(), options.end());
565 return ret;
566}
567
1e59de90
TL		 568/**
569 * Parses the specific double option and fills in the value.
570 * Returns true if the option is found.
571 * Returns false if the option is not found or if there is an error parsing the
572 * value. If there is an error, the specified exec_state is also
573 * updated.
574 */
575bool LDBCommand::ParseDoubleOption(
576 const std::map<std::string, std::string>& /*options*/,
577 const std::string& option, double& value,
578 LDBCommandExecuteResult& exec_state) {
579 auto itr = option_map_.find(option);
580 if (itr != option_map_.end()) {
581#if defined(CYGWIN)
582 char* str_end = nullptr;
583 value = std::strtod(itr->second.c_str(), &str_end);
584 if (str_end == itr->second.c_str()) {
585 exec_state =
586 LDBCommandExecuteResult::Failed(option + " has an invalid value.");
587 } else if (errno == ERANGE) {
588 exec_state = LDBCommandExecuteResult::Failed(
589 option + " has a value out-of-range.");
590 } else {
591 return true;
592 }
593#else
594 try {
595 value = std::stod(itr->second);
596 return true;
597 } catch (const std::invalid_argument&) {
598 exec_state =
599 LDBCommandExecuteResult::Failed(option + " has an invalid value.");
600 } catch (const std::out_of_range&) {
601 exec_state = LDBCommandExecuteResult::Failed(
602 option + " has a value out-of-range.");
603 }
604#endif
605 }
606 return false;
607}
608
7c673cae
FG		 609/**
610 * Parses the specific integer option and fills in the value.
611 * Returns true if the option is found.
612 * Returns false if the option is not found or if there is an error parsing the
613 * value. If there is an error, the specified exec_state is also
614 * updated.
615 */
616bool LDBCommand::ParseIntOption(
11fdf7f2 617 const std::map<std::string, std::string>& /*options*/,
7c673cae
FG		 618 const std::string& option, int& value,
619 LDBCommandExecuteResult& exec_state) {
1e59de90 620 auto itr = option_map_.find(option);
7c673cae 621 if (itr != option_map_.end()) {
7c673cae 622#if defined(CYGWIN)
1e59de90
TL		 623 char* str_end = nullptr;
624 value = strtol(itr->second.c_str(), &str_end, 10);
625 if (str_end == itr->second.c_str()) {
626 exec_state =
627 LDBCommandExecuteResult::Failed(option + " has an invalid value.");
628 } else if (errno == ERANGE) {
629 exec_state = LDBCommandExecuteResult::Failed(
630 option + " has a value out-of-range.");
631 } else {
632 return true;
633 }
7c673cae 634#else
1e59de90 635 try {
7c673cae 636 value = std::stoi(itr->second);
7c673cae
FG		 637 return true;
638 } catch (const std::invalid_argument&) {
639 exec_state =
640 LDBCommandExecuteResult::Failed(option + " has an invalid value.");
641 } catch (const std::out_of_range&) {
642 exec_state = LDBCommandExecuteResult::Failed(
643 option + " has a value out-of-range.");
644 }
1e59de90 645#endif
7c673cae
FG		 646 }
647 return false;
648}
649
650/**
651 * Parses the specified option and fills in the value.
652 * Returns true if the option is found.
653 * Returns false otherwise.
654 */
655bool LDBCommand::ParseStringOption(
11fdf7f2 656 const std::map<std::string, std::string>& /*options*/,
7c673cae
FG		 657 const std::string& option, std::string* value) {
658 auto itr = option_map_.find(option);
659 if (itr != option_map_.end()) {
660 *value = itr->second;
661 return true;
662 }
663 return false;
664}
665
1e59de90
TL		 666/**
667 * Parses the specified compression type and fills in the value.
668 * Returns true if the compression type is found.
669 * Returns false otherwise.
670 */
671bool LDBCommand::ParseCompressionTypeOption(
672 const std::map<std::string, std::string>& /*options*/,
673 const std::string& option, CompressionType& value,
674 LDBCommandExecuteResult& exec_state) {
675 auto itr = option_map_.find(option);
676 if (itr != option_map_.end()) {
677 const std::string& comp = itr->second;
678 if (comp == "no") {
679 value = kNoCompression;
680 return true;
681 } else if (comp == "snappy") {
682 value = kSnappyCompression;
683 return true;
684 } else if (comp == "zlib") {
685 value = kZlibCompression;
686 return true;
687 } else if (comp == "bzip2") {
688 value = kBZip2Compression;
689 return true;
690 } else if (comp == "lz4") {
691 value = kLZ4Compression;
692 return true;
693 } else if (comp == "lz4hc") {
694 value = kLZ4HCCompression;
695 return true;
696 } else if (comp == "xpress") {
697 value = kXpressCompression;
698 return true;
699 } else if (comp == "zstd") {
700 value = kZSTD;
701 return true;
702 } else {
703 // Unknown compression.
704 exec_state = LDBCommandExecuteResult::Failed(
705 "Unknown compression algorithm: " + comp);
706 }
707 }
708 return false;
709}
710
20effc67
TL		 711void LDBCommand::OverrideBaseOptions() {
712 options_.create_if_missing = false;
7c673cae 713
1e59de90
TL		 714 int db_write_buffer_size;
715 if (ParseIntOption(option_map_, ARG_DB_WRITE_BUFFER_SIZE,
716 db_write_buffer_size, exec_state_)) {
717 if (db_write_buffer_size >= 0) {
718 options_.db_write_buffer_size = db_write_buffer_size;
719 } else {
720 exec_state_ = LDBCommandExecuteResult::Failed(ARG_DB_WRITE_BUFFER_SIZE +
721 " must be >= 0.");
722 }
723 }
724
725 if (options_.db_paths.size() == 0) {
726 options_.db_paths.emplace_back(db_path_,
727 std::numeric_limits<uint64_t>::max());
728 }
729
730 OverrideBaseCFOptions(static_cast<ColumnFamilyOptions*>(&options_));
731}
732
733void LDBCommand::OverrideBaseCFOptions(ColumnFamilyOptions* cf_opts) {
7c673cae
FG		 734 BlockBasedTableOptions table_options;
735 bool use_table_options = false;
736 int bits;
737 if (ParseIntOption(option_map_, ARG_BLOOM_BITS, bits, exec_state_)) {
738 if (bits > 0) {
739 use_table_options = true;
740 table_options.filter_policy.reset(NewBloomFilterPolicy(bits));
741 } else {
742 exec_state_ =
743 LDBCommandExecuteResult::Failed(ARG_BLOOM_BITS + " must be > 0.");
744 }
745 }
746
747 int block_size;
748 if (ParseIntOption(option_map_, ARG_BLOCK_SIZE, block_size, exec_state_)) {
749 if (block_size > 0) {
750 use_table_options = true;
751 table_options.block_size = block_size;
752 } else {
753 exec_state_ =
754 LDBCommandExecuteResult::Failed(ARG_BLOCK_SIZE + " must be > 0.");
755 }
756 }
757
1e59de90 758 cf_opts->force_consistency_checks = force_consistency_checks_;
7c673cae 759 if (use_table_options) {
1e59de90 760 cf_opts->table_factory.reset(NewBlockBasedTableFactory(table_options));
7c673cae
FG		 761 }
762
1e59de90
TL		 763 cf_opts->enable_blob_files = enable_blob_files_;
764
765 int min_blob_size;
766 if (ParseIntOption(option_map_, ARG_MIN_BLOB_SIZE, min_blob_size,
767 exec_state_)) {
768 if (min_blob_size >= 0) {
769 cf_opts->min_blob_size = min_blob_size;
770 } else {
771 exec_state_ =
772 LDBCommandExecuteResult::Failed(ARG_MIN_BLOB_SIZE + " must be >= 0.");
773 }
7c673cae
FG		 774 }
775
1e59de90
TL		 776 int blob_file_size;
777 if (ParseIntOption(option_map_, ARG_BLOB_FILE_SIZE, blob_file_size,
778 exec_state_)) {
779 if (blob_file_size > 0) {
780 cf_opts->blob_file_size = blob_file_size;
7c673cae 781 } else {
7c673cae 782 exec_state_ =
1e59de90 783 LDBCommandExecuteResult::Failed(ARG_BLOB_FILE_SIZE + " must be > 0.");
7c673cae
FG		 784 }
785 }
786
1e59de90
TL		 787 cf_opts->enable_blob_garbage_collection = enable_blob_garbage_collection_;
788
789 double blob_garbage_collection_age_cutoff;
790 if (ParseDoubleOption(option_map_, ARG_BLOB_GARBAGE_COLLECTION_AGE_CUTOFF,
791 blob_garbage_collection_age_cutoff, exec_state_)) {
792 if (blob_garbage_collection_age_cutoff >= 0 &&
793 blob_garbage_collection_age_cutoff <= 1) {
794 cf_opts->blob_garbage_collection_age_cutoff =
795 blob_garbage_collection_age_cutoff;
796 } else {
797 exec_state_ = LDBCommandExecuteResult::Failed(
798 ARG_BLOB_GARBAGE_COLLECTION_AGE_CUTOFF + " must be >= 0 and <= 1.");
799 }
800 }
801
802 double blob_garbage_collection_force_threshold;
803 if (ParseDoubleOption(option_map_,
804 ARG_BLOB_GARBAGE_COLLECTION_FORCE_THRESHOLD,
805 blob_garbage_collection_force_threshold, exec_state_)) {
806 if (blob_garbage_collection_force_threshold >= 0 &&
807 blob_garbage_collection_force_threshold <= 1) {
808 cf_opts->blob_garbage_collection_force_threshold =
809 blob_garbage_collection_force_threshold;
810 } else {
811 exec_state_ = LDBCommandExecuteResult::Failed(
812 ARG_BLOB_GARBAGE_COLLECTION_FORCE_THRESHOLD +
813 " must be >= 0 and <= 1.");
814 }
815 }
816
817 int blob_compaction_readahead_size;
818 if (ParseIntOption(option_map_, ARG_BLOB_COMPACTION_READAHEAD_SIZE,
819 blob_compaction_readahead_size, exec_state_)) {
820 if (blob_compaction_readahead_size > 0) {
821 cf_opts->blob_compaction_readahead_size = blob_compaction_readahead_size;
822 } else {
823 exec_state_ = LDBCommandExecuteResult::Failed(
824 ARG_BLOB_COMPACTION_READAHEAD_SIZE + " must be > 0.");
825 }
826 }
827
828 int blob_file_starting_level;
829 if (ParseIntOption(option_map_, ARG_BLOB_FILE_STARTING_LEVEL,
830 blob_file_starting_level, exec_state_)) {
831 if (blob_file_starting_level >= 0) {
832 cf_opts->blob_file_starting_level = blob_file_starting_level;
833 } else {
834 exec_state_ = LDBCommandExecuteResult::Failed(
835 ARG_BLOB_FILE_STARTING_LEVEL + " must be >= 0.");
836 }
837 }
838
839 int prepopulate_blob_cache;
840 if (ParseIntOption(option_map_, ARG_PREPOPULATE_BLOB_CACHE,
841 prepopulate_blob_cache, exec_state_)) {
842 switch (prepopulate_blob_cache) {
843 case 0:
844 cf_opts->prepopulate_blob_cache = PrepopulateBlobCache::kDisable;
845 break;
846 case 1:
847 cf_opts->prepopulate_blob_cache = PrepopulateBlobCache::kFlushOnly;
848 break;
849 default:
850 exec_state_ = LDBCommandExecuteResult::Failed(
851 ARG_PREPOPULATE_BLOB_CACHE +
852 " must be 0 (disable) or 1 (flush only).");
853 }
854 }
855
856 auto itr = option_map_.find(ARG_AUTO_COMPACTION);
857 if (itr != option_map_.end()) {
858 cf_opts->disable_auto_compactions = !StringToBool(itr->second);
859 }
860
861 CompressionType compression_type;
862 if (ParseCompressionTypeOption(option_map_, ARG_COMPRESSION_TYPE,
863 compression_type, exec_state_)) {
864 cf_opts->compression = compression_type;
865 }
866
867 CompressionType blob_compression_type;
868 if (ParseCompressionTypeOption(option_map_, ARG_BLOB_COMPRESSION_TYPE,
869 blob_compression_type, exec_state_)) {
870 cf_opts->blob_compression_type = blob_compression_type;
871 }
872
7c673cae
FG		 873 int compression_max_dict_bytes;
874 if (ParseIntOption(option_map_, ARG_COMPRESSION_MAX_DICT_BYTES,
875 compression_max_dict_bytes, exec_state_)) {
876 if (compression_max_dict_bytes >= 0) {
1e59de90 877 cf_opts->compression_opts.max_dict_bytes = compression_max_dict_bytes;
7c673cae
FG		 878 } else {
879 exec_state_ = LDBCommandExecuteResult::Failed(
880 ARG_COMPRESSION_MAX_DICT_BYTES + " must be >= 0.");
881 }
882 }
883
7c673cae
FG		 884 int write_buffer_size;
885 if (ParseIntOption(option_map_, ARG_WRITE_BUFFER_SIZE, write_buffer_size,
1e59de90 886 exec_state_)) {
7c673cae 887 if (write_buffer_size > 0) {
1e59de90 888 cf_opts->write_buffer_size = write_buffer_size;
7c673cae
FG		 889 } else {
890 exec_state_ = LDBCommandExecuteResult::Failed(ARG_WRITE_BUFFER_SIZE +
891 " must be > 0.");
892 }
893 }
894
895 int file_size;
896 if (ParseIntOption(option_map_, ARG_FILE_SIZE, file_size, exec_state_)) {
897 if (file_size > 0) {
1e59de90 898 cf_opts->target_file_size_base = file_size;
7c673cae
FG		 899 } else {
900 exec_state_ =
901 LDBCommandExecuteResult::Failed(ARG_FILE_SIZE + " must be > 0.");
902 }
903 }
904
7c673cae
FG		 905 int fix_prefix_len;
906 if (ParseIntOption(option_map_, ARG_FIX_PREFIX_LEN, fix_prefix_len,
907 exec_state_)) {
908 if (fix_prefix_len > 0) {
1e59de90 909 cf_opts->prefix_extractor.reset(
7c673cae
FG		 910 NewFixedPrefixTransform(static_cast<size_t>(fix_prefix_len)));
911 } else {
912 exec_state_ =
913 LDBCommandExecuteResult::Failed(ARG_FIX_PREFIX_LEN + " must be > 0.");
914 }
915 }
20effc67
TL		 916}
917
918// First, initializes the options state using the OPTIONS file when enabled.
919// Second, overrides the options according to the CLI arguments and the
920// specific subcommand being run.
921void LDBCommand::PrepareOptions() {
922 if (!create_if_missing_ && try_load_options_) {
923 config_options_.env = options_.env;
924 Status s = LoadLatestOptions(config_options_, db_path_, &options_,
925 &column_families_);
926 if (!s.ok() && !s.IsNotFound()) {
927 // Option file exists but load option file error.
928 std::string msg = s.ToString();
929 exec_state_ = LDBCommandExecuteResult::Failed(msg);
930 db_ = nullptr;
931 return;
932 }
1e59de90
TL		 933 if (!options_.wal_dir.empty()) {
934 if (options_.env->FileExists(options_.wal_dir).IsNotFound()) {
935 options_.wal_dir = db_path_;
936 fprintf(
937 stderr,
938 "wal_dir loaded from the option file doesn't exist. Ignore it.\n");
939 }
20effc67
TL		 940 }
941
942 // If merge operator is not set, set a string append operator.
943 for (auto& cf_entry : column_families_) {
944 if (!cf_entry.options.merge_operator) {
945 cf_entry.options.merge_operator =
946 MergeOperators::CreateStringAppendOperator(':');
947 }
948 }
949 }
950
1e59de90
TL		 951 if (options_.env == Env::Default()) {
952 options_.env = config_options_.env;
953 }
954
20effc67
TL		 955 OverrideBaseOptions();
956 if (exec_state_.IsFailed()) {
957 return;
958 }
f67539c2 959
20effc67
TL		 960 if (column_families_.empty()) {
961 // Reads the MANIFEST to figure out what column families exist. In this
962 // case, the option overrides from the CLI argument/specific subcommand
963 // apply to all column families.
964 std::vector<std::string> cf_list;
965 Status st = DB::ListColumnFamilies(options_, db_path_, &cf_list);
966 // It is possible the DB doesn't exist yet, for "create if not
967 // existing" case. The failure is ignored here. We rely on DB::Open()
968 // to give us the correct error message for problem with opening
969 // existing DB.
970 if (st.ok() && cf_list.size() > 1) {
971 // Ignore single column family DB.
972 for (auto cf_name : cf_list) {
973 column_families_.emplace_back(cf_name, options_);
974 }
975 }
976 } else {
977 // We got column families from the OPTIONS file. In this case, the option
978 // overrides from the CLI argument/specific subcommand only apply to the
979 // column family specified by `--column_family_name`.
980 auto column_families_iter =
981 std::find_if(column_families_.begin(), column_families_.end(),
982 [this](const ColumnFamilyDescriptor& cf_desc) {
983 return cf_desc.name == column_family_name_;
984 });
985 if (column_families_iter == column_families_.end()) {
986 exec_state_ = LDBCommandExecuteResult::Failed(
987 "Non-existing column family " + column_family_name_);
988 return;
989 }
1e59de90 990 OverrideBaseCFOptions(&column_families_iter->options);
20effc67 991 }
7c673cae
FG		 992}
993
994bool LDBCommand::ParseKeyValue(const std::string& line, std::string* key,
995 std::string* value, bool is_key_hex,
996 bool is_value_hex) {
997 size_t pos = line.find(DELIM);
998 if (pos != std::string::npos) {
999 *key = line.substr(0, pos);
1000 *value = line.substr(pos + strlen(DELIM));
1001 if (is_key_hex) {
1002 *key = HexToString(*key);
1003 }
1004 if (is_value_hex) {
1005 *value = HexToString(*value);
1006 }
1007 return true;
1008 } else {
1009 return false;
1010 }
1011}
1012
1013/**
1014 * Make sure that ONLY the command-line options and flags expected by this
1015 * command are specified on the command-line. Extraneous options are usually
1016 * the result of user error.
1017 * Returns true if all checks pass. Else returns false, and prints an
1018 * appropriate error msg to stderr.
1019 */
1020bool LDBCommand::ValidateCmdLineOptions() {
1e59de90 1021 for (auto itr = option_map_.begin(); itr != option_map_.end(); ++itr) {
7c673cae
FG		 1022 if (std::find(valid_cmd_line_options_.begin(),
1023 valid_cmd_line_options_.end(),
1024 itr->first) == valid_cmd_line_options_.end()) {
1025 fprintf(stderr, "Invalid command-line option %s\n", itr->first.c_str());
1026 return false;
1027 }
1028 }
1029
1030 for (std::vector<std::string>::const_iterator itr = flags_.begin();
1031 itr != flags_.end(); ++itr) {
1032 if (std::find(valid_cmd_line_options_.begin(),
1033 valid_cmd_line_options_.end(),
1034 *itr) == valid_cmd_line_options_.end()) {
1035 fprintf(stderr, "Invalid command-line flag %s\n", itr->c_str());
1036 return false;
1037 }
1038 }
1039
1040 if (!NoDBOpen() && option_map_.find(ARG_DB) == option_map_.end() &&
1041 option_map_.find(ARG_PATH) == option_map_.end()) {
1042 fprintf(stderr, "Either %s or %s must be specified.\n", ARG_DB.c_str(),
1043 ARG_PATH.c_str());
1044 return false;
1045 }
1046
1047 return true;
1048}
1049
1050std::string LDBCommand::HexToString(const std::string& str) {
1051 std::string result;
1052 std::string::size_type len = str.length();
1053 if (len < 2 || str[0] != '0' || str[1] != 'x') {
1054 fprintf(stderr, "Invalid hex input %s. Must start with 0x\n", str.c_str());
1055 throw "Invalid hex input";
1056 }
1057 if (!Slice(str.data() + 2, len - 2).DecodeHex(&result)) {
1058 throw "Invalid hex input";
1059 }
1060 return result;
1061}
1062
1063std::string LDBCommand::StringToHex(const std::string& str) {
1064 std::string result("0x");
1065 result.append(Slice(str).ToString(true));
1066 return result;
1067}
1068
1069std::string LDBCommand::PrintKeyValue(const std::string& key,
1070 const std::string& value, bool is_key_hex,
1071 bool is_value_hex) {
1072 std::string result;
1073 result.append(is_key_hex ? StringToHex(key) : key);
1074 result.append(DELIM);
1075 result.append(is_value_hex ? StringToHex(value) : value);
1076 return result;
1077}
1078
1079std::string LDBCommand::PrintKeyValue(const std::string& key,
1080 const std::string& value, bool is_hex) {
1081 return PrintKeyValue(key, value, is_hex, is_hex);
1082}
1083
1084std::string LDBCommand::HelpRangeCmdArgs() {
1085 std::ostringstream str_stream;
1086 str_stream << " ";
1087 str_stream << "[--" << ARG_FROM << "] ";
1088 str_stream << "[--" << ARG_TO << "] ";
1089 return str_stream.str();
1090}
1091
1092bool LDBCommand::IsKeyHex(const std::map<std::string, std::string>& options,
1093 const std::vector<std::string>& flags) {
1094 return (IsFlagPresent(flags, ARG_HEX) || IsFlagPresent(flags, ARG_KEY_HEX) ||
1095 ParseBooleanOption(options, ARG_HEX, false) ||
1096 ParseBooleanOption(options, ARG_KEY_HEX, false));
1097}
1098
1099bool LDBCommand::IsValueHex(const std::map<std::string, std::string>& options,
1100 const std::vector<std::string>& flags) {
1101 return (IsFlagPresent(flags, ARG_HEX) ||
1102 IsFlagPresent(flags, ARG_VALUE_HEX) ||
1103 ParseBooleanOption(options, ARG_HEX, false) ||
1104 ParseBooleanOption(options, ARG_VALUE_HEX, false));
1105}
1106
1e59de90
TL		 1107bool LDBCommand::IsTryLoadOptions(
1108 const std::map<std::string, std::string>& options,
1109 const std::vector<std::string>& flags) {
1110 if (IsFlagPresent(flags, ARG_TRY_LOAD_OPTIONS)) {
1111 return true;
1112 }
1113 // if `DB` is specified and not explicitly to create a new db, default
1114 // `try_load_options` to true. The user could still disable that by set
1115 // `try_load_options=false`.
1116 // Note: Opening as TTL DB doesn't support `try_load_options`, so it's default
1117 // to false. TODO: TTL_DB may need to fix that, otherwise it's unable to open
1118 // DB which has incompatible setting with default options.
1119 bool default_val = (options.find(ARG_DB) != options.end()) &&
1120 !IsFlagPresent(flags, ARG_CREATE_IF_MISSING) &&
1121 !IsFlagPresent(flags, ARG_TTL);
1122 return ParseBooleanOption(options, ARG_TRY_LOAD_OPTIONS, default_val);
1123}
1124
7c673cae
FG		 1125bool LDBCommand::ParseBooleanOption(
1126 const std::map<std::string, std::string>& options,
1127 const std::string& option, bool default_val) {
1e59de90 1128 auto itr = options.find(option);
7c673cae
FG		 1129 if (itr != options.end()) {
1130 std::string option_val = itr->second;
1131 return StringToBool(itr->second);
1132 }
1133 return default_val;
1134}
1135
1136bool LDBCommand::StringToBool(std::string val) {
1137 std::transform(val.begin(), val.end(), val.begin(),
1138 [](char ch) -> char { return (char)::tolower(ch); });
1139
1140 if (val == "true") {
1141 return true;
1142 } else if (val == "false") {
1143 return false;
1144 } else {
1145 throw "Invalid value for boolean argument";
1146 }
1147}
1148
1149CompactorCommand::CompactorCommand(
11fdf7f2 1150 const std::vector<std::string>& /*params*/,
7c673cae
FG		 1151 const std::map<std::string, std::string>& options,
1152 const std::vector<std::string>& flags)
1153 : LDBCommand(options, flags, false,
1154 BuildCmdLineOptions({ARG_FROM, ARG_TO, ARG_HEX, ARG_KEY_HEX,
1155 ARG_VALUE_HEX, ARG_TTL})),
1156 null_from_(true),
1157 null_to_(true) {
1e59de90 1158 auto itr = options.find(ARG_FROM);
7c673cae
FG		 1159 if (itr != options.end()) {
1160 null_from_ = false;
1161 from_ = itr->second;
1162 }
1163
1164 itr = options.find(ARG_TO);
1165 if (itr != options.end()) {
1166 null_to_ = false;
1167 to_ = itr->second;
1168 }
1169
1170 if (is_key_hex_) {
1171 if (!null_from_) {
1172 from_ = HexToString(from_);
1173 }
1174 if (!null_to_) {
1175 to_ = HexToString(to_);
1176 }
1177 }
1178}
1179
1180void CompactorCommand::Help(std::string& ret) {
1181 ret.append(" ");
1182 ret.append(CompactorCommand::Name());
1183 ret.append(HelpRangeCmdArgs());
1184 ret.append("\n");
1185}
1186
1187void CompactorCommand::DoCommand() {
1188 if (!db_) {
1189 assert(GetExecuteState().IsFailed());
1190 return;
1191 }
1192
1193 Slice* begin = nullptr;
1194 Slice* end = nullptr;
1195 if (!null_from_) {
1196 begin = new Slice(from_);
1197 }
1198 if (!null_to_) {
1199 end = new Slice(to_);
1200 }
1201
1202 CompactRangeOptions cro;
f67539c2 1203 cro.bottommost_level_compaction = BottommostLevelCompaction::kForceOptimized;
7c673cae 1204
1e59de90
TL		 1205 Status s = db_->CompactRange(cro, GetCfHandle(), begin, end);
1206 if (!s.ok()) {
1207 std::stringstream oss;
1208 oss << "Compaction failed: " << s.ToString();
1209 exec_state_ = LDBCommandExecuteResult::Failed(oss.str());
1210 } else {
1211 exec_state_ = LDBCommandExecuteResult::Succeed("");
1212 }
7c673cae
FG		 1213
1214 delete begin;
1215 delete end;
1216}
1217
f67539c2 1218// ---------------------------------------------------------------------------
7c673cae
FG		 1219const std::string DBLoaderCommand::ARG_DISABLE_WAL = "disable_wal";
1220const std::string DBLoaderCommand::ARG_BULK_LOAD = "bulk_load";
1221const std::string DBLoaderCommand::ARG_COMPACT = "compact";
1222
1223DBLoaderCommand::DBLoaderCommand(
11fdf7f2 1224 const std::vector<std::string>& /*params*/,
7c673cae
FG		 1225 const std::map<std::string, std::string>& options,
1226 const std::vector<std::string>& flags)
1227 : LDBCommand(
1228 options, flags, false,
1229 BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM,
1230 ARG_TO, ARG_CREATE_IF_MISSING, ARG_DISABLE_WAL,
1231 ARG_BULK_LOAD, ARG_COMPACT})),
1232 disable_wal_(false),
1233 bulk_load_(false),
1234 compact_(false) {
1235 create_if_missing_ = IsFlagPresent(flags, ARG_CREATE_IF_MISSING);
1236 disable_wal_ = IsFlagPresent(flags, ARG_DISABLE_WAL);
1237 bulk_load_ = IsFlagPresent(flags, ARG_BULK_LOAD);
1238 compact_ = IsFlagPresent(flags, ARG_COMPACT);
1239}
1240
1241void DBLoaderCommand::Help(std::string& ret) {
1242 ret.append(" ");
1243 ret.append(DBLoaderCommand::Name());
1244 ret.append(" [--" + ARG_CREATE_IF_MISSING + "]");
1245 ret.append(" [--" + ARG_DISABLE_WAL + "]");
1246 ret.append(" [--" + ARG_BULK_LOAD + "]");
1247 ret.append(" [--" + ARG_COMPACT + "]");
1248 ret.append("\n");
1249}
1250
20effc67
TL		 1251void DBLoaderCommand::OverrideBaseOptions() {
1252 LDBCommand::OverrideBaseOptions();
1253 options_.create_if_missing = create_if_missing_;
7c673cae 1254 if (bulk_load_) {
20effc67 1255 options_.PrepareForBulkLoad();
7c673cae 1256 }
7c673cae
FG		 1257}
1258
1259void DBLoaderCommand::DoCommand() {
1260 if (!db_) {
1261 assert(GetExecuteState().IsFailed());
1262 return;
1263 }
1264
1265 WriteOptions write_options;
1266 if (disable_wal_) {
1267 write_options.disableWAL = true;
1268 }
1269
1270 int bad_lines = 0;
1271 std::string line;
1272 // prefer ifstream getline performance vs that from std::cin istream
1273 std::ifstream ifs_stdin("/dev/stdin");
1274 std::istream* istream_p = ifs_stdin.is_open() ? &ifs_stdin : &std::cin;
1e59de90
TL		 1275 Status s;
1276 while (s.ok() && getline(*istream_p, line, '\n')) {
7c673cae
FG		 1277 std::string key;
1278 std::string value;
1279 if (ParseKeyValue(line, &key, &value, is_key_hex_, is_value_hex_)) {
1e59de90 1280 s = db_->Put(write_options, GetCfHandle(), Slice(key), Slice(value));
7c673cae
FG		 1281 } else if (0 == line.find("Keys in range:")) {
1282 // ignore this line
1283 } else if (0 == line.find("Created bg thread 0x")) {
1284 // ignore this line
1285 } else {
1e59de90 1286 bad_lines++;
7c673cae
FG		 1287 }
1288 }
1289
1290 if (bad_lines > 0) {
1291 std::cout << "Warning: " << bad_lines << " bad lines ignored." << std::endl;
1292 }
1e59de90
TL		 1293 if (!s.ok()) {
1294 std::stringstream oss;
1295 oss << "Load failed: " << s.ToString();
1296 exec_state_ = LDBCommandExecuteResult::Failed(oss.str());
1297 }
1298 if (compact_ && s.ok()) {
1299 s = db_->CompactRange(CompactRangeOptions(), GetCfHandle(), nullptr,
1300 nullptr);
1301 }
1302 if (!s.ok()) {
1303 std::stringstream oss;
1304 oss << "Compaction failed: " << s.ToString();
1305 exec_state_ = LDBCommandExecuteResult::Failed(oss.str());
7c673cae
FG		 1306 }
1307}
1308
1309// ----------------------------------------------------------------------------
1310
1311namespace {
1312
494da23a
TL		 1313void DumpManifestFile(Options options, std::string file, bool verbose, bool hex,
1314 bool json) {
7c673cae
FG		 1315 EnvOptions sopt;
1316 std::string dbname("dummy");
1317 std::shared_ptr<Cache> tc(NewLRUCache(options.max_open_files - 10,
1318 options.table_cache_numshardbits));
1319 // Notice we are using the default options not through SanitizeOptions(),
1320 // if VersionSet::DumpManifest() depends on any option done by
1321 // SanitizeOptions(), we need to initialize it manually.
1322 options.db_paths.emplace_back("dummy", 0);
1323 options.num_levels = 64;
1324 WriteController wc(options.delayed_write_rate);
1325 WriteBufferManager wb(options.db_write_buffer_size);
1326 ImmutableDBOptions immutable_db_options(options);
f67539c2 1327 VersionSet versions(dbname, &immutable_db_options, sopt, tc.get(), &wb, &wc,
1e59de90
TL		 1328 /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
1329 /*db_id*/ "", /*db_session_id*/ "");
7c673cae
FG		 1330 Status s = versions.DumpManifest(options, file, verbose, hex, json);
1331 if (!s.ok()) {
f67539c2
TL		 1332 fprintf(stderr, "Error in processing file %s %s\n", file.c_str(),
1333 s.ToString().c_str());
7c673cae
FG		 1334 }
1335}
1336
1337} // namespace
1338
1339const std::string ManifestDumpCommand::ARG_VERBOSE = "verbose";
1340const std::string ManifestDumpCommand::ARG_JSON = "json";
1341const std::string ManifestDumpCommand::ARG_PATH = "path";
1342
1343void ManifestDumpCommand::Help(std::string& ret) {
1344 ret.append(" ");
1345 ret.append(ManifestDumpCommand::Name());
1346 ret.append(" [--" + ARG_VERBOSE + "]");
1347 ret.append(" [--" + ARG_JSON + "]");
1348 ret.append(" [--" + ARG_PATH + "=<path_to_manifest_file>]");
1349 ret.append("\n");
1350}
1351
1352ManifestDumpCommand::ManifestDumpCommand(
11fdf7f2 1353 const std::vector<std::string>& /*params*/,
7c673cae
FG		 1354 const std::map<std::string, std::string>& options,
1355 const std::vector<std::string>& flags)
1356 : LDBCommand(
1357 options, flags, false,
1358 BuildCmdLineOptions({ARG_VERBOSE, ARG_PATH, ARG_HEX, ARG_JSON})),
1359 verbose_(false),
1360 json_(false),
1361 path_("") {
1362 verbose_ = IsFlagPresent(flags, ARG_VERBOSE);
1363 json_ = IsFlagPresent(flags, ARG_JSON);
1364
1e59de90 1365 auto itr = options.find(ARG_PATH);
7c673cae
FG		 1366 if (itr != options.end()) {
1367 path_ = itr->second;
1368 if (path_.empty()) {
1369 exec_state_ = LDBCommandExecuteResult::Failed("--path: missing pathname");
1370 }
1371 }
1372}
1373
1374void ManifestDumpCommand::DoCommand() {
7c673cae
FG		 1375 std::string manifestfile;
1376
1377 if (!path_.empty()) {
1378 manifestfile = path_;
1379 } else {
7c673cae
FG		 1380 // We need to find the manifest file by searching the directory
1381 // containing the db for files of the form MANIFEST_[0-9]+
1382
f67539c2
TL		 1383 std::vector<std::string> files;
1384 Status s = options_.env->GetChildren(db_path_, &files);
1385 if (!s.ok()) {
1386 std::string err_msg = s.ToString();
1387 err_msg.append(": Failed to list the content of ");
1388 err_msg.append(db_path_);
1389 exec_state_ = LDBCommandExecuteResult::Failed(err_msg);
7c673cae
FG		 1390 return;
1391 }
f67539c2
TL		 1392 const std::string kManifestNamePrefix = "MANIFEST-";
1393 std::string matched_file;
1394#ifdef OS_WIN
1395 const char kPathDelim = '\\';
1396#else
1397 const char kPathDelim = '/';
1398#endif
1399 for (const auto& file_path : files) {
1400 // Some Env::GetChildren() return absolute paths. Some directories' path
1401 // end with path delim, e.g. '/' or '\\'.
1402 size_t pos = file_path.find_last_of(kPathDelim);
1403 if (pos == file_path.size() - 1) {
1404 continue;
1405 }
1406 std::string fname;
1407 if (pos != std::string::npos) {
1408 // Absolute path.
1409 fname.assign(file_path, pos + 1, file_path.size() - pos - 1);
1410 } else {
1411 fname = file_path;
1412 }
1413 uint64_t file_num = 0;
20effc67 1414 FileType file_type = kWalFile; // Just for initialization
f67539c2
TL		 1415 if (ParseFileName(fname, &file_num, &file_type) &&
1416 file_type == kDescriptorFile) {
1417 if (!matched_file.empty()) {
7c673cae
FG		 1418 exec_state_ = LDBCommandExecuteResult::Failed(
1419 "Multiple MANIFEST files found; use --path to select one");
1420 return;
f67539c2
TL		 1421 } else {
1422 matched_file.swap(fname);
7c673cae
FG		 1423 }
1424 }
1425 }
f67539c2
TL		 1426 if (matched_file.empty()) {
1427 std::string err_msg("No MANIFEST found in ");
1428 err_msg.append(db_path_);
1429 exec_state_ = LDBCommandExecuteResult::Failed(err_msg);
1430 return;
1431 }
1e59de90 1432 if (db_path_.back() != '/') {
f67539c2
TL		 1433 db_path_.append("/");
1434 }
1435 manifestfile = db_path_ + matched_file;
7c673cae
FG		 1436 }
1437
1438 if (verbose_) {
f67539c2 1439 fprintf(stdout, "Processing Manifest file %s\n", manifestfile.c_str());
7c673cae
FG		 1440 }
1441
494da23a 1442 DumpManifestFile(options_, manifestfile, verbose_, is_key_hex_, json_);
7c673cae
FG		 1443
1444 if (verbose_) {
f67539c2
TL		 1445 fprintf(stdout, "Processing Manifest file %s done\n", manifestfile.c_str());
1446 }
1447}
1448
1449// ----------------------------------------------------------------------------
1450namespace {
1451
1e59de90
TL		 1452Status GetLiveFilesChecksumInfoFromVersionSet(Options options,
1453 const std::string& db_path,
1454 FileChecksumList* checksum_list) {
f67539c2
TL		 1455 EnvOptions sopt;
1456 Status s;
1457 std::string dbname(db_path);
1458 std::shared_ptr<Cache> tc(NewLRUCache(options.max_open_files - 10,
1459 options.table_cache_numshardbits));
1460 // Notice we are using the default options not through SanitizeOptions(),
1461 // if VersionSet::GetLiveFilesChecksumInfo depends on any option done by
1462 // SanitizeOptions(), we need to initialize it manually.
1463 options.db_paths.emplace_back(db_path, 0);
1464 options.num_levels = 64;
1465 WriteController wc(options.delayed_write_rate);
1466 WriteBufferManager wb(options.db_write_buffer_size);
1467 ImmutableDBOptions immutable_db_options(options);
1468 VersionSet versions(dbname, &immutable_db_options, sopt, tc.get(), &wb, &wc,
1e59de90
TL		 1469 /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
1470 /*db_id*/ "", /*db_session_id*/ "");
f67539c2
TL		 1471 std::vector<std::string> cf_name_list;
1472 s = versions.ListColumnFamilies(&cf_name_list, db_path,
20effc67 1473 immutable_db_options.fs.get());
f67539c2
TL		 1474 if (s.ok()) {
1475 std::vector<ColumnFamilyDescriptor> cf_list;
1476 for (const auto& name : cf_name_list) {
1477 cf_list.emplace_back(name, ColumnFamilyOptions(options));
1478 }
1479 s = versions.Recover(cf_list, true);
1480 }
1481 if (s.ok()) {
1482 s = versions.GetLiveFilesChecksumInfo(checksum_list);
1483 }
1e59de90 1484 return s;
f67539c2
TL		 1485}
1486
1487} // namespace
1488
1489const std::string FileChecksumDumpCommand::ARG_PATH = "path";
1490
1491void FileChecksumDumpCommand::Help(std::string& ret) {
1492 ret.append(" ");
1493 ret.append(FileChecksumDumpCommand::Name());
1494 ret.append(" [--" + ARG_PATH + "=<path_to_manifest_file>]");
1495 ret.append("\n");
1496}
1497
1498FileChecksumDumpCommand::FileChecksumDumpCommand(
1499 const std::vector<std::string>& /*params*/,
1500 const std::map<std::string, std::string>& options,
1501 const std::vector<std::string>& flags)
1e59de90
TL		 1502 : LDBCommand(options, flags, false,
1503 BuildCmdLineOptions({ARG_PATH, ARG_HEX})),
f67539c2 1504 path_("") {
1e59de90 1505 auto itr = options.find(ARG_PATH);
f67539c2
TL		 1506 if (itr != options.end()) {
1507 path_ = itr->second;
1508 if (path_.empty()) {
1509 exec_state_ = LDBCommandExecuteResult::Failed("--path: missing pathname");
1510 }
1511 }
1e59de90 1512 is_checksum_hex_ = IsFlagPresent(flags, ARG_HEX);
f67539c2
TL		 1513}
1514
1515void FileChecksumDumpCommand::DoCommand() {
1516 // print out the checksum information in the following format:
1517 // sst file number, checksum function name, checksum value
1518 // sst file number, checksum function name, checksum value
1519 // ......
1520
1521 std::unique_ptr<FileChecksumList> checksum_list(NewFileChecksumList());
1e59de90
TL		 1522 Status s = GetLiveFilesChecksumInfoFromVersionSet(options_, db_path_,
1523 checksum_list.get());
1524 if (s.ok() && checksum_list != nullptr) {
f67539c2
TL		 1525 std::vector<uint64_t> file_numbers;
1526 std::vector<std::string> checksums;
1527 std::vector<std::string> checksum_func_names;
1e59de90
TL		 1528 s = checksum_list->GetAllFileChecksums(&file_numbers, &checksums,
1529 &checksum_func_names);
f67539c2
TL		 1530 if (s.ok()) {
1531 for (size_t i = 0; i < file_numbers.size(); i++) {
1532 assert(i < file_numbers.size());
1533 assert(i < checksums.size());
1534 assert(i < checksum_func_names.size());
1e59de90
TL		 1535 std::string checksum;
1536 if (is_checksum_hex_) {
1537 checksum = StringToHex(checksums[i]);
1538 } else {
1539 checksum = std::move(checksums[i]);
1540 }
f67539c2 1541 fprintf(stdout, "%" PRId64 ", %s, %s\n", file_numbers[i],
1e59de90 1542 checksum_func_names[i].c_str(), checksum.c_str());
f67539c2 1543 }
1e59de90 1544 fprintf(stdout, "Print SST file checksum information finished \n");
f67539c2 1545 }
1e59de90
TL		 1546 }
1547
1548 if (!s.ok()) {
1549 exec_state_ = LDBCommandExecuteResult::Failed(s.ToString());
1550 }
1551}
1552
1553// ----------------------------------------------------------------------------
1554
1555void GetPropertyCommand::Help(std::string& ret) {
1556 ret.append(" ");
1557 ret.append(GetPropertyCommand::Name());
1558 ret.append(" <property_name>");
1559 ret.append("\n");
1560}
1561
1562GetPropertyCommand::GetPropertyCommand(
1563 const std::vector<std::string>& params,
1564 const std::map<std::string, std::string>& options,
1565 const std::vector<std::string>& flags)
1566 : LDBCommand(options, flags, true, BuildCmdLineOptions({})) {
1567 if (params.size() != 1) {
1568 exec_state_ =
1569 LDBCommandExecuteResult::Failed("property name must be specified");
1570 } else {
1571 property_ = params[0];
1572 }
1573}
1574
1575void GetPropertyCommand::DoCommand() {
1576 if (!db_) {
1577 assert(GetExecuteState().IsFailed());
1578 return;
1579 }
1580
1581 std::map<std::string, std::string> value_map;
1582 std::string value;
1583
1584 // Rather than having different ldb command for map properties vs. string
1585 // properties, we simply try Map property first. (This order only chosen
1586 // because I prefer the map-style output for
1587 // "rocksdb.aggregated-table-properties".)
1588 if (db_->GetMapProperty(GetCfHandle(), property_, &value_map)) {
1589 if (value_map.empty()) {
1590 fprintf(stdout, "%s: <empty map>\n", property_.c_str());
1591 } else {
1592 for (auto& e : value_map) {
1593 fprintf(stdout, "%s.%s: %s\n", property_.c_str(), e.first.c_str(),
1594 e.second.c_str());
1595 }
1596 }
1597 } else if (db_->GetProperty(GetCfHandle(), property_, &value)) {
1598 fprintf(stdout, "%s: %s\n", property_.c_str(), value.c_str());
1599 } else {
1600 exec_state_ =
1601 LDBCommandExecuteResult::Failed("failed to get property: " + property_);
7c673cae
FG		 1602 }
1603}
1604
1605// ----------------------------------------------------------------------------
1606
1607void ListColumnFamiliesCommand::Help(std::string& ret) {
1608 ret.append(" ");
1609 ret.append(ListColumnFamiliesCommand::Name());
7c673cae
FG		 1610 ret.append("\n");
1611}
1612
1613ListColumnFamiliesCommand::ListColumnFamiliesCommand(
f67539c2 1614 const std::vector<std::string>& /*params*/,
7c673cae
FG		 1615 const std::map<std::string, std::string>& options,
1616 const std::vector<std::string>& flags)
f67539c2 1617 : LDBCommand(options, flags, false, BuildCmdLineOptions({})) {}
7c673cae
FG		 1618
1619void ListColumnFamiliesCommand::DoCommand() {
1620 std::vector<std::string> column_families;
f67539c2 1621 Status s = DB::ListColumnFamilies(options_, db_path_, &column_families);
7c673cae 1622 if (!s.ok()) {
f67539c2
TL		 1623 fprintf(stderr, "Error in processing db %s %s\n", db_path_.c_str(),
1624 s.ToString().c_str());
7c673cae 1625 } else {
f67539c2 1626 fprintf(stdout, "Column families in %s: \n{", db_path_.c_str());
7c673cae
FG		 1627 bool first = true;
1628 for (auto cf : column_families) {
1629 if (!first) {
f67539c2 1630 fprintf(stdout, ", ");
7c673cae
FG		 1631 }
1632 first = false;
f67539c2 1633 fprintf(stdout, "%s", cf.c_str());
7c673cae 1634 }
f67539c2 1635 fprintf(stdout, "}\n");
7c673cae
FG		 1636 }
1637}
1638
1639void CreateColumnFamilyCommand::Help(std::string& ret) {
1640 ret.append(" ");
1641 ret.append(CreateColumnFamilyCommand::Name());
1642 ret.append(" --db=<db_path> <new_column_family_name>");
1643 ret.append("\n");
1644}
1645
1646CreateColumnFamilyCommand::CreateColumnFamilyCommand(
1647 const std::vector<std::string>& params,
1648 const std::map<std::string, std::string>& options,
1649 const std::vector<std::string>& flags)
1650 : LDBCommand(options, flags, true, {ARG_DB}) {
1651 if (params.size() != 1) {
1652 exec_state_ = LDBCommandExecuteResult::Failed(
1653 "new column family name must be specified");
1654 } else {
1655 new_cf_name_ = params[0];
1656 }
1657}
1658
1659void CreateColumnFamilyCommand::DoCommand() {
20effc67
TL		 1660 if (!db_) {
1661 assert(GetExecuteState().IsFailed());
1662 return;
1663 }
11fdf7f2 1664 ColumnFamilyHandle* new_cf_handle = nullptr;
7c673cae
FG		 1665 Status st = db_->CreateColumnFamily(options_, new_cf_name_, &new_cf_handle);
1666 if (st.ok()) {
1667 fprintf(stdout, "OK\n");
1668 } else {
1669 exec_state_ = LDBCommandExecuteResult::Failed(
1670 "Fail to create new column family: " + st.ToString());
1671 }
1672 delete new_cf_handle;
1673 CloseDB();
1674}
1675
f67539c2
TL		 1676void DropColumnFamilyCommand::Help(std::string& ret) {
1677 ret.append(" ");
1678 ret.append(DropColumnFamilyCommand::Name());
1679 ret.append(" --db=<db_path> <column_family_name_to_drop>");
1680 ret.append("\n");
1681}
7c673cae 1682
f67539c2
TL		 1683DropColumnFamilyCommand::DropColumnFamilyCommand(
1684 const std::vector<std::string>& params,
1685 const std::map<std::string, std::string>& options,
1686 const std::vector<std::string>& flags)
1687 : LDBCommand(options, flags, true, {ARG_DB}) {
1688 if (params.size() != 1) {
1689 exec_state_ = LDBCommandExecuteResult::Failed(
1690 "The name of column family to drop must be specified");
1691 } else {
1692 cf_name_to_drop_ = params[0];
1693 }
1694}
7c673cae 1695
f67539c2 1696void DropColumnFamilyCommand::DoCommand() {
20effc67
TL		 1697 if (!db_) {
1698 assert(GetExecuteState().IsFailed());
1699 return;
1700 }
f67539c2
TL		 1701 auto iter = cf_handles_.find(cf_name_to_drop_);
1702 if (iter == cf_handles_.end()) {
1703 exec_state_ = LDBCommandExecuteResult::Failed(
1704 "Column family: " + cf_name_to_drop_ + " doesn't exist in db.");
1705 return;
1706 }
1707 ColumnFamilyHandle* cf_handle_to_drop = iter->second;
1708 Status st = db_->DropColumnFamily(cf_handle_to_drop);
1709 if (st.ok()) {
1710 fprintf(stdout, "OK\n");
1711 } else {
1712 exec_state_ = LDBCommandExecuteResult::Failed(
1713 "Fail to drop column family: " + st.ToString());
1714 }
1715 CloseDB();
7c673cae
FG		 1716}
1717
f67539c2
TL		 1718// ----------------------------------------------------------------------------
1719namespace {
1720
7c673cae
FG		 1721// This function only called when it's the sane case of >1 buckets in time-range
1722// Also called only when timekv falls between ttl_start and ttl_end provided
1723void IncBucketCounts(std::vector<uint64_t>& bucket_counts, int ttl_start,
1724 int time_range, int bucket_size, int timekv,
1725 int num_buckets) {
11fdf7f2
TL		 1726#ifdef NDEBUG
1727 (void)time_range;
1728 (void)num_buckets;
1729#endif
7c673cae 1730 assert(time_range > 0 && timekv >= ttl_start && bucket_size > 0 &&
1e59de90 1731 timekv < (ttl_start + time_range) && num_buckets > 1);
7c673cae
FG		 1732 int bucket = (timekv - ttl_start) / bucket_size;
1733 bucket_counts[bucket]++;
1734}
1735
1736void PrintBucketCounts(const std::vector<uint64_t>& bucket_counts,
1737 int ttl_start, int ttl_end, int bucket_size,
1738 int num_buckets) {
1739 int time_point = ttl_start;
1e59de90 1740 for (int i = 0; i < num_buckets - 1; i++, time_point += bucket_size) {
7c673cae 1741 fprintf(stdout, "Keys in range %s to %s : %lu\n",
f67539c2
TL		 1742 TimeToHumanString(time_point).c_str(),
1743 TimeToHumanString(time_point + bucket_size).c_str(),
7c673cae
FG		 1744 (unsigned long)bucket_counts[i]);
1745 }
1746 fprintf(stdout, "Keys in range %s to %s : %lu\n",
f67539c2
TL		 1747 TimeToHumanString(time_point).c_str(),
1748 TimeToHumanString(ttl_end).c_str(),
7c673cae
FG		 1749 (unsigned long)bucket_counts[num_buckets - 1]);
1750}
1751
1752} // namespace
1753
1754const std::string InternalDumpCommand::ARG_COUNT_ONLY = "count_only";
1755const std::string InternalDumpCommand::ARG_COUNT_DELIM = "count_delim";
1756const std::string InternalDumpCommand::ARG_STATS = "stats";
1757const std::string InternalDumpCommand::ARG_INPUT_KEY_HEX = "input_key_hex";
1758
1759InternalDumpCommand::InternalDumpCommand(
11fdf7f2 1760 const std::vector<std::string>& /*params*/,
7c673cae
FG		 1761 const std::map<std::string, std::string>& options,
1762 const std::vector<std::string>& flags)
1e59de90
TL		 1763 : LDBCommand(options, flags, true,
1764 BuildCmdLineOptions(
1765 {ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM, ARG_TO,
1766 ARG_MAX_KEYS, ARG_COUNT_ONLY, ARG_COUNT_DELIM, ARG_STATS,
1767 ARG_INPUT_KEY_HEX, ARG_DECODE_BLOB_INDEX})),
7c673cae
FG		 1768 has_from_(false),
1769 has_to_(false),
1770 max_keys_(-1),
1771 delim_("."),
1772 count_only_(false),
1773 count_delim_(false),
1774 print_stats_(false),
1e59de90
TL		 1775 is_input_key_hex_(false),
1776 decode_blob_index_(false) {
7c673cae
FG		 1777 has_from_ = ParseStringOption(options, ARG_FROM, &from_);
1778 has_to_ = ParseStringOption(options, ARG_TO, &to_);
1779
1780 ParseIntOption(options, ARG_MAX_KEYS, max_keys_, exec_state_);
1e59de90 1781 auto itr = options.find(ARG_COUNT_DELIM);
7c673cae
FG		 1782 if (itr != options.end()) {
1783 delim_ = itr->second;
1784 count_delim_ = true;
1e59de90 1785 // fprintf(stdout,"delim = %c\n",delim_[0]);
7c673cae
FG		 1786 } else {
1787 count_delim_ = IsFlagPresent(flags, ARG_COUNT_DELIM);
1e59de90 1788 delim_ = ".";
7c673cae
FG		 1789 }
1790
1791 print_stats_ = IsFlagPresent(flags, ARG_STATS);
1792 count_only_ = IsFlagPresent(flags, ARG_COUNT_ONLY);
1793 is_input_key_hex_ = IsFlagPresent(flags, ARG_INPUT_KEY_HEX);
1e59de90 1794 decode_blob_index_ = IsFlagPresent(flags, ARG_DECODE_BLOB_INDEX);
7c673cae
FG		 1795
1796 if (is_input_key_hex_) {
1797 if (has_from_) {
1798 from_ = HexToString(from_);
1799 }
1800 if (has_to_) {
1801 to_ = HexToString(to_);
1802 }
1803 }
1804}
1805
1806void InternalDumpCommand::Help(std::string& ret) {
1807 ret.append(" ");
1808 ret.append(InternalDumpCommand::Name());
1809 ret.append(HelpRangeCmdArgs());
1810 ret.append(" [--" + ARG_INPUT_KEY_HEX + "]");
1811 ret.append(" [--" + ARG_MAX_KEYS + "=<N>]");
1812 ret.append(" [--" + ARG_COUNT_ONLY + "]");
1813 ret.append(" [--" + ARG_COUNT_DELIM + "=<char>]");
1814 ret.append(" [--" + ARG_STATS + "]");
1e59de90 1815 ret.append(" [--" + ARG_DECODE_BLOB_INDEX + "]");
7c673cae
FG		 1816 ret.append("\n");
1817}
1818
1819void InternalDumpCommand::DoCommand() {
1820 if (!db_) {
1821 assert(GetExecuteState().IsFailed());
1822 return;
1823 }
1824
1825 if (print_stats_) {
1826 std::string stats;
1827 if (db_->GetProperty(GetCfHandle(), "rocksdb.stats", &stats)) {
1828 fprintf(stdout, "%s\n", stats.c_str());
1829 }
1830 }
1831
1832 // Cast as DBImpl to get internal iterator
11fdf7f2 1833 std::vector<KeyVersion> key_versions;
f67539c2
TL		 1834 Status st = GetAllKeyVersions(db_, GetCfHandle(), from_, to_, max_keys_,
1835 &key_versions);
11fdf7f2
TL		 1836 if (!st.ok()) {
1837 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
7c673cae
FG		 1838 return;
1839 }
1840 std::string rtype1, rtype2, row, val;
1841 rtype2 = "";
1e59de90
TL		 1842 uint64_t c = 0;
1843 uint64_t s1 = 0, s2 = 0;
7c673cae
FG		 1844
1845 long long count = 0;
11fdf7f2 1846 for (auto& key_version : key_versions) {
1e59de90
TL		 1847 ValueType value_type = static_cast<ValueType>(key_version.type);
1848 InternalKey ikey(key_version.user_key, key_version.sequence, value_type);
11fdf7f2
TL		 1849 if (has_to_ && ikey.user_key() == to_) {
1850 // GetAllKeyVersions() includes keys with user key `to_`, but idump has
1851 // traditionally excluded such keys.
7c673cae
FG		 1852 break;
1853 }
7c673cae
FG		 1854 ++count;
1855 int k;
1856 if (count_delim_) {
1857 rtype1 = "";
1e59de90 1858 s1 = 0;
11fdf7f2
TL		 1859 row = ikey.Encode().ToString();
1860 val = key_version.value;
1e59de90
TL		 1861 for (k = 0; row[k] != '\x01' && row[k] != '\0'; k++) s1++;
1862 for (k = 0; val[k] != '\x01' && val[k] != '\0'; k++) s1++;
1863 for (int j = 0; row[j] != delim_[0] && row[j] != '\0' && row[j] != '\x01';
1864 j++)
1865 rtype1 += row[j];
1866 if (rtype2.compare("") && rtype2.compare(rtype1) != 0) {
11fdf7f2
TL		 1867 fprintf(stdout, "%s => count:%" PRIu64 "\tsize:%" PRIu64 "\n",
1868 rtype2.c_str(), c, s2);
1e59de90
TL		 1869 c = 1;
1870 s2 = s1;
7c673cae
FG		 1871 rtype2 = rtype1;
1872 } else {
1873 c++;
1e59de90
TL		 1874 s2 += s1;
1875 rtype2 = rtype1;
11fdf7f2 1876 }
7c673cae 1877 }
7c673cae
FG		 1878
1879 if (!count_only_ && !count_delim_) {
1880 std::string key = ikey.DebugString(is_key_hex_);
1e59de90
TL		 1881 Slice value(key_version.value);
1882 if (!decode_blob_index_ || value_type != kTypeBlobIndex) {
1883 fprintf(stdout, "%s => %s\n", key.c_str(),
1884 value.ToString(is_value_hex_).c_str());
1885 } else {
1886 BlobIndex blob_index;
1887
1888 const Status s = blob_index.DecodeFrom(value);
1889 if (!s.ok()) {
1890 fprintf(stderr, "%s => error decoding blob index =>\n", key.c_str());
1891 } else {
1892 fprintf(stdout, "%s => %s\n", key.c_str(),
1893 blob_index.DebugString(is_value_hex_).c_str());
1894 }
1895 }
7c673cae
FG		 1896 }
1897
1898 // Terminate if maximum number of keys have been dumped
1899 if (max_keys_ > 0 && count >= max_keys_) break;
1900 }
1e59de90 1901 if (count_delim_) {
11fdf7f2
TL		 1902 fprintf(stdout, "%s => count:%" PRIu64 "\tsize:%" PRIu64 "\n",
1903 rtype2.c_str(), c, s2);
1904 } else {
1905 fprintf(stdout, "Internal keys in range: %lld\n", count);
1906 }
7c673cae
FG		 1907}
1908
1909const std::string DBDumperCommand::ARG_COUNT_ONLY = "count_only";
1910const std::string DBDumperCommand::ARG_COUNT_DELIM = "count_delim";
1911const std::string DBDumperCommand::ARG_STATS = "stats";
1912const std::string DBDumperCommand::ARG_TTL_BUCKET = "bucket";
1913
1914DBDumperCommand::DBDumperCommand(
11fdf7f2 1915 const std::vector<std::string>& /*params*/,
7c673cae
FG		 1916 const std::map<std::string, std::string>& options,
1917 const std::vector<std::string>& flags)
1e59de90
TL		 1918 : LDBCommand(
1919 options, flags, true,
1920 BuildCmdLineOptions(
1921 {ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM, ARG_TO,
1922 ARG_MAX_KEYS, ARG_COUNT_ONLY, ARG_COUNT_DELIM, ARG_STATS,
1923 ARG_TTL_START, ARG_TTL_END, ARG_TTL_BUCKET, ARG_TIMESTAMP,
1924 ARG_PATH, ARG_DECODE_BLOB_INDEX, ARG_DUMP_UNCOMPRESSED_BLOBS})),
7c673cae
FG		 1925 null_from_(true),
1926 null_to_(true),
1927 max_keys_(-1),
1928 count_only_(false),
1929 count_delim_(false),
1e59de90
TL		 1930 print_stats_(false),
1931 decode_blob_index_(false) {
1932 auto itr = options.find(ARG_FROM);
7c673cae
FG		 1933 if (itr != options.end()) {
1934 null_from_ = false;
1935 from_ = itr->second;
1936 }
1937
1938 itr = options.find(ARG_TO);
1939 if (itr != options.end()) {
1940 null_to_ = false;
1941 to_ = itr->second;
1942 }
1943
1944 itr = options.find(ARG_MAX_KEYS);
1945 if (itr != options.end()) {
1946 try {
1947#if defined(CYGWIN)
1948 max_keys_ = strtol(itr->second.c_str(), 0, 10);
1949#else
1950 max_keys_ = std::stoi(itr->second);
1951#endif
1952 } catch (const std::invalid_argument&) {
1953 exec_state_ = LDBCommandExecuteResult::Failed(ARG_MAX_KEYS +
1954 " has an invalid value");
1955 } catch (const std::out_of_range&) {
1956 exec_state_ = LDBCommandExecuteResult::Failed(
1957 ARG_MAX_KEYS + " has a value out-of-range");
1958 }
1959 }
1960 itr = options.find(ARG_COUNT_DELIM);
1961 if (itr != options.end()) {
1962 delim_ = itr->second;
1963 count_delim_ = true;
1964 } else {
1965 count_delim_ = IsFlagPresent(flags, ARG_COUNT_DELIM);
1e59de90 1966 delim_ = ".";
7c673cae
FG		 1967 }
1968
1969 print_stats_ = IsFlagPresent(flags, ARG_STATS);
1970 count_only_ = IsFlagPresent(flags, ARG_COUNT_ONLY);
1e59de90
TL		 1971 decode_blob_index_ = IsFlagPresent(flags, ARG_DECODE_BLOB_INDEX);
1972 dump_uncompressed_blobs_ = IsFlagPresent(flags, ARG_DUMP_UNCOMPRESSED_BLOBS);
7c673cae
FG		 1973
1974 if (is_key_hex_) {
1975 if (!null_from_) {
1976 from_ = HexToString(from_);
1977 }
1978 if (!null_to_) {
1979 to_ = HexToString(to_);
1980 }
1981 }
1982
1983 itr = options.find(ARG_PATH);
1984 if (itr != options.end()) {
1985 path_ = itr->second;
11fdf7f2
TL		 1986 if (db_path_.empty()) {
1987 db_path_ = path_;
1988 }
7c673cae
FG		 1989 }
1990}
1991
1992void DBDumperCommand::Help(std::string& ret) {
1993 ret.append(" ");
1994 ret.append(DBDumperCommand::Name());
1995 ret.append(HelpRangeCmdArgs());
1996 ret.append(" [--" + ARG_TTL + "]");
1997 ret.append(" [--" + ARG_MAX_KEYS + "=<N>]");
1998 ret.append(" [--" + ARG_TIMESTAMP + "]");
1999 ret.append(" [--" + ARG_COUNT_ONLY + "]");
2000 ret.append(" [--" + ARG_COUNT_DELIM + "=<char>]");
2001 ret.append(" [--" + ARG_STATS + "]");
2002 ret.append(" [--" + ARG_TTL_BUCKET + "=<N>]");
2003 ret.append(" [--" + ARG_TTL_START + "=<N>:- is inclusive]");
2004 ret.append(" [--" + ARG_TTL_END + "=<N>:- is exclusive]");
2005 ret.append(" [--" + ARG_PATH + "=<path_to_a_file>]");
1e59de90
TL		 2006 ret.append(" [--" + ARG_DECODE_BLOB_INDEX + "]");
2007 ret.append(" [--" + ARG_DUMP_UNCOMPRESSED_BLOBS + "]");
7c673cae
FG		 2008 ret.append("\n");
2009}
2010
2011/**
2012 * Handles two separate cases:
2013 *
2014 * 1) --db is specified - just dump the database.
2015 *
2016 * 2) --path is specified - determine based on file extension what dumping
2017 * function to call. Please note that we intentionally use the extension
2018 * and avoid probing the file contents under the assumption that renaming
2019 * the files is not a supported scenario.
2020 *
2021 */
2022void DBDumperCommand::DoCommand() {
2023 if (!db_) {
2024 assert(!path_.empty());
2025 std::string fileName = GetFileNameFromPath(path_);
2026 uint64_t number;
2027 FileType type;
2028
2029 exec_state_ = LDBCommandExecuteResult::Succeed("");
2030
2031 if (!ParseFileName(fileName, &number, &type)) {
2032 exec_state_ =
2033 LDBCommandExecuteResult::Failed("Can't parse file type: " + path_);
2034 return;
2035 }
2036
2037 switch (type) {
20effc67 2038 case kWalFile:
11fdf7f2 2039 // TODO(myabandeh): allow configuring is_write_commited
494da23a
TL		 2040 DumpWalFile(options_, path_, /* print_header_ */ true,
2041 /* print_values_ */ true, true /* is_write_commited */,
2042 &exec_state_);
7c673cae
FG		 2043 break;
2044 case kTableFile:
1e59de90
TL		 2045 DumpSstFile(options_, path_, is_key_hex_, /* show_properties */ true,
2046 decode_blob_index_, from_, to_);
7c673cae
FG		 2047 break;
2048 case kDescriptorFile:
494da23a 2049 DumpManifestFile(options_, path_, /* verbose_ */ false, is_key_hex_,
7c673cae
FG		 2050 /* json_ */ false);
2051 break;
1e59de90
TL		 2052 case kBlobFile:
2053 DumpBlobFile(path_, is_key_hex_, is_value_hex_,
2054 dump_uncompressed_blobs_);
2055 break;
7c673cae
FG		 2056 default:
2057 exec_state_ = LDBCommandExecuteResult::Failed(
2058 "File type not supported: " + path_);
2059 break;
2060 }
2061
2062 } else {
2063 DoDumpCommand();
2064 }
2065}
2066
2067void DBDumperCommand::DoDumpCommand() {
2068 assert(nullptr != db_);
2069 assert(path_.empty());
2070
2071 // Parse command line args
2072 uint64_t count = 0;
2073 if (print_stats_) {
2074 std::string stats;
2075 if (db_->GetProperty("rocksdb.stats", &stats)) {
2076 fprintf(stdout, "%s\n", stats.c_str());
2077 }
2078 }
2079
2080 // Setup key iterator
494da23a
TL		 2081 ReadOptions scan_read_opts;
2082 scan_read_opts.total_order_seek = true;
2083 Iterator* iter = db_->NewIterator(scan_read_opts, GetCfHandle());
7c673cae
FG		 2084 Status st = iter->status();
2085 if (!st.ok()) {
2086 exec_state_ =
2087 LDBCommandExecuteResult::Failed("Iterator error." + st.ToString());
2088 }
2089
2090 if (!null_from_) {
2091 iter->Seek(from_);
2092 } else {
2093 iter->SeekToFirst();
2094 }
2095
2096 int max_keys = max_keys_;
2097 int ttl_start;
2098 if (!ParseIntOption(option_map_, ARG_TTL_START, ttl_start, exec_state_)) {
2099 ttl_start = DBWithTTLImpl::kMinTimestamp; // TTL introduction time
2100 }
2101 int ttl_end;
2102 if (!ParseIntOption(option_map_, ARG_TTL_END, ttl_end, exec_state_)) {
2103 ttl_end = DBWithTTLImpl::kMaxTimestamp; // Max time allowed by TTL feature
2104 }
2105 if (ttl_end < ttl_start) {
2106 fprintf(stderr, "Error: End time can't be less than start time\n");
2107 delete iter;
2108 return;
2109 }
2110 int time_range = ttl_end - ttl_start;
2111 int bucket_size;
2112 if (!ParseIntOption(option_map_, ARG_TTL_BUCKET, bucket_size, exec_state_) ||
2113 bucket_size <= 0) {
1e59de90 2114 bucket_size = time_range; // Will have just 1 bucket by default
7c673cae 2115 }
1e59de90 2116 // cretaing variables for row count of each type
7c673cae
FG		 2117 std::string rtype1, rtype2, row, val;
2118 rtype2 = "";
1e59de90
TL		 2119 uint64_t c = 0;
2120 uint64_t s1 = 0, s2 = 0;
7c673cae
FG		 2121
2122 // At this point, bucket_size=0 => time_range=0
2123 int num_buckets = (bucket_size >= time_range)
2124 ? 1
2125 : ((time_range + bucket_size - 1) / bucket_size);
2126 std::vector<uint64_t> bucket_counts(num_buckets, 0);
2127 if (is_db_ttl_ && !count_only_ && timestamp_ && !count_delim_) {
2128 fprintf(stdout, "Dumping key-values from %s to %s\n",
f67539c2
TL		 2129 TimeToHumanString(ttl_start).c_str(),
2130 TimeToHumanString(ttl_end).c_str());
7c673cae
FG		 2131 }
2132
11fdf7f2
TL		 2133 HistogramImpl vsize_hist;
2134
7c673cae
FG		 2135 for (; iter->Valid(); iter->Next()) {
2136 int rawtime = 0;
2137 // If end marker was specified, we stop before it
1e59de90 2138 if (!null_to_ && (iter->key().ToString() >= to_)) break;
7c673cae 2139 // Terminate if maximum number of keys have been dumped
1e59de90 2140 if (max_keys == 0) break;
7c673cae 2141 if (is_db_ttl_) {
20effc67
TL		 2142 TtlIterator* it_ttl = static_cast_with_check<TtlIterator>(iter);
2143 rawtime = it_ttl->ttl_timestamp();
7c673cae
FG		 2144 if (rawtime < ttl_start || rawtime >= ttl_end) {
2145 continue;
2146 }
2147 }
2148 if (max_keys > 0) {
2149 --max_keys;
2150 }
2151 if (is_db_ttl_ && num_buckets > 1) {
2152 IncBucketCounts(bucket_counts, ttl_start, time_range, bucket_size,
2153 rawtime, num_buckets);
2154 }
2155 ++count;
2156 if (count_delim_) {
2157 rtype1 = "";
2158 row = iter->key().ToString();
2159 val = iter->value().ToString();
1e59de90
TL		 2160 s1 = row.size() + val.size();
2161 for (int j = 0; row[j] != delim_[0] && row[j] != '\0'; j++)
2162 rtype1 += row[j];
2163 if (rtype2.compare("") && rtype2.compare(rtype1) != 0) {
11fdf7f2
TL		 2164 fprintf(stdout, "%s => count:%" PRIu64 "\tsize:%" PRIu64 "\n",
2165 rtype2.c_str(), c, s2);
1e59de90
TL		 2166 c = 1;
2167 s2 = s1;
7c673cae
FG		 2168 rtype2 = rtype1;
2169 } else {
1e59de90
TL		 2170 c++;
2171 s2 += s1;
2172 rtype2 = rtype1;
7c673cae 2173 }
7c673cae
FG		 2174 }
2175
11fdf7f2
TL		 2176 if (count_only_) {
2177 vsize_hist.Add(iter->value().size());
2178 }
7c673cae
FG		 2179
2180 if (!count_only_ && !count_delim_) {
2181 if (is_db_ttl_ && timestamp_) {
f67539c2 2182 fprintf(stdout, "%s ", TimeToHumanString(rawtime).c_str());
7c673cae
FG		 2183 }
2184 std::string str =
2185 PrintKeyValue(iter->key().ToString(), iter->value().ToString(),
2186 is_key_hex_, is_value_hex_);
2187 fprintf(stdout, "%s\n", str.c_str());
2188 }
2189 }
2190
2191 if (num_buckets > 1 && is_db_ttl_) {
2192 PrintBucketCounts(bucket_counts, ttl_start, ttl_end, bucket_size,
2193 num_buckets);
1e59de90 2194 } else if (count_delim_) {
11fdf7f2
TL		 2195 fprintf(stdout, "%s => count:%" PRIu64 "\tsize:%" PRIu64 "\n",
2196 rtype2.c_str(), c, s2);
7c673cae 2197 } else {
11fdf7f2
TL		 2198 fprintf(stdout, "Keys in range: %" PRIu64 "\n", count);
2199 }
2200
2201 if (count_only_) {
2202 fprintf(stdout, "Value size distribution: \n");
2203 fprintf(stdout, "%s\n", vsize_hist.ToString().c_str());
7c673cae
FG		 2204 }
2205 // Clean up
2206 delete iter;
2207}
2208
2209const std::string ReduceDBLevelsCommand::ARG_NEW_LEVELS = "new_levels";
2210const std::string ReduceDBLevelsCommand::ARG_PRINT_OLD_LEVELS =
2211 "print_old_levels";
2212
2213ReduceDBLevelsCommand::ReduceDBLevelsCommand(
11fdf7f2 2214 const std::vector<std::string>& /*params*/,
7c673cae
FG		 2215 const std::map<std::string, std::string>& options,
2216 const std::vector<std::string>& flags)
2217 : LDBCommand(options, flags, false,
2218 BuildCmdLineOptions({ARG_NEW_LEVELS, ARG_PRINT_OLD_LEVELS})),
2219 old_levels_(1 << 7),
2220 new_levels_(-1),
2221 print_old_levels_(false) {
2222 ParseIntOption(option_map_, ARG_NEW_LEVELS, new_levels_, exec_state_);
2223 print_old_levels_ = IsFlagPresent(flags, ARG_PRINT_OLD_LEVELS);
2224
1e59de90 2225 if (new_levels_ <= 0) {
7c673cae
FG		 2226 exec_state_ = LDBCommandExecuteResult::Failed(
2227 " Use --" + ARG_NEW_LEVELS + " to specify a new level number\n");
2228 }
2229}
2230
2231std::vector<std::string> ReduceDBLevelsCommand::PrepareArgs(
2232 const std::string& db_path, int new_levels, bool print_old_level) {
2233 std::vector<std::string> ret;
2234 ret.push_back("reduce_levels");
2235 ret.push_back("--" + ARG_DB + "=" + db_path);
1e59de90
TL		 2236 ret.push_back("--" + ARG_NEW_LEVELS + "=" + std::to_string(new_levels));
2237 if (print_old_level) {
7c673cae
FG		 2238 ret.push_back("--" + ARG_PRINT_OLD_LEVELS);
2239 }
2240 return ret;
2241}
2242
2243void ReduceDBLevelsCommand::Help(std::string& ret) {
2244 ret.append(" ");
2245 ret.append(ReduceDBLevelsCommand::Name());
2246 ret.append(" --" + ARG_NEW_LEVELS + "=<New number of levels>");
2247 ret.append(" [--" + ARG_PRINT_OLD_LEVELS + "]");
2248 ret.append("\n");
2249}
2250
1e59de90
TL		 2251void ReduceDBLevelsCommand::OverrideBaseCFOptions(
2252 ColumnFamilyOptions* cf_opts) {
2253 LDBCommand::OverrideBaseCFOptions(cf_opts);
2254 cf_opts->num_levels = old_levels_;
2255 cf_opts->max_bytes_for_level_multiplier_additional.resize(cf_opts->num_levels,
20effc67 2256 1);
7c673cae 2257 // Disable size compaction
1e59de90
TL		 2258 cf_opts->max_bytes_for_level_base = 1ULL << 50;
2259 cf_opts->max_bytes_for_level_multiplier = 1;
7c673cae
FG		 2260}
2261
1e59de90 2262Status ReduceDBLevelsCommand::GetOldNumOfLevels(Options& opt, int* levels) {
7c673cae
FG		 2263 ImmutableDBOptions db_options(opt);
2264 EnvOptions soptions;
2265 std::shared_ptr<Cache> tc(
2266 NewLRUCache(opt.max_open_files - 10, opt.table_cache_numshardbits));
2267 const InternalKeyComparator cmp(opt.comparator);
2268 WriteController wc(opt.delayed_write_rate);
2269 WriteBufferManager wb(opt.db_write_buffer_size);
f67539c2 2270 VersionSet versions(db_path_, &db_options, soptions, tc.get(), &wb, &wc,
1e59de90
TL		 2271 /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
2272 /*db_id*/ "", /*db_session_id*/ "");
7c673cae
FG		 2273 std::vector<ColumnFamilyDescriptor> dummy;
2274 ColumnFamilyDescriptor dummy_descriptor(kDefaultColumnFamilyName,
2275 ColumnFamilyOptions(opt));
2276 dummy.push_back(dummy_descriptor);
2277 // We rely the VersionSet::Recover to tell us the internal data structures
2278 // in the db. And the Recover() should never do any change
2279 // (like LogAndApply) to the manifest file.
2280 Status st = versions.Recover(dummy);
2281 if (!st.ok()) {
2282 return st;
2283 }
2284 int max = -1;
2285 auto default_cfd = versions.GetColumnFamilySet()->GetDefault();
2286 for (int i = 0; i < default_cfd->NumberLevels(); i++) {
2287 if (default_cfd->current()->storage_info()->NumLevelFiles(i)) {
2288 max = i;
2289 }
2290 }
2291
2292 *levels = max + 1;
2293 return st;
2294}
2295
2296void ReduceDBLevelsCommand::DoCommand() {
2297 if (new_levels_ <= 1) {
2298 exec_state_ =
2299 LDBCommandExecuteResult::Failed("Invalid number of levels.\n");
2300 return;
2301 }
2302
2303 Status st;
20effc67 2304 PrepareOptions();
7c673cae 2305 int old_level_num = -1;
20effc67 2306 st = GetOldNumOfLevels(options_, &old_level_num);
7c673cae
FG		 2307 if (!st.ok()) {
2308 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2309 return;
2310 }
2311
2312 if (print_old_levels_) {
2313 fprintf(stdout, "The old number of levels in use is %d\n", old_level_num);
2314 }
2315
2316 if (old_level_num <= new_levels_) {
2317 return;
2318 }
2319
2320 old_levels_ = old_level_num;
2321
2322 OpenDB();
2323 if (exec_state_.IsFailed()) {
2324 return;
2325 }
11fdf7f2 2326 assert(db_ != nullptr);
7c673cae
FG		 2327 // Compact the whole DB to put all files to the highest level.
2328 fprintf(stdout, "Compacting the db...\n");
1e59de90
TL		 2329 st =
2330 db_->CompactRange(CompactRangeOptions(), GetCfHandle(), nullptr, nullptr);
2331
7c673cae
FG		 2332 CloseDB();
2333
1e59de90
TL		 2334 if (st.ok()) {
2335 EnvOptions soptions;
2336 st = VersionSet::ReduceNumberOfLevels(db_path_, &options_, soptions,
2337 new_levels_);
2338 }
7c673cae
FG		 2339 if (!st.ok()) {
2340 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2341 return;
2342 }
2343}
2344
2345const std::string ChangeCompactionStyleCommand::ARG_OLD_COMPACTION_STYLE =
2346 "old_compaction_style";
2347const std::string ChangeCompactionStyleCommand::ARG_NEW_COMPACTION_STYLE =
2348 "new_compaction_style";
2349
2350ChangeCompactionStyleCommand::ChangeCompactionStyleCommand(
11fdf7f2 2351 const std::vector<std::string>& /*params*/,
7c673cae
FG		 2352 const std::map<std::string, std::string>& options,
2353 const std::vector<std::string>& flags)
2354 : LDBCommand(options, flags, false,
2355 BuildCmdLineOptions(
2356 {ARG_OLD_COMPACTION_STYLE, ARG_NEW_COMPACTION_STYLE})),
2357 old_compaction_style_(-1),
2358 new_compaction_style_(-1) {
2359 ParseIntOption(option_map_, ARG_OLD_COMPACTION_STYLE, old_compaction_style_,
1e59de90 2360 exec_state_);
7c673cae 2361 if (old_compaction_style_ != kCompactionStyleLevel &&
1e59de90 2362 old_compaction_style_ != kCompactionStyleUniversal) {
7c673cae
FG		 2363 exec_state_ = LDBCommandExecuteResult::Failed(
2364 "Use --" + ARG_OLD_COMPACTION_STYLE + " to specify old compaction " +
2365 "style. Check ldb help for proper compaction style value.\n");
2366 return;
2367 }
2368
2369 ParseIntOption(option_map_, ARG_NEW_COMPACTION_STYLE, new_compaction_style_,
1e59de90 2370 exec_state_);
7c673cae 2371 if (new_compaction_style_ != kCompactionStyleLevel &&
1e59de90 2372 new_compaction_style_ != kCompactionStyleUniversal) {
7c673cae
FG		 2373 exec_state_ = LDBCommandExecuteResult::Failed(
2374 "Use --" + ARG_NEW_COMPACTION_STYLE + " to specify new compaction " +
2375 "style. Check ldb help for proper compaction style value.\n");
2376 return;
2377 }
2378
2379 if (new_compaction_style_ == old_compaction_style_) {
2380 exec_state_ = LDBCommandExecuteResult::Failed(
2381 "Old compaction style is the same as new compaction style. "
2382 "Nothing to do.\n");
2383 return;
2384 }
2385
2386 if (old_compaction_style_ == kCompactionStyleUniversal &&
2387 new_compaction_style_ == kCompactionStyleLevel) {
2388 exec_state_ = LDBCommandExecuteResult::Failed(
2389 "Convert from universal compaction to level compaction. "
2390 "Nothing to do.\n");
2391 return;
2392 }
2393}
2394
2395void ChangeCompactionStyleCommand::Help(std::string& ret) {
2396 ret.append(" ");
2397 ret.append(ChangeCompactionStyleCommand::Name());
2398 ret.append(" --" + ARG_OLD_COMPACTION_STYLE + "=<Old compaction style: 0 " +
2399 "for level compaction, 1 for universal compaction>");
2400 ret.append(" --" + ARG_NEW_COMPACTION_STYLE + "=<New compaction style: 0 " +
2401 "for level compaction, 1 for universal compaction>");
2402 ret.append("\n");
2403}
2404
1e59de90
TL		 2405void ChangeCompactionStyleCommand::OverrideBaseCFOptions(
2406 ColumnFamilyOptions* cf_opts) {
2407 LDBCommand::OverrideBaseCFOptions(cf_opts);
7c673cae
FG		 2408 if (old_compaction_style_ == kCompactionStyleLevel &&
2409 new_compaction_style_ == kCompactionStyleUniversal) {
2410 // In order to convert from level compaction to universal compaction, we
2411 // need to compact all data into a single file and move it to level 0.
1e59de90
TL		 2412 cf_opts->disable_auto_compactions = true;
2413 cf_opts->target_file_size_base = INT_MAX;
2414 cf_opts->target_file_size_multiplier = 1;
2415 cf_opts->max_bytes_for_level_base = INT_MAX;
2416 cf_opts->max_bytes_for_level_multiplier = 1;
7c673cae 2417 }
7c673cae
FG		 2418}
2419
2420void ChangeCompactionStyleCommand::DoCommand() {
20effc67
TL		 2421 if (!db_) {
2422 assert(GetExecuteState().IsFailed());
2423 return;
2424 }
7c673cae
FG		 2425 // print db stats before we have made any change
2426 std::string property;
2427 std::string files_per_level;
2428 for (int i = 0; i < db_->NumberLevels(GetCfHandle()); i++) {
2429 db_->GetProperty(GetCfHandle(),
1e59de90 2430 "rocksdb.num-files-at-level" + std::to_string(i),
7c673cae
FG		 2431 &property);
2432
2433 // format print string
2434 char buf[100];
2435 snprintf(buf, sizeof(buf), "%s%s", (i ? "," : ""), property.c_str());
2436 files_per_level += buf;
2437 }
2438 fprintf(stdout, "files per level before compaction: %s\n",
2439 files_per_level.c_str());
2440
2441 // manual compact into a single file and move the file to level 0
2442 CompactRangeOptions compact_options;
2443 compact_options.change_level = true;
2444 compact_options.target_level = 0;
1e59de90
TL		 2445 Status s =
2446 db_->CompactRange(compact_options, GetCfHandle(), nullptr, nullptr);
2447 if (!s.ok()) {
2448 std::stringstream oss;
2449 oss << "Compaction failed: " << s.ToString();
2450 exec_state_ = LDBCommandExecuteResult::Failed(oss.str());
2451 return;
2452 }
7c673cae
FG		 2453
2454 // verify compaction result
2455 files_per_level = "";
2456 int num_files = 0;
2457 for (int i = 0; i < db_->NumberLevels(GetCfHandle()); i++) {
2458 db_->GetProperty(GetCfHandle(),
1e59de90 2459 "rocksdb.num-files-at-level" + std::to_string(i),
7c673cae
FG		 2460 &property);
2461
2462 // format print string
2463 char buf[100];
2464 snprintf(buf, sizeof(buf), "%s%s", (i ? "," : ""), property.c_str());
2465 files_per_level += buf;
2466
2467 num_files = atoi(property.c_str());
2468
2469 // level 0 should have only 1 file
2470 if (i == 0 && num_files != 1) {
2471 exec_state_ = LDBCommandExecuteResult::Failed(
2472 "Number of db files at "
2473 "level 0 after compaction is " +
1e59de90 2474 std::to_string(num_files) + ", not 1.\n");
7c673cae
FG		 2475 return;
2476 }
2477 // other levels should have no file
2478 if (i > 0 && num_files != 0) {
2479 exec_state_ = LDBCommandExecuteResult::Failed(
2480 "Number of db files at "
2481 "level " +
1e59de90
TL		 2482 std::to_string(i) + " after compaction is " +
2483 std::to_string(num_files) + ", not 0.\n");
7c673cae
FG		 2484 return;
2485 }
2486 }
2487
2488 fprintf(stdout, "files per level after compaction: %s\n",
2489 files_per_level.c_str());
2490}
2491
2492// ----------------------------------------------------------------------------
2493
2494namespace {
2495
2496struct StdErrReporter : public log::Reader::Reporter {
494da23a 2497 void Corruption(size_t /*bytes*/, const Status& s) override {
7c673cae
FG		 2498 std::cerr << "Corruption detected in log file " << s.ToString() << "\n";
2499 }
2500};
2501
2502class InMemoryHandler : public WriteBatch::Handler {
2503 public:
11fdf7f2
TL		 2504 InMemoryHandler(std::stringstream& row, bool print_values,
2505 bool write_after_commit = false)
2506 : Handler(),
2507 row_(row),
2508 print_values_(print_values),
2509 write_after_commit_(write_after_commit) {}
7c673cae
FG		 2510
2511 void commonPutMerge(const Slice& key, const Slice& value) {
2512 std::string k = LDBCommand::StringToHex(key.ToString());
2513 if (print_values_) {
2514 std::string v = LDBCommand::StringToHex(value.ToString());
2515 row_ << k << " : ";
2516 row_ << v << " ";
2517 } else {
2518 row_ << k << " ";
2519 }
2520 }
2521
494da23a 2522 Status PutCF(uint32_t cf, const Slice& key, const Slice& value) override {
7c673cae
FG		 2523 row_ << "PUT(" << cf << ") : ";
2524 commonPutMerge(key, value);
2525 return Status::OK();
2526 }
2527
494da23a 2528 Status MergeCF(uint32_t cf, const Slice& key, const Slice& value) override {
7c673cae
FG		 2529 row_ << "MERGE(" << cf << ") : ";
2530 commonPutMerge(key, value);
2531 return Status::OK();
2532 }
2533
494da23a 2534 Status MarkNoop(bool) override {
11fdf7f2
TL		 2535 row_ << "NOOP ";
2536 return Status::OK();
2537 }
2538
494da23a 2539 Status DeleteCF(uint32_t cf, const Slice& key) override {
7c673cae
FG		 2540 row_ << "DELETE(" << cf << ") : ";
2541 row_ << LDBCommand::StringToHex(key.ToString()) << " ";
2542 return Status::OK();
2543 }
2544
494da23a 2545 Status SingleDeleteCF(uint32_t cf, const Slice& key) override {
7c673cae
FG		 2546 row_ << "SINGLE_DELETE(" << cf << ") : ";
2547 row_ << LDBCommand::StringToHex(key.ToString()) << " ";
2548 return Status::OK();
2549 }
2550
494da23a
TL		 2551 Status DeleteRangeCF(uint32_t cf, const Slice& begin_key,
2552 const Slice& end_key) override {
7c673cae
FG		 2553 row_ << "DELETE_RANGE(" << cf << ") : ";
2554 row_ << LDBCommand::StringToHex(begin_key.ToString()) << " ";
2555 row_ << LDBCommand::StringToHex(end_key.ToString()) << " ";
2556 return Status::OK();
2557 }
2558
494da23a 2559 Status MarkBeginPrepare(bool unprepare) override {
11fdf7f2
TL		 2560 row_ << "BEGIN_PREPARE(";
2561 row_ << (unprepare ? "true" : "false") << ") ";
7c673cae
FG		 2562 return Status::OK();
2563 }
2564
494da23a 2565 Status MarkEndPrepare(const Slice& xid) override {
7c673cae
FG		 2566 row_ << "END_PREPARE(";
2567 row_ << LDBCommand::StringToHex(xid.ToString()) << ") ";
2568 return Status::OK();
2569 }
2570
494da23a 2571 Status MarkRollback(const Slice& xid) override {
7c673cae
FG		 2572 row_ << "ROLLBACK(";
2573 row_ << LDBCommand::StringToHex(xid.ToString()) << ") ";
2574 return Status::OK();
2575 }
2576
494da23a 2577 Status MarkCommit(const Slice& xid) override {
7c673cae
FG		 2578 row_ << "COMMIT(";
2579 row_ << LDBCommand::StringToHex(xid.ToString()) << ") ";
2580 return Status::OK();
2581 }
2582
1e59de90
TL		 2583 Status MarkCommitWithTimestamp(const Slice& xid,
2584 const Slice& commit_ts) override {
2585 row_ << "COMMIT_WITH_TIMESTAMP(";
2586 row_ << LDBCommand::StringToHex(xid.ToString()) << ", ";
2587 row_ << LDBCommand::StringToHex(commit_ts.ToString()) << ") ";
2588 return Status::OK();
2589 }
2590
494da23a 2591 ~InMemoryHandler() override {}
7c673cae 2592
11fdf7f2 2593 protected:
1e59de90
TL		 2594 Handler::OptionState WriteAfterCommit() const override {
2595 return write_after_commit_ ? Handler::OptionState::kEnabled
2596 : Handler::OptionState::kDisabled;
2597 }
11fdf7f2 2598
7c673cae
FG		 2599 private:
2600 std::stringstream& row_;
2601 bool print_values_;
11fdf7f2 2602 bool write_after_commit_;
7c673cae
FG		 2603};
2604
494da23a
TL		 2605void DumpWalFile(Options options, std::string wal_file, bool print_header,
2606 bool print_values, bool is_write_committed,
2607 LDBCommandExecuteResult* exec_state) {
1e59de90
TL		 2608 const auto& fs = options.env->GetFileSystem();
2609 FileOptions soptions(options);
494da23a 2610 std::unique_ptr<SequentialFileReader> wal_file_reader;
1e59de90
TL		 2611 Status status = SequentialFileReader::Create(
2612 fs, wal_file, soptions, &wal_file_reader, nullptr /* dbg */,
2613 nullptr /* rate_limiter */);
7c673cae
FG		 2614 if (!status.ok()) {
2615 if (exec_state) {
2616 *exec_state = LDBCommandExecuteResult::Failed("Failed to open WAL file " +
2617 status.ToString());
2618 } else {
2619 std::cerr << "Error: Failed to open WAL file " << status.ToString()
2620 << std::endl;
2621 }
2622 } else {
2623 StdErrReporter reporter;
2624 uint64_t log_number;
2625 FileType type;
2626
2627 // we need the log number, but ParseFilename expects dbname/NNN.log.
2628 std::string sanitized = wal_file;
2629 size_t lastslash = sanitized.rfind('/');
2630 if (lastslash != std::string::npos)
2631 sanitized = sanitized.substr(lastslash + 1);
2632 if (!ParseFileName(sanitized, &log_number, &type)) {
2633 // bogus input, carry on as best we can
2634 log_number = 0;
2635 }
494da23a
TL		 2636 log::Reader reader(options.info_log, std::move(wal_file_reader), &reporter,
2637 true /* checksum */, log_number);
7c673cae
FG		 2638 std::string scratch;
2639 WriteBatch batch;
2640 Slice record;
2641 std::stringstream row;
2642 if (print_header) {
2643 std::cout << "Sequence,Count,ByteSize,Physical Offset,Key(s)";
2644 if (print_values) {
2645 std::cout << " : value ";
2646 }
2647 std::cout << "\n";
2648 }
1e59de90 2649 while (status.ok() && reader.ReadRecord(&record, &scratch)) {
7c673cae
FG		 2650 row.str("");
2651 if (record.size() < WriteBatchInternal::kHeader) {
2652 reporter.Corruption(record.size(),
2653 Status::Corruption("log record too small"));
2654 } else {
1e59de90
TL		 2655 status = WriteBatchInternal::SetContents(&batch, record);
2656 if (!status.ok()) {
2657 std::stringstream oss;
2658 oss << "Parsing write batch failed: " << status.ToString();
2659 if (exec_state) {
2660 *exec_state = LDBCommandExecuteResult::Failed(oss.str());
2661 } else {
2662 std::cerr << oss.str() << std::endl;
2663 }
2664 break;
2665 }
7c673cae
FG		 2666 row << WriteBatchInternal::Sequence(&batch) << ",";
2667 row << WriteBatchInternal::Count(&batch) << ",";
2668 row << WriteBatchInternal::ByteSize(&batch) << ",";
2669 row << reader.LastRecordOffset() << ",";
11fdf7f2 2670 InMemoryHandler handler(row, print_values, is_write_committed);
1e59de90
TL		 2671 status = batch.Iterate(&handler);
2672 if (!status.ok()) {
2673 if (exec_state) {
2674 std::stringstream oss;
2675 oss << "Print write batch error: " << status.ToString();
2676 *exec_state = LDBCommandExecuteResult::Failed(oss.str());
2677 }
2678 row << "error: " << status.ToString();
2679 break;
2680 }
7c673cae
FG		 2681 row << "\n";
2682 }
2683 std::cout << row.str();
2684 }
2685 }
2686}
2687
2688} // namespace
2689
2690const std::string WALDumperCommand::ARG_WAL_FILE = "walfile";
11fdf7f2 2691const std::string WALDumperCommand::ARG_WRITE_COMMITTED = "write_committed";
7c673cae
FG		 2692const std::string WALDumperCommand::ARG_PRINT_VALUE = "print_value";
2693const std::string WALDumperCommand::ARG_PRINT_HEADER = "header";
2694
2695WALDumperCommand::WALDumperCommand(
11fdf7f2 2696 const std::vector<std::string>& /*params*/,
7c673cae
FG		 2697 const std::map<std::string, std::string>& options,
2698 const std::vector<std::string>& flags)
2699 : LDBCommand(options, flags, true,
11fdf7f2
TL		 2700 BuildCmdLineOptions({ARG_WAL_FILE, ARG_WRITE_COMMITTED,
2701 ARG_PRINT_HEADER, ARG_PRINT_VALUE})),
7c673cae 2702 print_header_(false),
11fdf7f2
TL		 2703 print_values_(false),
2704 is_write_committed_(false) {
7c673cae
FG		 2705 wal_file_.clear();
2706
1e59de90 2707 auto itr = options.find(ARG_WAL_FILE);
7c673cae
FG		 2708 if (itr != options.end()) {
2709 wal_file_ = itr->second;
2710 }
2711
7c673cae
FG		 2712 print_header_ = IsFlagPresent(flags, ARG_PRINT_HEADER);
2713 print_values_ = IsFlagPresent(flags, ARG_PRINT_VALUE);
11fdf7f2
TL		 2714 is_write_committed_ = ParseBooleanOption(options, ARG_WRITE_COMMITTED, true);
2715
7c673cae
FG		 2716 if (wal_file_.empty()) {
2717 exec_state_ = LDBCommandExecuteResult::Failed("Argument " + ARG_WAL_FILE +
2718 " must be specified.");
2719 }
2720}
2721
2722void WALDumperCommand::Help(std::string& ret) {
2723 ret.append(" ");
2724 ret.append(WALDumperCommand::Name());
2725 ret.append(" --" + ARG_WAL_FILE + "=<write_ahead_log_file_path>");
2726 ret.append(" [--" + ARG_PRINT_HEADER + "] ");
2727 ret.append(" [--" + ARG_PRINT_VALUE + "] ");
11fdf7f2 2728 ret.append(" [--" + ARG_WRITE_COMMITTED + "=true|false] ");
7c673cae
FG		 2729 ret.append("\n");
2730}
2731
2732void WALDumperCommand::DoCommand() {
494da23a
TL		 2733 DumpWalFile(options_, wal_file_, print_header_, print_values_,
2734 is_write_committed_, &exec_state_);
7c673cae
FG		 2735}
2736
2737// ----------------------------------------------------------------------------
2738
2739GetCommand::GetCommand(const std::vector<std::string>& params,
2740 const std::map<std::string, std::string>& options,
2741 const std::vector<std::string>& flags)
2742 : LDBCommand(
2743 options, flags, true,
2744 BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
2745 if (params.size() != 1) {
2746 exec_state_ = LDBCommandExecuteResult::Failed(
2747 "<key> must be specified for the get command");
2748 } else {
2749 key_ = params.at(0);
2750 }
2751
2752 if (is_key_hex_) {
2753 key_ = HexToString(key_);
2754 }
2755}
2756
2757void GetCommand::Help(std::string& ret) {
2758 ret.append(" ");
2759 ret.append(GetCommand::Name());
2760 ret.append(" <key>");
2761 ret.append(" [--" + ARG_TTL + "]");
2762 ret.append("\n");
2763}
2764
2765void GetCommand::DoCommand() {
2766 if (!db_) {
2767 assert(GetExecuteState().IsFailed());
2768 return;
2769 }
2770 std::string value;
2771 Status st = db_->Get(ReadOptions(), GetCfHandle(), key_, &value);
2772 if (st.ok()) {
2773 fprintf(stdout, "%s\n",
1e59de90 2774 (is_value_hex_ ? StringToHex(value) : value).c_str());
7c673cae 2775 } else {
1e59de90
TL		 2776 std::stringstream oss;
2777 oss << "Get failed: " << st.ToString();
2778 exec_state_ = LDBCommandExecuteResult::Failed(oss.str());
7c673cae
FG		 2779 }
2780}
2781
2782// ----------------------------------------------------------------------------
2783
2784ApproxSizeCommand::ApproxSizeCommand(
11fdf7f2 2785 const std::vector<std::string>& /*params*/,
7c673cae
FG		 2786 const std::map<std::string, std::string>& options,
2787 const std::vector<std::string>& flags)
2788 : LDBCommand(options, flags, true,
2789 BuildCmdLineOptions(
2790 {ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM, ARG_TO})) {
2791 if (options.find(ARG_FROM) != options.end()) {
2792 start_key_ = options.find(ARG_FROM)->second;
2793 } else {
2794 exec_state_ = LDBCommandExecuteResult::Failed(
2795 ARG_FROM + " must be specified for approxsize command");
2796 return;
2797 }
2798
2799 if (options.find(ARG_TO) != options.end()) {
2800 end_key_ = options.find(ARG_TO)->second;
2801 } else {
2802 exec_state_ = LDBCommandExecuteResult::Failed(
2803 ARG_TO + " must be specified for approxsize command");
2804 return;
2805 }
2806
2807 if (is_key_hex_) {
2808 start_key_ = HexToString(start_key_);
2809 end_key_ = HexToString(end_key_);
2810 }
2811}
2812
2813void ApproxSizeCommand::Help(std::string& ret) {
2814 ret.append(" ");
2815 ret.append(ApproxSizeCommand::Name());
2816 ret.append(HelpRangeCmdArgs());
2817 ret.append("\n");
2818}
2819
2820void ApproxSizeCommand::DoCommand() {
2821 if (!db_) {
2822 assert(GetExecuteState().IsFailed());
2823 return;
2824 }
2825 Range ranges[1];
2826 ranges[0] = Range(start_key_, end_key_);
2827 uint64_t sizes[1];
1e59de90
TL		 2828 Status s = db_->GetApproximateSizes(GetCfHandle(), ranges, 1, sizes);
2829 if (!s.ok()) {
2830 std::stringstream oss;
2831 oss << "ApproximateSize failed: " << s.ToString();
2832 exec_state_ = LDBCommandExecuteResult::Failed(oss.str());
2833 } else {
2834 fprintf(stdout, "%lu\n", (unsigned long)sizes[0]);
7c673cae 2835 }
7c673cae
FG		 2836}
2837
2838// ----------------------------------------------------------------------------
2839
2840BatchPutCommand::BatchPutCommand(
2841 const std::vector<std::string>& params,
2842 const std::map<std::string, std::string>& options,
2843 const std::vector<std::string>& flags)
2844 : LDBCommand(options, flags, false,
2845 BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX,
2846 ARG_VALUE_HEX, ARG_CREATE_IF_MISSING})) {
2847 if (params.size() < 2) {
2848 exec_state_ = LDBCommandExecuteResult::Failed(
2849 "At least one <key> <value> pair must be specified batchput.");
2850 } else if (params.size() % 2 != 0) {
2851 exec_state_ = LDBCommandExecuteResult::Failed(
2852 "Equal number of <key>s and <value>s must be specified for batchput.");
2853 } else {
2854 for (size_t i = 0; i < params.size(); i += 2) {
2855 std::string key = params.at(i);
2856 std::string value = params.at(i + 1);
2857 key_values_.push_back(std::pair<std::string, std::string>(
2858 is_key_hex_ ? HexToString(key) : key,
2859 is_value_hex_ ? HexToString(value) : value));
2860 }
2861 }
2862 create_if_missing_ = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING);
2863}
2864
2865void BatchPutCommand::Help(std::string& ret) {
2866 ret.append(" ");
2867 ret.append(BatchPutCommand::Name());
2868 ret.append(" <key> <value> [<key> <value>] [..]");
20effc67 2869 ret.append(" [--" + ARG_CREATE_IF_MISSING + "]");
7c673cae
FG		 2870 ret.append(" [--" + ARG_TTL + "]");
2871 ret.append("\n");
2872}
2873
2874void BatchPutCommand::DoCommand() {
2875 if (!db_) {
2876 assert(GetExecuteState().IsFailed());
2877 return;
2878 }
2879 WriteBatch batch;
2880
1e59de90
TL		 2881 Status st;
2882 std::stringstream oss;
7c673cae
FG		 2883 for (std::vector<std::pair<std::string, std::string>>::const_iterator itr =
2884 key_values_.begin();
2885 itr != key_values_.end(); ++itr) {
1e59de90
TL		 2886 st = batch.Put(GetCfHandle(), itr->first, itr->second);
2887 if (!st.ok()) {
2888 oss << "Put to write batch failed: " << itr->first << "=>" << itr->second
2889 << " error: " << st.ToString();
2890 break;
2891 }
2892 }
2893 if (st.ok()) {
2894 st = db_->Write(WriteOptions(), &batch);
2895 if (!st.ok()) {
2896 oss << "Write failed: " << st.ToString();
2897 }
7c673cae 2898 }
7c673cae
FG		 2899 if (st.ok()) {
2900 fprintf(stdout, "OK\n");
2901 } else {
1e59de90 2902 exec_state_ = LDBCommandExecuteResult::Failed(oss.str());
7c673cae
FG		 2903 }
2904}
2905
20effc67
TL		 2906void BatchPutCommand::OverrideBaseOptions() {
2907 LDBCommand::OverrideBaseOptions();
2908 options_.create_if_missing = create_if_missing_;
7c673cae
FG		 2909}
2910
2911// ----------------------------------------------------------------------------
2912
11fdf7f2 2913ScanCommand::ScanCommand(const std::vector<std::string>& /*params*/,
7c673cae
FG		 2914 const std::map<std::string, std::string>& options,
2915 const std::vector<std::string>& flags)
2916 : LDBCommand(
2917 options, flags, true,
2918 BuildCmdLineOptions({ARG_TTL, ARG_NO_VALUE, ARG_HEX, ARG_KEY_HEX,
2919 ARG_TO, ARG_VALUE_HEX, ARG_FROM, ARG_TIMESTAMP,
2920 ARG_MAX_KEYS, ARG_TTL_START, ARG_TTL_END})),
2921 start_key_specified_(false),
2922 end_key_specified_(false),
2923 max_keys_scanned_(-1),
2924 no_value_(false) {
1e59de90 2925 auto itr = options.find(ARG_FROM);
7c673cae
FG		 2926 if (itr != options.end()) {
2927 start_key_ = itr->second;
2928 if (is_key_hex_) {
2929 start_key_ = HexToString(start_key_);
2930 }
2931 start_key_specified_ = true;
2932 }
2933 itr = options.find(ARG_TO);
2934 if (itr != options.end()) {
2935 end_key_ = itr->second;
2936 if (is_key_hex_) {
2937 end_key_ = HexToString(end_key_);
2938 }
2939 end_key_specified_ = true;
2940 }
2941
2942 std::vector<std::string>::const_iterator vitr =
2943 std::find(flags.begin(), flags.end(), ARG_NO_VALUE);
2944 if (vitr != flags.end()) {
2945 no_value_ = true;
2946 }
2947
2948 itr = options.find(ARG_MAX_KEYS);
2949 if (itr != options.end()) {
2950 try {
2951#if defined(CYGWIN)
2952 max_keys_scanned_ = strtol(itr->second.c_str(), 0, 10);
2953#else
2954 max_keys_scanned_ = std::stoi(itr->second);
2955#endif
2956 } catch (const std::invalid_argument&) {
2957 exec_state_ = LDBCommandExecuteResult::Failed(ARG_MAX_KEYS +
2958 " has an invalid value");
2959 } catch (const std::out_of_range&) {
2960 exec_state_ = LDBCommandExecuteResult::Failed(
2961 ARG_MAX_KEYS + " has a value out-of-range");
2962 }
2963 }
2964}
2965
2966void ScanCommand::Help(std::string& ret) {
2967 ret.append(" ");
2968 ret.append(ScanCommand::Name());
2969 ret.append(HelpRangeCmdArgs());
2970 ret.append(" [--" + ARG_TTL + "]");
2971 ret.append(" [--" + ARG_TIMESTAMP + "]");
2972 ret.append(" [--" + ARG_MAX_KEYS + "=<N>q] ");
2973 ret.append(" [--" + ARG_TTL_START + "=<N>:- is inclusive]");
2974 ret.append(" [--" + ARG_TTL_END + "=<N>:- is exclusive]");
2975 ret.append(" [--" + ARG_NO_VALUE + "]");
2976 ret.append("\n");
2977}
2978
2979void ScanCommand::DoCommand() {
2980 if (!db_) {
2981 assert(GetExecuteState().IsFailed());
2982 return;
2983 }
2984
2985 int num_keys_scanned = 0;
494da23a
TL		 2986 ReadOptions scan_read_opts;
2987 scan_read_opts.total_order_seek = true;
2988 Iterator* it = db_->NewIterator(scan_read_opts, GetCfHandle());
7c673cae
FG		 2989 if (start_key_specified_) {
2990 it->Seek(start_key_);
2991 } else {
2992 it->SeekToFirst();
2993 }
2994 int ttl_start;
2995 if (!ParseIntOption(option_map_, ARG_TTL_START, ttl_start, exec_state_)) {
2996 ttl_start = DBWithTTLImpl::kMinTimestamp; // TTL introduction time
2997 }
2998 int ttl_end;
2999 if (!ParseIntOption(option_map_, ARG_TTL_END, ttl_end, exec_state_)) {
3000 ttl_end = DBWithTTLImpl::kMaxTimestamp; // Max time allowed by TTL feature
3001 }
3002 if (ttl_end < ttl_start) {
3003 fprintf(stderr, "Error: End time can't be less than start time\n");
3004 delete it;
3005 return;
3006 }
3007 if (is_db_ttl_ && timestamp_) {
3008 fprintf(stdout, "Scanning key-values from %s to %s\n",
f67539c2
TL		 3009 TimeToHumanString(ttl_start).c_str(),
3010 TimeToHumanString(ttl_end).c_str());
7c673cae 3011 }
1e59de90
TL		 3012 for (;
3013 it->Valid() && (!end_key_specified_ || it->key().ToString() < end_key_);
3014 it->Next()) {
7c673cae 3015 if (is_db_ttl_) {
20effc67
TL		 3016 TtlIterator* it_ttl = static_cast_with_check<TtlIterator>(it);
3017 int rawtime = it_ttl->ttl_timestamp();
7c673cae
FG		 3018 if (rawtime < ttl_start || rawtime >= ttl_end) {
3019 continue;
3020 }
3021 if (timestamp_) {
f67539c2 3022 fprintf(stdout, "%s ", TimeToHumanString(rawtime).c_str());
7c673cae
FG		 3023 }
3024 }
3025
3026 Slice key_slice = it->key();
3027
3028 std::string formatted_key;
3029 if (is_key_hex_) {
3030 formatted_key = "0x" + key_slice.ToString(true /* hex */);
3031 key_slice = formatted_key;
3032 } else if (ldb_options_.key_formatter) {
3033 formatted_key = ldb_options_.key_formatter->Format(key_slice);
3034 key_slice = formatted_key;
3035 }
3036
3037 if (no_value_) {
3038 fprintf(stdout, "%.*s\n", static_cast<int>(key_slice.size()),
3039 key_slice.data());
3040 } else {
3041 Slice val_slice = it->value();
3042 std::string formatted_value;
3043 if (is_value_hex_) {
3044 formatted_value = "0x" + val_slice.ToString(true /* hex */);
3045 val_slice = formatted_value;
3046 }
3047 fprintf(stdout, "%.*s : %.*s\n", static_cast<int>(key_slice.size()),
3048 key_slice.data(), static_cast<int>(val_slice.size()),
3049 val_slice.data());
3050 }
3051
3052 num_keys_scanned++;
3053 if (max_keys_scanned_ >= 0 && num_keys_scanned >= max_keys_scanned_) {
3054 break;
3055 }
3056 }
3057 if (!it->status().ok()) { // Check for any errors found during the scan
3058 exec_state_ = LDBCommandExecuteResult::Failed(it->status().ToString());
3059 }
3060 delete it;
3061}
3062
3063// ----------------------------------------------------------------------------
3064
3065DeleteCommand::DeleteCommand(const std::vector<std::string>& params,
3066 const std::map<std::string, std::string>& options,
3067 const std::vector<std::string>& flags)
3068 : LDBCommand(options, flags, false,
3069 BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
3070 if (params.size() != 1) {
3071 exec_state_ = LDBCommandExecuteResult::Failed(
3072 "KEY must be specified for the delete command");
3073 } else {
3074 key_ = params.at(0);
3075 if (is_key_hex_) {
3076 key_ = HexToString(key_);
3077 }
3078 }
3079}
3080
3081void DeleteCommand::Help(std::string& ret) {
3082 ret.append(" ");
3083 ret.append(DeleteCommand::Name() + " <key>");
3084 ret.append("\n");
3085}
3086
3087void DeleteCommand::DoCommand() {
3088 if (!db_) {
3089 assert(GetExecuteState().IsFailed());
3090 return;
3091 }
3092 Status st = db_->Delete(WriteOptions(), GetCfHandle(), key_);
3093 if (st.ok()) {
3094 fprintf(stdout, "OK\n");
3095 } else {
3096 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
3097 }
3098}
3099
1e59de90
TL		 3100SingleDeleteCommand::SingleDeleteCommand(
3101 const std::vector<std::string>& params,
3102 const std::map<std::string, std::string>& options,
3103 const std::vector<std::string>& flags)
3104 : LDBCommand(options, flags, false,
3105 BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
3106 if (params.size() != 1) {
3107 exec_state_ = LDBCommandExecuteResult::Failed(
3108 "KEY must be specified for the single delete command");
3109 } else {
3110 key_ = params.at(0);
3111 if (is_key_hex_) {
3112 key_ = HexToString(key_);
3113 }
3114 }
3115}
3116
3117void SingleDeleteCommand::Help(std::string& ret) {
3118 ret.append(" ");
3119 ret.append(SingleDeleteCommand::Name() + " <key>");
3120 ret.append("\n");
3121}
3122
3123void SingleDeleteCommand::DoCommand() {
3124 if (!db_) {
3125 assert(GetExecuteState().IsFailed());
3126 return;
3127 }
3128 Status st = db_->SingleDelete(WriteOptions(), GetCfHandle(), key_);
3129 if (st.ok()) {
3130 fprintf(stdout, "OK\n");
3131 } else {
3132 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
3133 }
3134}
3135
7c673cae
FG		 3136DeleteRangeCommand::DeleteRangeCommand(
3137 const std::vector<std::string>& params,
3138 const std::map<std::string, std::string>& options,
3139 const std::vector<std::string>& flags)
3140 : LDBCommand(options, flags, false,
3141 BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
3142 if (params.size() != 2) {
3143 exec_state_ = LDBCommandExecuteResult::Failed(
3144 "begin and end keys must be specified for the delete command");
3145 } else {
3146 begin_key_ = params.at(0);
3147 end_key_ = params.at(1);
3148 if (is_key_hex_) {
3149 begin_key_ = HexToString(begin_key_);
3150 end_key_ = HexToString(end_key_);
3151 }
3152 }
3153}
3154
3155void DeleteRangeCommand::Help(std::string& ret) {
3156 ret.append(" ");
3157 ret.append(DeleteRangeCommand::Name() + " <begin key> <end key>");
3158 ret.append("\n");
3159}
3160
3161void DeleteRangeCommand::DoCommand() {
3162 if (!db_) {
3163 assert(GetExecuteState().IsFailed());
3164 return;
3165 }
3166 Status st =
3167 db_->DeleteRange(WriteOptions(), GetCfHandle(), begin_key_, end_key_);
3168 if (st.ok()) {
3169 fprintf(stdout, "OK\n");
3170 } else {
3171 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
3172 }
3173}
3174
3175PutCommand::PutCommand(const std::vector<std::string>& params,
3176 const std::map<std::string, std::string>& options,
3177 const std::vector<std::string>& flags)
3178 : LDBCommand(options, flags, false,
3179 BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX,
3180 ARG_VALUE_HEX, ARG_CREATE_IF_MISSING})) {
3181 if (params.size() != 2) {
3182 exec_state_ = LDBCommandExecuteResult::Failed(
3183 "<key> and <value> must be specified for the put command");
3184 } else {
3185 key_ = params.at(0);
3186 value_ = params.at(1);
3187 }
3188
3189 if (is_key_hex_) {
3190 key_ = HexToString(key_);
3191 }
3192
3193 if (is_value_hex_) {
3194 value_ = HexToString(value_);
3195 }
3196 create_if_missing_ = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING);
3197}
3198
3199void PutCommand::Help(std::string& ret) {
3200 ret.append(" ");
3201 ret.append(PutCommand::Name());
20effc67
TL		 3202 ret.append(" <key> <value>");
3203 ret.append(" [--" + ARG_CREATE_IF_MISSING + "]");
7c673cae
FG		 3204 ret.append(" [--" + ARG_TTL + "]");
3205 ret.append("\n");
3206}
3207
3208void PutCommand::DoCommand() {
3209 if (!db_) {
3210 assert(GetExecuteState().IsFailed());
3211 return;
3212 }
3213 Status st = db_->Put(WriteOptions(), GetCfHandle(), key_, value_);
3214 if (st.ok()) {
3215 fprintf(stdout, "OK\n");
3216 } else {
3217 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
3218 }
3219}
3220
20effc67
TL		 3221void PutCommand::OverrideBaseOptions() {
3222 LDBCommand::OverrideBaseOptions();
3223 options_.create_if_missing = create_if_missing_;
7c673cae
FG		 3224}
3225
3226// ----------------------------------------------------------------------------
3227
3228const char* DBQuerierCommand::HELP_CMD = "help";
3229const char* DBQuerierCommand::GET_CMD = "get";
3230const char* DBQuerierCommand::PUT_CMD = "put";
3231const char* DBQuerierCommand::DELETE_CMD = "delete";
3232
3233DBQuerierCommand::DBQuerierCommand(
11fdf7f2 3234 const std::vector<std::string>& /*params*/,
7c673cae
FG		 3235 const std::map<std::string, std::string>& options,
3236 const std::vector<std::string>& flags)
3237 : LDBCommand(
3238 options, flags, false,
3239 BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
3240
3241}
3242
3243void DBQuerierCommand::Help(std::string& ret) {
3244 ret.append(" ");
3245 ret.append(DBQuerierCommand::Name());
3246 ret.append(" [--" + ARG_TTL + "]");
3247 ret.append("\n");
1e59de90
TL		 3248 ret.append(
3249 " Starts a REPL shell. Type help for list of available "
3250 "commands.");
7c673cae
FG		 3251 ret.append("\n");
3252}
3253
3254void DBQuerierCommand::DoCommand() {
3255 if (!db_) {
3256 assert(GetExecuteState().IsFailed());
3257 return;
3258 }
3259
3260 ReadOptions read_options;
3261 WriteOptions write_options;
3262
3263 std::string line;
3264 std::string key;
3265 std::string value;
1e59de90
TL		 3266 Status s;
3267 std::stringstream oss;
3268 while (s.ok() && getline(std::cin, line, '\n')) {
7c673cae
FG		 3269 // Parse line into std::vector<std::string>
3270 std::vector<std::string> tokens;
3271 size_t pos = 0;
3272 while (true) {
3273 size_t pos2 = line.find(' ', pos);
3274 if (pos2 == std::string::npos) {
3275 break;
3276 }
1e59de90 3277 tokens.push_back(line.substr(pos, pos2 - pos));
7c673cae
FG		 3278 pos = pos2 + 1;
3279 }
3280 tokens.push_back(line.substr(pos));
3281
3282 const std::string& cmd = tokens[0];
3283
3284 if (cmd == HELP_CMD) {
3285 fprintf(stdout,
3286 "get <key>\n"
3287 "put <key> <value>\n"
3288 "delete <key>\n");
3289 } else if (cmd == DELETE_CMD && tokens.size() == 2) {
3290 key = (is_key_hex_ ? HexToString(tokens[1]) : tokens[1]);
1e59de90
TL		 3291 s = db_->Delete(write_options, GetCfHandle(), Slice(key));
3292 if (s.ok()) {
3293 fprintf(stdout, "Successfully deleted %s\n", tokens[1].c_str());
3294 } else {
3295 oss << "delete " << key << " failed: " << s.ToString();
3296 }
7c673cae
FG		 3297 } else if (cmd == PUT_CMD && tokens.size() == 3) {
3298 key = (is_key_hex_ ? HexToString(tokens[1]) : tokens[1]);
3299 value = (is_value_hex_ ? HexToString(tokens[2]) : tokens[2]);
1e59de90
TL		 3300 s = db_->Put(write_options, GetCfHandle(), Slice(key), Slice(value));
3301 if (s.ok()) {
3302 fprintf(stdout, "Successfully put %s %s\n", tokens[1].c_str(),
3303 tokens[2].c_str());
3304 } else {
3305 oss << "put " << key << "=>" << value << " failed: " << s.ToString();
3306 }
7c673cae
FG		 3307 } else if (cmd == GET_CMD && tokens.size() == 2) {
3308 key = (is_key_hex_ ? HexToString(tokens[1]) : tokens[1]);
1e59de90
TL		 3309 s = db_->Get(read_options, GetCfHandle(), Slice(key), &value);
3310 if (s.ok()) {
3311 fprintf(stdout, "%s\n",
3312 PrintKeyValue(key, value, is_key_hex_, is_value_hex_).c_str());
7c673cae 3313 } else {
1e59de90
TL		 3314 if (s.IsNotFound()) {
3315 fprintf(stdout, "Not found %s\n", tokens[1].c_str());
3316 } else {
3317 oss << "get " << key << " error: " << s.ToString();
3318 }
7c673cae
FG		 3319 }
3320 } else {
3321 fprintf(stdout, "Unknown command %s\n", line.c_str());
3322 }
3323 }
1e59de90
TL		 3324 if (!s.ok()) {
3325 exec_state_ = LDBCommandExecuteResult::Failed(oss.str());
3326 }
7c673cae
FG		 3327}
3328
3329// ----------------------------------------------------------------------------
3330
3331CheckConsistencyCommand::CheckConsistencyCommand(
11fdf7f2 3332 const std::vector<std::string>& /*params*/,
7c673cae
FG		 3333 const std::map<std::string, std::string>& options,
3334 const std::vector<std::string>& flags)
20effc67 3335 : LDBCommand(options, flags, true, BuildCmdLineOptions({})) {}
7c673cae
FG		 3336
3337void CheckConsistencyCommand::Help(std::string& ret) {
3338 ret.append(" ");
3339 ret.append(CheckConsistencyCommand::Name());
3340 ret.append("\n");
3341}
3342
3343void CheckConsistencyCommand::DoCommand() {
20effc67
TL		 3344 options_.paranoid_checks = true;
3345 options_.num_levels = 64;
3346 OpenDB();
3347 if (exec_state_.IsSucceed() || exec_state_.IsNotStarted()) {
7c673cae 3348 fprintf(stdout, "OK\n");
7c673cae 3349 }
20effc67 3350 CloseDB();
7c673cae
FG		 3351}
3352
3353// ----------------------------------------------------------------------------
3354
3355const std::string CheckPointCommand::ARG_CHECKPOINT_DIR = "checkpoint_dir";
3356
3357CheckPointCommand::CheckPointCommand(
11fdf7f2 3358 const std::vector<std::string>& /*params*/,
7c673cae
FG		 3359 const std::map<std::string, std::string>& options,
3360 const std::vector<std::string>& flags)
3361 : LDBCommand(options, flags, false /* is_read_only */,
3362 BuildCmdLineOptions({ARG_CHECKPOINT_DIR})) {
3363 auto itr = options.find(ARG_CHECKPOINT_DIR);
11fdf7f2 3364 if (itr != options.end()) {
7c673cae
FG		 3365 checkpoint_dir_ = itr->second;
3366 }
3367}
3368
3369void CheckPointCommand::Help(std::string& ret) {
3370 ret.append(" ");
3371 ret.append(CheckPointCommand::Name());
3372 ret.append(" [--" + ARG_CHECKPOINT_DIR + "] ");
3373 ret.append("\n");
3374}
3375
3376void CheckPointCommand::DoCommand() {
3377 if (!db_) {
3378 assert(GetExecuteState().IsFailed());
3379 return;
3380 }
3381 Checkpoint* checkpoint;
3382 Status status = Checkpoint::Create(db_, &checkpoint);
3383 status = checkpoint->CreateCheckpoint(checkpoint_dir_);
3384 if (status.ok()) {
f67539c2 3385 fprintf(stdout, "OK\n");
7c673cae
FG		 3386 } else {
3387 exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
3388 }
3389}
3390
3391// ----------------------------------------------------------------------------
3392
1e59de90
TL		 3393const std::string RepairCommand::ARG_VERBOSE = "verbose";
3394
11fdf7f2 3395RepairCommand::RepairCommand(const std::vector<std::string>& /*params*/,
7c673cae
FG		 3396 const std::map<std::string, std::string>& options,
3397 const std::vector<std::string>& flags)
1e59de90
TL		 3398 : LDBCommand(options, flags, false, BuildCmdLineOptions({ARG_VERBOSE})) {
3399 verbose_ = IsFlagPresent(flags, ARG_VERBOSE);
3400}
7c673cae
FG		 3401
3402void RepairCommand::Help(std::string& ret) {
3403 ret.append(" ");
3404 ret.append(RepairCommand::Name());
1e59de90 3405 ret.append(" [--" + ARG_VERBOSE + "]");
7c673cae
FG		 3406 ret.append("\n");
3407}
3408
20effc67
TL		 3409void RepairCommand::OverrideBaseOptions() {
3410 LDBCommand::OverrideBaseOptions();
1e59de90
TL		 3411 auto level = verbose_ ? InfoLogLevel::INFO_LEVEL : InfoLogLevel::WARN_LEVEL;
3412 options_.info_log.reset(new StderrLogger(level));
20effc67
TL		 3413}
3414
7c673cae 3415void RepairCommand::DoCommand() {
20effc67
TL		 3416 PrepareOptions();
3417 Status status = RepairDB(db_path_, options_);
7c673cae 3418 if (status.ok()) {
f67539c2 3419 fprintf(stdout, "OK\n");
7c673cae
FG		 3420 } else {
3421 exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
3422 }
3423}
3424
3425// ----------------------------------------------------------------------------
3426
1e59de90
TL		 3427const std::string BackupEngineCommand::ARG_NUM_THREADS = "num_threads";
3428const std::string BackupEngineCommand::ARG_BACKUP_ENV_URI = "backup_env_uri";
3429const std::string BackupEngineCommand::ARG_BACKUP_FS_URI = "backup_fs_uri";
3430const std::string BackupEngineCommand::ARG_BACKUP_DIR = "backup_dir";
3431const std::string BackupEngineCommand::ARG_STDERR_LOG_LEVEL =
3432 "stderr_log_level";
7c673cae 3433
1e59de90 3434BackupEngineCommand::BackupEngineCommand(
11fdf7f2 3435 const std::vector<std::string>& /*params*/,
7c673cae
FG		 3436 const std::map<std::string, std::string>& options,
3437 const std::vector<std::string>& flags)
3438 : LDBCommand(options, flags, false /* is_read_only */,
1e59de90
TL		 3439 BuildCmdLineOptions({ARG_BACKUP_ENV_URI, ARG_BACKUP_FS_URI,
3440 ARG_BACKUP_DIR, ARG_NUM_THREADS,
3441 ARG_STDERR_LOG_LEVEL})),
7c673cae
FG		 3442 num_threads_(1) {
3443 auto itr = options.find(ARG_NUM_THREADS);
3444 if (itr != options.end()) {
3445 num_threads_ = std::stoi(itr->second);
3446 }
3447 itr = options.find(ARG_BACKUP_ENV_URI);
3448 if (itr != options.end()) {
3449 backup_env_uri_ = itr->second;
3450 }
1e59de90
TL		 3451 itr = options.find(ARG_BACKUP_FS_URI);
3452 if (itr != options.end()) {
3453 backup_fs_uri_ = itr->second;
3454 }
3455 if (!backup_env_uri_.empty() && !backup_fs_uri_.empty()) {
3456 exec_state_ = LDBCommandExecuteResult::Failed(
3457 "you may not specity both --" + ARG_BACKUP_ENV_URI + " and --" +
3458 ARG_BACKUP_FS_URI);
3459 }
7c673cae
FG		 3460 itr = options.find(ARG_BACKUP_DIR);
3461 if (itr == options.end()) {
3462 exec_state_ = LDBCommandExecuteResult::Failed("--" + ARG_BACKUP_DIR +
3463 ": missing backup directory");
3464 } else {
3465 backup_dir_ = itr->second;
3466 }
3467
3468 itr = options.find(ARG_STDERR_LOG_LEVEL);
3469 if (itr != options.end()) {
3470 int stderr_log_level = std::stoi(itr->second);
3471 if (stderr_log_level < 0 ||
3472 stderr_log_level >= InfoLogLevel::NUM_INFO_LOG_LEVELS) {
3473 exec_state_ = LDBCommandExecuteResult::Failed(
3474 ARG_STDERR_LOG_LEVEL + " must be >= 0 and < " +
3475 std::to_string(InfoLogLevel::NUM_INFO_LOG_LEVELS) + ".");
3476 } else {
3477 logger_.reset(
3478 new StderrLogger(static_cast<InfoLogLevel>(stderr_log_level)));
3479 }
3480 }
3481}
3482
1e59de90 3483void BackupEngineCommand::Help(const std::string& name, std::string& ret) {
7c673cae
FG		 3484 ret.append(" ");
3485 ret.append(name);
1e59de90 3486 ret.append(" [--" + ARG_BACKUP_ENV_URI + " | --" + ARG_BACKUP_FS_URI + "] ");
7c673cae
FG		 3487 ret.append(" [--" + ARG_BACKUP_DIR + "] ");
3488 ret.append(" [--" + ARG_NUM_THREADS + "] ");
3489 ret.append(" [--" + ARG_STDERR_LOG_LEVEL + "=<int (InfoLogLevel)>] ");
3490 ret.append("\n");
3491}
3492
3493// ----------------------------------------------------------------------------
3494
3495BackupCommand::BackupCommand(const std::vector<std::string>& params,
3496 const std::map<std::string, std::string>& options,
3497 const std::vector<std::string>& flags)
1e59de90 3498 : BackupEngineCommand(params, options, flags) {}
7c673cae
FG		 3499
3500void BackupCommand::Help(std::string& ret) {
1e59de90 3501 BackupEngineCommand::Help(Name(), ret);
7c673cae
FG		 3502}
3503
3504void BackupCommand::DoCommand() {
3505 BackupEngine* backup_engine;
3506 Status status;
3507 if (!db_) {
3508 assert(GetExecuteState().IsFailed());
3509 return;
3510 }
f67539c2 3511 fprintf(stdout, "open db OK\n");
1e59de90
TL		 3512
3513 Env* custom_env = backup_env_guard_.get();
3514 if (custom_env == nullptr) {
3515 Status s =
3516 Env::CreateFromUri(config_options_, backup_env_uri_, backup_fs_uri_,
3517 &custom_env, &backup_env_guard_);
3518 if (!s.ok()) {
3519 exec_state_ = LDBCommandExecuteResult::Failed(s.ToString());
3520 return;
3521 }
3522 }
f67539c2
TL		 3523 assert(custom_env != nullptr);
3524
1e59de90
TL		 3525 BackupEngineOptions backup_options =
3526 BackupEngineOptions(backup_dir_, custom_env);
7c673cae
FG		 3527 backup_options.info_log = logger_.get();
3528 backup_options.max_background_operations = num_threads_;
1e59de90 3529 status = BackupEngine::Open(options_.env, backup_options, &backup_engine);
7c673cae 3530 if (status.ok()) {
f67539c2 3531 fprintf(stdout, "open backup engine OK\n");
7c673cae
FG		 3532 } else {
3533 exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
3534 return;
3535 }
3536 status = backup_engine->CreateNewBackup(db_);
3537 if (status.ok()) {
f67539c2 3538 fprintf(stdout, "create new backup OK\n");
7c673cae
FG		 3539 } else {
3540 exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
3541 return;
3542 }
3543}
3544
3545// ----------------------------------------------------------------------------
3546
3547RestoreCommand::RestoreCommand(
3548 const std::vector<std::string>& params,
3549 const std::map<std::string, std::string>& options,
3550 const std::vector<std::string>& flags)
1e59de90 3551 : BackupEngineCommand(params, options, flags) {}
7c673cae
FG		 3552
3553void RestoreCommand::Help(std::string& ret) {
1e59de90 3554 BackupEngineCommand::Help(Name(), ret);
7c673cae
FG		 3555}
3556
3557void RestoreCommand::DoCommand() {
1e59de90
TL		 3558 Env* custom_env = backup_env_guard_.get();
3559 if (custom_env == nullptr) {
3560 Status s =
3561 Env::CreateFromUri(config_options_, backup_env_uri_, backup_fs_uri_,
3562 &custom_env, &backup_env_guard_);
3563 if (!s.ok()) {
3564 exec_state_ = LDBCommandExecuteResult::Failed(s.ToString());
3565 return;
3566 }
3567 }
f67539c2
TL		 3568 assert(custom_env != nullptr);
3569
7c673cae
FG		 3570 std::unique_ptr<BackupEngineReadOnly> restore_engine;
3571 Status status;
3572 {
1e59de90 3573 BackupEngineOptions opts(backup_dir_, custom_env);
7c673cae
FG		 3574 opts.info_log = logger_.get();
3575 opts.max_background_operations = num_threads_;
3576 BackupEngineReadOnly* raw_restore_engine_ptr;
f67539c2 3577 status =
1e59de90 3578 BackupEngineReadOnly::Open(options_.env, opts, &raw_restore_engine_ptr);
7c673cae
FG		 3579 if (status.ok()) {
3580 restore_engine.reset(raw_restore_engine_ptr);
3581 }
3582 }
3583 if (status.ok()) {
f67539c2 3584 fprintf(stdout, "open restore engine OK\n");
7c673cae
FG		 3585 status = restore_engine->RestoreDBFromLatestBackup(db_path_, db_path_);
3586 }
3587 if (status.ok()) {
f67539c2 3588 fprintf(stdout, "restore from backup OK\n");
7c673cae
FG		 3589 } else {
3590 exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
3591 }
3592}
3593
3594// ----------------------------------------------------------------------------
3595
3596namespace {
3597
494da23a 3598void DumpSstFile(Options options, std::string filename, bool output_hex,
1e59de90
TL		 3599 bool show_properties, bool decode_blob_index,
3600 std::string from_key, std::string to_key) {
7c673cae
FG		 3601 if (filename.length() <= 4 ||
3602 filename.rfind(".sst") != filename.length() - 4) {
3603 std::cout << "Invalid sst file name." << std::endl;
3604 return;
3605 }
3606 // no verification
f67539c2 3607 ROCKSDB_NAMESPACE::SstFileDumper dumper(
1e59de90
TL		 3608 options, filename, Temperature::kUnknown,
3609 2 * 1024 * 1024 /* readahead_size */,
3610 /* verify_checksum */ false, output_hex, decode_blob_index);
494da23a 3611 Status st = dumper.ReadSequential(true, std::numeric_limits<uint64_t>::max(),
1e59de90
TL		 3612 !from_key.empty(), from_key,
3613 !to_key.empty(), to_key);
7c673cae
FG		 3614 if (!st.ok()) {
3615 std::cerr << "Error in reading SST file " << filename << st.ToString()
3616 << std::endl;
3617 return;
3618 }
3619
3620 if (show_properties) {
f67539c2 3621 const ROCKSDB_NAMESPACE::TableProperties* table_properties;
7c673cae 3622
f67539c2 3623 std::shared_ptr<const ROCKSDB_NAMESPACE::TableProperties>
7c673cae 3624 table_properties_from_reader;
494da23a 3625 st = dumper.ReadTableProperties(&table_properties_from_reader);
7c673cae
FG		 3626 if (!st.ok()) {
3627 std::cerr << filename << ": " << st.ToString()
3628 << ". Try to use initial table properties" << std::endl;
494da23a 3629 table_properties = dumper.GetInitTableProperties();
7c673cae
FG		 3630 } else {
3631 table_properties = table_properties_from_reader.get();
3632 }
3633 if (table_properties != nullptr) {
3634 std::cout << std::endl << "Table Properties:" << std::endl;
3635 std::cout << table_properties->ToString("\n") << std::endl;
7c673cae
FG		 3636 }
3637 }
3638}
3639
1e59de90
TL		 3640void DumpBlobFile(const std::string& filename, bool is_key_hex,
3641 bool is_value_hex, bool dump_uncompressed_blobs) {
3642 using ROCKSDB_NAMESPACE::blob_db::BlobDumpTool;
3643 BlobDumpTool tool;
3644 BlobDumpTool::DisplayType blob_type = is_value_hex
3645 ? BlobDumpTool::DisplayType::kHex
3646 : BlobDumpTool::DisplayType::kRaw;
3647 BlobDumpTool::DisplayType show_uncompressed_blob =
3648 dump_uncompressed_blobs ? blob_type : BlobDumpTool::DisplayType::kNone;
3649 BlobDumpTool::DisplayType show_blob =
3650 dump_uncompressed_blobs ? BlobDumpTool::DisplayType::kNone : blob_type;
3651
3652 BlobDumpTool::DisplayType show_key = is_key_hex
3653 ? BlobDumpTool::DisplayType::kHex
3654 : BlobDumpTool::DisplayType::kRaw;
3655 Status s = tool.Run(filename, show_key, show_blob, show_uncompressed_blob,
3656 /* show_summary */ true);
3657 if (!s.ok()) {
3658 fprintf(stderr, "Failed: %s\n", s.ToString().c_str());
3659 }
3660}
7c673cae
FG		 3661} // namespace
3662
3663DBFileDumperCommand::DBFileDumperCommand(
11fdf7f2 3664 const std::vector<std::string>& /*params*/,
7c673cae
FG		 3665 const std::map<std::string, std::string>& options,
3666 const std::vector<std::string>& flags)
1e59de90
TL		 3667 : LDBCommand(options, flags, true,
3668 BuildCmdLineOptions(
3669 {ARG_DECODE_BLOB_INDEX, ARG_DUMP_UNCOMPRESSED_BLOBS})),
3670 decode_blob_index_(IsFlagPresent(flags, ARG_DECODE_BLOB_INDEX)),
3671 dump_uncompressed_blobs_(
3672 IsFlagPresent(flags, ARG_DUMP_UNCOMPRESSED_BLOBS)) {}
7c673cae
FG		 3673
3674void DBFileDumperCommand::Help(std::string& ret) {
3675 ret.append(" ");
3676 ret.append(DBFileDumperCommand::Name());
1e59de90
TL		 3677 ret.append(" [--" + ARG_DECODE_BLOB_INDEX + "] ");
3678 ret.append(" [--" + ARG_DUMP_UNCOMPRESSED_BLOBS + "] ");
7c673cae
FG		 3679 ret.append("\n");
3680}
3681
3682void DBFileDumperCommand::DoCommand() {
3683 if (!db_) {
3684 assert(GetExecuteState().IsFailed());
3685 return;
3686 }
3687 Status s;
3688
1e59de90
TL		 3689 // TODO: Use --hex, --key_hex, --value_hex flags consistently for
3690 // dumping manifest file, sst files and blob files.
7c673cae
FG		 3691 std::cout << "Manifest File" << std::endl;
3692 std::cout << "==============================" << std::endl;
3693 std::string manifest_filename;
3694 s = ReadFileToString(db_->GetEnv(), CurrentFileName(db_->GetName()),
3695 &manifest_filename);
3696 if (!s.ok() || manifest_filename.empty() ||
3697 manifest_filename.back() != '\n') {
3698 std::cerr << "Error when reading CURRENT file "
3699 << CurrentFileName(db_->GetName()) << std::endl;
3700 }
3701 // remove the trailing '\n'
3702 manifest_filename.resize(manifest_filename.size() - 1);
3703 std::string manifest_filepath = db_->GetName() + "/" + manifest_filename;
1e59de90
TL		 3704 // Correct concatenation of filepath and filename:
3705 // Check that there is no double slashes (or more!) when concatenation
3706 // happens.
3707 manifest_filepath = NormalizePath(manifest_filepath);
3708
7c673cae 3709 std::cout << manifest_filepath << std::endl;
494da23a 3710 DumpManifestFile(options_, manifest_filepath, false, false, false);
7c673cae
FG		 3711 std::cout << std::endl;
3712
1e59de90
TL		 3713 std::vector<ColumnFamilyMetaData> column_families;
3714 db_->GetAllColumnFamilyMetaData(&column_families);
3715 for (const auto& column_family : column_families) {
3716 std::cout << "Column family name: " << column_family.name << std::endl;
3717 std::cout << "==============================" << std::endl;
7c673cae 3718 std::cout << std::endl;
1e59de90
TL		 3719 std::cout << "SST Files" << std::endl;
3720 std::cout << "==============================" << std::endl;
3721 for (const LevelMetaData& level : column_family.levels) {
3722 for (const SstFileMetaData& sst_file : level.files) {
3723 std::string filename = sst_file.db_path + "/" + sst_file.name;
3724 // Correct concatenation of filepath and filename:
3725 // Check that there is no double slashes (or more!) when concatenation
3726 // happens.
3727 filename = NormalizePath(filename);
3728 std::cout << filename << " level:" << level.level << std::endl;
3729 std::cout << "------------------------------" << std::endl;
3730 DumpSstFile(options_, filename, false, true, decode_blob_index_);
3731 std::cout << std::endl;
3732 }
3733 }
3734 std::cout << "Blob Files" << std::endl;
3735 std::cout << "==============================" << std::endl;
3736 for (const BlobMetaData& blob_file : column_family.blob_files) {
3737 std::string filename =
3738 blob_file.blob_file_path + "/" + blob_file.blob_file_name;
3739 // Correct concatenation of filepath and filename:
3740 // Check that there is no double slashes (or more!) when concatenation
3741 // happens.
3742 filename = NormalizePath(filename);
3743 std::cout << filename << std::endl;
3744 std::cout << "------------------------------" << std::endl;
3745 DumpBlobFile(filename, /* is_key_hex */ false, /* is_value_hex */ false,
3746 dump_uncompressed_blobs_);
3747 std::cout << std::endl;
3748 }
7c673cae
FG		 3749 }
3750 std::cout << std::endl;
3751
3752 std::cout << "Write Ahead Log Files" << std::endl;
3753 std::cout << "==============================" << std::endl;
f67539c2 3754 ROCKSDB_NAMESPACE::VectorLogPtr wal_files;
7c673cae
FG		 3755 s = db_->GetSortedWalFiles(wal_files);
3756 if (!s.ok()) {
3757 std::cerr << "Error when getting WAL files" << std::endl;
3758 } else {
1e59de90
TL		 3759 std::string wal_dir;
3760 if (options_.wal_dir.empty()) {
3761 wal_dir = db_->GetName();
3762 } else {
3763 wal_dir = NormalizePath(options_.wal_dir + "/");
3764 }
7c673cae
FG		 3765 for (auto& wal : wal_files) {
3766 // TODO(qyang): option.wal_dir should be passed into ldb command
1e59de90 3767 std::string filename = wal_dir + wal->PathName();
7c673cae 3768 std::cout << filename << std::endl;
11fdf7f2 3769 // TODO(myabandeh): allow configuring is_write_commited
494da23a 3770 DumpWalFile(options_, filename, true, true, true /* is_write_commited */,
11fdf7f2
TL		 3771 &exec_state_);
3772 }
3773 }
3774}
3775
1e59de90
TL		 3776const std::string DBLiveFilesMetadataDumperCommand::ARG_SORT_BY_FILENAME =
3777 "sort_by_filename";
3778
3779DBLiveFilesMetadataDumperCommand::DBLiveFilesMetadataDumperCommand(
3780 const std::vector<std::string>& /*params*/,
3781 const std::map<std::string, std::string>& options,
3782 const std::vector<std::string>& flags)
3783 : LDBCommand(options, flags, true,
3784 BuildCmdLineOptions({ARG_SORT_BY_FILENAME})) {
3785 sort_by_filename_ = IsFlagPresent(flags, ARG_SORT_BY_FILENAME);
3786}
3787
3788void DBLiveFilesMetadataDumperCommand::Help(std::string& ret) {
3789 ret.append(" ");
3790 ret.append(DBLiveFilesMetadataDumperCommand::Name());
3791 ret.append(" [--" + ARG_SORT_BY_FILENAME + "] ");
3792 ret.append("\n");
3793}
3794
3795void DBLiveFilesMetadataDumperCommand::DoCommand() {
3796 if (!db_) {
3797 assert(GetExecuteState().IsFailed());
3798 return;
3799 }
3800 Status s;
3801
3802 std::vector<ColumnFamilyMetaData> metadata;
3803 db_->GetAllColumnFamilyMetaData(&metadata);
3804 if (sort_by_filename_) {
3805 std::cout << "Live SST and Blob Files:" << std::endl;
3806 // tuple of <file path, level, column family name>
3807 std::vector<std::tuple<std::string, int, std::string>> all_files;
3808
3809 for (const auto& column_metadata : metadata) {
3810 // Iterate Levels
3811 const auto& levels = column_metadata.levels;
3812 const std::string& cf = column_metadata.name;
3813 for (const auto& level_metadata : levels) {
3814 // Iterate SST files
3815 const auto& sst_files = level_metadata.files;
3816 int level = level_metadata.level;
3817 for (const auto& sst_metadata : sst_files) {
3818 // The SstFileMetaData.name always starts with "/",
3819 // however SstFileMetaData.db_path is the string provided by
3820 // the user as an input. Therefore we check if we can
3821 // concantenate the two strings directly or if we need to
3822 // drop a possible extra "/" at the end of SstFileMetaData.db_path.
3823 std::string filename =
3824 NormalizePath(sst_metadata.db_path + "/" + sst_metadata.name);
3825 all_files.emplace_back(filename, level, cf);
3826 } // End of for-loop over sst files
3827 } // End of for-loop over levels
3828
3829 const auto& blob_files = column_metadata.blob_files;
3830 for (const auto& blob_metadata : blob_files) {
3831 // The BlobMetaData.blob_file_name always starts with "/",
3832 // however BlobMetaData.blob_file_path is the string provided by
3833 // the user as an input. Therefore we check if we can
3834 // concantenate the two strings directly or if we need to
3835 // drop a possible extra "/" at the end of BlobMetaData.blob_file_path.
3836 std::string filename = NormalizePath(
3837 blob_metadata.blob_file_path + "/" + blob_metadata.blob_file_name);
3838 // Level for blob files is encoded as -1
3839 all_files.emplace_back(filename, -1, cf);
3840 } // End of for-loop over blob files
3841 } // End of for-loop over column metadata
3842
3843 // Sort by filename (i.e. first entry in tuple)
3844 std::sort(all_files.begin(), all_files.end());
3845
3846 for (const auto& item : all_files) {
3847 const std::string& filename = std::get<0>(item);
3848 int level = std::get<1>(item);
3849 const std::string& cf = std::get<2>(item);
3850 if (level == -1) { // Blob File
3851 std::cout << filename << ", column family '" << cf << "'" << std::endl;
3852 } else { // SST file
3853 std::cout << filename << " : level " << level << ", column family '"
3854 << cf << "'" << std::endl;
3855 }
3856 }
3857 } else {
3858 for (const auto& column_metadata : metadata) {
3859 std::cout << "===== Column Family: " << column_metadata.name
3860 << " =====" << std::endl;
3861
3862 std::cout << "Live SST Files:" << std::endl;
3863 // Iterate levels
3864 const auto& levels = column_metadata.levels;
3865 for (const auto& level_metadata : levels) {
3866 std::cout << "---------- level " << level_metadata.level
3867 << " ----------" << std::endl;
3868 // Iterate SST files
3869 const auto& sst_files = level_metadata.files;
3870 for (const auto& sst_metadata : sst_files) {
3871 // The SstFileMetaData.name always starts with "/",
3872 // however SstFileMetaData.db_path is the string provided by
3873 // the user as an input. Therefore we check if we can
3874 // concantenate the two strings directly or if we need to
3875 // drop a possible extra "/" at the end of SstFileMetaData.db_path.
3876 std::string filename =
3877 NormalizePath(sst_metadata.db_path + "/" + sst_metadata.name);
3878 std::cout << filename << std::endl;
3879 } // End of for-loop over sst files
3880 } // End of for-loop over levels
3881
3882 std::cout << "Live Blob Files:" << std::endl;
3883 const auto& blob_files = column_metadata.blob_files;
3884 for (const auto& blob_metadata : blob_files) {
3885 // The BlobMetaData.blob_file_name always starts with "/",
3886 // however BlobMetaData.blob_file_path is the string provided by
3887 // the user as an input. Therefore we check if we can
3888 // concantenate the two strings directly or if we need to
3889 // drop a possible extra "/" at the end of BlobMetaData.blob_file_path.
3890 std::string filename = NormalizePath(
3891 blob_metadata.blob_file_path + "/" + blob_metadata.blob_file_name);
3892 std::cout << filename << std::endl;
3893 } // End of for-loop over blob files
3894 } // End of for-loop over column metadata
3895 } // End of else ("not sort_by_filename")
3896 std::cout << "------------------------------" << std::endl;
3897}
3898
11fdf7f2
TL		 3899void WriteExternalSstFilesCommand::Help(std::string& ret) {
3900 ret.append(" ");
3901 ret.append(WriteExternalSstFilesCommand::Name());
3902 ret.append(" <output_sst_path>");
3903 ret.append("\n");
3904}
3905
3906WriteExternalSstFilesCommand::WriteExternalSstFilesCommand(
3907 const std::vector<std::string>& params,
3908 const std::map<std::string, std::string>& options,
3909 const std::vector<std::string>& flags)
3910 : LDBCommand(
3911 options, flags, false /* is_read_only */,
3912 BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM,
3913 ARG_TO, ARG_CREATE_IF_MISSING})) {
3914 create_if_missing_ =
3915 IsFlagPresent(flags, ARG_CREATE_IF_MISSING) ||
3916 ParseBooleanOption(options, ARG_CREATE_IF_MISSING, false);
3917 if (params.size() != 1) {
3918 exec_state_ = LDBCommandExecuteResult::Failed(
3919 "output SST file path must be specified");
3920 } else {
3921 output_sst_path_ = params.at(0);
3922 }
3923}
3924
3925void WriteExternalSstFilesCommand::DoCommand() {
3926 if (!db_) {
3927 assert(GetExecuteState().IsFailed());
3928 return;
3929 }
3930 ColumnFamilyHandle* cfh = GetCfHandle();
3931 SstFileWriter sst_file_writer(EnvOptions(), db_->GetOptions(), cfh);
3932 Status status = sst_file_writer.Open(output_sst_path_);
3933 if (!status.ok()) {
3934 exec_state_ = LDBCommandExecuteResult::Failed("failed to open SST file: " +
3935 status.ToString());
3936 return;
3937 }
3938
3939 int bad_lines = 0;
3940 std::string line;
3941 std::ifstream ifs_stdin("/dev/stdin");
3942 std::istream* istream_p = ifs_stdin.is_open() ? &ifs_stdin : &std::cin;
3943 while (getline(*istream_p, line, '\n')) {
3944 std::string key;
3945 std::string value;
3946 if (ParseKeyValue(line, &key, &value, is_key_hex_, is_value_hex_)) {
3947 status = sst_file_writer.Put(key, value);
3948 if (!status.ok()) {
3949 exec_state_ = LDBCommandExecuteResult::Failed(
3950 "failed to write record to file: " + status.ToString());
3951 return;
3952 }
3953 } else if (0 == line.find("Keys in range:")) {
3954 // ignore this line
3955 } else if (0 == line.find("Created bg thread 0x")) {
3956 // ignore this line
3957 } else {
3958 bad_lines++;
7c673cae
FG		 3959 }
3960 }
11fdf7f2
TL		 3961
3962 status = sst_file_writer.Finish();
3963 if (!status.ok()) {
3964 exec_state_ = LDBCommandExecuteResult::Failed(
3965 "Failed to finish writing to file: " + status.ToString());
3966 return;
3967 }
3968
3969 if (bad_lines > 0) {
3970 fprintf(stderr, "Warning: %d bad lines ignored.\n", bad_lines);
3971 }
3972 exec_state_ = LDBCommandExecuteResult::Succeed(
3973 "external SST file written to " + output_sst_path_);
3974}
3975
20effc67
TL		 3976void WriteExternalSstFilesCommand::OverrideBaseOptions() {
3977 LDBCommand::OverrideBaseOptions();
3978 options_.create_if_missing = create_if_missing_;
11fdf7f2
TL		 3979}
3980
3981const std::string IngestExternalSstFilesCommand::ARG_MOVE_FILES = "move_files";
3982const std::string IngestExternalSstFilesCommand::ARG_SNAPSHOT_CONSISTENCY =
3983 "snapshot_consistency";
3984const std::string IngestExternalSstFilesCommand::ARG_ALLOW_GLOBAL_SEQNO =
3985 "allow_global_seqno";
3986const std::string IngestExternalSstFilesCommand::ARG_ALLOW_BLOCKING_FLUSH =
3987 "allow_blocking_flush";
3988const std::string IngestExternalSstFilesCommand::ARG_INGEST_BEHIND =
3989 "ingest_behind";
3990const std::string IngestExternalSstFilesCommand::ARG_WRITE_GLOBAL_SEQNO =
3991 "write_global_seqno";
3992
3993void IngestExternalSstFilesCommand::Help(std::string& ret) {
3994 ret.append(" ");
3995 ret.append(IngestExternalSstFilesCommand::Name());
3996 ret.append(" <input_sst_path>");
3997 ret.append(" [--" + ARG_MOVE_FILES + "] ");
3998 ret.append(" [--" + ARG_SNAPSHOT_CONSISTENCY + "] ");
3999 ret.append(" [--" + ARG_ALLOW_GLOBAL_SEQNO + "] ");
4000 ret.append(" [--" + ARG_ALLOW_BLOCKING_FLUSH + "] ");
4001 ret.append(" [--" + ARG_INGEST_BEHIND + "] ");
4002 ret.append(" [--" + ARG_WRITE_GLOBAL_SEQNO + "] ");
4003 ret.append("\n");
4004}
4005
4006IngestExternalSstFilesCommand::IngestExternalSstFilesCommand(
4007 const std::vector<std::string>& params,
4008 const std::map<std::string, std::string>& options,
4009 const std::vector<std::string>& flags)
4010 : LDBCommand(
4011 options, flags, false /* is_read_only */,
4012 BuildCmdLineOptions({ARG_MOVE_FILES, ARG_SNAPSHOT_CONSISTENCY,
4013 ARG_ALLOW_GLOBAL_SEQNO, ARG_CREATE_IF_MISSING,
4014 ARG_ALLOW_BLOCKING_FLUSH, ARG_INGEST_BEHIND,
4015 ARG_WRITE_GLOBAL_SEQNO})),
4016 move_files_(false),
4017 snapshot_consistency_(true),
4018 allow_global_seqno_(true),
4019 allow_blocking_flush_(true),
4020 ingest_behind_(false),
4021 write_global_seqno_(true) {
4022 create_if_missing_ =
4023 IsFlagPresent(flags, ARG_CREATE_IF_MISSING) ||
4024 ParseBooleanOption(options, ARG_CREATE_IF_MISSING, false);
4025 move_files_ = IsFlagPresent(flags, ARG_MOVE_FILES) ||
4026 ParseBooleanOption(options, ARG_MOVE_FILES, false);
4027 snapshot_consistency_ =
4028 IsFlagPresent(flags, ARG_SNAPSHOT_CONSISTENCY) ||
4029 ParseBooleanOption(options, ARG_SNAPSHOT_CONSISTENCY, true);
4030 allow_global_seqno_ =
4031 IsFlagPresent(flags, ARG_ALLOW_GLOBAL_SEQNO) ||
4032 ParseBooleanOption(options, ARG_ALLOW_GLOBAL_SEQNO, true);
4033 allow_blocking_flush_ =
4034 IsFlagPresent(flags, ARG_ALLOW_BLOCKING_FLUSH) ||
4035 ParseBooleanOption(options, ARG_ALLOW_BLOCKING_FLUSH, true);
4036 ingest_behind_ = IsFlagPresent(flags, ARG_INGEST_BEHIND) ||
4037 ParseBooleanOption(options, ARG_INGEST_BEHIND, false);
4038 write_global_seqno_ =
4039 IsFlagPresent(flags, ARG_WRITE_GLOBAL_SEQNO) ||
4040 ParseBooleanOption(options, ARG_WRITE_GLOBAL_SEQNO, true);
4041
4042 if (allow_global_seqno_) {
4043 if (!write_global_seqno_) {
4044 fprintf(stderr,
4045 "Warning: not writing global_seqno to the ingested SST can\n"
4046 "prevent older versions of RocksDB from being able to open it\n");
4047 }
4048 } else {
4049 if (write_global_seqno_) {
4050 exec_state_ = LDBCommandExecuteResult::Failed(
4051 "ldb cannot write global_seqno to the ingested SST when global_seqno "
4052 "is not allowed");
4053 }
4054 }
4055
4056 if (params.size() != 1) {
4057 exec_state_ =
4058 LDBCommandExecuteResult::Failed("input SST path must be specified");
4059 } else {
4060 input_sst_path_ = params.at(0);
4061 }
4062}
4063
4064void IngestExternalSstFilesCommand::DoCommand() {
4065 if (!db_) {
4066 assert(GetExecuteState().IsFailed());
4067 return;
4068 }
4069 if (GetExecuteState().IsFailed()) {
4070 return;
4071 }
4072 ColumnFamilyHandle* cfh = GetCfHandle();
4073 IngestExternalFileOptions ifo;
4074 ifo.move_files = move_files_;
4075 ifo.snapshot_consistency = snapshot_consistency_;
4076 ifo.allow_global_seqno = allow_global_seqno_;
4077 ifo.allow_blocking_flush = allow_blocking_flush_;
4078 ifo.ingest_behind = ingest_behind_;
4079 ifo.write_global_seqno = write_global_seqno_;
4080 Status status = db_->IngestExternalFile(cfh, {input_sst_path_}, ifo);
4081 if (!status.ok()) {
4082 exec_state_ = LDBCommandExecuteResult::Failed(
4083 "failed to ingest external SST: " + status.ToString());
4084 } else {
4085 exec_state_ =
4086 LDBCommandExecuteResult::Succeed("external SST files ingested");
4087 }
4088}
4089
20effc67
TL		 4090void IngestExternalSstFilesCommand::OverrideBaseOptions() {
4091 LDBCommand::OverrideBaseOptions();
4092 options_.create_if_missing = create_if_missing_;
7c673cae
FG		 4093}
4094
f67539c2
TL		 4095ListFileRangeDeletesCommand::ListFileRangeDeletesCommand(
4096 const std::map<std::string, std::string>& options,
4097 const std::vector<std::string>& flags)
4098 : LDBCommand(options, flags, true, BuildCmdLineOptions({ARG_MAX_KEYS})) {
1e59de90 4099 auto itr = options.find(ARG_MAX_KEYS);
f67539c2
TL		 4100 if (itr != options.end()) {
4101 try {
4102#if defined(CYGWIN)
4103 max_keys_ = strtol(itr->second.c_str(), 0, 10);
4104#else
4105 max_keys_ = std::stoi(itr->second);
4106#endif
4107 } catch (const std::invalid_argument&) {
4108 exec_state_ = LDBCommandExecuteResult::Failed(ARG_MAX_KEYS +
4109 " has an invalid value");
4110 } catch (const std::out_of_range&) {
4111 exec_state_ = LDBCommandExecuteResult::Failed(
4112 ARG_MAX_KEYS + " has a value out-of-range");
4113 }
4114 }
4115}
4116
4117void ListFileRangeDeletesCommand::Help(std::string& ret) {
4118 ret.append(" ");
4119 ret.append(ListFileRangeDeletesCommand::Name());
4120 ret.append(" [--" + ARG_MAX_KEYS + "=<N>]");
4121 ret.append(" : print tombstones in SST files.\n");
4122}
4123
4124void ListFileRangeDeletesCommand::DoCommand() {
4125 if (!db_) {
4126 assert(GetExecuteState().IsFailed());
4127 return;
4128 }
4129
20effc67 4130 DBImpl* db_impl = static_cast_with_check<DBImpl>(db_->GetRootDB());
f67539c2
TL		 4131
4132 std::string out_str;
4133
4134 Status st =
4135 db_impl->TablesRangeTombstoneSummary(GetCfHandle(), max_keys_, &out_str);
4136 if (st.ok()) {
4137 TEST_SYNC_POINT_CALLBACK(
4138 "ListFileRangeDeletesCommand::DoCommand:BeforePrint", &out_str);
4139 fprintf(stdout, "%s\n", out_str.c_str());
20effc67
TL		 4140 }
4141}
4142
4143void UnsafeRemoveSstFileCommand::Help(std::string& ret) {
4144 ret.append(" ");
4145 ret.append(UnsafeRemoveSstFileCommand::Name());
4146 ret.append(" <SST file number>");
1e59de90 4147 ret.append(" ");
20effc67
TL		 4148 ret.append(" MUST NOT be used on a live DB.");
4149 ret.append("\n");
4150}
4151
4152UnsafeRemoveSstFileCommand::UnsafeRemoveSstFileCommand(
4153 const std::vector<std::string>& params,
4154 const std::map<std::string, std::string>& options,
4155 const std::vector<std::string>& flags)
4156 : LDBCommand(options, flags, false /* is_read_only */,
4157 BuildCmdLineOptions({})) {
4158 if (params.size() != 1) {
4159 exec_state_ =
4160 LDBCommandExecuteResult::Failed("SST file number must be specified");
f67539c2 4161 } else {
20effc67
TL		 4162 char* endptr = nullptr;
4163 sst_file_number_ = strtoull(params.at(0).c_str(), &endptr, 10 /* base */);
4164 if (endptr == nullptr || *endptr != '\0') {
4165 exec_state_ = LDBCommandExecuteResult::Failed(
4166 "Failed to parse SST file number " + params.at(0));
4167 }
4168 }
4169}
4170
4171void UnsafeRemoveSstFileCommand::DoCommand() {
20effc67
TL		 4172 PrepareOptions();
4173
1e59de90
TL		 4174 OfflineManifestWriter w(options_, db_path_);
4175 if (column_families_.empty()) {
4176 column_families_.emplace_back(kDefaultColumnFamilyName, options_);
20effc67 4177 }
1e59de90 4178 Status s = w.Recover(column_families_);
20effc67
TL		 4179
4180 ColumnFamilyData* cfd = nullptr;
4181 int level = -1;
4182 if (s.ok()) {
4183 FileMetaData* metadata = nullptr;
1e59de90
TL		 4184 s = w.Versions().GetMetadataForFile(sst_file_number_, &level, &metadata,
4185 &cfd);
20effc67
TL		 4186 }
4187
4188 if (s.ok()) {
4189 VersionEdit edit;
4190 edit.SetColumnFamily(cfd->GetID());
4191 edit.DeleteFile(level, sst_file_number_);
1e59de90
TL		 4192 std::unique_ptr<FSDirectory> db_dir;
4193 s = options_.env->GetFileSystem()->NewDirectory(db_path_, IOOptions(),
4194 &db_dir, nullptr);
4195 if (s.ok()) {
4196 s = w.LogAndApply(cfd, &edit, db_dir.get());
4197 }
20effc67
TL		 4198 }
4199
4200 if (!s.ok()) {
4201 exec_state_ = LDBCommandExecuteResult::Failed(
4202 "failed to unsafely remove SST file: " + s.ToString());
4203 } else {
4204 exec_state_ = LDBCommandExecuteResult::Succeed("unsafely removed SST file");
f67539c2
TL		 4205 }
4206}
4207
1e59de90
TL		 4208const std::string UpdateManifestCommand::ARG_VERBOSE = "verbose";
4209const std::string UpdateManifestCommand::ARG_UPDATE_TEMPERATURES =
4210 "update_temperatures";
4211
4212void UpdateManifestCommand::Help(std::string& ret) {
4213 ret.append(" ");
4214 ret.append(UpdateManifestCommand::Name());
4215 ret.append(" [--update_temperatures]");
4216 ret.append(" ");
4217 ret.append(" MUST NOT be used on a live DB.");
4218 ret.append("\n");
4219}
4220
4221UpdateManifestCommand::UpdateManifestCommand(
4222 const std::vector<std::string>& /*params*/,
4223 const std::map<std::string, std::string>& options,
4224 const std::vector<std::string>& flags)
4225 : LDBCommand(options, flags, false /* is_read_only */,
4226 BuildCmdLineOptions({ARG_VERBOSE, ARG_UPDATE_TEMPERATURES})) {
4227 verbose_ = IsFlagPresent(flags, ARG_VERBOSE) ||
4228 ParseBooleanOption(options, ARG_VERBOSE, false);
4229 update_temperatures_ =
4230 IsFlagPresent(flags, ARG_UPDATE_TEMPERATURES) ||
4231 ParseBooleanOption(options, ARG_UPDATE_TEMPERATURES, false);
4232
4233 if (!update_temperatures_) {
4234 exec_state_ = LDBCommandExecuteResult::Failed(
4235 "No action like --update_temperatures specified for update_manifest");
4236 }
4237}
4238
4239void UpdateManifestCommand::DoCommand() {
4240 PrepareOptions();
4241
4242 auto level = verbose_ ? InfoLogLevel::INFO_LEVEL : InfoLogLevel::WARN_LEVEL;
4243 options_.info_log.reset(new StderrLogger(level));
4244
4245 experimental::UpdateManifestForFilesStateOptions opts;
4246 opts.update_temperatures = update_temperatures_;
4247 if (column_families_.empty()) {
4248 column_families_.emplace_back(kDefaultColumnFamilyName, options_);
4249 }
4250 Status s = experimental::UpdateManifestForFilesState(options_, db_path_,
4251 column_families_);
4252
4253 if (!s.ok()) {
4254 exec_state_ = LDBCommandExecuteResult::Failed(
4255 "failed to update manifest: " + s.ToString());
4256 } else {
4257 exec_state_ =
4258 LDBCommandExecuteResult::Succeed("Manifest updates successful");
4259 }
4260}
4261
f67539c2 4262} // namespace ROCKSDB_NAMESPACE
7c673cae 4263#endif // ROCKSDB_LITE
Ceph