2 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
3 // This source code is licensed under both the GPLv2 (found in the
4 // COPYING file in the root directory) and Apache 2.0 License
5 // (found in the LICENSE.Apache file in the root directory).
8 #include "rocksdb/utilities/ldb_cmd.h"
21 #include "db/blob/blob_index.h"
22 #include "db/db_impl/db_impl.h"
23 #include "db/dbformat.h"
24 #include "db/log_reader.h"
25 #include "db/version_util.h"
26 #include "db/write_batch_internal.h"
27 #include "file/filename.h"
28 #include "rocksdb/cache.h"
29 #include "rocksdb/experimental.h"
30 #include "rocksdb/file_checksum.h"
31 #include "rocksdb/filter_policy.h"
32 #include "rocksdb/options.h"
33 #include "rocksdb/table_properties.h"
34 #include "rocksdb/utilities/backup_engine.h"
35 #include "rocksdb/utilities/checkpoint.h"
36 #include "rocksdb/utilities/debug.h"
37 #include "rocksdb/utilities/options_util.h"
38 #include "rocksdb/write_batch.h"
39 #include "rocksdb/write_buffer_manager.h"
40 #include "table/scoped_arena_iterator.h"
41 #include "table/sst_file_dumper.h"
42 #include "tools/ldb_cmd_impl.h"
43 #include "util/cast_util.h"
44 #include "util/coding.h"
45 #include "util/file_checksum_helper.h"
46 #include "util/stderr_logger.h"
47 #include "util/string_util.h"
48 #include "utilities/blob_db/blob_dump_tool.h"
49 #include "utilities/merge_operators.h"
50 #include "utilities/ttl/db_ttl_impl.h"
52 namespace ROCKSDB_NAMESPACE
{
54 class FileChecksumGenCrc32c
;
55 class FileChecksumGenCrc32cFactory
;
57 const std::string
LDBCommand::ARG_ENV_URI
= "env_uri";
58 const std::string
LDBCommand::ARG_FS_URI
= "fs_uri";
59 const std::string
LDBCommand::ARG_DB
= "db";
60 const std::string
LDBCommand::ARG_PATH
= "path";
61 const std::string
LDBCommand::ARG_SECONDARY_PATH
= "secondary_path";
62 const std::string
LDBCommand::ARG_HEX
= "hex";
63 const std::string
LDBCommand::ARG_KEY_HEX
= "key_hex";
64 const std::string
LDBCommand::ARG_VALUE_HEX
= "value_hex";
65 const std::string
LDBCommand::ARG_CF_NAME
= "column_family";
66 const std::string
LDBCommand::ARG_TTL
= "ttl";
67 const std::string
LDBCommand::ARG_TTL_START
= "start_time";
68 const std::string
LDBCommand::ARG_TTL_END
= "end_time";
69 const std::string
LDBCommand::ARG_TIMESTAMP
= "timestamp";
70 const std::string
LDBCommand::ARG_TRY_LOAD_OPTIONS
= "try_load_options";
71 const std::string
LDBCommand::ARG_DISABLE_CONSISTENCY_CHECKS
=
72 "disable_consistency_checks";
73 const std::string
LDBCommand::ARG_IGNORE_UNKNOWN_OPTIONS
=
74 "ignore_unknown_options";
75 const std::string
LDBCommand::ARG_FROM
= "from";
76 const std::string
LDBCommand::ARG_TO
= "to";
77 const std::string
LDBCommand::ARG_MAX_KEYS
= "max_keys";
78 const std::string
LDBCommand::ARG_BLOOM_BITS
= "bloom_bits";
79 const std::string
LDBCommand::ARG_FIX_PREFIX_LEN
= "fix_prefix_len";
80 const std::string
LDBCommand::ARG_COMPRESSION_TYPE
= "compression_type";
81 const std::string
LDBCommand::ARG_COMPRESSION_MAX_DICT_BYTES
=
82 "compression_max_dict_bytes";
83 const std::string
LDBCommand::ARG_BLOCK_SIZE
= "block_size";
84 const std::string
LDBCommand::ARG_AUTO_COMPACTION
= "auto_compaction";
85 const std::string
LDBCommand::ARG_DB_WRITE_BUFFER_SIZE
= "db_write_buffer_size";
86 const std::string
LDBCommand::ARG_WRITE_BUFFER_SIZE
= "write_buffer_size";
87 const std::string
LDBCommand::ARG_FILE_SIZE
= "file_size";
88 const std::string
LDBCommand::ARG_CREATE_IF_MISSING
= "create_if_missing";
89 const std::string
LDBCommand::ARG_NO_VALUE
= "no_value";
90 const std::string
LDBCommand::ARG_ENABLE_BLOB_FILES
= "enable_blob_files";
91 const std::string
LDBCommand::ARG_MIN_BLOB_SIZE
= "min_blob_size";
92 const std::string
LDBCommand::ARG_BLOB_FILE_SIZE
= "blob_file_size";
93 const std::string
LDBCommand::ARG_BLOB_COMPRESSION_TYPE
=
94 "blob_compression_type";
95 const std::string
LDBCommand::ARG_ENABLE_BLOB_GARBAGE_COLLECTION
=
96 "enable_blob_garbage_collection";
97 const std::string
LDBCommand::ARG_BLOB_GARBAGE_COLLECTION_AGE_CUTOFF
=
98 "blob_garbage_collection_age_cutoff";
99 const std::string
LDBCommand::ARG_BLOB_GARBAGE_COLLECTION_FORCE_THRESHOLD
=
100 "blob_garbage_collection_force_threshold";
101 const std::string
LDBCommand::ARG_BLOB_COMPACTION_READAHEAD_SIZE
=
102 "blob_compaction_readahead_size";
103 const std::string
LDBCommand::ARG_BLOB_FILE_STARTING_LEVEL
=
104 "blob_file_starting_level";
105 const std::string
LDBCommand::ARG_PREPOPULATE_BLOB_CACHE
=
106 "prepopulate_blob_cache";
107 const std::string
LDBCommand::ARG_DECODE_BLOB_INDEX
= "decode_blob_index";
108 const std::string
LDBCommand::ARG_DUMP_UNCOMPRESSED_BLOBS
=
109 "dump_uncompressed_blobs";
111 const char* LDBCommand::DELIM
= " ==> ";
115 void DumpWalFile(Options options
, std::string wal_file
, bool print_header
,
116 bool print_values
, bool is_write_committed
,
117 LDBCommandExecuteResult
* exec_state
);
119 void DumpSstFile(Options options
, std::string filename
, bool output_hex
,
120 bool show_properties
, bool decode_blob_index
,
121 std::string from_key
= "", std::string to_key
= "");
123 void DumpBlobFile(const std::string
& filename
, bool is_key_hex
,
124 bool is_value_hex
, bool dump_uncompressed_blobs
);
127 LDBCommand
* LDBCommand::InitFromCmdLineArgs(
128 int argc
, char const* const* argv
, const Options
& options
,
129 const LDBOptions
& ldb_options
,
130 const std::vector
<ColumnFamilyDescriptor
>* column_families
) {
131 std::vector
<std::string
> args
;
132 for (int i
= 1; i
< argc
; i
++) {
133 args
.push_back(argv
[i
]);
135 return InitFromCmdLineArgs(args
, options
, ldb_options
, column_families
,
140 * Parse the command-line arguments and create the appropriate LDBCommand2
142 * The command line arguments must be in the following format:
143 * ./ldb --db=PATH_TO_DB [--commonOpt1=commonOpt1Val] ..
144 * COMMAND <PARAM1> <PARAM2> ... [-cmdSpecificOpt1=cmdSpecificOpt1Val] ..
145 * This is similar to the command line format used by HBaseClientTool.
146 * Command name is not included in args.
147 * Returns nullptr if the command-line cannot be parsed.
149 LDBCommand
* LDBCommand::InitFromCmdLineArgs(
150 const std::vector
<std::string
>& args
, const Options
& options
,
151 const LDBOptions
& ldb_options
,
152 const std::vector
<ColumnFamilyDescriptor
>* /*column_families*/,
153 const std::function
<LDBCommand
*(const ParsedParams
&)>& selector
) {
154 // --x=y command line arguments are added as x->y map entries in
155 // parsed_params.option_map.
157 // Command-line arguments of the form --hex end up in this array as hex to
158 // parsed_params.flags
159 ParsedParams parsed_params
;
161 // Everything other than option_map and flags. Represents commands
162 // and their parameters. For eg: put key1 value1 go into this vector.
163 std::vector
<std::string
> cmdTokens
;
165 const std::string OPTION_PREFIX
= "--";
167 for (const auto& arg
: args
) {
168 if (arg
[0] == '-' && arg
[1] == '-') {
169 std::vector
<std::string
> splits
= StringSplit(arg
, '=');
170 // --option_name=option_value
171 if (splits
.size() == 2) {
172 std::string optionKey
= splits
[0].substr(OPTION_PREFIX
.size());
173 parsed_params
.option_map
[optionKey
] = splits
[1];
174 } else if (splits
.size() == 1) {
176 std::string optionKey
= splits
[0].substr(OPTION_PREFIX
.size());
177 parsed_params
.flags
.push_back(optionKey
);
179 // --option_name=option_value, option_value contains '='
180 std::string optionKey
= splits
[0].substr(OPTION_PREFIX
.size());
181 parsed_params
.option_map
[optionKey
] =
182 arg
.substr(splits
[0].length() + 1);
185 cmdTokens
.push_back(arg
);
189 if (cmdTokens
.size() < 1) {
190 fprintf(stderr
, "Command not specified!");
194 parsed_params
.cmd
= cmdTokens
[0];
195 parsed_params
.cmd_params
.assign(cmdTokens
.begin() + 1, cmdTokens
.end());
197 LDBCommand
* command
= selector(parsed_params
);
200 command
->SetDBOptions(options
);
201 command
->SetLDBOptions(ldb_options
);
206 LDBCommand
* LDBCommand::SelectCommand(const ParsedParams
& parsed_params
) {
207 if (parsed_params
.cmd
== GetCommand::Name()) {
208 return new GetCommand(parsed_params
.cmd_params
, parsed_params
.option_map
,
209 parsed_params
.flags
);
210 } else if (parsed_params
.cmd
== PutCommand::Name()) {
211 return new PutCommand(parsed_params
.cmd_params
, parsed_params
.option_map
,
212 parsed_params
.flags
);
213 } else if (parsed_params
.cmd
== BatchPutCommand::Name()) {
214 return new BatchPutCommand(parsed_params
.cmd_params
,
215 parsed_params
.option_map
, parsed_params
.flags
);
216 } else if (parsed_params
.cmd
== ScanCommand::Name()) {
217 return new ScanCommand(parsed_params
.cmd_params
, parsed_params
.option_map
,
218 parsed_params
.flags
);
219 } else if (parsed_params
.cmd
== DeleteCommand::Name()) {
220 return new DeleteCommand(parsed_params
.cmd_params
, parsed_params
.option_map
,
221 parsed_params
.flags
);
222 } else if (parsed_params
.cmd
== SingleDeleteCommand::Name()) {
223 return new SingleDeleteCommand(parsed_params
.cmd_params
,
224 parsed_params
.option_map
,
225 parsed_params
.flags
);
226 } else if (parsed_params
.cmd
== DeleteRangeCommand::Name()) {
227 return new DeleteRangeCommand(parsed_params
.cmd_params
,
228 parsed_params
.option_map
,
229 parsed_params
.flags
);
230 } else if (parsed_params
.cmd
== ApproxSizeCommand::Name()) {
231 return new ApproxSizeCommand(parsed_params
.cmd_params
,
232 parsed_params
.option_map
, parsed_params
.flags
);
233 } else if (parsed_params
.cmd
== DBQuerierCommand::Name()) {
234 return new DBQuerierCommand(parsed_params
.cmd_params
,
235 parsed_params
.option_map
, parsed_params
.flags
);
236 } else if (parsed_params
.cmd
== CompactorCommand::Name()) {
237 return new CompactorCommand(parsed_params
.cmd_params
,
238 parsed_params
.option_map
, parsed_params
.flags
);
239 } else if (parsed_params
.cmd
== WALDumperCommand::Name()) {
240 return new WALDumperCommand(parsed_params
.cmd_params
,
241 parsed_params
.option_map
, parsed_params
.flags
);
242 } else if (parsed_params
.cmd
== ReduceDBLevelsCommand::Name()) {
243 return new ReduceDBLevelsCommand(parsed_params
.cmd_params
,
244 parsed_params
.option_map
,
245 parsed_params
.flags
);
246 } else if (parsed_params
.cmd
== ChangeCompactionStyleCommand::Name()) {
247 return new ChangeCompactionStyleCommand(parsed_params
.cmd_params
,
248 parsed_params
.option_map
,
249 parsed_params
.flags
);
250 } else if (parsed_params
.cmd
== DBDumperCommand::Name()) {
251 return new DBDumperCommand(parsed_params
.cmd_params
,
252 parsed_params
.option_map
, parsed_params
.flags
);
253 } else if (parsed_params
.cmd
== DBLoaderCommand::Name()) {
254 return new DBLoaderCommand(parsed_params
.cmd_params
,
255 parsed_params
.option_map
, parsed_params
.flags
);
256 } else if (parsed_params
.cmd
== ManifestDumpCommand::Name()) {
257 return new ManifestDumpCommand(parsed_params
.cmd_params
,
258 parsed_params
.option_map
,
259 parsed_params
.flags
);
260 } else if (parsed_params
.cmd
== FileChecksumDumpCommand::Name()) {
261 return new FileChecksumDumpCommand(parsed_params
.cmd_params
,
262 parsed_params
.option_map
,
263 parsed_params
.flags
);
264 } else if (parsed_params
.cmd
== GetPropertyCommand::Name()) {
265 return new GetPropertyCommand(parsed_params
.cmd_params
,
266 parsed_params
.option_map
,
267 parsed_params
.flags
);
268 } else if (parsed_params
.cmd
== ListColumnFamiliesCommand::Name()) {
269 return new ListColumnFamiliesCommand(parsed_params
.cmd_params
,
270 parsed_params
.option_map
,
271 parsed_params
.flags
);
272 } else if (parsed_params
.cmd
== CreateColumnFamilyCommand::Name()) {
273 return new CreateColumnFamilyCommand(parsed_params
.cmd_params
,
274 parsed_params
.option_map
,
275 parsed_params
.flags
);
276 } else if (parsed_params
.cmd
== DropColumnFamilyCommand::Name()) {
277 return new DropColumnFamilyCommand(parsed_params
.cmd_params
,
278 parsed_params
.option_map
,
279 parsed_params
.flags
);
280 } else if (parsed_params
.cmd
== DBFileDumperCommand::Name()) {
281 return new DBFileDumperCommand(parsed_params
.cmd_params
,
282 parsed_params
.option_map
,
283 parsed_params
.flags
);
284 } else if (parsed_params
.cmd
== DBLiveFilesMetadataDumperCommand::Name()) {
285 return new DBLiveFilesMetadataDumperCommand(parsed_params
.cmd_params
,
286 parsed_params
.option_map
,
287 parsed_params
.flags
);
288 } else if (parsed_params
.cmd
== InternalDumpCommand::Name()) {
289 return new InternalDumpCommand(parsed_params
.cmd_params
,
290 parsed_params
.option_map
,
291 parsed_params
.flags
);
292 } else if (parsed_params
.cmd
== CheckConsistencyCommand::Name()) {
293 return new CheckConsistencyCommand(parsed_params
.cmd_params
,
294 parsed_params
.option_map
,
295 parsed_params
.flags
);
296 } else if (parsed_params
.cmd
== CheckPointCommand::Name()) {
297 return new CheckPointCommand(parsed_params
.cmd_params
,
298 parsed_params
.option_map
, parsed_params
.flags
);
299 } else if (parsed_params
.cmd
== RepairCommand::Name()) {
300 return new RepairCommand(parsed_params
.cmd_params
, parsed_params
.option_map
,
301 parsed_params
.flags
);
302 } else if (parsed_params
.cmd
== BackupCommand::Name()) {
303 return new BackupCommand(parsed_params
.cmd_params
, parsed_params
.option_map
,
304 parsed_params
.flags
);
305 } else if (parsed_params
.cmd
== RestoreCommand::Name()) {
306 return new RestoreCommand(parsed_params
.cmd_params
,
307 parsed_params
.option_map
, parsed_params
.flags
);
308 } else if (parsed_params
.cmd
== WriteExternalSstFilesCommand::Name()) {
309 return new WriteExternalSstFilesCommand(parsed_params
.cmd_params
,
310 parsed_params
.option_map
,
311 parsed_params
.flags
);
312 } else if (parsed_params
.cmd
== IngestExternalSstFilesCommand::Name()) {
313 return new IngestExternalSstFilesCommand(parsed_params
.cmd_params
,
314 parsed_params
.option_map
,
315 parsed_params
.flags
);
316 } else if (parsed_params
.cmd
== ListFileRangeDeletesCommand::Name()) {
317 return new ListFileRangeDeletesCommand(parsed_params
.option_map
,
318 parsed_params
.flags
);
319 } else if (parsed_params
.cmd
== UnsafeRemoveSstFileCommand::Name()) {
320 return new UnsafeRemoveSstFileCommand(parsed_params
.cmd_params
,
321 parsed_params
.option_map
,
322 parsed_params
.flags
);
323 } else if (parsed_params
.cmd
== UpdateManifestCommand::Name()) {
324 return new UpdateManifestCommand(parsed_params
.cmd_params
,
325 parsed_params
.option_map
,
326 parsed_params
.flags
);
331 /* Run the command, and return the execute result. */
332 void LDBCommand::Run() {
333 if (!exec_state_
.IsNotStarted()) {
337 if (!options_
.env
|| options_
.env
== Env::Default()) {
338 Env
* env
= Env::Default();
339 Status s
= Env::CreateFromUri(config_options_
, env_uri_
, fs_uri_
, &env
,
342 fprintf(stderr
, "%s\n", s
.ToString().c_str());
343 exec_state_
= LDBCommandExecuteResult::Failed(s
.ToString());
349 if (db_
== nullptr && !NoDBOpen()) {
351 if (exec_state_
.IsFailed() && try_load_options_
) {
352 // We don't always return if there is a failure because a WAL file or
353 // manifest file can be given to "dump" command so we should continue.
354 // --try_load_options is not valid in those cases.
359 // We'll intentionally proceed even if the DB can't be opened because users
360 // can also specify a filename, not just a directory.
363 if (exec_state_
.IsNotStarted()) {
364 exec_state_
= LDBCommandExecuteResult::Succeed("");
367 if (db_
!= nullptr) {
372 LDBCommand::LDBCommand(const std::map
<std::string
, std::string
>& options
,
373 const std::vector
<std::string
>& flags
, bool is_read_only
,
374 const std::vector
<std::string
>& valid_cmd_line_options
)
377 is_read_only_(is_read_only
),
379 is_value_hex_(false),
382 try_load_options_(false),
383 create_if_missing_(false),
384 option_map_(options
),
386 valid_cmd_line_options_(valid_cmd_line_options
) {
387 auto itr
= options
.find(ARG_DB
);
388 if (itr
!= options
.end()) {
389 db_path_
= itr
->second
;
392 itr
= options
.find(ARG_ENV_URI
);
393 if (itr
!= options
.end()) {
394 env_uri_
= itr
->second
;
397 itr
= options
.find(ARG_FS_URI
);
398 if (itr
!= options
.end()) {
399 fs_uri_
= itr
->second
;
402 itr
= options
.find(ARG_CF_NAME
);
403 if (itr
!= options
.end()) {
404 column_family_name_
= itr
->second
;
406 column_family_name_
= kDefaultColumnFamilyName
;
409 itr
= options
.find(ARG_SECONDARY_PATH
);
410 secondary_path_
= "";
411 if (itr
!= options
.end()) {
412 secondary_path_
= itr
->second
;
415 is_key_hex_
= IsKeyHex(options
, flags
);
416 is_value_hex_
= IsValueHex(options
, flags
);
417 is_db_ttl_
= IsFlagPresent(flags
, ARG_TTL
);
418 timestamp_
= IsFlagPresent(flags
, ARG_TIMESTAMP
);
419 try_load_options_
= IsTryLoadOptions(options
, flags
);
420 force_consistency_checks_
=
421 !IsFlagPresent(flags
, ARG_DISABLE_CONSISTENCY_CHECKS
);
422 enable_blob_files_
= IsFlagPresent(flags
, ARG_ENABLE_BLOB_FILES
);
423 enable_blob_garbage_collection_
=
424 IsFlagPresent(flags
, ARG_ENABLE_BLOB_GARBAGE_COLLECTION
);
425 config_options_
.ignore_unknown_options
=
426 IsFlagPresent(flags
, ARG_IGNORE_UNKNOWN_OPTIONS
);
429 void LDBCommand::OpenDB() {
431 if (!exec_state_
.IsNotStarted()) {
434 if (column_families_
.empty() && !options_
.merge_operator
) {
435 // No harm to add a general merge operator if it is not specified.
436 options_
.merge_operator
= MergeOperators::CreateStringAppendOperator(':');
440 std::vector
<ColumnFamilyHandle
*> handles_opened
;
442 // ldb doesn't yet support TTL DB with multiple column families
443 if (!column_family_name_
.empty() || !column_families_
.empty()) {
444 exec_state_
= LDBCommandExecuteResult::Failed(
445 "ldb doesn't support TTL DB with multiple column families");
447 if (!secondary_path_
.empty()) {
448 exec_state_
= LDBCommandExecuteResult::Failed(
449 "Open as secondary is not supported for TTL DB yet.");
452 st
= DBWithTTL::Open(options_
, db_path_
, &db_ttl_
, 0, true);
454 st
= DBWithTTL::Open(options_
, db_path_
, &db_ttl_
);
458 if (is_read_only_
&& secondary_path_
.empty()) {
459 if (column_families_
.empty()) {
460 st
= DB::OpenForReadOnly(options_
, db_path_
, &db_
);
462 st
= DB::OpenForReadOnly(options_
, db_path_
, column_families_
,
463 &handles_opened
, &db_
);
466 if (column_families_
.empty()) {
467 if (secondary_path_
.empty()) {
468 st
= DB::Open(options_
, db_path_
, &db_
);
470 st
= DB::OpenAsSecondary(options_
, db_path_
, secondary_path_
, &db_
);
473 if (secondary_path_
.empty()) {
474 st
= DB::Open(options_
, db_path_
, column_families_
, &handles_opened
,
477 st
= DB::OpenAsSecondary(options_
, db_path_
, secondary_path_
,
478 column_families_
, &handles_opened
, &db_
);
484 std::string msg
= st
.ToString();
485 exec_state_
= LDBCommandExecuteResult::Failed(msg
);
486 } else if (!handles_opened
.empty()) {
487 assert(handles_opened
.size() == column_families_
.size());
488 bool found_cf_name
= false;
489 for (size_t i
= 0; i
< handles_opened
.size(); i
++) {
490 cf_handles_
[column_families_
[i
].name
] = handles_opened
[i
];
491 if (column_family_name_
== column_families_
[i
].name
) {
492 found_cf_name
= true;
495 if (!found_cf_name
) {
496 exec_state_
= LDBCommandExecuteResult::Failed(
497 "Non-existing column family " + column_family_name_
);
501 // We successfully opened DB in single column family mode.
502 assert(column_families_
.empty());
503 if (column_family_name_
!= kDefaultColumnFamilyName
) {
504 exec_state_
= LDBCommandExecuteResult::Failed(
505 "Non-existing column family " + column_family_name_
);
511 void LDBCommand::CloseDB() {
512 if (db_
!= nullptr) {
513 for (auto& pair
: cf_handles_
) {
516 Status s
= db_
->Close();
517 s
.PermitUncheckedError();
523 ColumnFamilyHandle
* LDBCommand::GetCfHandle() {
524 if (!cf_handles_
.empty()) {
525 auto it
= cf_handles_
.find(column_family_name_
);
526 if (it
== cf_handles_
.end()) {
527 exec_state_
= LDBCommandExecuteResult::Failed(
528 "Cannot find column family " + column_family_name_
);
533 return db_
->DefaultColumnFamily();
536 std::vector
<std::string
> LDBCommand::BuildCmdLineOptions(
537 std::vector
<std::string
> options
) {
538 std::vector
<std::string
> ret
= {ARG_ENV_URI
,
545 ARG_COMPRESSION_TYPE
,
546 ARG_COMPRESSION_MAX_DICT_BYTES
,
547 ARG_WRITE_BUFFER_SIZE
,
550 ARG_TRY_LOAD_OPTIONS
,
551 ARG_DISABLE_CONSISTENCY_CHECKS
,
552 ARG_ENABLE_BLOB_FILES
,
555 ARG_BLOB_COMPRESSION_TYPE
,
556 ARG_ENABLE_BLOB_GARBAGE_COLLECTION
,
557 ARG_BLOB_GARBAGE_COLLECTION_AGE_CUTOFF
,
558 ARG_BLOB_GARBAGE_COLLECTION_FORCE_THRESHOLD
,
559 ARG_BLOB_COMPACTION_READAHEAD_SIZE
,
560 ARG_BLOB_FILE_STARTING_LEVEL
,
561 ARG_PREPOPULATE_BLOB_CACHE
,
562 ARG_IGNORE_UNKNOWN_OPTIONS
,
564 ret
.insert(ret
.end(), options
.begin(), options
.end());
569 * Parses the specific double option and fills in the value.
570 * Returns true if the option is found.
571 * Returns false if the option is not found or if there is an error parsing the
572 * value. If there is an error, the specified exec_state is also
575 bool LDBCommand::ParseDoubleOption(
576 const std::map
<std::string
, std::string
>& /*options*/,
577 const std::string
& option
, double& value
,
578 LDBCommandExecuteResult
& exec_state
) {
579 auto itr
= option_map_
.find(option
);
580 if (itr
!= option_map_
.end()) {
582 char* str_end
= nullptr;
583 value
= std::strtod(itr
->second
.c_str(), &str_end
);
584 if (str_end
== itr
->second
.c_str()) {
586 LDBCommandExecuteResult::Failed(option
+ " has an invalid value.");
587 } else if (errno
== ERANGE
) {
588 exec_state
= LDBCommandExecuteResult::Failed(
589 option
+ " has a value out-of-range.");
595 value
= std::stod(itr
->second
);
597 } catch (const std::invalid_argument
&) {
599 LDBCommandExecuteResult::Failed(option
+ " has an invalid value.");
600 } catch (const std::out_of_range
&) {
601 exec_state
= LDBCommandExecuteResult::Failed(
602 option
+ " has a value out-of-range.");
610 * Parses the specific integer option and fills in the value.
611 * Returns true if the option is found.
612 * Returns false if the option is not found or if there is an error parsing the
613 * value. If there is an error, the specified exec_state is also
616 bool LDBCommand::ParseIntOption(
617 const std::map
<std::string
, std::string
>& /*options*/,
618 const std::string
& option
, int& value
,
619 LDBCommandExecuteResult
& exec_state
) {
620 auto itr
= option_map_
.find(option
);
621 if (itr
!= option_map_
.end()) {
623 char* str_end
= nullptr;
624 value
= strtol(itr
->second
.c_str(), &str_end
, 10);
625 if (str_end
== itr
->second
.c_str()) {
627 LDBCommandExecuteResult::Failed(option
+ " has an invalid value.");
628 } else if (errno
== ERANGE
) {
629 exec_state
= LDBCommandExecuteResult::Failed(
630 option
+ " has a value out-of-range.");
636 value
= std::stoi(itr
->second
);
638 } catch (const std::invalid_argument
&) {
640 LDBCommandExecuteResult::Failed(option
+ " has an invalid value.");
641 } catch (const std::out_of_range
&) {
642 exec_state
= LDBCommandExecuteResult::Failed(
643 option
+ " has a value out-of-range.");
651 * Parses the specified option and fills in the value.
652 * Returns true if the option is found.
653 * Returns false otherwise.
655 bool LDBCommand::ParseStringOption(
656 const std::map
<std::string
, std::string
>& /*options*/,
657 const std::string
& option
, std::string
* value
) {
658 auto itr
= option_map_
.find(option
);
659 if (itr
!= option_map_
.end()) {
660 *value
= itr
->second
;
667 * Parses the specified compression type and fills in the value.
668 * Returns true if the compression type is found.
669 * Returns false otherwise.
671 bool LDBCommand::ParseCompressionTypeOption(
672 const std::map
<std::string
, std::string
>& /*options*/,
673 const std::string
& option
, CompressionType
& value
,
674 LDBCommandExecuteResult
& exec_state
) {
675 auto itr
= option_map_
.find(option
);
676 if (itr
!= option_map_
.end()) {
677 const std::string
& comp
= itr
->second
;
679 value
= kNoCompression
;
681 } else if (comp
== "snappy") {
682 value
= kSnappyCompression
;
684 } else if (comp
== "zlib") {
685 value
= kZlibCompression
;
687 } else if (comp
== "bzip2") {
688 value
= kBZip2Compression
;
690 } else if (comp
== "lz4") {
691 value
= kLZ4Compression
;
693 } else if (comp
== "lz4hc") {
694 value
= kLZ4HCCompression
;
696 } else if (comp
== "xpress") {
697 value
= kXpressCompression
;
699 } else if (comp
== "zstd") {
703 // Unknown compression.
704 exec_state
= LDBCommandExecuteResult::Failed(
705 "Unknown compression algorithm: " + comp
);
711 void LDBCommand::OverrideBaseOptions() {
712 options_
.create_if_missing
= false;
714 int db_write_buffer_size
;
715 if (ParseIntOption(option_map_
, ARG_DB_WRITE_BUFFER_SIZE
,
716 db_write_buffer_size
, exec_state_
)) {
717 if (db_write_buffer_size
>= 0) {
718 options_
.db_write_buffer_size
= db_write_buffer_size
;
720 exec_state_
= LDBCommandExecuteResult::Failed(ARG_DB_WRITE_BUFFER_SIZE
+
725 if (options_
.db_paths
.size() == 0) {
726 options_
.db_paths
.emplace_back(db_path_
,
727 std::numeric_limits
<uint64_t>::max());
730 OverrideBaseCFOptions(static_cast<ColumnFamilyOptions
*>(&options_
));
733 void LDBCommand::OverrideBaseCFOptions(ColumnFamilyOptions
* cf_opts
) {
734 BlockBasedTableOptions table_options
;
735 bool use_table_options
= false;
737 if (ParseIntOption(option_map_
, ARG_BLOOM_BITS
, bits
, exec_state_
)) {
739 use_table_options
= true;
740 table_options
.filter_policy
.reset(NewBloomFilterPolicy(bits
));
743 LDBCommandExecuteResult::Failed(ARG_BLOOM_BITS
+ " must be > 0.");
748 if (ParseIntOption(option_map_
, ARG_BLOCK_SIZE
, block_size
, exec_state_
)) {
749 if (block_size
> 0) {
750 use_table_options
= true;
751 table_options
.block_size
= block_size
;
754 LDBCommandExecuteResult::Failed(ARG_BLOCK_SIZE
+ " must be > 0.");
758 cf_opts
->force_consistency_checks
= force_consistency_checks_
;
759 if (use_table_options
) {
760 cf_opts
->table_factory
.reset(NewBlockBasedTableFactory(table_options
));
763 cf_opts
->enable_blob_files
= enable_blob_files_
;
766 if (ParseIntOption(option_map_
, ARG_MIN_BLOB_SIZE
, min_blob_size
,
768 if (min_blob_size
>= 0) {
769 cf_opts
->min_blob_size
= min_blob_size
;
772 LDBCommandExecuteResult::Failed(ARG_MIN_BLOB_SIZE
+ " must be >= 0.");
777 if (ParseIntOption(option_map_
, ARG_BLOB_FILE_SIZE
, blob_file_size
,
779 if (blob_file_size
> 0) {
780 cf_opts
->blob_file_size
= blob_file_size
;
783 LDBCommandExecuteResult::Failed(ARG_BLOB_FILE_SIZE
+ " must be > 0.");
787 cf_opts
->enable_blob_garbage_collection
= enable_blob_garbage_collection_
;
789 double blob_garbage_collection_age_cutoff
;
790 if (ParseDoubleOption(option_map_
, ARG_BLOB_GARBAGE_COLLECTION_AGE_CUTOFF
,
791 blob_garbage_collection_age_cutoff
, exec_state_
)) {
792 if (blob_garbage_collection_age_cutoff
>= 0 &&
793 blob_garbage_collection_age_cutoff
<= 1) {
794 cf_opts
->blob_garbage_collection_age_cutoff
=
795 blob_garbage_collection_age_cutoff
;
797 exec_state_
= LDBCommandExecuteResult::Failed(
798 ARG_BLOB_GARBAGE_COLLECTION_AGE_CUTOFF
+ " must be >= 0 and <= 1.");
802 double blob_garbage_collection_force_threshold
;
803 if (ParseDoubleOption(option_map_
,
804 ARG_BLOB_GARBAGE_COLLECTION_FORCE_THRESHOLD
,
805 blob_garbage_collection_force_threshold
, exec_state_
)) {
806 if (blob_garbage_collection_force_threshold
>= 0 &&
807 blob_garbage_collection_force_threshold
<= 1) {
808 cf_opts
->blob_garbage_collection_force_threshold
=
809 blob_garbage_collection_force_threshold
;
811 exec_state_
= LDBCommandExecuteResult::Failed(
812 ARG_BLOB_GARBAGE_COLLECTION_FORCE_THRESHOLD
+
813 " must be >= 0 and <= 1.");
817 int blob_compaction_readahead_size
;
818 if (ParseIntOption(option_map_
, ARG_BLOB_COMPACTION_READAHEAD_SIZE
,
819 blob_compaction_readahead_size
, exec_state_
)) {
820 if (blob_compaction_readahead_size
> 0) {
821 cf_opts
->blob_compaction_readahead_size
= blob_compaction_readahead_size
;
823 exec_state_
= LDBCommandExecuteResult::Failed(
824 ARG_BLOB_COMPACTION_READAHEAD_SIZE
+ " must be > 0.");
828 int blob_file_starting_level
;
829 if (ParseIntOption(option_map_
, ARG_BLOB_FILE_STARTING_LEVEL
,
830 blob_file_starting_level
, exec_state_
)) {
831 if (blob_file_starting_level
>= 0) {
832 cf_opts
->blob_file_starting_level
= blob_file_starting_level
;
834 exec_state_
= LDBCommandExecuteResult::Failed(
835 ARG_BLOB_FILE_STARTING_LEVEL
+ " must be >= 0.");
839 int prepopulate_blob_cache
;
840 if (ParseIntOption(option_map_
, ARG_PREPOPULATE_BLOB_CACHE
,
841 prepopulate_blob_cache
, exec_state_
)) {
842 switch (prepopulate_blob_cache
) {
844 cf_opts
->prepopulate_blob_cache
= PrepopulateBlobCache::kDisable
;
847 cf_opts
->prepopulate_blob_cache
= PrepopulateBlobCache::kFlushOnly
;
850 exec_state_
= LDBCommandExecuteResult::Failed(
851 ARG_PREPOPULATE_BLOB_CACHE
+
852 " must be 0 (disable) or 1 (flush only).");
856 auto itr
= option_map_
.find(ARG_AUTO_COMPACTION
);
857 if (itr
!= option_map_
.end()) {
858 cf_opts
->disable_auto_compactions
= !StringToBool(itr
->second
);
861 CompressionType compression_type
;
862 if (ParseCompressionTypeOption(option_map_
, ARG_COMPRESSION_TYPE
,
863 compression_type
, exec_state_
)) {
864 cf_opts
->compression
= compression_type
;
867 CompressionType blob_compression_type
;
868 if (ParseCompressionTypeOption(option_map_
, ARG_BLOB_COMPRESSION_TYPE
,
869 blob_compression_type
, exec_state_
)) {
870 cf_opts
->blob_compression_type
= blob_compression_type
;
873 int compression_max_dict_bytes
;
874 if (ParseIntOption(option_map_
, ARG_COMPRESSION_MAX_DICT_BYTES
,
875 compression_max_dict_bytes
, exec_state_
)) {
876 if (compression_max_dict_bytes
>= 0) {
877 cf_opts
->compression_opts
.max_dict_bytes
= compression_max_dict_bytes
;
879 exec_state_
= LDBCommandExecuteResult::Failed(
880 ARG_COMPRESSION_MAX_DICT_BYTES
+ " must be >= 0.");
884 int write_buffer_size
;
885 if (ParseIntOption(option_map_
, ARG_WRITE_BUFFER_SIZE
, write_buffer_size
,
887 if (write_buffer_size
> 0) {
888 cf_opts
->write_buffer_size
= write_buffer_size
;
890 exec_state_
= LDBCommandExecuteResult::Failed(ARG_WRITE_BUFFER_SIZE
+
896 if (ParseIntOption(option_map_
, ARG_FILE_SIZE
, file_size
, exec_state_
)) {
898 cf_opts
->target_file_size_base
= file_size
;
901 LDBCommandExecuteResult::Failed(ARG_FILE_SIZE
+ " must be > 0.");
906 if (ParseIntOption(option_map_
, ARG_FIX_PREFIX_LEN
, fix_prefix_len
,
908 if (fix_prefix_len
> 0) {
909 cf_opts
->prefix_extractor
.reset(
910 NewFixedPrefixTransform(static_cast<size_t>(fix_prefix_len
)));
913 LDBCommandExecuteResult::Failed(ARG_FIX_PREFIX_LEN
+ " must be > 0.");
918 // First, initializes the options state using the OPTIONS file when enabled.
919 // Second, overrides the options according to the CLI arguments and the
920 // specific subcommand being run.
921 void LDBCommand::PrepareOptions() {
922 if (!create_if_missing_
&& try_load_options_
) {
923 config_options_
.env
= options_
.env
;
924 Status s
= LoadLatestOptions(config_options_
, db_path_
, &options_
,
926 if (!s
.ok() && !s
.IsNotFound()) {
927 // Option file exists but load option file error.
928 std::string msg
= s
.ToString();
929 exec_state_
= LDBCommandExecuteResult::Failed(msg
);
933 if (!options_
.wal_dir
.empty()) {
934 if (options_
.env
->FileExists(options_
.wal_dir
).IsNotFound()) {
935 options_
.wal_dir
= db_path_
;
938 "wal_dir loaded from the option file doesn't exist. Ignore it.\n");
942 // If merge operator is not set, set a string append operator.
943 for (auto& cf_entry
: column_families_
) {
944 if (!cf_entry
.options
.merge_operator
) {
945 cf_entry
.options
.merge_operator
=
946 MergeOperators::CreateStringAppendOperator(':');
951 if (options_
.env
== Env::Default()) {
952 options_
.env
= config_options_
.env
;
955 OverrideBaseOptions();
956 if (exec_state_
.IsFailed()) {
960 if (column_families_
.empty()) {
961 // Reads the MANIFEST to figure out what column families exist. In this
962 // case, the option overrides from the CLI argument/specific subcommand
963 // apply to all column families.
964 std::vector
<std::string
> cf_list
;
965 Status st
= DB::ListColumnFamilies(options_
, db_path_
, &cf_list
);
966 // It is possible the DB doesn't exist yet, for "create if not
967 // existing" case. The failure is ignored here. We rely on DB::Open()
968 // to give us the correct error message for problem with opening
970 if (st
.ok() && cf_list
.size() > 1) {
971 // Ignore single column family DB.
972 for (auto cf_name
: cf_list
) {
973 column_families_
.emplace_back(cf_name
, options_
);
977 // We got column families from the OPTIONS file. In this case, the option
978 // overrides from the CLI argument/specific subcommand only apply to the
979 // column family specified by `--column_family_name`.
980 auto column_families_iter
=
981 std::find_if(column_families_
.begin(), column_families_
.end(),
982 [this](const ColumnFamilyDescriptor
& cf_desc
) {
983 return cf_desc
.name
== column_family_name_
;
985 if (column_families_iter
== column_families_
.end()) {
986 exec_state_
= LDBCommandExecuteResult::Failed(
987 "Non-existing column family " + column_family_name_
);
990 OverrideBaseCFOptions(&column_families_iter
->options
);
994 bool LDBCommand::ParseKeyValue(const std::string
& line
, std::string
* key
,
995 std::string
* value
, bool is_key_hex
,
997 size_t pos
= line
.find(DELIM
);
998 if (pos
!= std::string::npos
) {
999 *key
= line
.substr(0, pos
);
1000 *value
= line
.substr(pos
+ strlen(DELIM
));
1002 *key
= HexToString(*key
);
1005 *value
= HexToString(*value
);
1014 * Make sure that ONLY the command-line options and flags expected by this
1015 * command are specified on the command-line. Extraneous options are usually
1016 * the result of user error.
1017 * Returns true if all checks pass. Else returns false, and prints an
1018 * appropriate error msg to stderr.
1020 bool LDBCommand::ValidateCmdLineOptions() {
1021 for (auto itr
= option_map_
.begin(); itr
!= option_map_
.end(); ++itr
) {
1022 if (std::find(valid_cmd_line_options_
.begin(),
1023 valid_cmd_line_options_
.end(),
1024 itr
->first
) == valid_cmd_line_options_
.end()) {
1025 fprintf(stderr
, "Invalid command-line option %s\n", itr
->first
.c_str());
1030 for (std::vector
<std::string
>::const_iterator itr
= flags_
.begin();
1031 itr
!= flags_
.end(); ++itr
) {
1032 if (std::find(valid_cmd_line_options_
.begin(),
1033 valid_cmd_line_options_
.end(),
1034 *itr
) == valid_cmd_line_options_
.end()) {
1035 fprintf(stderr
, "Invalid command-line flag %s\n", itr
->c_str());
1040 if (!NoDBOpen() && option_map_
.find(ARG_DB
) == option_map_
.end() &&
1041 option_map_
.find(ARG_PATH
) == option_map_
.end()) {
1042 fprintf(stderr
, "Either %s or %s must be specified.\n", ARG_DB
.c_str(),
1050 std::string
LDBCommand::HexToString(const std::string
& str
) {
1052 std::string::size_type len
= str
.length();
1053 if (len
< 2 || str
[0] != '0' || str
[1] != 'x') {
1054 fprintf(stderr
, "Invalid hex input %s. Must start with 0x\n", str
.c_str());
1055 throw "Invalid hex input";
1057 if (!Slice(str
.data() + 2, len
- 2).DecodeHex(&result
)) {
1058 throw "Invalid hex input";
1063 std::string
LDBCommand::StringToHex(const std::string
& str
) {
1064 std::string
result("0x");
1065 result
.append(Slice(str
).ToString(true));
1069 std::string
LDBCommand::PrintKeyValue(const std::string
& key
,
1070 const std::string
& value
, bool is_key_hex
,
1071 bool is_value_hex
) {
1073 result
.append(is_key_hex
? StringToHex(key
) : key
);
1074 result
.append(DELIM
);
1075 result
.append(is_value_hex
? StringToHex(value
) : value
);
1079 std::string
LDBCommand::PrintKeyValue(const std::string
& key
,
1080 const std::string
& value
, bool is_hex
) {
1081 return PrintKeyValue(key
, value
, is_hex
, is_hex
);
1084 std::string
LDBCommand::HelpRangeCmdArgs() {
1085 std::ostringstream str_stream
;
1087 str_stream
<< "[--" << ARG_FROM
<< "] ";
1088 str_stream
<< "[--" << ARG_TO
<< "] ";
1089 return str_stream
.str();
1092 bool LDBCommand::IsKeyHex(const std::map
<std::string
, std::string
>& options
,
1093 const std::vector
<std::string
>& flags
) {
1094 return (IsFlagPresent(flags
, ARG_HEX
) || IsFlagPresent(flags
, ARG_KEY_HEX
) ||
1095 ParseBooleanOption(options
, ARG_HEX
, false) ||
1096 ParseBooleanOption(options
, ARG_KEY_HEX
, false));
1099 bool LDBCommand::IsValueHex(const std::map
<std::string
, std::string
>& options
,
1100 const std::vector
<std::string
>& flags
) {
1101 return (IsFlagPresent(flags
, ARG_HEX
) ||
1102 IsFlagPresent(flags
, ARG_VALUE_HEX
) ||
1103 ParseBooleanOption(options
, ARG_HEX
, false) ||
1104 ParseBooleanOption(options
, ARG_VALUE_HEX
, false));
1107 bool LDBCommand::IsTryLoadOptions(
1108 const std::map
<std::string
, std::string
>& options
,
1109 const std::vector
<std::string
>& flags
) {
1110 if (IsFlagPresent(flags
, ARG_TRY_LOAD_OPTIONS
)) {
1113 // if `DB` is specified and not explicitly to create a new db, default
1114 // `try_load_options` to true. The user could still disable that by set
1115 // `try_load_options=false`.
1116 // Note: Opening as TTL DB doesn't support `try_load_options`, so it's default
1117 // to false. TODO: TTL_DB may need to fix that, otherwise it's unable to open
1118 // DB which has incompatible setting with default options.
1119 bool default_val
= (options
.find(ARG_DB
) != options
.end()) &&
1120 !IsFlagPresent(flags
, ARG_CREATE_IF_MISSING
) &&
1121 !IsFlagPresent(flags
, ARG_TTL
);
1122 return ParseBooleanOption(options
, ARG_TRY_LOAD_OPTIONS
, default_val
);
1125 bool LDBCommand::ParseBooleanOption(
1126 const std::map
<std::string
, std::string
>& options
,
1127 const std::string
& option
, bool default_val
) {
1128 auto itr
= options
.find(option
);
1129 if (itr
!= options
.end()) {
1130 std::string option_val
= itr
->second
;
1131 return StringToBool(itr
->second
);
1136 bool LDBCommand::StringToBool(std::string val
) {
1137 std::transform(val
.begin(), val
.end(), val
.begin(),
1138 [](char ch
) -> char { return (char)::tolower(ch
); });
1140 if (val
== "true") {
1142 } else if (val
== "false") {
1145 throw "Invalid value for boolean argument";
1149 CompactorCommand::CompactorCommand(
1150 const std::vector
<std::string
>& /*params*/,
1151 const std::map
<std::string
, std::string
>& options
,
1152 const std::vector
<std::string
>& flags
)
1153 : LDBCommand(options
, flags
, false,
1154 BuildCmdLineOptions({ARG_FROM
, ARG_TO
, ARG_HEX
, ARG_KEY_HEX
,
1155 ARG_VALUE_HEX
, ARG_TTL
})),
1158 auto itr
= options
.find(ARG_FROM
);
1159 if (itr
!= options
.end()) {
1161 from_
= itr
->second
;
1164 itr
= options
.find(ARG_TO
);
1165 if (itr
!= options
.end()) {
1172 from_
= HexToString(from_
);
1175 to_
= HexToString(to_
);
1180 void CompactorCommand::Help(std::string
& ret
) {
1182 ret
.append(CompactorCommand::Name());
1183 ret
.append(HelpRangeCmdArgs());
1187 void CompactorCommand::DoCommand() {
1189 assert(GetExecuteState().IsFailed());
1193 Slice
* begin
= nullptr;
1194 Slice
* end
= nullptr;
1196 begin
= new Slice(from_
);
1199 end
= new Slice(to_
);
1202 CompactRangeOptions cro
;
1203 cro
.bottommost_level_compaction
= BottommostLevelCompaction::kForceOptimized
;
1205 Status s
= db_
->CompactRange(cro
, GetCfHandle(), begin
, end
);
1207 std::stringstream oss
;
1208 oss
<< "Compaction failed: " << s
.ToString();
1209 exec_state_
= LDBCommandExecuteResult::Failed(oss
.str());
1211 exec_state_
= LDBCommandExecuteResult::Succeed("");
1218 // ---------------------------------------------------------------------------
1219 const std::string
DBLoaderCommand::ARG_DISABLE_WAL
= "disable_wal";
1220 const std::string
DBLoaderCommand::ARG_BULK_LOAD
= "bulk_load";
1221 const std::string
DBLoaderCommand::ARG_COMPACT
= "compact";
1223 DBLoaderCommand::DBLoaderCommand(
1224 const std::vector
<std::string
>& /*params*/,
1225 const std::map
<std::string
, std::string
>& options
,
1226 const std::vector
<std::string
>& flags
)
1228 options
, flags
, false,
1229 BuildCmdLineOptions({ARG_HEX
, ARG_KEY_HEX
, ARG_VALUE_HEX
, ARG_FROM
,
1230 ARG_TO
, ARG_CREATE_IF_MISSING
, ARG_DISABLE_WAL
,
1231 ARG_BULK_LOAD
, ARG_COMPACT
})),
1232 disable_wal_(false),
1235 create_if_missing_
= IsFlagPresent(flags
, ARG_CREATE_IF_MISSING
);
1236 disable_wal_
= IsFlagPresent(flags
, ARG_DISABLE_WAL
);
1237 bulk_load_
= IsFlagPresent(flags
, ARG_BULK_LOAD
);
1238 compact_
= IsFlagPresent(flags
, ARG_COMPACT
);
1241 void DBLoaderCommand::Help(std::string
& ret
) {
1243 ret
.append(DBLoaderCommand::Name());
1244 ret
.append(" [--" + ARG_CREATE_IF_MISSING
+ "]");
1245 ret
.append(" [--" + ARG_DISABLE_WAL
+ "]");
1246 ret
.append(" [--" + ARG_BULK_LOAD
+ "]");
1247 ret
.append(" [--" + ARG_COMPACT
+ "]");
1251 void DBLoaderCommand::OverrideBaseOptions() {
1252 LDBCommand::OverrideBaseOptions();
1253 options_
.create_if_missing
= create_if_missing_
;
1255 options_
.PrepareForBulkLoad();
1259 void DBLoaderCommand::DoCommand() {
1261 assert(GetExecuteState().IsFailed());
1265 WriteOptions write_options
;
1267 write_options
.disableWAL
= true;
1272 // prefer ifstream getline performance vs that from std::cin istream
1273 std::ifstream
ifs_stdin("/dev/stdin");
1274 std::istream
* istream_p
= ifs_stdin
.is_open() ? &ifs_stdin
: &std::cin
;
1276 while (s
.ok() && getline(*istream_p
, line
, '\n')) {
1279 if (ParseKeyValue(line
, &key
, &value
, is_key_hex_
, is_value_hex_
)) {
1280 s
= db_
->Put(write_options
, GetCfHandle(), Slice(key
), Slice(value
));
1281 } else if (0 == line
.find("Keys in range:")) {
1283 } else if (0 == line
.find("Created bg thread 0x")) {
1290 if (bad_lines
> 0) {
1291 std::cout
<< "Warning: " << bad_lines
<< " bad lines ignored." << std::endl
;
1294 std::stringstream oss
;
1295 oss
<< "Load failed: " << s
.ToString();
1296 exec_state_
= LDBCommandExecuteResult::Failed(oss
.str());
1298 if (compact_
&& s
.ok()) {
1299 s
= db_
->CompactRange(CompactRangeOptions(), GetCfHandle(), nullptr,
1303 std::stringstream oss
;
1304 oss
<< "Compaction failed: " << s
.ToString();
1305 exec_state_
= LDBCommandExecuteResult::Failed(oss
.str());
1309 // ----------------------------------------------------------------------------
1313 void DumpManifestFile(Options options
, std::string file
, bool verbose
, bool hex
,
1316 std::string
dbname("dummy");
1317 std::shared_ptr
<Cache
> tc(NewLRUCache(options
.max_open_files
- 10,
1318 options
.table_cache_numshardbits
));
1319 // Notice we are using the default options not through SanitizeOptions(),
1320 // if VersionSet::DumpManifest() depends on any option done by
1321 // SanitizeOptions(), we need to initialize it manually.
1322 options
.db_paths
.emplace_back("dummy", 0);
1323 options
.num_levels
= 64;
1324 WriteController
wc(options
.delayed_write_rate
);
1325 WriteBufferManager
wb(options
.db_write_buffer_size
);
1326 ImmutableDBOptions
immutable_db_options(options
);
1327 VersionSet
versions(dbname
, &immutable_db_options
, sopt
, tc
.get(), &wb
, &wc
,
1328 /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
1329 /*db_id*/ "", /*db_session_id*/ "");
1330 Status s
= versions
.DumpManifest(options
, file
, verbose
, hex
, json
);
1332 fprintf(stderr
, "Error in processing file %s %s\n", file
.c_str(),
1333 s
.ToString().c_str());
1339 const std::string
ManifestDumpCommand::ARG_VERBOSE
= "verbose";
1340 const std::string
ManifestDumpCommand::ARG_JSON
= "json";
1341 const std::string
ManifestDumpCommand::ARG_PATH
= "path";
1343 void ManifestDumpCommand::Help(std::string
& ret
) {
1345 ret
.append(ManifestDumpCommand::Name());
1346 ret
.append(" [--" + ARG_VERBOSE
+ "]");
1347 ret
.append(" [--" + ARG_JSON
+ "]");
1348 ret
.append(" [--" + ARG_PATH
+ "=<path_to_manifest_file>]");
1352 ManifestDumpCommand::ManifestDumpCommand(
1353 const std::vector
<std::string
>& /*params*/,
1354 const std::map
<std::string
, std::string
>& options
,
1355 const std::vector
<std::string
>& flags
)
1357 options
, flags
, false,
1358 BuildCmdLineOptions({ARG_VERBOSE
, ARG_PATH
, ARG_HEX
, ARG_JSON
})),
1362 verbose_
= IsFlagPresent(flags
, ARG_VERBOSE
);
1363 json_
= IsFlagPresent(flags
, ARG_JSON
);
1365 auto itr
= options
.find(ARG_PATH
);
1366 if (itr
!= options
.end()) {
1367 path_
= itr
->second
;
1368 if (path_
.empty()) {
1369 exec_state_
= LDBCommandExecuteResult::Failed("--path: missing pathname");
1374 void ManifestDumpCommand::DoCommand() {
1375 std::string manifestfile
;
1377 if (!path_
.empty()) {
1378 manifestfile
= path_
;
1380 // We need to find the manifest file by searching the directory
1381 // containing the db for files of the form MANIFEST_[0-9]+
1383 std::vector
<std::string
> files
;
1384 Status s
= options_
.env
->GetChildren(db_path_
, &files
);
1386 std::string err_msg
= s
.ToString();
1387 err_msg
.append(": Failed to list the content of ");
1388 err_msg
.append(db_path_
);
1389 exec_state_
= LDBCommandExecuteResult::Failed(err_msg
);
1392 const std::string kManifestNamePrefix
= "MANIFEST-";
1393 std::string matched_file
;
1395 const char kPathDelim
= '\\';
1397 const char kPathDelim
= '/';
1399 for (const auto& file_path
: files
) {
1400 // Some Env::GetChildren() return absolute paths. Some directories' path
1401 // end with path delim, e.g. '/' or '\\'.
1402 size_t pos
= file_path
.find_last_of(kPathDelim
);
1403 if (pos
== file_path
.size() - 1) {
1407 if (pos
!= std::string::npos
) {
1409 fname
.assign(file_path
, pos
+ 1, file_path
.size() - pos
- 1);
1413 uint64_t file_num
= 0;
1414 FileType file_type
= kWalFile
; // Just for initialization
1415 if (ParseFileName(fname
, &file_num
, &file_type
) &&
1416 file_type
== kDescriptorFile
) {
1417 if (!matched_file
.empty()) {
1418 exec_state_
= LDBCommandExecuteResult::Failed(
1419 "Multiple MANIFEST files found; use --path to select one");
1422 matched_file
.swap(fname
);
1426 if (matched_file
.empty()) {
1427 std::string
err_msg("No MANIFEST found in ");
1428 err_msg
.append(db_path_
);
1429 exec_state_
= LDBCommandExecuteResult::Failed(err_msg
);
1432 if (db_path_
.back() != '/') {
1433 db_path_
.append("/");
1435 manifestfile
= db_path_
+ matched_file
;
1439 fprintf(stdout
, "Processing Manifest file %s\n", manifestfile
.c_str());
1442 DumpManifestFile(options_
, manifestfile
, verbose_
, is_key_hex_
, json_
);
1445 fprintf(stdout
, "Processing Manifest file %s done\n", manifestfile
.c_str());
1449 // ----------------------------------------------------------------------------
1452 Status
GetLiveFilesChecksumInfoFromVersionSet(Options options
,
1453 const std::string
& db_path
,
1454 FileChecksumList
* checksum_list
) {
1457 std::string
dbname(db_path
);
1458 std::shared_ptr
<Cache
> tc(NewLRUCache(options
.max_open_files
- 10,
1459 options
.table_cache_numshardbits
));
1460 // Notice we are using the default options not through SanitizeOptions(),
1461 // if VersionSet::GetLiveFilesChecksumInfo depends on any option done by
1462 // SanitizeOptions(), we need to initialize it manually.
1463 options
.db_paths
.emplace_back(db_path
, 0);
1464 options
.num_levels
= 64;
1465 WriteController
wc(options
.delayed_write_rate
);
1466 WriteBufferManager
wb(options
.db_write_buffer_size
);
1467 ImmutableDBOptions
immutable_db_options(options
);
1468 VersionSet
versions(dbname
, &immutable_db_options
, sopt
, tc
.get(), &wb
, &wc
,
1469 /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
1470 /*db_id*/ "", /*db_session_id*/ "");
1471 std::vector
<std::string
> cf_name_list
;
1472 s
= versions
.ListColumnFamilies(&cf_name_list
, db_path
,
1473 immutable_db_options
.fs
.get());
1475 std::vector
<ColumnFamilyDescriptor
> cf_list
;
1476 for (const auto& name
: cf_name_list
) {
1477 cf_list
.emplace_back(name
, ColumnFamilyOptions(options
));
1479 s
= versions
.Recover(cf_list
, true);
1482 s
= versions
.GetLiveFilesChecksumInfo(checksum_list
);
1489 const std::string
FileChecksumDumpCommand::ARG_PATH
= "path";
1491 void FileChecksumDumpCommand::Help(std::string
& ret
) {
1493 ret
.append(FileChecksumDumpCommand::Name());
1494 ret
.append(" [--" + ARG_PATH
+ "=<path_to_manifest_file>]");
1498 FileChecksumDumpCommand::FileChecksumDumpCommand(
1499 const std::vector
<std::string
>& /*params*/,
1500 const std::map
<std::string
, std::string
>& options
,
1501 const std::vector
<std::string
>& flags
)
1502 : LDBCommand(options
, flags
, false,
1503 BuildCmdLineOptions({ARG_PATH
, ARG_HEX
})),
1505 auto itr
= options
.find(ARG_PATH
);
1506 if (itr
!= options
.end()) {
1507 path_
= itr
->second
;
1508 if (path_
.empty()) {
1509 exec_state_
= LDBCommandExecuteResult::Failed("--path: missing pathname");
1512 is_checksum_hex_
= IsFlagPresent(flags
, ARG_HEX
);
1515 void FileChecksumDumpCommand::DoCommand() {
1516 // print out the checksum information in the following format:
1517 // sst file number, checksum function name, checksum value
1518 // sst file number, checksum function name, checksum value
1521 std::unique_ptr
<FileChecksumList
> checksum_list(NewFileChecksumList());
1522 Status s
= GetLiveFilesChecksumInfoFromVersionSet(options_
, db_path_
,
1523 checksum_list
.get());
1524 if (s
.ok() && checksum_list
!= nullptr) {
1525 std::vector
<uint64_t> file_numbers
;
1526 std::vector
<std::string
> checksums
;
1527 std::vector
<std::string
> checksum_func_names
;
1528 s
= checksum_list
->GetAllFileChecksums(&file_numbers
, &checksums
,
1529 &checksum_func_names
);
1531 for (size_t i
= 0; i
< file_numbers
.size(); i
++) {
1532 assert(i
< file_numbers
.size());
1533 assert(i
< checksums
.size());
1534 assert(i
< checksum_func_names
.size());
1535 std::string checksum
;
1536 if (is_checksum_hex_
) {
1537 checksum
= StringToHex(checksums
[i
]);
1539 checksum
= std::move(checksums
[i
]);
1541 fprintf(stdout
, "%" PRId64
", %s, %s\n", file_numbers
[i
],
1542 checksum_func_names
[i
].c_str(), checksum
.c_str());
1544 fprintf(stdout
, "Print SST file checksum information finished \n");
1549 exec_state_
= LDBCommandExecuteResult::Failed(s
.ToString());
1553 // ----------------------------------------------------------------------------
1555 void GetPropertyCommand::Help(std::string
& ret
) {
1557 ret
.append(GetPropertyCommand::Name());
1558 ret
.append(" <property_name>");
1562 GetPropertyCommand::GetPropertyCommand(
1563 const std::vector
<std::string
>& params
,
1564 const std::map
<std::string
, std::string
>& options
,
1565 const std::vector
<std::string
>& flags
)
1566 : LDBCommand(options
, flags
, true, BuildCmdLineOptions({})) {
1567 if (params
.size() != 1) {
1569 LDBCommandExecuteResult::Failed("property name must be specified");
1571 property_
= params
[0];
1575 void GetPropertyCommand::DoCommand() {
1577 assert(GetExecuteState().IsFailed());
1581 std::map
<std::string
, std::string
> value_map
;
1584 // Rather than having different ldb command for map properties vs. string
1585 // properties, we simply try Map property first. (This order only chosen
1586 // because I prefer the map-style output for
1587 // "rocksdb.aggregated-table-properties".)
1588 if (db_
->GetMapProperty(GetCfHandle(), property_
, &value_map
)) {
1589 if (value_map
.empty()) {
1590 fprintf(stdout
, "%s: <empty map>\n", property_
.c_str());
1592 for (auto& e
: value_map
) {
1593 fprintf(stdout
, "%s.%s: %s\n", property_
.c_str(), e
.first
.c_str(),
1597 } else if (db_
->GetProperty(GetCfHandle(), property_
, &value
)) {
1598 fprintf(stdout
, "%s: %s\n", property_
.c_str(), value
.c_str());
1601 LDBCommandExecuteResult::Failed("failed to get property: " + property_
);
1605 // ----------------------------------------------------------------------------
1607 void ListColumnFamiliesCommand::Help(std::string
& ret
) {
1609 ret
.append(ListColumnFamiliesCommand::Name());
1613 ListColumnFamiliesCommand::ListColumnFamiliesCommand(
1614 const std::vector
<std::string
>& /*params*/,
1615 const std::map
<std::string
, std::string
>& options
,
1616 const std::vector
<std::string
>& flags
)
1617 : LDBCommand(options
, flags
, false, BuildCmdLineOptions({})) {}
1619 void ListColumnFamiliesCommand::DoCommand() {
1620 std::vector
<std::string
> column_families
;
1621 Status s
= DB::ListColumnFamilies(options_
, db_path_
, &column_families
);
1623 fprintf(stderr
, "Error in processing db %s %s\n", db_path_
.c_str(),
1624 s
.ToString().c_str());
1626 fprintf(stdout
, "Column families in %s: \n{", db_path_
.c_str());
1628 for (auto cf
: column_families
) {
1630 fprintf(stdout
, ", ");
1633 fprintf(stdout
, "%s", cf
.c_str());
1635 fprintf(stdout
, "}\n");
1639 void CreateColumnFamilyCommand::Help(std::string
& ret
) {
1641 ret
.append(CreateColumnFamilyCommand::Name());
1642 ret
.append(" --db=<db_path> <new_column_family_name>");
1646 CreateColumnFamilyCommand::CreateColumnFamilyCommand(
1647 const std::vector
<std::string
>& params
,
1648 const std::map
<std::string
, std::string
>& options
,
1649 const std::vector
<std::string
>& flags
)
1650 : LDBCommand(options
, flags
, true, {ARG_DB
}) {
1651 if (params
.size() != 1) {
1652 exec_state_
= LDBCommandExecuteResult::Failed(
1653 "new column family name must be specified");
1655 new_cf_name_
= params
[0];
1659 void CreateColumnFamilyCommand::DoCommand() {
1661 assert(GetExecuteState().IsFailed());
1664 ColumnFamilyHandle
* new_cf_handle
= nullptr;
1665 Status st
= db_
->CreateColumnFamily(options_
, new_cf_name_
, &new_cf_handle
);
1667 fprintf(stdout
, "OK\n");
1669 exec_state_
= LDBCommandExecuteResult::Failed(
1670 "Fail to create new column family: " + st
.ToString());
1672 delete new_cf_handle
;
1676 void DropColumnFamilyCommand::Help(std::string
& ret
) {
1678 ret
.append(DropColumnFamilyCommand::Name());
1679 ret
.append(" --db=<db_path> <column_family_name_to_drop>");
1683 DropColumnFamilyCommand::DropColumnFamilyCommand(
1684 const std::vector
<std::string
>& params
,
1685 const std::map
<std::string
, std::string
>& options
,
1686 const std::vector
<std::string
>& flags
)
1687 : LDBCommand(options
, flags
, true, {ARG_DB
}) {
1688 if (params
.size() != 1) {
1689 exec_state_
= LDBCommandExecuteResult::Failed(
1690 "The name of column family to drop must be specified");
1692 cf_name_to_drop_
= params
[0];
1696 void DropColumnFamilyCommand::DoCommand() {
1698 assert(GetExecuteState().IsFailed());
1701 auto iter
= cf_handles_
.find(cf_name_to_drop_
);
1702 if (iter
== cf_handles_
.end()) {
1703 exec_state_
= LDBCommandExecuteResult::Failed(
1704 "Column family: " + cf_name_to_drop_
+ " doesn't exist in db.");
1707 ColumnFamilyHandle
* cf_handle_to_drop
= iter
->second
;
1708 Status st
= db_
->DropColumnFamily(cf_handle_to_drop
);
1710 fprintf(stdout
, "OK\n");
1712 exec_state_
= LDBCommandExecuteResult::Failed(
1713 "Fail to drop column family: " + st
.ToString());
1718 // ----------------------------------------------------------------------------
1721 // This function only called when it's the sane case of >1 buckets in time-range
1722 // Also called only when timekv falls between ttl_start and ttl_end provided
1723 void IncBucketCounts(std::vector
<uint64_t>& bucket_counts
, int ttl_start
,
1724 int time_range
, int bucket_size
, int timekv
,
1730 assert(time_range
> 0 && timekv
>= ttl_start
&& bucket_size
> 0 &&
1731 timekv
< (ttl_start
+ time_range
) && num_buckets
> 1);
1732 int bucket
= (timekv
- ttl_start
) / bucket_size
;
1733 bucket_counts
[bucket
]++;
1736 void PrintBucketCounts(const std::vector
<uint64_t>& bucket_counts
,
1737 int ttl_start
, int ttl_end
, int bucket_size
,
1739 int time_point
= ttl_start
;
1740 for (int i
= 0; i
< num_buckets
- 1; i
++, time_point
+= bucket_size
) {
1741 fprintf(stdout
, "Keys in range %s to %s : %lu\n",
1742 TimeToHumanString(time_point
).c_str(),
1743 TimeToHumanString(time_point
+ bucket_size
).c_str(),
1744 (unsigned long)bucket_counts
[i
]);
1746 fprintf(stdout
, "Keys in range %s to %s : %lu\n",
1747 TimeToHumanString(time_point
).c_str(),
1748 TimeToHumanString(ttl_end
).c_str(),
1749 (unsigned long)bucket_counts
[num_buckets
- 1]);
1754 const std::string
InternalDumpCommand::ARG_COUNT_ONLY
= "count_only";
1755 const std::string
InternalDumpCommand::ARG_COUNT_DELIM
= "count_delim";
1756 const std::string
InternalDumpCommand::ARG_STATS
= "stats";
1757 const std::string
InternalDumpCommand::ARG_INPUT_KEY_HEX
= "input_key_hex";
1759 InternalDumpCommand::InternalDumpCommand(
1760 const std::vector
<std::string
>& /*params*/,
1761 const std::map
<std::string
, std::string
>& options
,
1762 const std::vector
<std::string
>& flags
)
1763 : LDBCommand(options
, flags
, true,
1764 BuildCmdLineOptions(
1765 {ARG_HEX
, ARG_KEY_HEX
, ARG_VALUE_HEX
, ARG_FROM
, ARG_TO
,
1766 ARG_MAX_KEYS
, ARG_COUNT_ONLY
, ARG_COUNT_DELIM
, ARG_STATS
,
1767 ARG_INPUT_KEY_HEX
, ARG_DECODE_BLOB_INDEX
})),
1773 count_delim_(false),
1774 print_stats_(false),
1775 is_input_key_hex_(false),
1776 decode_blob_index_(false) {
1777 has_from_
= ParseStringOption(options
, ARG_FROM
, &from_
);
1778 has_to_
= ParseStringOption(options
, ARG_TO
, &to_
);
1780 ParseIntOption(options
, ARG_MAX_KEYS
, max_keys_
, exec_state_
);
1781 auto itr
= options
.find(ARG_COUNT_DELIM
);
1782 if (itr
!= options
.end()) {
1783 delim_
= itr
->second
;
1784 count_delim_
= true;
1785 // fprintf(stdout,"delim = %c\n",delim_[0]);
1787 count_delim_
= IsFlagPresent(flags
, ARG_COUNT_DELIM
);
1791 print_stats_
= IsFlagPresent(flags
, ARG_STATS
);
1792 count_only_
= IsFlagPresent(flags
, ARG_COUNT_ONLY
);
1793 is_input_key_hex_
= IsFlagPresent(flags
, ARG_INPUT_KEY_HEX
);
1794 decode_blob_index_
= IsFlagPresent(flags
, ARG_DECODE_BLOB_INDEX
);
1796 if (is_input_key_hex_
) {
1798 from_
= HexToString(from_
);
1801 to_
= HexToString(to_
);
1806 void InternalDumpCommand::Help(std::string
& ret
) {
1808 ret
.append(InternalDumpCommand::Name());
1809 ret
.append(HelpRangeCmdArgs());
1810 ret
.append(" [--" + ARG_INPUT_KEY_HEX
+ "]");
1811 ret
.append(" [--" + ARG_MAX_KEYS
+ "=<N>]");
1812 ret
.append(" [--" + ARG_COUNT_ONLY
+ "]");
1813 ret
.append(" [--" + ARG_COUNT_DELIM
+ "=<char>]");
1814 ret
.append(" [--" + ARG_STATS
+ "]");
1815 ret
.append(" [--" + ARG_DECODE_BLOB_INDEX
+ "]");
1819 void InternalDumpCommand::DoCommand() {
1821 assert(GetExecuteState().IsFailed());
1827 if (db_
->GetProperty(GetCfHandle(), "rocksdb.stats", &stats
)) {
1828 fprintf(stdout
, "%s\n", stats
.c_str());
1832 // Cast as DBImpl to get internal iterator
1833 std::vector
<KeyVersion
> key_versions
;
1834 Status st
= GetAllKeyVersions(db_
, GetCfHandle(), from_
, to_
, max_keys_
,
1837 exec_state_
= LDBCommandExecuteResult::Failed(st
.ToString());
1840 std::string rtype1
, rtype2
, row
, val
;
1843 uint64_t s1
= 0, s2
= 0;
1845 long long count
= 0;
1846 for (auto& key_version
: key_versions
) {
1847 ValueType value_type
= static_cast<ValueType
>(key_version
.type
);
1848 InternalKey
ikey(key_version
.user_key
, key_version
.sequence
, value_type
);
1849 if (has_to_
&& ikey
.user_key() == to_
) {
1850 // GetAllKeyVersions() includes keys with user key `to_`, but idump has
1851 // traditionally excluded such keys.
1859 row
= ikey
.Encode().ToString();
1860 val
= key_version
.value
;
1861 for (k
= 0; row
[k
] != '\x01' && row
[k
] != '\0'; k
++) s1
++;
1862 for (k
= 0; val
[k
] != '\x01' && val
[k
] != '\0'; k
++) s1
++;
1863 for (int j
= 0; row
[j
] != delim_
[0] && row
[j
] != '\0' && row
[j
] != '\x01';
1866 if (rtype2
.compare("") && rtype2
.compare(rtype1
) != 0) {
1867 fprintf(stdout
, "%s => count:%" PRIu64
"\tsize:%" PRIu64
"\n",
1868 rtype2
.c_str(), c
, s2
);
1879 if (!count_only_
&& !count_delim_
) {
1880 std::string key
= ikey
.DebugString(is_key_hex_
);
1881 Slice
value(key_version
.value
);
1882 if (!decode_blob_index_
|| value_type
!= kTypeBlobIndex
) {
1883 fprintf(stdout
, "%s => %s\n", key
.c_str(),
1884 value
.ToString(is_value_hex_
).c_str());
1886 BlobIndex blob_index
;
1888 const Status s
= blob_index
.DecodeFrom(value
);
1890 fprintf(stderr
, "%s => error decoding blob index =>\n", key
.c_str());
1892 fprintf(stdout
, "%s => %s\n", key
.c_str(),
1893 blob_index
.DebugString(is_value_hex_
).c_str());
1898 // Terminate if maximum number of keys have been dumped
1899 if (max_keys_
> 0 && count
>= max_keys_
) break;
1902 fprintf(stdout
, "%s => count:%" PRIu64
"\tsize:%" PRIu64
"\n",
1903 rtype2
.c_str(), c
, s2
);
1905 fprintf(stdout
, "Internal keys in range: %lld\n", count
);
1909 const std::string
DBDumperCommand::ARG_COUNT_ONLY
= "count_only";
1910 const std::string
DBDumperCommand::ARG_COUNT_DELIM
= "count_delim";
1911 const std::string
DBDumperCommand::ARG_STATS
= "stats";
1912 const std::string
DBDumperCommand::ARG_TTL_BUCKET
= "bucket";
1914 DBDumperCommand::DBDumperCommand(
1915 const std::vector
<std::string
>& /*params*/,
1916 const std::map
<std::string
, std::string
>& options
,
1917 const std::vector
<std::string
>& flags
)
1919 options
, flags
, true,
1920 BuildCmdLineOptions(
1921 {ARG_TTL
, ARG_HEX
, ARG_KEY_HEX
, ARG_VALUE_HEX
, ARG_FROM
, ARG_TO
,
1922 ARG_MAX_KEYS
, ARG_COUNT_ONLY
, ARG_COUNT_DELIM
, ARG_STATS
,
1923 ARG_TTL_START
, ARG_TTL_END
, ARG_TTL_BUCKET
, ARG_TIMESTAMP
,
1924 ARG_PATH
, ARG_DECODE_BLOB_INDEX
, ARG_DUMP_UNCOMPRESSED_BLOBS
})),
1929 count_delim_(false),
1930 print_stats_(false),
1931 decode_blob_index_(false) {
1932 auto itr
= options
.find(ARG_FROM
);
1933 if (itr
!= options
.end()) {
1935 from_
= itr
->second
;
1938 itr
= options
.find(ARG_TO
);
1939 if (itr
!= options
.end()) {
1944 itr
= options
.find(ARG_MAX_KEYS
);
1945 if (itr
!= options
.end()) {
1948 max_keys_
= strtol(itr
->second
.c_str(), 0, 10);
1950 max_keys_
= std::stoi(itr
->second
);
1952 } catch (const std::invalid_argument
&) {
1953 exec_state_
= LDBCommandExecuteResult::Failed(ARG_MAX_KEYS
+
1954 " has an invalid value");
1955 } catch (const std::out_of_range
&) {
1956 exec_state_
= LDBCommandExecuteResult::Failed(
1957 ARG_MAX_KEYS
+ " has a value out-of-range");
1960 itr
= options
.find(ARG_COUNT_DELIM
);
1961 if (itr
!= options
.end()) {
1962 delim_
= itr
->second
;
1963 count_delim_
= true;
1965 count_delim_
= IsFlagPresent(flags
, ARG_COUNT_DELIM
);
1969 print_stats_
= IsFlagPresent(flags
, ARG_STATS
);
1970 count_only_
= IsFlagPresent(flags
, ARG_COUNT_ONLY
);
1971 decode_blob_index_
= IsFlagPresent(flags
, ARG_DECODE_BLOB_INDEX
);
1972 dump_uncompressed_blobs_
= IsFlagPresent(flags
, ARG_DUMP_UNCOMPRESSED_BLOBS
);
1976 from_
= HexToString(from_
);
1979 to_
= HexToString(to_
);
1983 itr
= options
.find(ARG_PATH
);
1984 if (itr
!= options
.end()) {
1985 path_
= itr
->second
;
1986 if (db_path_
.empty()) {
1992 void DBDumperCommand::Help(std::string
& ret
) {
1994 ret
.append(DBDumperCommand::Name());
1995 ret
.append(HelpRangeCmdArgs());
1996 ret
.append(" [--" + ARG_TTL
+ "]");
1997 ret
.append(" [--" + ARG_MAX_KEYS
+ "=<N>]");
1998 ret
.append(" [--" + ARG_TIMESTAMP
+ "]");
1999 ret
.append(" [--" + ARG_COUNT_ONLY
+ "]");
2000 ret
.append(" [--" + ARG_COUNT_DELIM
+ "=<char>]");
2001 ret
.append(" [--" + ARG_STATS
+ "]");
2002 ret
.append(" [--" + ARG_TTL_BUCKET
+ "=<N>]");
2003 ret
.append(" [--" + ARG_TTL_START
+ "=<N>:- is inclusive]");
2004 ret
.append(" [--" + ARG_TTL_END
+ "=<N>:- is exclusive]");
2005 ret
.append(" [--" + ARG_PATH
+ "=<path_to_a_file>]");
2006 ret
.append(" [--" + ARG_DECODE_BLOB_INDEX
+ "]");
2007 ret
.append(" [--" + ARG_DUMP_UNCOMPRESSED_BLOBS
+ "]");
2012 * Handles two separate cases:
2014 * 1) --db is specified - just dump the database.
2016 * 2) --path is specified - determine based on file extension what dumping
2017 * function to call. Please note that we intentionally use the extension
2018 * and avoid probing the file contents under the assumption that renaming
2019 * the files is not a supported scenario.
2022 void DBDumperCommand::DoCommand() {
2024 assert(!path_
.empty());
2025 std::string fileName
= GetFileNameFromPath(path_
);
2029 exec_state_
= LDBCommandExecuteResult::Succeed("");
2031 if (!ParseFileName(fileName
, &number
, &type
)) {
2033 LDBCommandExecuteResult::Failed("Can't parse file type: " + path_
);
2039 // TODO(myabandeh): allow configuring is_write_commited
2040 DumpWalFile(options_
, path_
, /* print_header_ */ true,
2041 /* print_values_ */ true, true /* is_write_commited */,
2045 DumpSstFile(options_
, path_
, is_key_hex_
, /* show_properties */ true,
2046 decode_blob_index_
, from_
, to_
);
2048 case kDescriptorFile
:
2049 DumpManifestFile(options_
, path_
, /* verbose_ */ false, is_key_hex_
,
2053 DumpBlobFile(path_
, is_key_hex_
, is_value_hex_
,
2054 dump_uncompressed_blobs_
);
2057 exec_state_
= LDBCommandExecuteResult::Failed(
2058 "File type not supported: " + path_
);
2067 void DBDumperCommand::DoDumpCommand() {
2068 assert(nullptr != db_
);
2069 assert(path_
.empty());
2071 // Parse command line args
2075 if (db_
->GetProperty("rocksdb.stats", &stats
)) {
2076 fprintf(stdout
, "%s\n", stats
.c_str());
2080 // Setup key iterator
2081 ReadOptions scan_read_opts
;
2082 scan_read_opts
.total_order_seek
= true;
2083 Iterator
* iter
= db_
->NewIterator(scan_read_opts
, GetCfHandle());
2084 Status st
= iter
->status();
2087 LDBCommandExecuteResult::Failed("Iterator error." + st
.ToString());
2093 iter
->SeekToFirst();
2096 int max_keys
= max_keys_
;
2098 if (!ParseIntOption(option_map_
, ARG_TTL_START
, ttl_start
, exec_state_
)) {
2099 ttl_start
= DBWithTTLImpl::kMinTimestamp
; // TTL introduction time
2102 if (!ParseIntOption(option_map_
, ARG_TTL_END
, ttl_end
, exec_state_
)) {
2103 ttl_end
= DBWithTTLImpl::kMaxTimestamp
; // Max time allowed by TTL feature
2105 if (ttl_end
< ttl_start
) {
2106 fprintf(stderr
, "Error: End time can't be less than start time\n");
2110 int time_range
= ttl_end
- ttl_start
;
2112 if (!ParseIntOption(option_map_
, ARG_TTL_BUCKET
, bucket_size
, exec_state_
) ||
2114 bucket_size
= time_range
; // Will have just 1 bucket by default
2116 // cretaing variables for row count of each type
2117 std::string rtype1
, rtype2
, row
, val
;
2120 uint64_t s1
= 0, s2
= 0;
2122 // At this point, bucket_size=0 => time_range=0
2123 int num_buckets
= (bucket_size
>= time_range
)
2125 : ((time_range
+ bucket_size
- 1) / bucket_size
);
2126 std::vector
<uint64_t> bucket_counts(num_buckets
, 0);
2127 if (is_db_ttl_
&& !count_only_
&& timestamp_
&& !count_delim_
) {
2128 fprintf(stdout
, "Dumping key-values from %s to %s\n",
2129 TimeToHumanString(ttl_start
).c_str(),
2130 TimeToHumanString(ttl_end
).c_str());
2133 HistogramImpl vsize_hist
;
2135 for (; iter
->Valid(); iter
->Next()) {
2137 // If end marker was specified, we stop before it
2138 if (!null_to_
&& (iter
->key().ToString() >= to_
)) break;
2139 // Terminate if maximum number of keys have been dumped
2140 if (max_keys
== 0) break;
2142 TtlIterator
* it_ttl
= static_cast_with_check
<TtlIterator
>(iter
);
2143 rawtime
= it_ttl
->ttl_timestamp();
2144 if (rawtime
< ttl_start
|| rawtime
>= ttl_end
) {
2151 if (is_db_ttl_
&& num_buckets
> 1) {
2152 IncBucketCounts(bucket_counts
, ttl_start
, time_range
, bucket_size
,
2153 rawtime
, num_buckets
);
2158 row
= iter
->key().ToString();
2159 val
= iter
->value().ToString();
2160 s1
= row
.size() + val
.size();
2161 for (int j
= 0; row
[j
] != delim_
[0] && row
[j
] != '\0'; j
++)
2163 if (rtype2
.compare("") && rtype2
.compare(rtype1
) != 0) {
2164 fprintf(stdout
, "%s => count:%" PRIu64
"\tsize:%" PRIu64
"\n",
2165 rtype2
.c_str(), c
, s2
);
2177 vsize_hist
.Add(iter
->value().size());
2180 if (!count_only_
&& !count_delim_
) {
2181 if (is_db_ttl_
&& timestamp_
) {
2182 fprintf(stdout
, "%s ", TimeToHumanString(rawtime
).c_str());
2185 PrintKeyValue(iter
->key().ToString(), iter
->value().ToString(),
2186 is_key_hex_
, is_value_hex_
);
2187 fprintf(stdout
, "%s\n", str
.c_str());
2191 if (num_buckets
> 1 && is_db_ttl_
) {
2192 PrintBucketCounts(bucket_counts
, ttl_start
, ttl_end
, bucket_size
,
2194 } else if (count_delim_
) {
2195 fprintf(stdout
, "%s => count:%" PRIu64
"\tsize:%" PRIu64
"\n",
2196 rtype2
.c_str(), c
, s2
);
2198 fprintf(stdout
, "Keys in range: %" PRIu64
"\n", count
);
2202 fprintf(stdout
, "Value size distribution: \n");
2203 fprintf(stdout
, "%s\n", vsize_hist
.ToString().c_str());
2209 const std::string
ReduceDBLevelsCommand::ARG_NEW_LEVELS
= "new_levels";
2210 const std::string
ReduceDBLevelsCommand::ARG_PRINT_OLD_LEVELS
=
2213 ReduceDBLevelsCommand::ReduceDBLevelsCommand(
2214 const std::vector
<std::string
>& /*params*/,
2215 const std::map
<std::string
, std::string
>& options
,
2216 const std::vector
<std::string
>& flags
)
2217 : LDBCommand(options
, flags
, false,
2218 BuildCmdLineOptions({ARG_NEW_LEVELS
, ARG_PRINT_OLD_LEVELS
})),
2219 old_levels_(1 << 7),
2221 print_old_levels_(false) {
2222 ParseIntOption(option_map_
, ARG_NEW_LEVELS
, new_levels_
, exec_state_
);
2223 print_old_levels_
= IsFlagPresent(flags
, ARG_PRINT_OLD_LEVELS
);
2225 if (new_levels_
<= 0) {
2226 exec_state_
= LDBCommandExecuteResult::Failed(
2227 " Use --" + ARG_NEW_LEVELS
+ " to specify a new level number\n");
2231 std::vector
<std::string
> ReduceDBLevelsCommand::PrepareArgs(
2232 const std::string
& db_path
, int new_levels
, bool print_old_level
) {
2233 std::vector
<std::string
> ret
;
2234 ret
.push_back("reduce_levels");
2235 ret
.push_back("--" + ARG_DB
+ "=" + db_path
);
2236 ret
.push_back("--" + ARG_NEW_LEVELS
+ "=" + std::to_string(new_levels
));
2237 if (print_old_level
) {
2238 ret
.push_back("--" + ARG_PRINT_OLD_LEVELS
);
2243 void ReduceDBLevelsCommand::Help(std::string
& ret
) {
2245 ret
.append(ReduceDBLevelsCommand::Name());
2246 ret
.append(" --" + ARG_NEW_LEVELS
+ "=<New number of levels>");
2247 ret
.append(" [--" + ARG_PRINT_OLD_LEVELS
+ "]");
2251 void ReduceDBLevelsCommand::OverrideBaseCFOptions(
2252 ColumnFamilyOptions
* cf_opts
) {
2253 LDBCommand::OverrideBaseCFOptions(cf_opts
);
2254 cf_opts
->num_levels
= old_levels_
;
2255 cf_opts
->max_bytes_for_level_multiplier_additional
.resize(cf_opts
->num_levels
,
2257 // Disable size compaction
2258 cf_opts
->max_bytes_for_level_base
= 1ULL << 50;
2259 cf_opts
->max_bytes_for_level_multiplier
= 1;
2262 Status
ReduceDBLevelsCommand::GetOldNumOfLevels(Options
& opt
, int* levels
) {
2263 ImmutableDBOptions
db_options(opt
);
2264 EnvOptions soptions
;
2265 std::shared_ptr
<Cache
> tc(
2266 NewLRUCache(opt
.max_open_files
- 10, opt
.table_cache_numshardbits
));
2267 const InternalKeyComparator
cmp(opt
.comparator
);
2268 WriteController
wc(opt
.delayed_write_rate
);
2269 WriteBufferManager
wb(opt
.db_write_buffer_size
);
2270 VersionSet
versions(db_path_
, &db_options
, soptions
, tc
.get(), &wb
, &wc
,
2271 /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
2272 /*db_id*/ "", /*db_session_id*/ "");
2273 std::vector
<ColumnFamilyDescriptor
> dummy
;
2274 ColumnFamilyDescriptor
dummy_descriptor(kDefaultColumnFamilyName
,
2275 ColumnFamilyOptions(opt
));
2276 dummy
.push_back(dummy_descriptor
);
2277 // We rely the VersionSet::Recover to tell us the internal data structures
2278 // in the db. And the Recover() should never do any change
2279 // (like LogAndApply) to the manifest file.
2280 Status st
= versions
.Recover(dummy
);
2285 auto default_cfd
= versions
.GetColumnFamilySet()->GetDefault();
2286 for (int i
= 0; i
< default_cfd
->NumberLevels(); i
++) {
2287 if (default_cfd
->current()->storage_info()->NumLevelFiles(i
)) {
2296 void ReduceDBLevelsCommand::DoCommand() {
2297 if (new_levels_
<= 1) {
2299 LDBCommandExecuteResult::Failed("Invalid number of levels.\n");
2305 int old_level_num
= -1;
2306 st
= GetOldNumOfLevels(options_
, &old_level_num
);
2308 exec_state_
= LDBCommandExecuteResult::Failed(st
.ToString());
2312 if (print_old_levels_
) {
2313 fprintf(stdout
, "The old number of levels in use is %d\n", old_level_num
);
2316 if (old_level_num
<= new_levels_
) {
2320 old_levels_
= old_level_num
;
2323 if (exec_state_
.IsFailed()) {
2326 assert(db_
!= nullptr);
2327 // Compact the whole DB to put all files to the highest level.
2328 fprintf(stdout
, "Compacting the db...\n");
2330 db_
->CompactRange(CompactRangeOptions(), GetCfHandle(), nullptr, nullptr);
2335 EnvOptions soptions
;
2336 st
= VersionSet::ReduceNumberOfLevels(db_path_
, &options_
, soptions
,
2340 exec_state_
= LDBCommandExecuteResult::Failed(st
.ToString());
2345 const std::string
ChangeCompactionStyleCommand::ARG_OLD_COMPACTION_STYLE
=
2346 "old_compaction_style";
2347 const std::string
ChangeCompactionStyleCommand::ARG_NEW_COMPACTION_STYLE
=
2348 "new_compaction_style";
2350 ChangeCompactionStyleCommand::ChangeCompactionStyleCommand(
2351 const std::vector
<std::string
>& /*params*/,
2352 const std::map
<std::string
, std::string
>& options
,
2353 const std::vector
<std::string
>& flags
)
2354 : LDBCommand(options
, flags
, false,
2355 BuildCmdLineOptions(
2356 {ARG_OLD_COMPACTION_STYLE
, ARG_NEW_COMPACTION_STYLE
})),
2357 old_compaction_style_(-1),
2358 new_compaction_style_(-1) {
2359 ParseIntOption(option_map_
, ARG_OLD_COMPACTION_STYLE
, old_compaction_style_
,
2361 if (old_compaction_style_
!= kCompactionStyleLevel
&&
2362 old_compaction_style_
!= kCompactionStyleUniversal
) {
2363 exec_state_
= LDBCommandExecuteResult::Failed(
2364 "Use --" + ARG_OLD_COMPACTION_STYLE
+ " to specify old compaction " +
2365 "style. Check ldb help for proper compaction style value.\n");
2369 ParseIntOption(option_map_
, ARG_NEW_COMPACTION_STYLE
, new_compaction_style_
,
2371 if (new_compaction_style_
!= kCompactionStyleLevel
&&
2372 new_compaction_style_
!= kCompactionStyleUniversal
) {
2373 exec_state_
= LDBCommandExecuteResult::Failed(
2374 "Use --" + ARG_NEW_COMPACTION_STYLE
+ " to specify new compaction " +
2375 "style. Check ldb help for proper compaction style value.\n");
2379 if (new_compaction_style_
== old_compaction_style_
) {
2380 exec_state_
= LDBCommandExecuteResult::Failed(
2381 "Old compaction style is the same as new compaction style. "
2382 "Nothing to do.\n");
2386 if (old_compaction_style_
== kCompactionStyleUniversal
&&
2387 new_compaction_style_
== kCompactionStyleLevel
) {
2388 exec_state_
= LDBCommandExecuteResult::Failed(
2389 "Convert from universal compaction to level compaction. "
2390 "Nothing to do.\n");
2395 void ChangeCompactionStyleCommand::Help(std::string
& ret
) {
2397 ret
.append(ChangeCompactionStyleCommand::Name());
2398 ret
.append(" --" + ARG_OLD_COMPACTION_STYLE
+ "=<Old compaction style: 0 " +
2399 "for level compaction, 1 for universal compaction>");
2400 ret
.append(" --" + ARG_NEW_COMPACTION_STYLE
+ "=<New compaction style: 0 " +
2401 "for level compaction, 1 for universal compaction>");
2405 void ChangeCompactionStyleCommand::OverrideBaseCFOptions(
2406 ColumnFamilyOptions
* cf_opts
) {
2407 LDBCommand::OverrideBaseCFOptions(cf_opts
);
2408 if (old_compaction_style_
== kCompactionStyleLevel
&&
2409 new_compaction_style_
== kCompactionStyleUniversal
) {
2410 // In order to convert from level compaction to universal compaction, we
2411 // need to compact all data into a single file and move it to level 0.
2412 cf_opts
->disable_auto_compactions
= true;
2413 cf_opts
->target_file_size_base
= INT_MAX
;
2414 cf_opts
->target_file_size_multiplier
= 1;
2415 cf_opts
->max_bytes_for_level_base
= INT_MAX
;
2416 cf_opts
->max_bytes_for_level_multiplier
= 1;
2420 void ChangeCompactionStyleCommand::DoCommand() {
2422 assert(GetExecuteState().IsFailed());
2425 // print db stats before we have made any change
2426 std::string property
;
2427 std::string files_per_level
;
2428 for (int i
= 0; i
< db_
->NumberLevels(GetCfHandle()); i
++) {
2429 db_
->GetProperty(GetCfHandle(),
2430 "rocksdb.num-files-at-level" + std::to_string(i
),
2433 // format print string
2435 snprintf(buf
, sizeof(buf
), "%s%s", (i
? "," : ""), property
.c_str());
2436 files_per_level
+= buf
;
2438 fprintf(stdout
, "files per level before compaction: %s\n",
2439 files_per_level
.c_str());
2441 // manual compact into a single file and move the file to level 0
2442 CompactRangeOptions compact_options
;
2443 compact_options
.change_level
= true;
2444 compact_options
.target_level
= 0;
2446 db_
->CompactRange(compact_options
, GetCfHandle(), nullptr, nullptr);
2448 std::stringstream oss
;
2449 oss
<< "Compaction failed: " << s
.ToString();
2450 exec_state_
= LDBCommandExecuteResult::Failed(oss
.str());
2454 // verify compaction result
2455 files_per_level
= "";
2457 for (int i
= 0; i
< db_
->NumberLevels(GetCfHandle()); i
++) {
2458 db_
->GetProperty(GetCfHandle(),
2459 "rocksdb.num-files-at-level" + std::to_string(i
),
2462 // format print string
2464 snprintf(buf
, sizeof(buf
), "%s%s", (i
? "," : ""), property
.c_str());
2465 files_per_level
+= buf
;
2467 num_files
= atoi(property
.c_str());
2469 // level 0 should have only 1 file
2470 if (i
== 0 && num_files
!= 1) {
2471 exec_state_
= LDBCommandExecuteResult::Failed(
2472 "Number of db files at "
2473 "level 0 after compaction is " +
2474 std::to_string(num_files
) + ", not 1.\n");
2477 // other levels should have no file
2478 if (i
> 0 && num_files
!= 0) {
2479 exec_state_
= LDBCommandExecuteResult::Failed(
2480 "Number of db files at "
2482 std::to_string(i
) + " after compaction is " +
2483 std::to_string(num_files
) + ", not 0.\n");
2488 fprintf(stdout
, "files per level after compaction: %s\n",
2489 files_per_level
.c_str());
2492 // ----------------------------------------------------------------------------
2496 struct StdErrReporter
: public log::Reader::Reporter
{
2497 void Corruption(size_t /*bytes*/, const Status
& s
) override
{
2498 std::cerr
<< "Corruption detected in log file " << s
.ToString() << "\n";
2502 class InMemoryHandler
: public WriteBatch::Handler
{
2504 InMemoryHandler(std::stringstream
& row
, bool print_values
,
2505 bool write_after_commit
= false)
2508 print_values_(print_values
),
2509 write_after_commit_(write_after_commit
) {}
2511 void commonPutMerge(const Slice
& key
, const Slice
& value
) {
2512 std::string k
= LDBCommand::StringToHex(key
.ToString());
2513 if (print_values_
) {
2514 std::string v
= LDBCommand::StringToHex(value
.ToString());
2522 Status
PutCF(uint32_t cf
, const Slice
& key
, const Slice
& value
) override
{
2523 row_
<< "PUT(" << cf
<< ") : ";
2524 commonPutMerge(key
, value
);
2525 return Status::OK();
2528 Status
MergeCF(uint32_t cf
, const Slice
& key
, const Slice
& value
) override
{
2529 row_
<< "MERGE(" << cf
<< ") : ";
2530 commonPutMerge(key
, value
);
2531 return Status::OK();
2534 Status
MarkNoop(bool) override
{
2536 return Status::OK();
2539 Status
DeleteCF(uint32_t cf
, const Slice
& key
) override
{
2540 row_
<< "DELETE(" << cf
<< ") : ";
2541 row_
<< LDBCommand::StringToHex(key
.ToString()) << " ";
2542 return Status::OK();
2545 Status
SingleDeleteCF(uint32_t cf
, const Slice
& key
) override
{
2546 row_
<< "SINGLE_DELETE(" << cf
<< ") : ";
2547 row_
<< LDBCommand::StringToHex(key
.ToString()) << " ";
2548 return Status::OK();
2551 Status
DeleteRangeCF(uint32_t cf
, const Slice
& begin_key
,
2552 const Slice
& end_key
) override
{
2553 row_
<< "DELETE_RANGE(" << cf
<< ") : ";
2554 row_
<< LDBCommand::StringToHex(begin_key
.ToString()) << " ";
2555 row_
<< LDBCommand::StringToHex(end_key
.ToString()) << " ";
2556 return Status::OK();
2559 Status
MarkBeginPrepare(bool unprepare
) override
{
2560 row_
<< "BEGIN_PREPARE(";
2561 row_
<< (unprepare
? "true" : "false") << ") ";
2562 return Status::OK();
2565 Status
MarkEndPrepare(const Slice
& xid
) override
{
2566 row_
<< "END_PREPARE(";
2567 row_
<< LDBCommand::StringToHex(xid
.ToString()) << ") ";
2568 return Status::OK();
2571 Status
MarkRollback(const Slice
& xid
) override
{
2572 row_
<< "ROLLBACK(";
2573 row_
<< LDBCommand::StringToHex(xid
.ToString()) << ") ";
2574 return Status::OK();
2577 Status
MarkCommit(const Slice
& xid
) override
{
2579 row_
<< LDBCommand::StringToHex(xid
.ToString()) << ") ";
2580 return Status::OK();
2583 Status
MarkCommitWithTimestamp(const Slice
& xid
,
2584 const Slice
& commit_ts
) override
{
2585 row_
<< "COMMIT_WITH_TIMESTAMP(";
2586 row_
<< LDBCommand::StringToHex(xid
.ToString()) << ", ";
2587 row_
<< LDBCommand::StringToHex(commit_ts
.ToString()) << ") ";
2588 return Status::OK();
2591 ~InMemoryHandler() override
{}
2594 Handler::OptionState
WriteAfterCommit() const override
{
2595 return write_after_commit_
? Handler::OptionState::kEnabled
2596 : Handler::OptionState::kDisabled
;
2600 std::stringstream
& row_
;
2602 bool write_after_commit_
;
2605 void DumpWalFile(Options options
, std::string wal_file
, bool print_header
,
2606 bool print_values
, bool is_write_committed
,
2607 LDBCommandExecuteResult
* exec_state
) {
2608 const auto& fs
= options
.env
->GetFileSystem();
2609 FileOptions
soptions(options
);
2610 std::unique_ptr
<SequentialFileReader
> wal_file_reader
;
2611 Status status
= SequentialFileReader::Create(
2612 fs
, wal_file
, soptions
, &wal_file_reader
, nullptr /* dbg */,
2613 nullptr /* rate_limiter */);
2616 *exec_state
= LDBCommandExecuteResult::Failed("Failed to open WAL file " +
2619 std::cerr
<< "Error: Failed to open WAL file " << status
.ToString()
2623 StdErrReporter reporter
;
2624 uint64_t log_number
;
2627 // we need the log number, but ParseFilename expects dbname/NNN.log.
2628 std::string sanitized
= wal_file
;
2629 size_t lastslash
= sanitized
.rfind('/');
2630 if (lastslash
!= std::string::npos
)
2631 sanitized
= sanitized
.substr(lastslash
+ 1);
2632 if (!ParseFileName(sanitized
, &log_number
, &type
)) {
2633 // bogus input, carry on as best we can
2636 log::Reader
reader(options
.info_log
, std::move(wal_file_reader
), &reporter
,
2637 true /* checksum */, log_number
);
2638 std::string scratch
;
2641 std::stringstream row
;
2643 std::cout
<< "Sequence,Count,ByteSize,Physical Offset,Key(s)";
2645 std::cout
<< " : value ";
2649 while (status
.ok() && reader
.ReadRecord(&record
, &scratch
)) {
2651 if (record
.size() < WriteBatchInternal::kHeader
) {
2652 reporter
.Corruption(record
.size(),
2653 Status::Corruption("log record too small"));
2655 status
= WriteBatchInternal::SetContents(&batch
, record
);
2657 std::stringstream oss
;
2658 oss
<< "Parsing write batch failed: " << status
.ToString();
2660 *exec_state
= LDBCommandExecuteResult::Failed(oss
.str());
2662 std::cerr
<< oss
.str() << std::endl
;
2666 row
<< WriteBatchInternal::Sequence(&batch
) << ",";
2667 row
<< WriteBatchInternal::Count(&batch
) << ",";
2668 row
<< WriteBatchInternal::ByteSize(&batch
) << ",";
2669 row
<< reader
.LastRecordOffset() << ",";
2670 InMemoryHandler
handler(row
, print_values
, is_write_committed
);
2671 status
= batch
.Iterate(&handler
);
2674 std::stringstream oss
;
2675 oss
<< "Print write batch error: " << status
.ToString();
2676 *exec_state
= LDBCommandExecuteResult::Failed(oss
.str());
2678 row
<< "error: " << status
.ToString();
2683 std::cout
<< row
.str();
2690 const std::string
WALDumperCommand::ARG_WAL_FILE
= "walfile";
2691 const std::string
WALDumperCommand::ARG_WRITE_COMMITTED
= "write_committed";
2692 const std::string
WALDumperCommand::ARG_PRINT_VALUE
= "print_value";
2693 const std::string
WALDumperCommand::ARG_PRINT_HEADER
= "header";
2695 WALDumperCommand::WALDumperCommand(
2696 const std::vector
<std::string
>& /*params*/,
2697 const std::map
<std::string
, std::string
>& options
,
2698 const std::vector
<std::string
>& flags
)
2699 : LDBCommand(options
, flags
, true,
2700 BuildCmdLineOptions({ARG_WAL_FILE
, ARG_WRITE_COMMITTED
,
2701 ARG_PRINT_HEADER
, ARG_PRINT_VALUE
})),
2702 print_header_(false),
2703 print_values_(false),
2704 is_write_committed_(false) {
2707 auto itr
= options
.find(ARG_WAL_FILE
);
2708 if (itr
!= options
.end()) {
2709 wal_file_
= itr
->second
;
2712 print_header_
= IsFlagPresent(flags
, ARG_PRINT_HEADER
);
2713 print_values_
= IsFlagPresent(flags
, ARG_PRINT_VALUE
);
2714 is_write_committed_
= ParseBooleanOption(options
, ARG_WRITE_COMMITTED
, true);
2716 if (wal_file_
.empty()) {
2717 exec_state_
= LDBCommandExecuteResult::Failed("Argument " + ARG_WAL_FILE
+
2718 " must be specified.");
2722 void WALDumperCommand::Help(std::string
& ret
) {
2724 ret
.append(WALDumperCommand::Name());
2725 ret
.append(" --" + ARG_WAL_FILE
+ "=<write_ahead_log_file_path>");
2726 ret
.append(" [--" + ARG_PRINT_HEADER
+ "] ");
2727 ret
.append(" [--" + ARG_PRINT_VALUE
+ "] ");
2728 ret
.append(" [--" + ARG_WRITE_COMMITTED
+ "=true|false] ");
2732 void WALDumperCommand::DoCommand() {
2733 DumpWalFile(options_
, wal_file_
, print_header_
, print_values_
,
2734 is_write_committed_
, &exec_state_
);
2737 // ----------------------------------------------------------------------------
2739 GetCommand::GetCommand(const std::vector
<std::string
>& params
,
2740 const std::map
<std::string
, std::string
>& options
,
2741 const std::vector
<std::string
>& flags
)
2743 options
, flags
, true,
2744 BuildCmdLineOptions({ARG_TTL
, ARG_HEX
, ARG_KEY_HEX
, ARG_VALUE_HEX
})) {
2745 if (params
.size() != 1) {
2746 exec_state_
= LDBCommandExecuteResult::Failed(
2747 "<key> must be specified for the get command");
2749 key_
= params
.at(0);
2753 key_
= HexToString(key_
);
2757 void GetCommand::Help(std::string
& ret
) {
2759 ret
.append(GetCommand::Name());
2760 ret
.append(" <key>");
2761 ret
.append(" [--" + ARG_TTL
+ "]");
2765 void GetCommand::DoCommand() {
2767 assert(GetExecuteState().IsFailed());
2771 Status st
= db_
->Get(ReadOptions(), GetCfHandle(), key_
, &value
);
2773 fprintf(stdout
, "%s\n",
2774 (is_value_hex_
? StringToHex(value
) : value
).c_str());
2776 std::stringstream oss
;
2777 oss
<< "Get failed: " << st
.ToString();
2778 exec_state_
= LDBCommandExecuteResult::Failed(oss
.str());
2782 // ----------------------------------------------------------------------------
2784 ApproxSizeCommand::ApproxSizeCommand(
2785 const std::vector
<std::string
>& /*params*/,
2786 const std::map
<std::string
, std::string
>& options
,
2787 const std::vector
<std::string
>& flags
)
2788 : LDBCommand(options
, flags
, true,
2789 BuildCmdLineOptions(
2790 {ARG_HEX
, ARG_KEY_HEX
, ARG_VALUE_HEX
, ARG_FROM
, ARG_TO
})) {
2791 if (options
.find(ARG_FROM
) != options
.end()) {
2792 start_key_
= options
.find(ARG_FROM
)->second
;
2794 exec_state_
= LDBCommandExecuteResult::Failed(
2795 ARG_FROM
+ " must be specified for approxsize command");
2799 if (options
.find(ARG_TO
) != options
.end()) {
2800 end_key_
= options
.find(ARG_TO
)->second
;
2802 exec_state_
= LDBCommandExecuteResult::Failed(
2803 ARG_TO
+ " must be specified for approxsize command");
2808 start_key_
= HexToString(start_key_
);
2809 end_key_
= HexToString(end_key_
);
2813 void ApproxSizeCommand::Help(std::string
& ret
) {
2815 ret
.append(ApproxSizeCommand::Name());
2816 ret
.append(HelpRangeCmdArgs());
2820 void ApproxSizeCommand::DoCommand() {
2822 assert(GetExecuteState().IsFailed());
2826 ranges
[0] = Range(start_key_
, end_key_
);
2828 Status s
= db_
->GetApproximateSizes(GetCfHandle(), ranges
, 1, sizes
);
2830 std::stringstream oss
;
2831 oss
<< "ApproximateSize failed: " << s
.ToString();
2832 exec_state_
= LDBCommandExecuteResult::Failed(oss
.str());
2834 fprintf(stdout
, "%lu\n", (unsigned long)sizes
[0]);
2838 // ----------------------------------------------------------------------------
2840 BatchPutCommand::BatchPutCommand(
2841 const std::vector
<std::string
>& params
,
2842 const std::map
<std::string
, std::string
>& options
,
2843 const std::vector
<std::string
>& flags
)
2844 : LDBCommand(options
, flags
, false,
2845 BuildCmdLineOptions({ARG_TTL
, ARG_HEX
, ARG_KEY_HEX
,
2846 ARG_VALUE_HEX
, ARG_CREATE_IF_MISSING
})) {
2847 if (params
.size() < 2) {
2848 exec_state_
= LDBCommandExecuteResult::Failed(
2849 "At least one <key> <value> pair must be specified batchput.");
2850 } else if (params
.size() % 2 != 0) {
2851 exec_state_
= LDBCommandExecuteResult::Failed(
2852 "Equal number of <key>s and <value>s must be specified for batchput.");
2854 for (size_t i
= 0; i
< params
.size(); i
+= 2) {
2855 std::string key
= params
.at(i
);
2856 std::string value
= params
.at(i
+ 1);
2857 key_values_
.push_back(std::pair
<std::string
, std::string
>(
2858 is_key_hex_
? HexToString(key
) : key
,
2859 is_value_hex_
? HexToString(value
) : value
));
2862 create_if_missing_
= IsFlagPresent(flags_
, ARG_CREATE_IF_MISSING
);
2865 void BatchPutCommand::Help(std::string
& ret
) {
2867 ret
.append(BatchPutCommand::Name());
2868 ret
.append(" <key> <value> [<key> <value>] [..]");
2869 ret
.append(" [--" + ARG_CREATE_IF_MISSING
+ "]");
2870 ret
.append(" [--" + ARG_TTL
+ "]");
2874 void BatchPutCommand::DoCommand() {
2876 assert(GetExecuteState().IsFailed());
2882 std::stringstream oss
;
2883 for (std::vector
<std::pair
<std::string
, std::string
>>::const_iterator itr
=
2884 key_values_
.begin();
2885 itr
!= key_values_
.end(); ++itr
) {
2886 st
= batch
.Put(GetCfHandle(), itr
->first
, itr
->second
);
2888 oss
<< "Put to write batch failed: " << itr
->first
<< "=>" << itr
->second
2889 << " error: " << st
.ToString();
2894 st
= db_
->Write(WriteOptions(), &batch
);
2896 oss
<< "Write failed: " << st
.ToString();
2900 fprintf(stdout
, "OK\n");
2902 exec_state_
= LDBCommandExecuteResult::Failed(oss
.str());
2906 void BatchPutCommand::OverrideBaseOptions() {
2907 LDBCommand::OverrideBaseOptions();
2908 options_
.create_if_missing
= create_if_missing_
;
2911 // ----------------------------------------------------------------------------
2913 ScanCommand::ScanCommand(const std::vector
<std::string
>& /*params*/,
2914 const std::map
<std::string
, std::string
>& options
,
2915 const std::vector
<std::string
>& flags
)
2917 options
, flags
, true,
2918 BuildCmdLineOptions({ARG_TTL
, ARG_NO_VALUE
, ARG_HEX
, ARG_KEY_HEX
,
2919 ARG_TO
, ARG_VALUE_HEX
, ARG_FROM
, ARG_TIMESTAMP
,
2920 ARG_MAX_KEYS
, ARG_TTL_START
, ARG_TTL_END
})),
2921 start_key_specified_(false),
2922 end_key_specified_(false),
2923 max_keys_scanned_(-1),
2925 auto itr
= options
.find(ARG_FROM
);
2926 if (itr
!= options
.end()) {
2927 start_key_
= itr
->second
;
2929 start_key_
= HexToString(start_key_
);
2931 start_key_specified_
= true;
2933 itr
= options
.find(ARG_TO
);
2934 if (itr
!= options
.end()) {
2935 end_key_
= itr
->second
;
2937 end_key_
= HexToString(end_key_
);
2939 end_key_specified_
= true;
2942 std::vector
<std::string
>::const_iterator vitr
=
2943 std::find(flags
.begin(), flags
.end(), ARG_NO_VALUE
);
2944 if (vitr
!= flags
.end()) {
2948 itr
= options
.find(ARG_MAX_KEYS
);
2949 if (itr
!= options
.end()) {
2952 max_keys_scanned_
= strtol(itr
->second
.c_str(), 0, 10);
2954 max_keys_scanned_
= std::stoi(itr
->second
);
2956 } catch (const std::invalid_argument
&) {
2957 exec_state_
= LDBCommandExecuteResult::Failed(ARG_MAX_KEYS
+
2958 " has an invalid value");
2959 } catch (const std::out_of_range
&) {
2960 exec_state_
= LDBCommandExecuteResult::Failed(
2961 ARG_MAX_KEYS
+ " has a value out-of-range");
2966 void ScanCommand::Help(std::string
& ret
) {
2968 ret
.append(ScanCommand::Name());
2969 ret
.append(HelpRangeCmdArgs());
2970 ret
.append(" [--" + ARG_TTL
+ "]");
2971 ret
.append(" [--" + ARG_TIMESTAMP
+ "]");
2972 ret
.append(" [--" + ARG_MAX_KEYS
+ "=<N>q] ");
2973 ret
.append(" [--" + ARG_TTL_START
+ "=<N>:- is inclusive]");
2974 ret
.append(" [--" + ARG_TTL_END
+ "=<N>:- is exclusive]");
2975 ret
.append(" [--" + ARG_NO_VALUE
+ "]");
2979 void ScanCommand::DoCommand() {
2981 assert(GetExecuteState().IsFailed());
2985 int num_keys_scanned
= 0;
2986 ReadOptions scan_read_opts
;
2987 scan_read_opts
.total_order_seek
= true;
2988 Iterator
* it
= db_
->NewIterator(scan_read_opts
, GetCfHandle());
2989 if (start_key_specified_
) {
2990 it
->Seek(start_key_
);
2995 if (!ParseIntOption(option_map_
, ARG_TTL_START
, ttl_start
, exec_state_
)) {
2996 ttl_start
= DBWithTTLImpl::kMinTimestamp
; // TTL introduction time
2999 if (!ParseIntOption(option_map_
, ARG_TTL_END
, ttl_end
, exec_state_
)) {
3000 ttl_end
= DBWithTTLImpl::kMaxTimestamp
; // Max time allowed by TTL feature
3002 if (ttl_end
< ttl_start
) {
3003 fprintf(stderr
, "Error: End time can't be less than start time\n");
3007 if (is_db_ttl_
&& timestamp_
) {
3008 fprintf(stdout
, "Scanning key-values from %s to %s\n",
3009 TimeToHumanString(ttl_start
).c_str(),
3010 TimeToHumanString(ttl_end
).c_str());
3013 it
->Valid() && (!end_key_specified_
|| it
->key().ToString() < end_key_
);
3016 TtlIterator
* it_ttl
= static_cast_with_check
<TtlIterator
>(it
);
3017 int rawtime
= it_ttl
->ttl_timestamp();
3018 if (rawtime
< ttl_start
|| rawtime
>= ttl_end
) {
3022 fprintf(stdout
, "%s ", TimeToHumanString(rawtime
).c_str());
3026 Slice key_slice
= it
->key();
3028 std::string formatted_key
;
3030 formatted_key
= "0x" + key_slice
.ToString(true /* hex */);
3031 key_slice
= formatted_key
;
3032 } else if (ldb_options_
.key_formatter
) {
3033 formatted_key
= ldb_options_
.key_formatter
->Format(key_slice
);
3034 key_slice
= formatted_key
;
3038 fprintf(stdout
, "%.*s\n", static_cast<int>(key_slice
.size()),
3041 Slice val_slice
= it
->value();
3042 std::string formatted_value
;
3043 if (is_value_hex_
) {
3044 formatted_value
= "0x" + val_slice
.ToString(true /* hex */);
3045 val_slice
= formatted_value
;
3047 fprintf(stdout
, "%.*s : %.*s\n", static_cast<int>(key_slice
.size()),
3048 key_slice
.data(), static_cast<int>(val_slice
.size()),
3053 if (max_keys_scanned_
>= 0 && num_keys_scanned
>= max_keys_scanned_
) {
3057 if (!it
->status().ok()) { // Check for any errors found during the scan
3058 exec_state_
= LDBCommandExecuteResult::Failed(it
->status().ToString());
3063 // ----------------------------------------------------------------------------
3065 DeleteCommand::DeleteCommand(const std::vector
<std::string
>& params
,
3066 const std::map
<std::string
, std::string
>& options
,
3067 const std::vector
<std::string
>& flags
)
3068 : LDBCommand(options
, flags
, false,
3069 BuildCmdLineOptions({ARG_HEX
, ARG_KEY_HEX
, ARG_VALUE_HEX
})) {
3070 if (params
.size() != 1) {
3071 exec_state_
= LDBCommandExecuteResult::Failed(
3072 "KEY must be specified for the delete command");
3074 key_
= params
.at(0);
3076 key_
= HexToString(key_
);
3081 void DeleteCommand::Help(std::string
& ret
) {
3083 ret
.append(DeleteCommand::Name() + " <key>");
3087 void DeleteCommand::DoCommand() {
3089 assert(GetExecuteState().IsFailed());
3092 Status st
= db_
->Delete(WriteOptions(), GetCfHandle(), key_
);
3094 fprintf(stdout
, "OK\n");
3096 exec_state_
= LDBCommandExecuteResult::Failed(st
.ToString());
3100 SingleDeleteCommand::SingleDeleteCommand(
3101 const std::vector
<std::string
>& params
,
3102 const std::map
<std::string
, std::string
>& options
,
3103 const std::vector
<std::string
>& flags
)
3104 : LDBCommand(options
, flags
, false,
3105 BuildCmdLineOptions({ARG_HEX
, ARG_KEY_HEX
, ARG_VALUE_HEX
})) {
3106 if (params
.size() != 1) {
3107 exec_state_
= LDBCommandExecuteResult::Failed(
3108 "KEY must be specified for the single delete command");
3110 key_
= params
.at(0);
3112 key_
= HexToString(key_
);
3117 void SingleDeleteCommand::Help(std::string
& ret
) {
3119 ret
.append(SingleDeleteCommand::Name() + " <key>");
3123 void SingleDeleteCommand::DoCommand() {
3125 assert(GetExecuteState().IsFailed());
3128 Status st
= db_
->SingleDelete(WriteOptions(), GetCfHandle(), key_
);
3130 fprintf(stdout
, "OK\n");
3132 exec_state_
= LDBCommandExecuteResult::Failed(st
.ToString());
3136 DeleteRangeCommand::DeleteRangeCommand(
3137 const std::vector
<std::string
>& params
,
3138 const std::map
<std::string
, std::string
>& options
,
3139 const std::vector
<std::string
>& flags
)
3140 : LDBCommand(options
, flags
, false,
3141 BuildCmdLineOptions({ARG_HEX
, ARG_KEY_HEX
, ARG_VALUE_HEX
})) {
3142 if (params
.size() != 2) {
3143 exec_state_
= LDBCommandExecuteResult::Failed(
3144 "begin and end keys must be specified for the delete command");
3146 begin_key_
= params
.at(0);
3147 end_key_
= params
.at(1);
3149 begin_key_
= HexToString(begin_key_
);
3150 end_key_
= HexToString(end_key_
);
3155 void DeleteRangeCommand::Help(std::string
& ret
) {
3157 ret
.append(DeleteRangeCommand::Name() + " <begin key> <end key>");
3161 void DeleteRangeCommand::DoCommand() {
3163 assert(GetExecuteState().IsFailed());
3167 db_
->DeleteRange(WriteOptions(), GetCfHandle(), begin_key_
, end_key_
);
3169 fprintf(stdout
, "OK\n");
3171 exec_state_
= LDBCommandExecuteResult::Failed(st
.ToString());
3175 PutCommand::PutCommand(const std::vector
<std::string
>& params
,
3176 const std::map
<std::string
, std::string
>& options
,
3177 const std::vector
<std::string
>& flags
)
3178 : LDBCommand(options
, flags
, false,
3179 BuildCmdLineOptions({ARG_TTL
, ARG_HEX
, ARG_KEY_HEX
,
3180 ARG_VALUE_HEX
, ARG_CREATE_IF_MISSING
})) {
3181 if (params
.size() != 2) {
3182 exec_state_
= LDBCommandExecuteResult::Failed(
3183 "<key> and <value> must be specified for the put command");
3185 key_
= params
.at(0);
3186 value_
= params
.at(1);
3190 key_
= HexToString(key_
);
3193 if (is_value_hex_
) {
3194 value_
= HexToString(value_
);
3196 create_if_missing_
= IsFlagPresent(flags_
, ARG_CREATE_IF_MISSING
);
3199 void PutCommand::Help(std::string
& ret
) {
3201 ret
.append(PutCommand::Name());
3202 ret
.append(" <key> <value>");
3203 ret
.append(" [--" + ARG_CREATE_IF_MISSING
+ "]");
3204 ret
.append(" [--" + ARG_TTL
+ "]");
3208 void PutCommand::DoCommand() {
3210 assert(GetExecuteState().IsFailed());
3213 Status st
= db_
->Put(WriteOptions(), GetCfHandle(), key_
, value_
);
3215 fprintf(stdout
, "OK\n");
3217 exec_state_
= LDBCommandExecuteResult::Failed(st
.ToString());
3221 void PutCommand::OverrideBaseOptions() {
3222 LDBCommand::OverrideBaseOptions();
3223 options_
.create_if_missing
= create_if_missing_
;
3226 // ----------------------------------------------------------------------------
3228 const char* DBQuerierCommand::HELP_CMD
= "help";
3229 const char* DBQuerierCommand::GET_CMD
= "get";
3230 const char* DBQuerierCommand::PUT_CMD
= "put";
3231 const char* DBQuerierCommand::DELETE_CMD
= "delete";
3233 DBQuerierCommand::DBQuerierCommand(
3234 const std::vector
<std::string
>& /*params*/,
3235 const std::map
<std::string
, std::string
>& options
,
3236 const std::vector
<std::string
>& flags
)
3238 options
, flags
, false,
3239 BuildCmdLineOptions({ARG_TTL
, ARG_HEX
, ARG_KEY_HEX
, ARG_VALUE_HEX
})) {
3243 void DBQuerierCommand::Help(std::string
& ret
) {
3245 ret
.append(DBQuerierCommand::Name());
3246 ret
.append(" [--" + ARG_TTL
+ "]");
3249 " Starts a REPL shell. Type help for list of available "
3254 void DBQuerierCommand::DoCommand() {
3256 assert(GetExecuteState().IsFailed());
3260 ReadOptions read_options
;
3261 WriteOptions write_options
;
3267 std::stringstream oss
;
3268 while (s
.ok() && getline(std::cin
, line
, '\n')) {
3269 // Parse line into std::vector<std::string>
3270 std::vector
<std::string
> tokens
;
3273 size_t pos2
= line
.find(' ', pos
);
3274 if (pos2
== std::string::npos
) {
3277 tokens
.push_back(line
.substr(pos
, pos2
- pos
));
3280 tokens
.push_back(line
.substr(pos
));
3282 const std::string
& cmd
= tokens
[0];
3284 if (cmd
== HELP_CMD
) {
3287 "put <key> <value>\n"
3289 } else if (cmd
== DELETE_CMD
&& tokens
.size() == 2) {
3290 key
= (is_key_hex_
? HexToString(tokens
[1]) : tokens
[1]);
3291 s
= db_
->Delete(write_options
, GetCfHandle(), Slice(key
));
3293 fprintf(stdout
, "Successfully deleted %s\n", tokens
[1].c_str());
3295 oss
<< "delete " << key
<< " failed: " << s
.ToString();
3297 } else if (cmd
== PUT_CMD
&& tokens
.size() == 3) {
3298 key
= (is_key_hex_
? HexToString(tokens
[1]) : tokens
[1]);
3299 value
= (is_value_hex_
? HexToString(tokens
[2]) : tokens
[2]);
3300 s
= db_
->Put(write_options
, GetCfHandle(), Slice(key
), Slice(value
));
3302 fprintf(stdout
, "Successfully put %s %s\n", tokens
[1].c_str(),
3305 oss
<< "put " << key
<< "=>" << value
<< " failed: " << s
.ToString();
3307 } else if (cmd
== GET_CMD
&& tokens
.size() == 2) {
3308 key
= (is_key_hex_
? HexToString(tokens
[1]) : tokens
[1]);
3309 s
= db_
->Get(read_options
, GetCfHandle(), Slice(key
), &value
);
3311 fprintf(stdout
, "%s\n",
3312 PrintKeyValue(key
, value
, is_key_hex_
, is_value_hex_
).c_str());
3314 if (s
.IsNotFound()) {
3315 fprintf(stdout
, "Not found %s\n", tokens
[1].c_str());
3317 oss
<< "get " << key
<< " error: " << s
.ToString();
3321 fprintf(stdout
, "Unknown command %s\n", line
.c_str());
3325 exec_state_
= LDBCommandExecuteResult::Failed(oss
.str());
3329 // ----------------------------------------------------------------------------
3331 CheckConsistencyCommand::CheckConsistencyCommand(
3332 const std::vector
<std::string
>& /*params*/,
3333 const std::map
<std::string
, std::string
>& options
,
3334 const std::vector
<std::string
>& flags
)
3335 : LDBCommand(options
, flags
, true, BuildCmdLineOptions({})) {}
3337 void CheckConsistencyCommand::Help(std::string
& ret
) {
3339 ret
.append(CheckConsistencyCommand::Name());
3343 void CheckConsistencyCommand::DoCommand() {
3344 options_
.paranoid_checks
= true;
3345 options_
.num_levels
= 64;
3347 if (exec_state_
.IsSucceed() || exec_state_
.IsNotStarted()) {
3348 fprintf(stdout
, "OK\n");
3353 // ----------------------------------------------------------------------------
3355 const std::string
CheckPointCommand::ARG_CHECKPOINT_DIR
= "checkpoint_dir";
3357 CheckPointCommand::CheckPointCommand(
3358 const std::vector
<std::string
>& /*params*/,
3359 const std::map
<std::string
, std::string
>& options
,
3360 const std::vector
<std::string
>& flags
)
3361 : LDBCommand(options
, flags
, false /* is_read_only */,
3362 BuildCmdLineOptions({ARG_CHECKPOINT_DIR
})) {
3363 auto itr
= options
.find(ARG_CHECKPOINT_DIR
);
3364 if (itr
!= options
.end()) {
3365 checkpoint_dir_
= itr
->second
;
3369 void CheckPointCommand::Help(std::string
& ret
) {
3371 ret
.append(CheckPointCommand::Name());
3372 ret
.append(" [--" + ARG_CHECKPOINT_DIR
+ "] ");
3376 void CheckPointCommand::DoCommand() {
3378 assert(GetExecuteState().IsFailed());
3381 Checkpoint
* checkpoint
;
3382 Status status
= Checkpoint::Create(db_
, &checkpoint
);
3383 status
= checkpoint
->CreateCheckpoint(checkpoint_dir_
);
3385 fprintf(stdout
, "OK\n");
3387 exec_state_
= LDBCommandExecuteResult::Failed(status
.ToString());
3391 // ----------------------------------------------------------------------------
3393 const std::string
RepairCommand::ARG_VERBOSE
= "verbose";
3395 RepairCommand::RepairCommand(const std::vector
<std::string
>& /*params*/,
3396 const std::map
<std::string
, std::string
>& options
,
3397 const std::vector
<std::string
>& flags
)
3398 : LDBCommand(options
, flags
, false, BuildCmdLineOptions({ARG_VERBOSE
})) {
3399 verbose_
= IsFlagPresent(flags
, ARG_VERBOSE
);
3402 void RepairCommand::Help(std::string
& ret
) {
3404 ret
.append(RepairCommand::Name());
3405 ret
.append(" [--" + ARG_VERBOSE
+ "]");
3409 void RepairCommand::OverrideBaseOptions() {
3410 LDBCommand::OverrideBaseOptions();
3411 auto level
= verbose_
? InfoLogLevel::INFO_LEVEL
: InfoLogLevel::WARN_LEVEL
;
3412 options_
.info_log
.reset(new StderrLogger(level
));
3415 void RepairCommand::DoCommand() {
3417 Status status
= RepairDB(db_path_
, options_
);
3419 fprintf(stdout
, "OK\n");
3421 exec_state_
= LDBCommandExecuteResult::Failed(status
.ToString());
3425 // ----------------------------------------------------------------------------
3427 const std::string
BackupEngineCommand::ARG_NUM_THREADS
= "num_threads";
3428 const std::string
BackupEngineCommand::ARG_BACKUP_ENV_URI
= "backup_env_uri";
3429 const std::string
BackupEngineCommand::ARG_BACKUP_FS_URI
= "backup_fs_uri";
3430 const std::string
BackupEngineCommand::ARG_BACKUP_DIR
= "backup_dir";
3431 const std::string
BackupEngineCommand::ARG_STDERR_LOG_LEVEL
=
3434 BackupEngineCommand::BackupEngineCommand(
3435 const std::vector
<std::string
>& /*params*/,
3436 const std::map
<std::string
, std::string
>& options
,
3437 const std::vector
<std::string
>& flags
)
3438 : LDBCommand(options
, flags
, false /* is_read_only */,
3439 BuildCmdLineOptions({ARG_BACKUP_ENV_URI
, ARG_BACKUP_FS_URI
,
3440 ARG_BACKUP_DIR
, ARG_NUM_THREADS
,
3441 ARG_STDERR_LOG_LEVEL
})),
3443 auto itr
= options
.find(ARG_NUM_THREADS
);
3444 if (itr
!= options
.end()) {
3445 num_threads_
= std::stoi(itr
->second
);
3447 itr
= options
.find(ARG_BACKUP_ENV_URI
);
3448 if (itr
!= options
.end()) {
3449 backup_env_uri_
= itr
->second
;
3451 itr
= options
.find(ARG_BACKUP_FS_URI
);
3452 if (itr
!= options
.end()) {
3453 backup_fs_uri_
= itr
->second
;
3455 if (!backup_env_uri_
.empty() && !backup_fs_uri_
.empty()) {
3456 exec_state_
= LDBCommandExecuteResult::Failed(
3457 "you may not specity both --" + ARG_BACKUP_ENV_URI
+ " and --" +
3460 itr
= options
.find(ARG_BACKUP_DIR
);
3461 if (itr
== options
.end()) {
3462 exec_state_
= LDBCommandExecuteResult::Failed("--" + ARG_BACKUP_DIR
+
3463 ": missing backup directory");
3465 backup_dir_
= itr
->second
;
3468 itr
= options
.find(ARG_STDERR_LOG_LEVEL
);
3469 if (itr
!= options
.end()) {
3470 int stderr_log_level
= std::stoi(itr
->second
);
3471 if (stderr_log_level
< 0 ||
3472 stderr_log_level
>= InfoLogLevel::NUM_INFO_LOG_LEVELS
) {
3473 exec_state_
= LDBCommandExecuteResult::Failed(
3474 ARG_STDERR_LOG_LEVEL
+ " must be >= 0 and < " +
3475 std::to_string(InfoLogLevel::NUM_INFO_LOG_LEVELS
) + ".");
3478 new StderrLogger(static_cast<InfoLogLevel
>(stderr_log_level
)));
3483 void BackupEngineCommand::Help(const std::string
& name
, std::string
& ret
) {
3486 ret
.append(" [--" + ARG_BACKUP_ENV_URI
+ " | --" + ARG_BACKUP_FS_URI
+ "] ");
3487 ret
.append(" [--" + ARG_BACKUP_DIR
+ "] ");
3488 ret
.append(" [--" + ARG_NUM_THREADS
+ "] ");
3489 ret
.append(" [--" + ARG_STDERR_LOG_LEVEL
+ "=<int (InfoLogLevel)>] ");
3493 // ----------------------------------------------------------------------------
3495 BackupCommand::BackupCommand(const std::vector
<std::string
>& params
,
3496 const std::map
<std::string
, std::string
>& options
,
3497 const std::vector
<std::string
>& flags
)
3498 : BackupEngineCommand(params
, options
, flags
) {}
3500 void BackupCommand::Help(std::string
& ret
) {
3501 BackupEngineCommand::Help(Name(), ret
);
3504 void BackupCommand::DoCommand() {
3505 BackupEngine
* backup_engine
;
3508 assert(GetExecuteState().IsFailed());
3511 fprintf(stdout
, "open db OK\n");
3513 Env
* custom_env
= backup_env_guard_
.get();
3514 if (custom_env
== nullptr) {
3516 Env::CreateFromUri(config_options_
, backup_env_uri_
, backup_fs_uri_
,
3517 &custom_env
, &backup_env_guard_
);
3519 exec_state_
= LDBCommandExecuteResult::Failed(s
.ToString());
3523 assert(custom_env
!= nullptr);
3525 BackupEngineOptions backup_options
=
3526 BackupEngineOptions(backup_dir_
, custom_env
);
3527 backup_options
.info_log
= logger_
.get();
3528 backup_options
.max_background_operations
= num_threads_
;
3529 status
= BackupEngine::Open(options_
.env
, backup_options
, &backup_engine
);
3531 fprintf(stdout
, "open backup engine OK\n");
3533 exec_state_
= LDBCommandExecuteResult::Failed(status
.ToString());
3536 status
= backup_engine
->CreateNewBackup(db_
);
3538 fprintf(stdout
, "create new backup OK\n");
3540 exec_state_
= LDBCommandExecuteResult::Failed(status
.ToString());
3545 // ----------------------------------------------------------------------------
3547 RestoreCommand::RestoreCommand(
3548 const std::vector
<std::string
>& params
,
3549 const std::map
<std::string
, std::string
>& options
,
3550 const std::vector
<std::string
>& flags
)
3551 : BackupEngineCommand(params
, options
, flags
) {}
3553 void RestoreCommand::Help(std::string
& ret
) {
3554 BackupEngineCommand::Help(Name(), ret
);
3557 void RestoreCommand::DoCommand() {
3558 Env
* custom_env
= backup_env_guard_
.get();
3559 if (custom_env
== nullptr) {
3561 Env::CreateFromUri(config_options_
, backup_env_uri_
, backup_fs_uri_
,
3562 &custom_env
, &backup_env_guard_
);
3564 exec_state_
= LDBCommandExecuteResult::Failed(s
.ToString());
3568 assert(custom_env
!= nullptr);
3570 std::unique_ptr
<BackupEngineReadOnly
> restore_engine
;
3573 BackupEngineOptions
opts(backup_dir_
, custom_env
);
3574 opts
.info_log
= logger_
.get();
3575 opts
.max_background_operations
= num_threads_
;
3576 BackupEngineReadOnly
* raw_restore_engine_ptr
;
3578 BackupEngineReadOnly::Open(options_
.env
, opts
, &raw_restore_engine_ptr
);
3580 restore_engine
.reset(raw_restore_engine_ptr
);
3584 fprintf(stdout
, "open restore engine OK\n");
3585 status
= restore_engine
->RestoreDBFromLatestBackup(db_path_
, db_path_
);
3588 fprintf(stdout
, "restore from backup OK\n");
3590 exec_state_
= LDBCommandExecuteResult::Failed(status
.ToString());
3594 // ----------------------------------------------------------------------------
3598 void DumpSstFile(Options options
, std::string filename
, bool output_hex
,
3599 bool show_properties
, bool decode_blob_index
,
3600 std::string from_key
, std::string to_key
) {
3601 if (filename
.length() <= 4 ||
3602 filename
.rfind(".sst") != filename
.length() - 4) {
3603 std::cout
<< "Invalid sst file name." << std::endl
;
3607 ROCKSDB_NAMESPACE::SstFileDumper
dumper(
3608 options
, filename
, Temperature::kUnknown
,
3609 2 * 1024 * 1024 /* readahead_size */,
3610 /* verify_checksum */ false, output_hex
, decode_blob_index
);
3611 Status st
= dumper
.ReadSequential(true, std::numeric_limits
<uint64_t>::max(),
3612 !from_key
.empty(), from_key
,
3613 !to_key
.empty(), to_key
);
3615 std::cerr
<< "Error in reading SST file " << filename
<< st
.ToString()
3620 if (show_properties
) {
3621 const ROCKSDB_NAMESPACE::TableProperties
* table_properties
;
3623 std::shared_ptr
<const ROCKSDB_NAMESPACE::TableProperties
>
3624 table_properties_from_reader
;
3625 st
= dumper
.ReadTableProperties(&table_properties_from_reader
);
3627 std::cerr
<< filename
<< ": " << st
.ToString()
3628 << ". Try to use initial table properties" << std::endl
;
3629 table_properties
= dumper
.GetInitTableProperties();
3631 table_properties
= table_properties_from_reader
.get();
3633 if (table_properties
!= nullptr) {
3634 std::cout
<< std::endl
<< "Table Properties:" << std::endl
;
3635 std::cout
<< table_properties
->ToString("\n") << std::endl
;
3640 void DumpBlobFile(const std::string
& filename
, bool is_key_hex
,
3641 bool is_value_hex
, bool dump_uncompressed_blobs
) {
3642 using ROCKSDB_NAMESPACE::blob_db::BlobDumpTool
;
3644 BlobDumpTool::DisplayType blob_type
= is_value_hex
3645 ? BlobDumpTool::DisplayType::kHex
3646 : BlobDumpTool::DisplayType::kRaw
;
3647 BlobDumpTool::DisplayType show_uncompressed_blob
=
3648 dump_uncompressed_blobs
? blob_type
: BlobDumpTool::DisplayType::kNone
;
3649 BlobDumpTool::DisplayType show_blob
=
3650 dump_uncompressed_blobs
? BlobDumpTool::DisplayType::kNone
: blob_type
;
3652 BlobDumpTool::DisplayType show_key
= is_key_hex
3653 ? BlobDumpTool::DisplayType::kHex
3654 : BlobDumpTool::DisplayType::kRaw
;
3655 Status s
= tool
.Run(filename
, show_key
, show_blob
, show_uncompressed_blob
,
3656 /* show_summary */ true);
3658 fprintf(stderr
, "Failed: %s\n", s
.ToString().c_str());
3663 DBFileDumperCommand::DBFileDumperCommand(
3664 const std::vector
<std::string
>& /*params*/,
3665 const std::map
<std::string
, std::string
>& options
,
3666 const std::vector
<std::string
>& flags
)
3667 : LDBCommand(options
, flags
, true,
3668 BuildCmdLineOptions(
3669 {ARG_DECODE_BLOB_INDEX
, ARG_DUMP_UNCOMPRESSED_BLOBS
})),
3670 decode_blob_index_(IsFlagPresent(flags
, ARG_DECODE_BLOB_INDEX
)),
3671 dump_uncompressed_blobs_(
3672 IsFlagPresent(flags
, ARG_DUMP_UNCOMPRESSED_BLOBS
)) {}
3674 void DBFileDumperCommand::Help(std::string
& ret
) {
3676 ret
.append(DBFileDumperCommand::Name());
3677 ret
.append(" [--" + ARG_DECODE_BLOB_INDEX
+ "] ");
3678 ret
.append(" [--" + ARG_DUMP_UNCOMPRESSED_BLOBS
+ "] ");
3682 void DBFileDumperCommand::DoCommand() {
3684 assert(GetExecuteState().IsFailed());
3689 // TODO: Use --hex, --key_hex, --value_hex flags consistently for
3690 // dumping manifest file, sst files and blob files.
3691 std::cout
<< "Manifest File" << std::endl
;
3692 std::cout
<< "==============================" << std::endl
;
3693 std::string manifest_filename
;
3694 s
= ReadFileToString(db_
->GetEnv(), CurrentFileName(db_
->GetName()),
3695 &manifest_filename
);
3696 if (!s
.ok() || manifest_filename
.empty() ||
3697 manifest_filename
.back() != '\n') {
3698 std::cerr
<< "Error when reading CURRENT file "
3699 << CurrentFileName(db_
->GetName()) << std::endl
;
3701 // remove the trailing '\n'
3702 manifest_filename
.resize(manifest_filename
.size() - 1);
3703 std::string manifest_filepath
= db_
->GetName() + "/" + manifest_filename
;
3704 // Correct concatenation of filepath and filename:
3705 // Check that there is no double slashes (or more!) when concatenation
3707 manifest_filepath
= NormalizePath(manifest_filepath
);
3709 std::cout
<< manifest_filepath
<< std::endl
;
3710 DumpManifestFile(options_
, manifest_filepath
, false, false, false);
3711 std::cout
<< std::endl
;
3713 std::vector
<ColumnFamilyMetaData
> column_families
;
3714 db_
->GetAllColumnFamilyMetaData(&column_families
);
3715 for (const auto& column_family
: column_families
) {
3716 std::cout
<< "Column family name: " << column_family
.name
<< std::endl
;
3717 std::cout
<< "==============================" << std::endl
;
3718 std::cout
<< std::endl
;
3719 std::cout
<< "SST Files" << std::endl
;
3720 std::cout
<< "==============================" << std::endl
;
3721 for (const LevelMetaData
& level
: column_family
.levels
) {
3722 for (const SstFileMetaData
& sst_file
: level
.files
) {
3723 std::string filename
= sst_file
.db_path
+ "/" + sst_file
.name
;
3724 // Correct concatenation of filepath and filename:
3725 // Check that there is no double slashes (or more!) when concatenation
3727 filename
= NormalizePath(filename
);
3728 std::cout
<< filename
<< " level:" << level
.level
<< std::endl
;
3729 std::cout
<< "------------------------------" << std::endl
;
3730 DumpSstFile(options_
, filename
, false, true, decode_blob_index_
);
3731 std::cout
<< std::endl
;
3734 std::cout
<< "Blob Files" << std::endl
;
3735 std::cout
<< "==============================" << std::endl
;
3736 for (const BlobMetaData
& blob_file
: column_family
.blob_files
) {
3737 std::string filename
=
3738 blob_file
.blob_file_path
+ "/" + blob_file
.blob_file_name
;
3739 // Correct concatenation of filepath and filename:
3740 // Check that there is no double slashes (or more!) when concatenation
3742 filename
= NormalizePath(filename
);
3743 std::cout
<< filename
<< std::endl
;
3744 std::cout
<< "------------------------------" << std::endl
;
3745 DumpBlobFile(filename
, /* is_key_hex */ false, /* is_value_hex */ false,
3746 dump_uncompressed_blobs_
);
3747 std::cout
<< std::endl
;
3750 std::cout
<< std::endl
;
3752 std::cout
<< "Write Ahead Log Files" << std::endl
;
3753 std::cout
<< "==============================" << std::endl
;
3754 ROCKSDB_NAMESPACE::VectorLogPtr wal_files
;
3755 s
= db_
->GetSortedWalFiles(wal_files
);
3757 std::cerr
<< "Error when getting WAL files" << std::endl
;
3759 std::string wal_dir
;
3760 if (options_
.wal_dir
.empty()) {
3761 wal_dir
= db_
->GetName();
3763 wal_dir
= NormalizePath(options_
.wal_dir
+ "/");
3765 for (auto& wal
: wal_files
) {
3766 // TODO(qyang): option.wal_dir should be passed into ldb command
3767 std::string filename
= wal_dir
+ wal
->PathName();
3768 std::cout
<< filename
<< std::endl
;
3769 // TODO(myabandeh): allow configuring is_write_commited
3770 DumpWalFile(options_
, filename
, true, true, true /* is_write_commited */,
3776 const std::string
DBLiveFilesMetadataDumperCommand::ARG_SORT_BY_FILENAME
=
3779 DBLiveFilesMetadataDumperCommand::DBLiveFilesMetadataDumperCommand(
3780 const std::vector
<std::string
>& /*params*/,
3781 const std::map
<std::string
, std::string
>& options
,
3782 const std::vector
<std::string
>& flags
)
3783 : LDBCommand(options
, flags
, true,
3784 BuildCmdLineOptions({ARG_SORT_BY_FILENAME
})) {
3785 sort_by_filename_
= IsFlagPresent(flags
, ARG_SORT_BY_FILENAME
);
3788 void DBLiveFilesMetadataDumperCommand::Help(std::string
& ret
) {
3790 ret
.append(DBLiveFilesMetadataDumperCommand::Name());
3791 ret
.append(" [--" + ARG_SORT_BY_FILENAME
+ "] ");
3795 void DBLiveFilesMetadataDumperCommand::DoCommand() {
3797 assert(GetExecuteState().IsFailed());
3802 std::vector
<ColumnFamilyMetaData
> metadata
;
3803 db_
->GetAllColumnFamilyMetaData(&metadata
);
3804 if (sort_by_filename_
) {
3805 std::cout
<< "Live SST and Blob Files:" << std::endl
;
3806 // tuple of <file path, level, column family name>
3807 std::vector
<std::tuple
<std::string
, int, std::string
>> all_files
;
3809 for (const auto& column_metadata
: metadata
) {
3811 const auto& levels
= column_metadata
.levels
;
3812 const std::string
& cf
= column_metadata
.name
;
3813 for (const auto& level_metadata
: levels
) {
3814 // Iterate SST files
3815 const auto& sst_files
= level_metadata
.files
;
3816 int level
= level_metadata
.level
;
3817 for (const auto& sst_metadata
: sst_files
) {
3818 // The SstFileMetaData.name always starts with "/",
3819 // however SstFileMetaData.db_path is the string provided by
3820 // the user as an input. Therefore we check if we can
3821 // concantenate the two strings directly or if we need to
3822 // drop a possible extra "/" at the end of SstFileMetaData.db_path.
3823 std::string filename
=
3824 NormalizePath(sst_metadata
.db_path
+ "/" + sst_metadata
.name
);
3825 all_files
.emplace_back(filename
, level
, cf
);
3826 } // End of for-loop over sst files
3827 } // End of for-loop over levels
3829 const auto& blob_files
= column_metadata
.blob_files
;
3830 for (const auto& blob_metadata
: blob_files
) {
3831 // The BlobMetaData.blob_file_name always starts with "/",
3832 // however BlobMetaData.blob_file_path is the string provided by
3833 // the user as an input. Therefore we check if we can
3834 // concantenate the two strings directly or if we need to
3835 // drop a possible extra "/" at the end of BlobMetaData.blob_file_path.
3836 std::string filename
= NormalizePath(
3837 blob_metadata
.blob_file_path
+ "/" + blob_metadata
.blob_file_name
);
3838 // Level for blob files is encoded as -1
3839 all_files
.emplace_back(filename
, -1, cf
);
3840 } // End of for-loop over blob files
3841 } // End of for-loop over column metadata
3843 // Sort by filename (i.e. first entry in tuple)
3844 std::sort(all_files
.begin(), all_files
.end());
3846 for (const auto& item
: all_files
) {
3847 const std::string
& filename
= std::get
<0>(item
);
3848 int level
= std::get
<1>(item
);
3849 const std::string
& cf
= std::get
<2>(item
);
3850 if (level
== -1) { // Blob File
3851 std::cout
<< filename
<< ", column family '" << cf
<< "'" << std::endl
;
3852 } else { // SST file
3853 std::cout
<< filename
<< " : level " << level
<< ", column family '"
3854 << cf
<< "'" << std::endl
;
3858 for (const auto& column_metadata
: metadata
) {
3859 std::cout
<< "===== Column Family: " << column_metadata
.name
3860 << " =====" << std::endl
;
3862 std::cout
<< "Live SST Files:" << std::endl
;
3864 const auto& levels
= column_metadata
.levels
;
3865 for (const auto& level_metadata
: levels
) {
3866 std::cout
<< "---------- level " << level_metadata
.level
3867 << " ----------" << std::endl
;
3868 // Iterate SST files
3869 const auto& sst_files
= level_metadata
.files
;
3870 for (const auto& sst_metadata
: sst_files
) {
3871 // The SstFileMetaData.name always starts with "/",
3872 // however SstFileMetaData.db_path is the string provided by
3873 // the user as an input. Therefore we check if we can
3874 // concantenate the two strings directly or if we need to
3875 // drop a possible extra "/" at the end of SstFileMetaData.db_path.
3876 std::string filename
=
3877 NormalizePath(sst_metadata
.db_path
+ "/" + sst_metadata
.name
);
3878 std::cout
<< filename
<< std::endl
;
3879 } // End of for-loop over sst files
3880 } // End of for-loop over levels
3882 std::cout
<< "Live Blob Files:" << std::endl
;
3883 const auto& blob_files
= column_metadata
.blob_files
;
3884 for (const auto& blob_metadata
: blob_files
) {
3885 // The BlobMetaData.blob_file_name always starts with "/",
3886 // however BlobMetaData.blob_file_path is the string provided by
3887 // the user as an input. Therefore we check if we can
3888 // concantenate the two strings directly or if we need to
3889 // drop a possible extra "/" at the end of BlobMetaData.blob_file_path.
3890 std::string filename
= NormalizePath(
3891 blob_metadata
.blob_file_path
+ "/" + blob_metadata
.blob_file_name
);
3892 std::cout
<< filename
<< std::endl
;
3893 } // End of for-loop over blob files
3894 } // End of for-loop over column metadata
3895 } // End of else ("not sort_by_filename")
3896 std::cout
<< "------------------------------" << std::endl
;
3899 void WriteExternalSstFilesCommand::Help(std::string
& ret
) {
3901 ret
.append(WriteExternalSstFilesCommand::Name());
3902 ret
.append(" <output_sst_path>");
3906 WriteExternalSstFilesCommand::WriteExternalSstFilesCommand(
3907 const std::vector
<std::string
>& params
,
3908 const std::map
<std::string
, std::string
>& options
,
3909 const std::vector
<std::string
>& flags
)
3911 options
, flags
, false /* is_read_only */,
3912 BuildCmdLineOptions({ARG_HEX
, ARG_KEY_HEX
, ARG_VALUE_HEX
, ARG_FROM
,
3913 ARG_TO
, ARG_CREATE_IF_MISSING
})) {
3914 create_if_missing_
=
3915 IsFlagPresent(flags
, ARG_CREATE_IF_MISSING
) ||
3916 ParseBooleanOption(options
, ARG_CREATE_IF_MISSING
, false);
3917 if (params
.size() != 1) {
3918 exec_state_
= LDBCommandExecuteResult::Failed(
3919 "output SST file path must be specified");
3921 output_sst_path_
= params
.at(0);
3925 void WriteExternalSstFilesCommand::DoCommand() {
3927 assert(GetExecuteState().IsFailed());
3930 ColumnFamilyHandle
* cfh
= GetCfHandle();
3931 SstFileWriter
sst_file_writer(EnvOptions(), db_
->GetOptions(), cfh
);
3932 Status status
= sst_file_writer
.Open(output_sst_path_
);
3934 exec_state_
= LDBCommandExecuteResult::Failed("failed to open SST file: " +
3941 std::ifstream
ifs_stdin("/dev/stdin");
3942 std::istream
* istream_p
= ifs_stdin
.is_open() ? &ifs_stdin
: &std::cin
;
3943 while (getline(*istream_p
, line
, '\n')) {
3946 if (ParseKeyValue(line
, &key
, &value
, is_key_hex_
, is_value_hex_
)) {
3947 status
= sst_file_writer
.Put(key
, value
);
3949 exec_state_
= LDBCommandExecuteResult::Failed(
3950 "failed to write record to file: " + status
.ToString());
3953 } else if (0 == line
.find("Keys in range:")) {
3955 } else if (0 == line
.find("Created bg thread 0x")) {
3962 status
= sst_file_writer
.Finish();
3964 exec_state_
= LDBCommandExecuteResult::Failed(
3965 "Failed to finish writing to file: " + status
.ToString());
3969 if (bad_lines
> 0) {
3970 fprintf(stderr
, "Warning: %d bad lines ignored.\n", bad_lines
);
3972 exec_state_
= LDBCommandExecuteResult::Succeed(
3973 "external SST file written to " + output_sst_path_
);
3976 void WriteExternalSstFilesCommand::OverrideBaseOptions() {
3977 LDBCommand::OverrideBaseOptions();
3978 options_
.create_if_missing
= create_if_missing_
;
3981 const std::string
IngestExternalSstFilesCommand::ARG_MOVE_FILES
= "move_files";
3982 const std::string
IngestExternalSstFilesCommand::ARG_SNAPSHOT_CONSISTENCY
=
3983 "snapshot_consistency";
3984 const std::string
IngestExternalSstFilesCommand::ARG_ALLOW_GLOBAL_SEQNO
=
3985 "allow_global_seqno";
3986 const std::string
IngestExternalSstFilesCommand::ARG_ALLOW_BLOCKING_FLUSH
=
3987 "allow_blocking_flush";
3988 const std::string
IngestExternalSstFilesCommand::ARG_INGEST_BEHIND
=
3990 const std::string
IngestExternalSstFilesCommand::ARG_WRITE_GLOBAL_SEQNO
=
3991 "write_global_seqno";
3993 void IngestExternalSstFilesCommand::Help(std::string
& ret
) {
3995 ret
.append(IngestExternalSstFilesCommand::Name());
3996 ret
.append(" <input_sst_path>");
3997 ret
.append(" [--" + ARG_MOVE_FILES
+ "] ");
3998 ret
.append(" [--" + ARG_SNAPSHOT_CONSISTENCY
+ "] ");
3999 ret
.append(" [--" + ARG_ALLOW_GLOBAL_SEQNO
+ "] ");
4000 ret
.append(" [--" + ARG_ALLOW_BLOCKING_FLUSH
+ "] ");
4001 ret
.append(" [--" + ARG_INGEST_BEHIND
+ "] ");
4002 ret
.append(" [--" + ARG_WRITE_GLOBAL_SEQNO
+ "] ");
4006 IngestExternalSstFilesCommand::IngestExternalSstFilesCommand(
4007 const std::vector
<std::string
>& params
,
4008 const std::map
<std::string
, std::string
>& options
,
4009 const std::vector
<std::string
>& flags
)
4011 options
, flags
, false /* is_read_only */,
4012 BuildCmdLineOptions({ARG_MOVE_FILES
, ARG_SNAPSHOT_CONSISTENCY
,
4013 ARG_ALLOW_GLOBAL_SEQNO
, ARG_CREATE_IF_MISSING
,
4014 ARG_ALLOW_BLOCKING_FLUSH
, ARG_INGEST_BEHIND
,
4015 ARG_WRITE_GLOBAL_SEQNO
})),
4017 snapshot_consistency_(true),
4018 allow_global_seqno_(true),
4019 allow_blocking_flush_(true),
4020 ingest_behind_(false),
4021 write_global_seqno_(true) {
4022 create_if_missing_
=
4023 IsFlagPresent(flags
, ARG_CREATE_IF_MISSING
) ||
4024 ParseBooleanOption(options
, ARG_CREATE_IF_MISSING
, false);
4025 move_files_
= IsFlagPresent(flags
, ARG_MOVE_FILES
) ||
4026 ParseBooleanOption(options
, ARG_MOVE_FILES
, false);
4027 snapshot_consistency_
=
4028 IsFlagPresent(flags
, ARG_SNAPSHOT_CONSISTENCY
) ||
4029 ParseBooleanOption(options
, ARG_SNAPSHOT_CONSISTENCY
, true);
4030 allow_global_seqno_
=
4031 IsFlagPresent(flags
, ARG_ALLOW_GLOBAL_SEQNO
) ||
4032 ParseBooleanOption(options
, ARG_ALLOW_GLOBAL_SEQNO
, true);
4033 allow_blocking_flush_
=
4034 IsFlagPresent(flags
, ARG_ALLOW_BLOCKING_FLUSH
) ||
4035 ParseBooleanOption(options
, ARG_ALLOW_BLOCKING_FLUSH
, true);
4036 ingest_behind_
= IsFlagPresent(flags
, ARG_INGEST_BEHIND
) ||
4037 ParseBooleanOption(options
, ARG_INGEST_BEHIND
, false);
4038 write_global_seqno_
=
4039 IsFlagPresent(flags
, ARG_WRITE_GLOBAL_SEQNO
) ||
4040 ParseBooleanOption(options
, ARG_WRITE_GLOBAL_SEQNO
, true);
4042 if (allow_global_seqno_
) {
4043 if (!write_global_seqno_
) {
4045 "Warning: not writing global_seqno to the ingested SST can\n"
4046 "prevent older versions of RocksDB from being able to open it\n");
4049 if (write_global_seqno_
) {
4050 exec_state_
= LDBCommandExecuteResult::Failed(
4051 "ldb cannot write global_seqno to the ingested SST when global_seqno "
4056 if (params
.size() != 1) {
4058 LDBCommandExecuteResult::Failed("input SST path must be specified");
4060 input_sst_path_
= params
.at(0);
4064 void IngestExternalSstFilesCommand::DoCommand() {
4066 assert(GetExecuteState().IsFailed());
4069 if (GetExecuteState().IsFailed()) {
4072 ColumnFamilyHandle
* cfh
= GetCfHandle();
4073 IngestExternalFileOptions ifo
;
4074 ifo
.move_files
= move_files_
;
4075 ifo
.snapshot_consistency
= snapshot_consistency_
;
4076 ifo
.allow_global_seqno
= allow_global_seqno_
;
4077 ifo
.allow_blocking_flush
= allow_blocking_flush_
;
4078 ifo
.ingest_behind
= ingest_behind_
;
4079 ifo
.write_global_seqno
= write_global_seqno_
;
4080 Status status
= db_
->IngestExternalFile(cfh
, {input_sst_path_
}, ifo
);
4082 exec_state_
= LDBCommandExecuteResult::Failed(
4083 "failed to ingest external SST: " + status
.ToString());
4086 LDBCommandExecuteResult::Succeed("external SST files ingested");
4090 void IngestExternalSstFilesCommand::OverrideBaseOptions() {
4091 LDBCommand::OverrideBaseOptions();
4092 options_
.create_if_missing
= create_if_missing_
;
4095 ListFileRangeDeletesCommand::ListFileRangeDeletesCommand(
4096 const std::map
<std::string
, std::string
>& options
,
4097 const std::vector
<std::string
>& flags
)
4098 : LDBCommand(options
, flags
, true, BuildCmdLineOptions({ARG_MAX_KEYS
})) {
4099 auto itr
= options
.find(ARG_MAX_KEYS
);
4100 if (itr
!= options
.end()) {
4103 max_keys_
= strtol(itr
->second
.c_str(), 0, 10);
4105 max_keys_
= std::stoi(itr
->second
);
4107 } catch (const std::invalid_argument
&) {
4108 exec_state_
= LDBCommandExecuteResult::Failed(ARG_MAX_KEYS
+
4109 " has an invalid value");
4110 } catch (const std::out_of_range
&) {
4111 exec_state_
= LDBCommandExecuteResult::Failed(
4112 ARG_MAX_KEYS
+ " has a value out-of-range");
4117 void ListFileRangeDeletesCommand::Help(std::string
& ret
) {
4119 ret
.append(ListFileRangeDeletesCommand::Name());
4120 ret
.append(" [--" + ARG_MAX_KEYS
+ "=<N>]");
4121 ret
.append(" : print tombstones in SST files.\n");
4124 void ListFileRangeDeletesCommand::DoCommand() {
4126 assert(GetExecuteState().IsFailed());
4130 DBImpl
* db_impl
= static_cast_with_check
<DBImpl
>(db_
->GetRootDB());
4132 std::string out_str
;
4135 db_impl
->TablesRangeTombstoneSummary(GetCfHandle(), max_keys_
, &out_str
);
4137 TEST_SYNC_POINT_CALLBACK(
4138 "ListFileRangeDeletesCommand::DoCommand:BeforePrint", &out_str
);
4139 fprintf(stdout
, "%s\n", out_str
.c_str());
4143 void UnsafeRemoveSstFileCommand::Help(std::string
& ret
) {
4145 ret
.append(UnsafeRemoveSstFileCommand::Name());
4146 ret
.append(" <SST file number>");
4148 ret
.append(" MUST NOT be used on a live DB.");
4152 UnsafeRemoveSstFileCommand::UnsafeRemoveSstFileCommand(
4153 const std::vector
<std::string
>& params
,
4154 const std::map
<std::string
, std::string
>& options
,
4155 const std::vector
<std::string
>& flags
)
4156 : LDBCommand(options
, flags
, false /* is_read_only */,
4157 BuildCmdLineOptions({})) {
4158 if (params
.size() != 1) {
4160 LDBCommandExecuteResult::Failed("SST file number must be specified");
4162 char* endptr
= nullptr;
4163 sst_file_number_
= strtoull(params
.at(0).c_str(), &endptr
, 10 /* base */);
4164 if (endptr
== nullptr || *endptr
!= '\0') {
4165 exec_state_
= LDBCommandExecuteResult::Failed(
4166 "Failed to parse SST file number " + params
.at(0));
4171 void UnsafeRemoveSstFileCommand::DoCommand() {
4174 OfflineManifestWriter
w(options_
, db_path_
);
4175 if (column_families_
.empty()) {
4176 column_families_
.emplace_back(kDefaultColumnFamilyName
, options_
);
4178 Status s
= w
.Recover(column_families_
);
4180 ColumnFamilyData
* cfd
= nullptr;
4183 FileMetaData
* metadata
= nullptr;
4184 s
= w
.Versions().GetMetadataForFile(sst_file_number_
, &level
, &metadata
,
4190 edit
.SetColumnFamily(cfd
->GetID());
4191 edit
.DeleteFile(level
, sst_file_number_
);
4192 std::unique_ptr
<FSDirectory
> db_dir
;
4193 s
= options_
.env
->GetFileSystem()->NewDirectory(db_path_
, IOOptions(),
4196 s
= w
.LogAndApply(cfd
, &edit
, db_dir
.get());
4201 exec_state_
= LDBCommandExecuteResult::Failed(
4202 "failed to unsafely remove SST file: " + s
.ToString());
4204 exec_state_
= LDBCommandExecuteResult::Succeed("unsafely removed SST file");
4208 const std::string
UpdateManifestCommand::ARG_VERBOSE
= "verbose";
4209 const std::string
UpdateManifestCommand::ARG_UPDATE_TEMPERATURES
=
4210 "update_temperatures";
4212 void UpdateManifestCommand::Help(std::string
& ret
) {
4214 ret
.append(UpdateManifestCommand::Name());
4215 ret
.append(" [--update_temperatures]");
4217 ret
.append(" MUST NOT be used on a live DB.");
4221 UpdateManifestCommand::UpdateManifestCommand(
4222 const std::vector
<std::string
>& /*params*/,
4223 const std::map
<std::string
, std::string
>& options
,
4224 const std::vector
<std::string
>& flags
)
4225 : LDBCommand(options
, flags
, false /* is_read_only */,
4226 BuildCmdLineOptions({ARG_VERBOSE
, ARG_UPDATE_TEMPERATURES
})) {
4227 verbose_
= IsFlagPresent(flags
, ARG_VERBOSE
) ||
4228 ParseBooleanOption(options
, ARG_VERBOSE
, false);
4229 update_temperatures_
=
4230 IsFlagPresent(flags
, ARG_UPDATE_TEMPERATURES
) ||
4231 ParseBooleanOption(options
, ARG_UPDATE_TEMPERATURES
, false);
4233 if (!update_temperatures_
) {
4234 exec_state_
= LDBCommandExecuteResult::Failed(
4235 "No action like --update_temperatures specified for update_manifest");
4239 void UpdateManifestCommand::DoCommand() {
4242 auto level
= verbose_
? InfoLogLevel::INFO_LEVEL
: InfoLogLevel::WARN_LEVEL
;
4243 options_
.info_log
.reset(new StderrLogger(level
));
4245 experimental::UpdateManifestForFilesStateOptions opts
;
4246 opts
.update_temperatures
= update_temperatures_
;
4247 if (column_families_
.empty()) {
4248 column_families_
.emplace_back(kDefaultColumnFamilyName
, options_
);
4250 Status s
= experimental::UpdateManifestForFilesState(options_
, db_path_
,
4254 exec_state_
= LDBCommandExecuteResult::Failed(
4255 "failed to update manifest: " + s
.ToString());
4258 LDBCommandExecuteResult::Succeed("Manifest updates successful");
4262 } // namespace ROCKSDB_NAMESPACE
4263 #endif // ROCKSDB_LITE