[ceph.git] / ceph / src / rocksdb / tools / dump / db_dump_tool.cc

//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
//  This source code is licensed under both the GPLv2 (found in the
//  COPYING file in the root directory) and Apache 2.0 License
//  (found in the LICENSE.Apache file in the root directory).

#ifndef ROCKSDB_LITE

#include "rocksdb/db_dump_tool.h"

#include <cinttypes>
#include <iostream>

#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "util/coding.h"

namespace ROCKSDB_NAMESPACE {

bool DbDumpTool::Run(const DumpOptions& dump_options,
                     ROCKSDB_NAMESPACE::Options options) {
  ROCKSDB_NAMESPACE::DB* dbptr;
  ROCKSDB_NAMESPACE::Status status;
  std::unique_ptr<ROCKSDB_NAMESPACE::WritableFile> dumpfile;
  char hostname[1024];
  int64_t timesec = 0;
  std::string abspath;
  char json[4096];

  static const char* magicstr = "ROCKDUMP";
  static const char versionstr[8] = {0, 0, 0, 0, 0, 0, 0, 1};

  ROCKSDB_NAMESPACE::Env* env = ROCKSDB_NAMESPACE::Env::Default();

  // Open the database
  options.create_if_missing = false;
  status = ROCKSDB_NAMESPACE::DB::OpenForReadOnly(options, dump_options.db_path,
                                                  &dbptr);
  if (!status.ok()) {
    std::cerr << "Unable to open database '" << dump_options.db_path
              << "' for reading: " << status.ToString() << std::endl;
    return false;
  }

  const std::unique_ptr<ROCKSDB_NAMESPACE::DB> db(dbptr);

  status = env->NewWritableFile(dump_options.dump_location, &dumpfile,
                                ROCKSDB_NAMESPACE::EnvOptions());
  if (!status.ok()) {
    std::cerr << "Unable to open dump file '" << dump_options.dump_location
              << "' for writing: " << status.ToString() << std::endl;
    return false;
  }

  ROCKSDB_NAMESPACE::Slice magicslice(magicstr, 8);
  status = dumpfile->Append(magicslice);
  if (!status.ok()) {
    std::cerr << "Append failed: " << status.ToString() << std::endl;
    return false;
  }

  ROCKSDB_NAMESPACE::Slice versionslice(versionstr, 8);
  status = dumpfile->Append(versionslice);
  if (!status.ok()) {
    std::cerr << "Append failed: " << status.ToString() << std::endl;
    return false;
  }

  if (dump_options.anonymous) {
    snprintf(json, sizeof(json), "{}");
  } else {
    status = env->GetHostName(hostname, sizeof(hostname));
    status = env->GetCurrentTime(&timesec);
    status = env->GetAbsolutePath(dump_options.db_path, &abspath);
    snprintf(json, sizeof(json),
             "{ \"database-path\": \"%s\", \"hostname\": \"%s\", "
             "\"creation-time\": %" PRIi64 " }",
             abspath.c_str(), hostname, timesec);
  }

  ROCKSDB_NAMESPACE::Slice infoslice(json, strlen(json));
  char infosize[4];
  ROCKSDB_NAMESPACE::EncodeFixed32(infosize, (uint32_t)infoslice.size());
  ROCKSDB_NAMESPACE::Slice infosizeslice(infosize, 4);
  status = dumpfile->Append(infosizeslice);
  if (!status.ok()) {
    std::cerr << "Append failed: " << status.ToString() << std::endl;
    return false;
  }
  status = dumpfile->Append(infoslice);
  if (!status.ok()) {
    std::cerr << "Append failed: " << status.ToString() << std::endl;
    return false;
  }

  const std::unique_ptr<ROCKSDB_NAMESPACE::Iterator> it(
      db->NewIterator(ROCKSDB_NAMESPACE::ReadOptions()));
  for (it->SeekToFirst(); it->Valid(); it->Next()) {
    char keysize[4];
    ROCKSDB_NAMESPACE::EncodeFixed32(keysize, (uint32_t)it->key().size());
    ROCKSDB_NAMESPACE::Slice keysizeslice(keysize, 4);
    status = dumpfile->Append(keysizeslice);
    if (!status.ok()) {
      std::cerr << "Append failed: " << status.ToString() << std::endl;
      return false;
    }
    status = dumpfile->Append(it->key());
    if (!status.ok()) {
      std::cerr << "Append failed: " << status.ToString() << std::endl;
      return false;
    }

    char valsize[4];
    ROCKSDB_NAMESPACE::EncodeFixed32(valsize, (uint32_t)it->value().size());
    ROCKSDB_NAMESPACE::Slice valsizeslice(valsize, 4);
    status = dumpfile->Append(valsizeslice);
    if (!status.ok()) {
      std::cerr << "Append failed: " << status.ToString() << std::endl;
      return false;
    }
    status = dumpfile->Append(it->value());
    if (!status.ok()) {
      std::cerr << "Append failed: " << status.ToString() << std::endl;
      return false;
    }
  }
  if (!it->status().ok()) {
    std::cerr << "Database iteration failed: " << status.ToString()
              << std::endl;
    return false;
  }
  return true;
}

bool DbUndumpTool::Run(const UndumpOptions& undump_options,
                       ROCKSDB_NAMESPACE::Options options) {
  ROCKSDB_NAMESPACE::DB* dbptr;
  ROCKSDB_NAMESPACE::Status status;
  ROCKSDB_NAMESPACE::Env* env;
  std::unique_ptr<ROCKSDB_NAMESPACE::SequentialFile> dumpfile;
  ROCKSDB_NAMESPACE::Slice slice;
  char scratch8[8];

  static const char* magicstr = "ROCKDUMP";
  static const char versionstr[8] = {0, 0, 0, 0, 0, 0, 0, 1};

  env = ROCKSDB_NAMESPACE::Env::Default();

  status = env->NewSequentialFile(undump_options.dump_location, &dumpfile,
                                  ROCKSDB_NAMESPACE::EnvOptions());
  if (!status.ok()) {
    std::cerr << "Unable to open dump file '" << undump_options.dump_location
              << "' for reading: " << status.ToString() << std::endl;
    return false;
  }

  status = dumpfile->Read(8, &slice, scratch8);
  if (!status.ok() || slice.size() != 8 ||
      memcmp(slice.data(), magicstr, 8) != 0) {
    std::cerr << "File '" << undump_options.dump_location
              << "' is not a recognizable dump file." << std::endl;
    return false;
  }

  status = dumpfile->Read(8, &slice, scratch8);
  if (!status.ok() || slice.size() != 8 ||
      memcmp(slice.data(), versionstr, 8) != 0) {
    std::cerr << "File '" << undump_options.dump_location
              << "' version not recognized." << std::endl;
    return false;
  }

  status = dumpfile->Read(4, &slice, scratch8);
  if (!status.ok() || slice.size() != 4) {
    std::cerr << "Unable to read info blob size." << std::endl;
    return false;
  }
  uint32_t infosize = ROCKSDB_NAMESPACE::DecodeFixed32(slice.data());
  status = dumpfile->Skip(infosize);
  if (!status.ok()) {
    std::cerr << "Unable to skip info blob: " << status.ToString() << std::endl;
    return false;
  }

  options.create_if_missing = true;
  status = ROCKSDB_NAMESPACE::DB::Open(options, undump_options.db_path, &dbptr);
  if (!status.ok()) {
    std::cerr << "Unable to open database '" << undump_options.db_path
              << "' for writing: " << status.ToString() << std::endl;
    return false;
  }

  const std::unique_ptr<ROCKSDB_NAMESPACE::DB> db(dbptr);

  uint32_t last_keysize = 64;
  size_t last_valsize = 1 << 20;
  std::unique_ptr<char[]> keyscratch(new char[last_keysize]);
  std::unique_ptr<char[]> valscratch(new char[last_valsize]);

  while (1) {
    uint32_t keysize, valsize;
    ROCKSDB_NAMESPACE::Slice keyslice;
    ROCKSDB_NAMESPACE::Slice valslice;

    status = dumpfile->Read(4, &slice, scratch8);
    if (!status.ok() || slice.size() != 4) break;
    keysize = ROCKSDB_NAMESPACE::DecodeFixed32(slice.data());
    if (keysize > last_keysize) {
      while (keysize > last_keysize) last_keysize *= 2;
      keyscratch = std::unique_ptr<char[]>(new char[last_keysize]);
    }

    status = dumpfile->Read(keysize, &keyslice, keyscratch.get());
    if (!status.ok() || keyslice.size() != keysize) {
      std::cerr << "Key read failure: "
                << (status.ok() ? "insufficient data" : status.ToString())
                << std::endl;
      return false;
    }

    status = dumpfile->Read(4, &slice, scratch8);
    if (!status.ok() || slice.size() != 4) {
      std::cerr << "Unable to read value size: "
                << (status.ok() ? "insufficient data" : status.ToString())
                << std::endl;
      return false;
    }
    valsize = ROCKSDB_NAMESPACE::DecodeFixed32(slice.data());
    if (valsize > last_valsize) {
      while (valsize > last_valsize) last_valsize *= 2;
      valscratch = std::unique_ptr<char[]>(new char[last_valsize]);
    }

    status = dumpfile->Read(valsize, &valslice, valscratch.get());
    if (!status.ok() || valslice.size() != valsize) {
      std::cerr << "Unable to read value: "
                << (status.ok() ? "insufficient data" : status.ToString())
                << std::endl;
      return false;
    }

    status = db->Put(ROCKSDB_NAMESPACE::WriteOptions(), keyslice, valslice);
    if (!status.ok()) {
      fprintf(stderr, "Unable to write database entry\n");
      return false;
    }
  }

  if (undump_options.compact_db) {
    status = db->CompactRange(ROCKSDB_NAMESPACE::CompactRangeOptions(), nullptr,
                              nullptr);
    if (!status.ok()) {
      fprintf(stderr,
              "Unable to compact the database after loading the dumped file\n");
      return false;
    }
  }
  return true;
}
}  // namespace ROCKSDB_NAMESPACE
#endif  // ROCKSDB_LITE
Commit	Line	Data
7c673cae	1	// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
11fdf7f2 TL	2	// This source code is licensed under both the GPLv2 (found in the
	3	// COPYING file in the root directory) and Apache 2.0 License
	4	// (found in the LICENSE.Apache file in the root directory).
7c673cae FG	5
	6	#ifndef ROCKSDB_LITE
	7
1e59de90 TL	8	#include "rocksdb/db_dump_tool.h"
1e59de90 TL	9
f67539c2	10	#include <cinttypes>
7c673cae FG	11	#include <iostream>
	12
	13	#include "rocksdb/db.h"
7c673cae FG	14	#include "rocksdb/env.h"
	15	#include "util/coding.h"
	16
f67539c2	17	namespace ROCKSDB_NAMESPACE {
7c673cae FG	18
7c673cae FG	19	bool DbDumpTool::Run(const DumpOptions& dump_options,
f67539c2 TL	20	ROCKSDB_NAMESPACE::Options options) {
	21	ROCKSDB_NAMESPACE::DB* dbptr;
	22	ROCKSDB_NAMESPACE::Status status;
	23	std::unique_ptr<ROCKSDB_NAMESPACE::WritableFile> dumpfile;
7c673cae	24	char hostname[1024];
11fdf7f2	25	int64_t timesec = 0;
7c673cae FG	26	std::string abspath;
	27	char json[4096];
	28
	29	static const char* magicstr = "ROCKDUMP";
	30	static const char versionstr[8] = {0, 0, 0, 0, 0, 0, 0, 1};
	31
f67539c2	32	ROCKSDB_NAMESPACE::Env* env = ROCKSDB_NAMESPACE::Env::Default();
7c673cae FG	33
	34	// Open the database
	35	options.create_if_missing = false;
f67539c2 TL	36	status = ROCKSDB_NAMESPACE::DB::OpenForReadOnly(options, dump_options.db_path,
f67539c2 TL	37	&dbptr);
7c673cae FG	38	if (!status.ok()) {
	39	std::cerr << "Unable to open database '" << dump_options.db_path
	40	<< "' for reading: " << status.ToString() << std::endl;
	41	return false;
	42	}
	43
f67539c2	44	const std::unique_ptr<ROCKSDB_NAMESPACE::DB> db(dbptr);
7c673cae FG	45
7c673cae FG	46	status = env->NewWritableFile(dump_options.dump_location, &dumpfile,
f67539c2	47	ROCKSDB_NAMESPACE::EnvOptions());
7c673cae FG	48	if (!status.ok()) {
	49	std::cerr << "Unable to open dump file '" << dump_options.dump_location
	50	<< "' for writing: " << status.ToString() << std::endl;
	51	return false;
	52	}
	53
f67539c2	54	ROCKSDB_NAMESPACE::Slice magicslice(magicstr, 8);
7c673cae FG	55	status = dumpfile->Append(magicslice);
	56	if (!status.ok()) {
	57	std::cerr << "Append failed: " << status.ToString() << std::endl;
	58	return false;
	59	}
	60
f67539c2	61	ROCKSDB_NAMESPACE::Slice versionslice(versionstr, 8);
7c673cae FG	62	status = dumpfile->Append(versionslice);
	63	if (!status.ok()) {
	64	std::cerr << "Append failed: " << status.ToString() << std::endl;
	65	return false;
	66	}
	67
	68	if (dump_options.anonymous) {
	69	snprintf(json, sizeof(json), "{}");
	70	} else {
	71	status = env->GetHostName(hostname, sizeof(hostname));
	72	status = env->GetCurrentTime(&timesec);
	73	status = env->GetAbsolutePath(dump_options.db_path, &abspath);
	74	snprintf(json, sizeof(json),
	75	"{ \"database-path\": \"%s\", \"hostname\": \"%s\", "
	76	"\"creation-time\": %" PRIi64 " }",
	77	abspath.c_str(), hostname, timesec);
	78	}
	79
f67539c2	80	ROCKSDB_NAMESPACE::Slice infoslice(json, strlen(json));
7c673cae	81	char infosize[4];
f67539c2 TL	82	ROCKSDB_NAMESPACE::EncodeFixed32(infosize, (uint32_t)infoslice.size());
f67539c2 TL	83	ROCKSDB_NAMESPACE::Slice infosizeslice(infosize, 4);
7c673cae FG	84	status = dumpfile->Append(infosizeslice);
	85	if (!status.ok()) {
	86	std::cerr << "Append failed: " << status.ToString() << std::endl;
	87	return false;
	88	}
	89	status = dumpfile->Append(infoslice);
	90	if (!status.ok()) {
	91	std::cerr << "Append failed: " << status.ToString() << std::endl;
	92	return false;
	93	}
	94
f67539c2 TL	95	const std::unique_ptr<ROCKSDB_NAMESPACE::Iterator> it(
f67539c2 TL	96	db->NewIterator(ROCKSDB_NAMESPACE::ReadOptions()));
7c673cae FG	97	for (it->SeekToFirst(); it->Valid(); it->Next()) {
7c673cae FG	98	char keysize[4];
f67539c2 TL	99	ROCKSDB_NAMESPACE::EncodeFixed32(keysize, (uint32_t)it->key().size());
f67539c2 TL	100	ROCKSDB_NAMESPACE::Slice keysizeslice(keysize, 4);
7c673cae FG	101	status = dumpfile->Append(keysizeslice);
	102	if (!status.ok()) {
	103	std::cerr << "Append failed: " << status.ToString() << std::endl;
	104	return false;
	105	}
	106	status = dumpfile->Append(it->key());
	107	if (!status.ok()) {
	108	std::cerr << "Append failed: " << status.ToString() << std::endl;
	109	return false;
	110	}
	111
	112	char valsize[4];
f67539c2 TL	113	ROCKSDB_NAMESPACE::EncodeFixed32(valsize, (uint32_t)it->value().size());
f67539c2 TL	114	ROCKSDB_NAMESPACE::Slice valsizeslice(valsize, 4);
7c673cae FG	115	status = dumpfile->Append(valsizeslice);
	116	if (!status.ok()) {
	117	std::cerr << "Append failed: " << status.ToString() << std::endl;
	118	return false;
	119	}
	120	status = dumpfile->Append(it->value());
	121	if (!status.ok()) {
	122	std::cerr << "Append failed: " << status.ToString() << std::endl;
	123	return false;
	124	}
	125	}
	126	if (!it->status().ok()) {
	127	std::cerr << "Database iteration failed: " << status.ToString()
	128	<< std::endl;
	129	return false;
	130	}
	131	return true;
	132	}
	133
	134	bool DbUndumpTool::Run(const UndumpOptions& undump_options,
f67539c2 TL	135	ROCKSDB_NAMESPACE::Options options) {
	136	ROCKSDB_NAMESPACE::DB* dbptr;
	137	ROCKSDB_NAMESPACE::Status status;
	138	ROCKSDB_NAMESPACE::Env* env;
	139	std::unique_ptr<ROCKSDB_NAMESPACE::SequentialFile> dumpfile;
	140	ROCKSDB_NAMESPACE::Slice slice;
7c673cae FG	141	char scratch8[8];
	142
	143	static const char* magicstr = "ROCKDUMP";
	144	static const char versionstr[8] = {0, 0, 0, 0, 0, 0, 0, 1};
	145
f67539c2	146	env = ROCKSDB_NAMESPACE::Env::Default();
7c673cae FG	147
7c673cae FG	148	status = env->NewSequentialFile(undump_options.dump_location, &dumpfile,
f67539c2	149	ROCKSDB_NAMESPACE::EnvOptions());
7c673cae FG	150	if (!status.ok()) {
	151	std::cerr << "Unable to open dump file '" << undump_options.dump_location
	152	<< "' for reading: " << status.ToString() << std::endl;
	153	return false;
	154	}
	155
	156	status = dumpfile->Read(8, &slice, scratch8);
	157	if (!status.ok() \|\| slice.size() != 8 \|\|
	158	memcmp(slice.data(), magicstr, 8) != 0) {
	159	std::cerr << "File '" << undump_options.dump_location
	160	<< "' is not a recognizable dump file." << std::endl;
	161	return false;
	162	}
	163
	164	status = dumpfile->Read(8, &slice, scratch8);
	165	if (!status.ok() \|\| slice.size() != 8 \|\|
	166	memcmp(slice.data(), versionstr, 8) != 0) {
	167	std::cerr << "File '" << undump_options.dump_location
	168	<< "' version not recognized." << std::endl;
	169	return false;
	170	}
	171
	172	status = dumpfile->Read(4, &slice, scratch8);
	173	if (!status.ok() \|\| slice.size() != 4) {
	174	std::cerr << "Unable to read info blob size." << std::endl;
	175	return false;
	176	}
f67539c2	177	uint32_t infosize = ROCKSDB_NAMESPACE::DecodeFixed32(slice.data());
7c673cae FG	178	status = dumpfile->Skip(infosize);
	179	if (!status.ok()) {
	180	std::cerr << "Unable to skip info blob: " << status.ToString() << std::endl;
	181	return false;
	182	}
	183
	184	options.create_if_missing = true;
f67539c2	185	status = ROCKSDB_NAMESPACE::DB::Open(options, undump_options.db_path, &dbptr);
7c673cae FG	186	if (!status.ok()) {
	187	std::cerr << "Unable to open database '" << undump_options.db_path
	188	<< "' for writing: " << status.ToString() << std::endl;
	189	return false;
	190	}
	191
f67539c2	192	const std::unique_ptr<ROCKSDB_NAMESPACE::DB> db(dbptr);
7c673cae FG	193
	194	uint32_t last_keysize = 64;
	195	size_t last_valsize = 1 << 20;
	196	std::unique_ptr<char[]> keyscratch(new char[last_keysize]);
	197	std::unique_ptr<char[]> valscratch(new char[last_valsize]);
	198
	199	while (1) {
	200	uint32_t keysize, valsize;
f67539c2 TL	201	ROCKSDB_NAMESPACE::Slice keyslice;
f67539c2 TL	202	ROCKSDB_NAMESPACE::Slice valslice;
7c673cae FG	203
	204	status = dumpfile->Read(4, &slice, scratch8);
	205	if (!status.ok() \|\| slice.size() != 4) break;
f67539c2	206	keysize = ROCKSDB_NAMESPACE::DecodeFixed32(slice.data());
7c673cae FG	207	if (keysize > last_keysize) {
	208	while (keysize > last_keysize) last_keysize *= 2;
	209	keyscratch = std::unique_ptr<char[]>(new char[last_keysize]);
	210	}
	211
	212	status = dumpfile->Read(keysize, &keyslice, keyscratch.get());
	213	if (!status.ok() \|\| keyslice.size() != keysize) {
	214	std::cerr << "Key read failure: "
	215	<< (status.ok() ? "insufficient data" : status.ToString())
	216	<< std::endl;
	217	return false;
	218	}
	219
	220	status = dumpfile->Read(4, &slice, scratch8);
	221	if (!status.ok() \|\| slice.size() != 4) {
	222	std::cerr << "Unable to read value size: "
	223	<< (status.ok() ? "insufficient data" : status.ToString())
	224	<< std::endl;
	225	return false;
	226	}
f67539c2	227	valsize = ROCKSDB_NAMESPACE::DecodeFixed32(slice.data());
7c673cae FG	228	if (valsize > last_valsize) {
	229	while (valsize > last_valsize) last_valsize *= 2;
	230	valscratch = std::unique_ptr<char[]>(new char[last_valsize]);
	231	}
	232
	233	status = dumpfile->Read(valsize, &valslice, valscratch.get());
	234	if (!status.ok() \|\| valslice.size() != valsize) {
	235	std::cerr << "Unable to read value: "
	236	<< (status.ok() ? "insufficient data" : status.ToString())
	237	<< std::endl;
	238	return false;
	239	}
	240
f67539c2	241	status = db->Put(ROCKSDB_NAMESPACE::WriteOptions(), keyslice, valslice);
7c673cae FG	242	if (!status.ok()) {
	243	fprintf(stderr, "Unable to write database entry\n");
	244	return false;
	245	}
	246	}
	247
	248	if (undump_options.compact_db) {
f67539c2 TL	249	status = db->CompactRange(ROCKSDB_NAMESPACE::CompactRangeOptions(), nullptr,
f67539c2 TL	250	nullptr);
7c673cae FG	251	if (!status.ok()) {
	252	fprintf(stderr,
	253	"Unable to compact the database after loading the dumped file\n");
	254	return false;
	255	}
	256	}
	257	return true;
	258	}
f67539c2	259	} // namespace ROCKSDB_NAMESPACE
7c673cae	260	#endif // ROCKSDB_LITE