[ceph.git] / ceph / src / rocksdb / tools / dump / db_dump_tool.cc

//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
//  This source code is licensed under both the GPLv2 (found in the
//  COPYING file in the root directory) and Apache 2.0 License
//  (found in the LICENSE.Apache file in the root directory).

#ifndef ROCKSDB_LITE

#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS
#endif

#include <inttypes.h>
#include <iostream>

#include "rocksdb/db.h"
#include "rocksdb/db_dump_tool.h"
#include "rocksdb/env.h"
#include "util/coding.h"

namespace rocksdb {

bool DbDumpTool::Run(const DumpOptions& dump_options,
                     rocksdb::Options options) {
  rocksdb::DB* dbptr;
  rocksdb::Status status;
  std::unique_ptr<rocksdb::WritableFile> dumpfile;
  char hostname[1024];
  int64_t timesec = 0;
  std::string abspath;
  char json[4096];

  static const char* magicstr = "ROCKDUMP";
  static const char versionstr[8] = {0, 0, 0, 0, 0, 0, 0, 1};

  rocksdb::Env* env = rocksdb::Env::Default();

  // Open the database
  options.create_if_missing = false;
  status = rocksdb::DB::OpenForReadOnly(options, dump_options.db_path, &dbptr);
  if (!status.ok()) {
    std::cerr << "Unable to open database '" << dump_options.db_path
              << "' for reading: " << status.ToString() << std::endl;
    return false;
  }

  const std::unique_ptr<rocksdb::DB> db(dbptr);

  status = env->NewWritableFile(dump_options.dump_location, &dumpfile,
                                rocksdb::EnvOptions());
  if (!status.ok()) {
    std::cerr << "Unable to open dump file '" << dump_options.dump_location
              << "' for writing: " << status.ToString() << std::endl;
    return false;
  }

  rocksdb::Slice magicslice(magicstr, 8);
  status = dumpfile->Append(magicslice);
  if (!status.ok()) {
    std::cerr << "Append failed: " << status.ToString() << std::endl;
    return false;
  }

  rocksdb::Slice versionslice(versionstr, 8);
  status = dumpfile->Append(versionslice);
  if (!status.ok()) {
    std::cerr << "Append failed: " << status.ToString() << std::endl;
    return false;
  }

  if (dump_options.anonymous) {
    snprintf(json, sizeof(json), "{}");
  } else {
    status = env->GetHostName(hostname, sizeof(hostname));
    status = env->GetCurrentTime(&timesec);
    status = env->GetAbsolutePath(dump_options.db_path, &abspath);
    snprintf(json, sizeof(json),
             "{ \"database-path\": \"%s\", \"hostname\": \"%s\", "
             "\"creation-time\": %" PRIi64 " }",
             abspath.c_str(), hostname, timesec);
  }

  rocksdb::Slice infoslice(json, strlen(json));
  char infosize[4];
  rocksdb::EncodeFixed32(infosize, (uint32_t)infoslice.size());
  rocksdb::Slice infosizeslice(infosize, 4);
  status = dumpfile->Append(infosizeslice);
  if (!status.ok()) {
    std::cerr << "Append failed: " << status.ToString() << std::endl;
    return false;
  }
  status = dumpfile->Append(infoslice);
  if (!status.ok()) {
    std::cerr << "Append failed: " << status.ToString() << std::endl;
    return false;
  }

  const std::unique_ptr<rocksdb::Iterator> it(
      db->NewIterator(rocksdb::ReadOptions()));
  for (it->SeekToFirst(); it->Valid(); it->Next()) {
    char keysize[4];
    rocksdb::EncodeFixed32(keysize, (uint32_t)it->key().size());
    rocksdb::Slice keysizeslice(keysize, 4);
    status = dumpfile->Append(keysizeslice);
    if (!status.ok()) {
      std::cerr << "Append failed: " << status.ToString() << std::endl;
      return false;
    }
    status = dumpfile->Append(it->key());
    if (!status.ok()) {
      std::cerr << "Append failed: " << status.ToString() << std::endl;
      return false;
    }

    char valsize[4];
    rocksdb::EncodeFixed32(valsize, (uint32_t)it->value().size());
    rocksdb::Slice valsizeslice(valsize, 4);
    status = dumpfile->Append(valsizeslice);
    if (!status.ok()) {
      std::cerr << "Append failed: " << status.ToString() << std::endl;
      return false;
    }
    status = dumpfile->Append(it->value());
    if (!status.ok()) {
      std::cerr << "Append failed: " << status.ToString() << std::endl;
      return false;
    }
  }
  if (!it->status().ok()) {
    std::cerr << "Database iteration failed: " << status.ToString()
              << std::endl;
    return false;
  }
  return true;
}

bool DbUndumpTool::Run(const UndumpOptions& undump_options,
                       rocksdb::Options options) {
  rocksdb::DB* dbptr;
  rocksdb::Status status;
  rocksdb::Env* env;
  std::unique_ptr<rocksdb::SequentialFile> dumpfile;
  rocksdb::Slice slice;
  char scratch8[8];

  static const char* magicstr = "ROCKDUMP";
  static const char versionstr[8] = {0, 0, 0, 0, 0, 0, 0, 1};

  env = rocksdb::Env::Default();

  status = env->NewSequentialFile(undump_options.dump_location, &dumpfile,
                                  rocksdb::EnvOptions());
  if (!status.ok()) {
    std::cerr << "Unable to open dump file '" << undump_options.dump_location
              << "' for reading: " << status.ToString() << std::endl;
    return false;
  }

  status = dumpfile->Read(8, &slice, scratch8);
  if (!status.ok() || slice.size() != 8 ||
      memcmp(slice.data(), magicstr, 8) != 0) {
    std::cerr << "File '" << undump_options.dump_location
              << "' is not a recognizable dump file." << std::endl;
    return false;
  }

  status = dumpfile->Read(8, &slice, scratch8);
  if (!status.ok() || slice.size() != 8 ||
      memcmp(slice.data(), versionstr, 8) != 0) {
    std::cerr << "File '" << undump_options.dump_location
              << "' version not recognized." << std::endl;
    return false;
  }

  status = dumpfile->Read(4, &slice, scratch8);
  if (!status.ok() || slice.size() != 4) {
    std::cerr << "Unable to read info blob size." << std::endl;
    return false;
  }
  uint32_t infosize = rocksdb::DecodeFixed32(slice.data());
  status = dumpfile->Skip(infosize);
  if (!status.ok()) {
    std::cerr << "Unable to skip info blob: " << status.ToString() << std::endl;
    return false;
  }

  options.create_if_missing = true;
  status = rocksdb::DB::Open(options, undump_options.db_path, &dbptr);
  if (!status.ok()) {
    std::cerr << "Unable to open database '" << undump_options.db_path
              << "' for writing: " << status.ToString() << std::endl;
    return false;
  }

  const std::unique_ptr<rocksdb::DB> db(dbptr);

  uint32_t last_keysize = 64;
  size_t last_valsize = 1 << 20;
  std::unique_ptr<char[]> keyscratch(new char[last_keysize]);
  std::unique_ptr<char[]> valscratch(new char[last_valsize]);

  while (1) {
    uint32_t keysize, valsize;
    rocksdb::Slice keyslice;
    rocksdb::Slice valslice;

    status = dumpfile->Read(4, &slice, scratch8);
    if (!status.ok() || slice.size() != 4) break;
    keysize = rocksdb::DecodeFixed32(slice.data());
    if (keysize > last_keysize) {
      while (keysize > last_keysize) last_keysize *= 2;
      keyscratch = std::unique_ptr<char[]>(new char[last_keysize]);
    }

    status = dumpfile->Read(keysize, &keyslice, keyscratch.get());
    if (!status.ok() || keyslice.size() != keysize) {
      std::cerr << "Key read failure: "
                << (status.ok() ? "insufficient data" : status.ToString())
                << std::endl;
      return false;
    }

    status = dumpfile->Read(4, &slice, scratch8);
    if (!status.ok() || slice.size() != 4) {
      std::cerr << "Unable to read value size: "
                << (status.ok() ? "insufficient data" : status.ToString())
                << std::endl;
      return false;
    }
    valsize = rocksdb::DecodeFixed32(slice.data());
    if (valsize > last_valsize) {
      while (valsize > last_valsize) last_valsize *= 2;
      valscratch = std::unique_ptr<char[]>(new char[last_valsize]);
    }

    status = dumpfile->Read(valsize, &valslice, valscratch.get());
    if (!status.ok() || valslice.size() != valsize) {
      std::cerr << "Unable to read value: "
                << (status.ok() ? "insufficient data" : status.ToString())
                << std::endl;
      return false;
    }

    status = db->Put(rocksdb::WriteOptions(), keyslice, valslice);
    if (!status.ok()) {
      fprintf(stderr, "Unable to write database entry\n");
      return false;
    }
  }

  if (undump_options.compact_db) {
    status = db->CompactRange(rocksdb::CompactRangeOptions(), nullptr, nullptr);
    if (!status.ok()) {
      fprintf(stderr,
              "Unable to compact the database after loading the dumped file\n");
      return false;
    }
  }
  return true;
}
}  // namespace rocksdb
#endif  // ROCKSDB_LITE
Commit	Line	Data
7c673cae	1	// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
11fdf7f2 TL	2	// This source code is licensed under both the GPLv2 (found in the
	3	// COPYING file in the root directory) and Apache 2.0 License
	4	// (found in the LICENSE.Apache file in the root directory).
7c673cae FG	5
	6	#ifndef ROCKSDB_LITE
	7
	8	#ifndef __STDC_FORMAT_MACROS
	9	#define __STDC_FORMAT_MACROS
	10	#endif
	11
	12	#include <inttypes.h>
	13	#include <iostream>
	14
	15	#include "rocksdb/db.h"
	16	#include "rocksdb/db_dump_tool.h"
	17	#include "rocksdb/env.h"
	18	#include "util/coding.h"
	19
	20	namespace rocksdb {
	21
	22	bool DbDumpTool::Run(const DumpOptions& dump_options,
	23	rocksdb::Options options) {
	24	rocksdb::DB* dbptr;
	25	rocksdb::Status status;
	26	std::unique_ptr<rocksdb::WritableFile> dumpfile;
	27	char hostname[1024];
11fdf7f2	28	int64_t timesec = 0;
7c673cae FG	29	std::string abspath;
	30	char json[4096];
	31
	32	static const char* magicstr = "ROCKDUMP";
	33	static const char versionstr[8] = {0, 0, 0, 0, 0, 0, 0, 1};
	34
	35	rocksdb::Env* env = rocksdb::Env::Default();
	36
	37	// Open the database
	38	options.create_if_missing = false;
	39	status = rocksdb::DB::OpenForReadOnly(options, dump_options.db_path, &dbptr);
	40	if (!status.ok()) {
	41	std::cerr << "Unable to open database '" << dump_options.db_path
	42	<< "' for reading: " << status.ToString() << std::endl;
	43	return false;
	44	}
	45
	46	const std::unique_ptr<rocksdb::DB> db(dbptr);
	47
	48	status = env->NewWritableFile(dump_options.dump_location, &dumpfile,
	49	rocksdb::EnvOptions());
	50	if (!status.ok()) {
	51	std::cerr << "Unable to open dump file '" << dump_options.dump_location
	52	<< "' for writing: " << status.ToString() << std::endl;
	53	return false;
	54	}
	55
	56	rocksdb::Slice magicslice(magicstr, 8);
	57	status = dumpfile->Append(magicslice);
	58	if (!status.ok()) {
	59	std::cerr << "Append failed: " << status.ToString() << std::endl;
	60	return false;
	61	}
	62
	63	rocksdb::Slice versionslice(versionstr, 8);
	64	status = dumpfile->Append(versionslice);
	65	if (!status.ok()) {
	66	std::cerr << "Append failed: " << status.ToString() << std::endl;
	67	return false;
	68	}
	69
	70	if (dump_options.anonymous) {
	71	snprintf(json, sizeof(json), "{}");
	72	} else {
	73	status = env->GetHostName(hostname, sizeof(hostname));
	74	status = env->GetCurrentTime(&timesec);
	75	status = env->GetAbsolutePath(dump_options.db_path, &abspath);
	76	snprintf(json, sizeof(json),
	77	"{ \"database-path\": \"%s\", \"hostname\": \"%s\", "
	78	"\"creation-time\": %" PRIi64 " }",
	79	abspath.c_str(), hostname, timesec);
	80	}
	81
	82	rocksdb::Slice infoslice(json, strlen(json));
	83	char infosize[4];
	84	rocksdb::EncodeFixed32(infosize, (uint32_t)infoslice.size());
	85	rocksdb::Slice infosizeslice(infosize, 4);
	86	status = dumpfile->Append(infosizeslice);
	87	if (!status.ok()) {
	88	std::cerr << "Append failed: " << status.ToString() << std::endl;
	89	return false;
	90	}
	91	status = dumpfile->Append(infoslice);
	92	if (!status.ok()) {
93	std::cerr << "Append failed: " << status.ToString() << std::endl;
94	return false;
95	}
96
97	const std::unique_ptr<rocksdb::Iterator> it(
98	db->NewIterator(rocksdb::ReadOptions()));
99	for (it->SeekToFirst(); it->Valid(); it->Next()) {
100	char keysize[4];
101	rocksdb::EncodeFixed32(keysize, (uint32_t)it->key().size());
102	rocksdb::Slice keysizeslice(keysize, 4);
103	status = dumpfile->Append(keysizeslice);
104	if (!status.ok()) {
105	std::cerr << "Append failed: " << status.ToString() << std::endl;
106	return false;
107	}
108	status = dumpfile->Append(it->key());
109	if (!status.ok()) {
110	std::cerr << "Append failed: " << status.ToString() << std::endl;
111	return false;
112	}
113
114	char valsize[4];
115	rocksdb::EncodeFixed32(valsize, (uint32_t)it->value().size());
116	rocksdb::Slice valsizeslice(valsize, 4);
117	status = dumpfile->Append(valsizeslice);
118	if (!status.ok()) {
119	std::cerr << "Append failed: " << status.ToString() << std::endl;
120	return false;
121	}
122	status = dumpfile->Append(it->value());
123	if (!status.ok()) {
124	std::cerr << "Append failed: " << status.ToString() << std::endl;
125	return false;
126	}
127	}
128	if (!it->status().ok()) {
129	std::cerr << "Database iteration failed: " << status.ToString()
130	<< std::endl;
131	return false;
132	}
133	return true;
134	}
135
136	bool DbUndumpTool::Run(const UndumpOptions& undump_options,
137	rocksdb::Options options) {
138	rocksdb::DB* dbptr;
139	rocksdb::Status status;
140	rocksdb::Env* env;
141	std::unique_ptr<rocksdb::SequentialFile> dumpfile;
142	rocksdb::Slice slice;
143	char scratch8[8];
144
145	static const char* magicstr = "ROCKDUMP";
146	static const char versionstr[8] = {0, 0, 0, 0, 0, 0, 0, 1};
147
148	env = rocksdb::Env::Default();
149
150	status = env->NewSequentialFile(undump_options.dump_location, &dumpfile,
151	rocksdb::EnvOptions());
152	if (!status.ok()) {
153	std::cerr << "Unable to open dump file '" << undump_options.dump_location
154	<< "' for reading: " << status.ToString() << std::endl;
155	return false;
156	}
157
158	status = dumpfile->Read(8, &slice, scratch8);
159	if (!status.ok() \|\| slice.size() != 8 \|\|
160	memcmp(slice.data(), magicstr, 8) != 0) {
161	std::cerr << "File '" << undump_options.dump_location
162	<< "' is not a recognizable dump file." << std::endl;
163	return false;
164	}
165
166	status = dumpfile->Read(8, &slice, scratch8);
167	if (!status.ok() \|\| slice.size() != 8 \|\|
168	memcmp(slice.data(), versionstr, 8) != 0) {
169	std::cerr << "File '" << undump_options.dump_location
170	<< "' version not recognized." << std::endl;
171	return false;
172	}
173
174	status = dumpfile->Read(4, &slice, scratch8);
175	if (!status.ok() \|\| slice.size() != 4) {
176	std::cerr << "Unable to read info blob size." << std::endl;
177	return false;
178	}
179	uint32_t infosize = rocksdb::DecodeFixed32(slice.data());
180	status = dumpfile->Skip(infosize);
181	if (!status.ok()) {
182	std::cerr << "Unable to skip info blob: " << status.ToString() << std::endl;
183	return false;
184	}
185
186	options.create_if_missing = true;
187	status = rocksdb::DB::Open(options, undump_options.db_path, &dbptr);
188	if (!status.ok()) {
189	std::cerr << "Unable to open database '" << undump_options.db_path
190	<< "' for writing: " << status.ToString() << std::endl;
191	return false;
192	}
193
194	const std::unique_ptr<rocksdb::DB> db(dbptr);
195
196	uint32_t last_keysize = 64;
197	size_t last_valsize = 1 << 20;
198	std::unique_ptr<char[]> keyscratch(new char[last_keysize]);
199	std::unique_ptr<char[]> valscratch(new char[last_valsize]);
200
201	while (1) {
202	uint32_t keysize, valsize;
203	rocksdb::Slice keyslice;
204	rocksdb::Slice valslice;
205
206	status = dumpfile->Read(4, &slice, scratch8);
207	if (!status.ok() \|\| slice.size() != 4) break;
208	keysize = rocksdb::DecodeFixed32(slice.data());
209	if (keysize > last_keysize) {
210	while (keysize > last_keysize) last_keysize *= 2;
211	keyscratch = std::unique_ptr<char[]>(new char[last_keysize]);
212	}
213
214	status = dumpfile->Read(keysize, &keyslice, keyscratch.get());
215	if (!status.ok() \|\| keyslice.size() != keysize) {
216	std::cerr << "Key read failure: "
217	<< (status.ok() ? "insufficient data" : status.ToString())
218	<< std::endl;
219	return false;
220	}
221
222	status = dumpfile->Read(4, &slice, scratch8);
223	if (!status.ok() \|\| slice.size() != 4) {
224	std::cerr << "Unable to read value size: "
225	<< (status.ok() ? "insufficient data" : status.ToString())
226	<< std::endl;
227	return false;
228	}
229	valsize = rocksdb::DecodeFixed32(slice.data());
230	if (valsize > last_valsize) {
231	while (valsize > last_valsize) last_valsize *= 2;
232	valscratch = std::unique_ptr<char[]>(new char[last_valsize]);
233	}
234
235	status = dumpfile->Read(valsize, &valslice, valscratch.get());
236	if (!status.ok() \|\| valslice.size() != valsize) {
237	std::cerr << "Unable to read value: "
238	<< (status.ok() ? "insufficient data" : status.ToString())
239	<< std::endl;
240	return false;
241	}
242
243	status = db->Put(rocksdb::WriteOptions(), keyslice, valslice);
244	if (!status.ok()) {
245	fprintf(stderr, "Unable to write database entry\n");
246	return false;
247	}
248	}
249
250	if (undump_options.compact_db) {
251	status = db->CompactRange(rocksdb::CompactRangeOptions(), nullptr, nullptr);
252	if (!status.ok()) {
253	fprintf(stderr,
254	"Unable to compact the database after loading the dumped file\n");
255	return false;
256	}
257	}
258	return true;
259	}
260	} // namespace rocksdb
261	#endif // ROCKSDB_LITE