ceph/src/rocksdb/utilities/blob_db/blob_dump_tool.cc

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5 #ifndef ROCKSDB_LITE
   6
   7 #ifndef __STDC_FORMAT_MACROS
   8 #define __STDC_FORMAT_MACROS
   9 #endif
  10
  11 #include "utilities/blob_db/blob_dump_tool.h"
  12 #include <inttypes.h>
  13 #include <stdio.h>
  14 #include <iostream>
  15 #include <memory>
  16 #include <string>
  17 #include "port/port.h"
  18 #include "rocksdb/convenience.h"
  19 #include "rocksdb/env.h"
  20 #include "table/format.h"
  21 #include "util/coding.h"
  22 #include "util/file_reader_writer.h"
  23 #include "util/string_util.h"
  24
  25 namespace rocksdb {
  26 namespace blob_db {
  27
  28 BlobDumpTool::BlobDumpTool()
  29     : reader_(nullptr), buffer_(nullptr), buffer_size_(0) {}
  30
  31 Status BlobDumpTool::Run(const std::string& filename, DisplayType show_key,
  32                          DisplayType show_blob,
  33                          DisplayType show_uncompressed_blob,
  34                          bool show_summary) {
  35   constexpr size_t kReadaheadSize = 2 * 1024 * 1024;
  36   Status s;
  37   Env* env = Env::Default();
  38   s = env->FileExists(filename);
  39   if (!s.ok()) {
  40     return s;
  41   }
  42   uint64_t file_size = 0;
  43   s = env->GetFileSize(filename, &file_size);
  44   if (!s.ok()) {
  45     return s;
  46   }
  47   std::unique_ptr<RandomAccessFile> file;
  48   s = env->NewRandomAccessFile(filename, &file, EnvOptions());
  49   if (!s.ok()) {
  50     return s;
  51   }
  52   file = NewReadaheadRandomAccessFile(std::move(file), kReadaheadSize);
  53   if (file_size == 0) {
  54     return Status::Corruption("File is empty.");
  55   }
  56   reader_.reset(new RandomAccessFileReader(std::move(file), filename));
  57   uint64_t offset = 0;
  58   uint64_t footer_offset = 0;
  59   CompressionType compression = kNoCompression;
  60   s = DumpBlobLogHeader(&offset, &compression);
  61   if (!s.ok()) {
  62     return s;
  63   }
  64   s = DumpBlobLogFooter(file_size, &footer_offset);
  65   if (!s.ok()) {
  66     return s;
  67   }
  68   uint64_t total_records = 0;
  69   uint64_t total_key_size = 0;
  70   uint64_t total_blob_size = 0;
  71   uint64_t total_uncompressed_blob_size = 0;
  72   if (show_key != DisplayType::kNone || show_summary) {
  73     while (offset < footer_offset) {
  74       s = DumpRecord(show_key, show_blob, show_uncompressed_blob, show_summary,
  75                      compression, &offset, &total_records, &total_key_size,
  76                      &total_blob_size, &total_uncompressed_blob_size);
  77       if (!s.ok()) {
  78         break;
  79       }
  80     }
  81   }
  82   if (show_summary) {
  83     fprintf(stdout, "Summary:\n");
  84     fprintf(stdout, "  total records: %" PRIu64 "\n", total_records);
  85     fprintf(stdout, "  total key size: %" PRIu64 "\n", total_key_size);
  86     fprintf(stdout, "  total blob size: %" PRIu64 "\n", total_blob_size);
  87     if (compression != kNoCompression) {
  88       fprintf(stdout, "  total raw blob size: %" PRIu64 "\n",
  89               total_uncompressed_blob_size);
  90     }
  91   }
  92   return s;
  93 }
  94
  95 Status BlobDumpTool::Read(uint64_t offset, size_t size, Slice* result) {
  96   if (buffer_size_ < size) {
  97     if (buffer_size_ == 0) {
  98       buffer_size_ = 4096;
  99     }
 100     while (buffer_size_ < size) {
 101       buffer_size_ *= 2;
 102     }
 103     buffer_.reset(new char[buffer_size_]);
 104   }
 105   Status s = reader_->Read(offset, size, result, buffer_.get());
 106   if (!s.ok()) {
 107     return s;
 108   }
 109   if (result->size() != size) {
 110     return Status::Corruption("Reach the end of the file unexpectedly.");
 111   }
 112   return s;
 113 }
 114
 115 Status BlobDumpTool::DumpBlobLogHeader(uint64_t* offset,
 116                                        CompressionType* compression) {
 117   Slice slice;
 118   Status s = Read(0, BlobLogHeader::kSize, &slice);
 119   if (!s.ok()) {
 120     return s;
 121   }
 122   BlobLogHeader header;
 123   s = header.DecodeFrom(slice);
 124   if (!s.ok()) {
 125     return s;
 126   }
 127   fprintf(stdout, "Blob log header:\n");
 128   fprintf(stdout, "  Version          : %" PRIu32 "\n", header.version);
 129   fprintf(stdout, "  Column Family ID : %" PRIu32 "\n",
 130           header.column_family_id);
 131   std::string compression_str;
 132   if (!GetStringFromCompressionType(&compression_str, header.compression)
 133            .ok()) {
 134     compression_str = "Unrecongnized compression type (" +
 135                       ToString((int)header.compression) + ")";
 136   }
 137   fprintf(stdout, "  Compression      : %s\n", compression_str.c_str());
 138   fprintf(stdout, "  Expiration range : %s\n",
 139           GetString(header.expiration_range).c_str());
 140   *offset = BlobLogHeader::kSize;
 141   *compression = header.compression;
 142   return s;
 143 }
 144
 145 Status BlobDumpTool::DumpBlobLogFooter(uint64_t file_size,
 146                                        uint64_t* footer_offset) {
 147   auto no_footer = [&]() {
 148     *footer_offset = file_size;
 149     fprintf(stdout, "No blob log footer.\n");
 150     return Status::OK();
 151   };
 152   if (file_size < BlobLogHeader::kSize + BlobLogFooter::kSize) {
 153     return no_footer();
 154   }
 155   Slice slice;
 156   *footer_offset = file_size - BlobLogFooter::kSize;
 157   Status s = Read(*footer_offset, BlobLogFooter::kSize, &slice);
 158   if (!s.ok()) {
 159     return s;
 160   }
 161   BlobLogFooter footer;
 162   s = footer.DecodeFrom(slice);
 163   if (!s.ok()) {
 164     return no_footer();
 165   }
 166   fprintf(stdout, "Blob log footer:\n");
 167   fprintf(stdout, "  Blob count       : %" PRIu64 "\n", footer.blob_count);
 168   fprintf(stdout, "  Expiration Range : %s\n",
 169           GetString(footer.expiration_range).c_str());
 170   return s;
 171 }
 172
 173 Status BlobDumpTool::DumpRecord(DisplayType show_key, DisplayType show_blob,
 174                                 DisplayType show_uncompressed_blob,
 175                                 bool show_summary, CompressionType compression,
 176                                 uint64_t* offset, uint64_t* total_records,
 177                                 uint64_t* total_key_size,
 178                                 uint64_t* total_blob_size,
 179                                 uint64_t* total_uncompressed_blob_size) {
 180   if (show_key != DisplayType::kNone) {
 181     fprintf(stdout, "Read record with offset 0x%" PRIx64 " (%" PRIu64 "):\n",
 182             *offset, *offset);
 183   }
 184   Slice slice;
 185   Status s = Read(*offset, BlobLogRecord::kHeaderSize, &slice);
 186   if (!s.ok()) {
 187     return s;
 188   }
 189   BlobLogRecord record;
 190   s = record.DecodeHeaderFrom(slice);
 191   if (!s.ok()) {
 192     return s;
 193   }
 194   uint64_t key_size = record.key_size;
 195   uint64_t value_size = record.value_size;
 196   if (show_key != DisplayType::kNone) {
 197     fprintf(stdout, "  key size   : %" PRIu64 "\n", key_size);
 198     fprintf(stdout, "  value size : %" PRIu64 "\n", value_size);
 199     fprintf(stdout, "  expiration : %" PRIu64 "\n", record.expiration);
 200   }
 201   *offset += BlobLogRecord::kHeaderSize;
 202   s = Read(*offset, static_cast<size_t>(key_size + value_size), &slice);
 203   if (!s.ok()) {
 204     return s;
 205   }
 206   // Decompress value
 207   std::string uncompressed_value;
 208   if (compression != kNoCompression &&
 209       (show_uncompressed_blob != DisplayType::kNone || show_summary)) {
 210     BlockContents contents;
 211     UncompressionContext context(compression);
 212     UncompressionInfo info(context, UncompressionDict::GetEmptyDict(),
 213                            compression);
 214     s = UncompressBlockContentsForCompressionType(
 215         info, slice.data() + key_size, static_cast<size_t>(value_size),
 216         &contents, 2 /*compress_format_version*/,
 217         ImmutableCFOptions(Options()));
 218     if (!s.ok()) {
 219       return s;
 220     }
 221     uncompressed_value = contents.data.ToString();
 222   }
 223   if (show_key != DisplayType::kNone) {
 224     fprintf(stdout, "  key        : ");
 225     DumpSlice(Slice(slice.data(), static_cast<size_t>(key_size)), show_key);
 226     if (show_blob != DisplayType::kNone) {
 227       fprintf(stdout, "  blob       : ");
 228       DumpSlice(Slice(slice.data() + static_cast<size_t>(key_size), static_cast<size_t>(value_size)), show_blob);
 229     }
 230     if (show_uncompressed_blob != DisplayType::kNone) {
 231       fprintf(stdout, "  raw blob   : ");
 232       DumpSlice(Slice(uncompressed_value), show_uncompressed_blob);
 233     }
 234   }
 235   *offset += key_size + value_size;
 236   *total_records += 1;
 237   *total_key_size += key_size;
 238   *total_blob_size += value_size;
 239   *total_uncompressed_blob_size += uncompressed_value.size();
 240   return s;
 241 }
 242
 243 void BlobDumpTool::DumpSlice(const Slice s, DisplayType type) {
 244   if (type == DisplayType::kRaw) {
 245     fprintf(stdout, "%s\n", s.ToString().c_str());
 246   } else if (type == DisplayType::kHex) {
 247     fprintf(stdout, "%s\n", s.ToString(true /*hex*/).c_str());
 248   } else if (type == DisplayType::kDetail) {
 249     char buf[100];
 250     for (size_t i = 0; i < s.size(); i += 16) {
 251       memset(buf, 0, sizeof(buf));
 252       for (size_t j = 0; j < 16 && i + j < s.size(); j++) {
 253         unsigned char c = s[i + j];
 254         snprintf(buf + j * 3 + 15, 2, "%x", c >> 4);
 255         snprintf(buf + j * 3 + 16, 2, "%x", c & 0xf);
 256         snprintf(buf + j + 65, 2, "%c", (0x20 <= c && c <= 0x7e) ? c : '.');
 257       }
 258       for (size_t p = 0; p < sizeof(buf) - 1; p++) {
 259         if (buf[p] == 0) {
 260           buf[p] = ' ';
 261         }
 262       }
 263       fprintf(stdout, "%s\n", i == 0 ? buf + 15 : buf);
 264     }
 265   }
 266 }
 267
 268 template <class T>
 269 std::string BlobDumpTool::GetString(std::pair<T, T> p) {
 270   if (p.first == 0 && p.second == 0) {
 271     return "nil";
 272   }
 273   return "(" + ToString(p.first) + ", " + ToString(p.second) + ")";
 274 }
 275
 276 }  // namespace blob_db
 277 }  // namespace rocksdb
 278
 279 #endif  // ROCKSDB_LITE