ceph/src/rocksdb/utilities/blob_db/blob_dump_tool.cc

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5 #ifndef ROCKSDB_LITE
   6
   7 #include "utilities/blob_db/blob_dump_tool.h"
   8
   9 #include <stdio.h>
  10
  11 #include <cinttypes>
  12 #include <iostream>
  13 #include <memory>
  14 #include <string>
  15
  16 #include "file/random_access_file_reader.h"
  17 #include "file/readahead_raf.h"
  18 #include "port/port.h"
  19 #include "rocksdb/convenience.h"
  20 #include "rocksdb/file_system.h"
  21 #include "table/format.h"
  22 #include "util/coding.h"
  23 #include "util/string_util.h"
  24
  25 namespace ROCKSDB_NAMESPACE {
  26 namespace blob_db {
  27
  28 BlobDumpTool::BlobDumpTool()
  29     : reader_(nullptr), buffer_(nullptr), buffer_size_(0) {}
  30
  31 Status BlobDumpTool::Run(const std::string& filename, DisplayType show_key,
  32                          DisplayType show_blob,
  33                          DisplayType show_uncompressed_blob,
  34                          bool show_summary) {
  35   constexpr size_t kReadaheadSize = 2 * 1024 * 1024;
  36   Status s;
  37   const auto fs = FileSystem::Default();
  38   IOOptions io_opts;
  39   s = fs->FileExists(filename, io_opts, nullptr);
  40   if (!s.ok()) {
  41     return s;
  42   }
  43   uint64_t file_size = 0;
  44   s = fs->GetFileSize(filename, io_opts, &file_size, nullptr);
  45   if (!s.ok()) {
  46     return s;
  47   }
  48   std::unique_ptr<FSRandomAccessFile> file;
  49   s = fs->NewRandomAccessFile(filename, FileOptions(), &file, nullptr);
  50   if (!s.ok()) {
  51     return s;
  52   }
  53   file = NewReadaheadRandomAccessFile(std::move(file), kReadaheadSize);
  54   if (file_size == 0) {
  55     return Status::Corruption("File is empty.");
  56   }
  57   reader_.reset(new RandomAccessFileReader(std::move(file), filename));
  58   uint64_t offset = 0;
  59   uint64_t footer_offset = 0;
  60   CompressionType compression = kNoCompression;
  61   s = DumpBlobLogHeader(&offset, &compression);
  62   if (!s.ok()) {
  63     return s;
  64   }
  65   s = DumpBlobLogFooter(file_size, &footer_offset);
  66   if (!s.ok()) {
  67     return s;
  68   }
  69   uint64_t total_records = 0;
  70   uint64_t total_key_size = 0;
  71   uint64_t total_blob_size = 0;
  72   uint64_t total_uncompressed_blob_size = 0;
  73   if (show_key != DisplayType::kNone || show_summary) {
  74     while (offset < footer_offset) {
  75       s = DumpRecord(show_key, show_blob, show_uncompressed_blob, show_summary,
  76                      compression, &offset, &total_records, &total_key_size,
  77                      &total_blob_size, &total_uncompressed_blob_size);
  78       if (!s.ok()) {
  79         break;
  80       }
  81     }
  82   }
  83   if (show_summary) {
  84     fprintf(stdout, "Summary:\n");
  85     fprintf(stdout, "  total records: %" PRIu64 "\n", total_records);
  86     fprintf(stdout, "  total key size: %" PRIu64 "\n", total_key_size);
  87     fprintf(stdout, "  total blob size: %" PRIu64 "\n", total_blob_size);
  88     if (compression != kNoCompression) {
  89       fprintf(stdout, "  total raw blob size: %" PRIu64 "\n",
  90               total_uncompressed_blob_size);
  91     }
  92   }
  93   return s;
  94 }
  95
  96 Status BlobDumpTool::Read(uint64_t offset, size_t size, Slice* result) {
  97   if (buffer_size_ < size) {
  98     if (buffer_size_ == 0) {
  99       buffer_size_ = 4096;
 100     }
 101     while (buffer_size_ < size) {
 102       buffer_size_ *= 2;
 103     }
 104     buffer_.reset(new char[buffer_size_]);
 105   }
 106   Status s = reader_->Read(IOOptions(), offset, size, result, buffer_.get(),
 107                            nullptr, Env::IO_TOTAL /* rate_limiter_priority */);
 108   if (!s.ok()) {
 109     return s;
 110   }
 111   if (result->size() != size) {
 112     return Status::Corruption("Reach the end of the file unexpectedly.");
 113   }
 114   return s;
 115 }
 116
 117 Status BlobDumpTool::DumpBlobLogHeader(uint64_t* offset,
 118                                        CompressionType* compression) {
 119   Slice slice;
 120   Status s = Read(0, BlobLogHeader::kSize, &slice);
 121   if (!s.ok()) {
 122     return s;
 123   }
 124   BlobLogHeader header;
 125   s = header.DecodeFrom(slice);
 126   if (!s.ok()) {
 127     return s;
 128   }
 129   fprintf(stdout, "Blob log header:\n");
 130   fprintf(stdout, "  Version          : %" PRIu32 "\n", header.version);
 131   fprintf(stdout, "  Column Family ID : %" PRIu32 "\n",
 132           header.column_family_id);
 133   std::string compression_str;
 134   if (!GetStringFromCompressionType(&compression_str, header.compression)
 135            .ok()) {
 136     compression_str = "Unrecongnized compression type (" +
 137                       std::to_string((int)header.compression) + ")";
 138   }
 139   fprintf(stdout, "  Compression      : %s\n", compression_str.c_str());
 140   fprintf(stdout, "  Expiration range : %s\n",
 141           GetString(header.expiration_range).c_str());
 142   *offset = BlobLogHeader::kSize;
 143   *compression = header.compression;
 144   return s;
 145 }
 146
 147 Status BlobDumpTool::DumpBlobLogFooter(uint64_t file_size,
 148                                        uint64_t* footer_offset) {
 149   auto no_footer = [&]() {
 150     *footer_offset = file_size;
 151     fprintf(stdout, "No blob log footer.\n");
 152     return Status::OK();
 153   };
 154   if (file_size < BlobLogHeader::kSize + BlobLogFooter::kSize) {
 155     return no_footer();
 156   }
 157   Slice slice;
 158   *footer_offset = file_size - BlobLogFooter::kSize;
 159   Status s = Read(*footer_offset, BlobLogFooter::kSize, &slice);
 160   if (!s.ok()) {
 161     return s;
 162   }
 163   BlobLogFooter footer;
 164   s = footer.DecodeFrom(slice);
 165   if (!s.ok()) {
 166     return no_footer();
 167   }
 168   fprintf(stdout, "Blob log footer:\n");
 169   fprintf(stdout, "  Blob count       : %" PRIu64 "\n", footer.blob_count);
 170   fprintf(stdout, "  Expiration Range : %s\n",
 171           GetString(footer.expiration_range).c_str());
 172   return s;
 173 }
 174
 175 Status BlobDumpTool::DumpRecord(DisplayType show_key, DisplayType show_blob,
 176                                 DisplayType show_uncompressed_blob,
 177                                 bool show_summary, CompressionType compression,
 178                                 uint64_t* offset, uint64_t* total_records,
 179                                 uint64_t* total_key_size,
 180                                 uint64_t* total_blob_size,
 181                                 uint64_t* total_uncompressed_blob_size) {
 182   if (show_key != DisplayType::kNone) {
 183     fprintf(stdout, "Read record with offset 0x%" PRIx64 " (%" PRIu64 "):\n",
 184             *offset, *offset);
 185   }
 186   Slice slice;
 187   Status s = Read(*offset, BlobLogRecord::kHeaderSize, &slice);
 188   if (!s.ok()) {
 189     return s;
 190   }
 191   BlobLogRecord record;
 192   s = record.DecodeHeaderFrom(slice);
 193   if (!s.ok()) {
 194     return s;
 195   }
 196   uint64_t key_size = record.key_size;
 197   uint64_t value_size = record.value_size;
 198   if (show_key != DisplayType::kNone) {
 199     fprintf(stdout, "  key size   : %" PRIu64 "\n", key_size);
 200     fprintf(stdout, "  value size : %" PRIu64 "\n", value_size);
 201     fprintf(stdout, "  expiration : %" PRIu64 "\n", record.expiration);
 202   }
 203   *offset += BlobLogRecord::kHeaderSize;
 204   s = Read(*offset, static_cast<size_t>(key_size + value_size), &slice);
 205   if (!s.ok()) {
 206     return s;
 207   }
 208   // Decompress value
 209   std::string uncompressed_value;
 210   if (compression != kNoCompression &&
 211       (show_uncompressed_blob != DisplayType::kNone || show_summary)) {
 212     BlockContents contents;
 213     UncompressionContext context(compression);
 214     UncompressionInfo info(context, UncompressionDict::GetEmptyDict(),
 215                            compression);
 216     s = UncompressBlockData(
 217         info, slice.data() + key_size, static_cast<size_t>(value_size),
 218         &contents, 2 /*compress_format_version*/, ImmutableOptions(Options()));
 219     if (!s.ok()) {
 220       return s;
 221     }
 222     uncompressed_value = contents.data.ToString();
 223   }
 224   if (show_key != DisplayType::kNone) {
 225     fprintf(stdout, "  key        : ");
 226     DumpSlice(Slice(slice.data(), static_cast<size_t>(key_size)), show_key);
 227     if (show_blob != DisplayType::kNone) {
 228       fprintf(stdout, "  blob       : ");
 229       DumpSlice(Slice(slice.data() + static_cast<size_t>(key_size),
 230                       static_cast<size_t>(value_size)),
 231                 show_blob);
 232     }
 233     if (show_uncompressed_blob != DisplayType::kNone) {
 234       fprintf(stdout, "  raw blob   : ");
 235       DumpSlice(Slice(uncompressed_value), show_uncompressed_blob);
 236     }
 237   }
 238   *offset += key_size + value_size;
 239   *total_records += 1;
 240   *total_key_size += key_size;
 241   *total_blob_size += value_size;
 242   *total_uncompressed_blob_size += uncompressed_value.size();
 243   return s;
 244 }
 245
 246 void BlobDumpTool::DumpSlice(const Slice s, DisplayType type) {
 247   if (type == DisplayType::kRaw) {
 248     fprintf(stdout, "%s\n", s.ToString().c_str());
 249   } else if (type == DisplayType::kHex) {
 250     fprintf(stdout, "%s\n", s.ToString(true /*hex*/).c_str());
 251   } else if (type == DisplayType::kDetail) {
 252     char buf[100];
 253     for (size_t i = 0; i < s.size(); i += 16) {
 254       memset(buf, 0, sizeof(buf));
 255       for (size_t j = 0; j < 16 && i + j < s.size(); j++) {
 256         unsigned char c = s[i + j];
 257         snprintf(buf + j * 3 + 15, 2, "%x", c >> 4);
 258         snprintf(buf + j * 3 + 16, 2, "%x", c & 0xf);
 259         snprintf(buf + j + 65, 2, "%c", (0x20 <= c && c <= 0x7e) ? c : '.');
 260       }
 261       for (size_t p = 0; p + 1 < sizeof(buf); p++) {
 262         if (buf[p] == 0) {
 263           buf[p] = ' ';
 264         }
 265       }
 266       fprintf(stdout, "%s\n", i == 0 ? buf + 15 : buf);
 267     }
 268   }
 269 }
 270
 271 template <class T>
 272 std::string BlobDumpTool::GetString(std::pair<T, T> p) {
 273   if (p.first == 0 && p.second == 0) {
 274     return "nil";
 275   }
 276   return "(" + std::to_string(p.first) + ", " + std::to_string(p.second) + ")";
 277 }
 278
 279 }  // namespace blob_db
 280 }  // namespace ROCKSDB_NAMESPACE
 281
 282 #endif  // ROCKSDB_LITE