ceph/src/rocksdb/utilities/blob_db/blob_log_format.h

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5 //
   6 // Log format information shared by reader and writer.
   7
   8 #pragma once
   9
  10 #ifndef ROCKSDB_LITE
  11
  12 #include <limits>
  13 #include <memory>
  14 #include <utility>
  15
  16 #include "rocksdb/options.h"
  17 #include "rocksdb/slice.h"
  18 #include "rocksdb/status.h"
  19 #include "rocksdb/types.h"
  20
  21 namespace rocksdb {
  22 namespace blob_db {
  23
  24 constexpr uint32_t kMagicNumber = 2395959;  // 0x00248f37
  25 constexpr uint32_t kVersion1 = 1;
  26 constexpr uint64_t kNoExpiration = std::numeric_limits<uint64_t>::max();
  27
  28 using ExpirationRange = std::pair<uint64_t, uint64_t>;
  29
  30 // Format of blob log file header (30 bytes):
  31 //
  32 //    +--------------+---------+---------+-------+-------------+-------------------+
  33 //    | magic number | version |  cf id  | flags | compression | expiration range  |
  34 //    +--------------+---------+---------+-------+-------------+-------------------+
  35 //    |   Fixed32    | Fixed32 | Fixed32 | char  |    char     | Fixed64   Fixed64 |
  36 //    +--------------+---------+---------+-------+-------------+-------------------+
  37 //
  38 // List of flags:
  39 //   has_ttl: Whether the file contain TTL data.
  40 //
  41 // Expiration range in the header is a rough range based on
  42 // blob_db_options.ttl_range_secs.
  43 struct BlobLogHeader {
  44   static constexpr size_t kSize = 30;
  45
  46   uint32_t version = kVersion1;
  47   uint32_t column_family_id = 0;
  48   CompressionType compression = kNoCompression;
  49   bool has_ttl = false;
  50   ExpirationRange expiration_range = std::make_pair(0, 0);
  51
  52   void EncodeTo(std::string* dst);
  53
  54   Status DecodeFrom(Slice slice);
  55 };
  56
  57 // Format of blob log file footer (32 bytes):
  58 //
  59 //    +--------------+------------+-------------------+------------+
  60 //    | magic number | blob count | expiration range  | footer CRC |
  61 //    +--------------+------------+-------------------+------------+
  62 //    |   Fixed32    |  Fixed64   | Fixed64 + Fixed64 |   Fixed32  |
  63 //    +--------------+------------+-------------------+------------+
  64 //
  65 // The footer will be presented only when the blob file is properly closed.
  66 //
  67 // Unlike the same field in file header, expiration range in the footer is the
  68 // range of smallest and largest expiration of the data in this file.
  69 struct BlobLogFooter {
  70   static constexpr size_t kSize = 32;
  71
  72   uint64_t blob_count = 0;
  73   ExpirationRange expiration_range = std::make_pair(0, 0);
  74   uint32_t crc = 0;
  75
  76   void EncodeTo(std::string* dst);
  77
  78   Status DecodeFrom(Slice slice);
  79 };
  80
  81 // Blob record format (32 bytes header + key + value):
  82 //
  83 //    +------------+--------------+------------+------------+----------+---------+-----------+
  84 //    | key length | value length | expiration | header CRC | blob CRC |   key   |   value   |
  85 //    +------------+--------------+------------+------------+----------+---------+-----------+
  86 //    |   Fixed64  |   Fixed64    |  Fixed64   |  Fixed32   | Fixed32  | key len | value len |
  87 //    +------------+--------------+------------+------------+----------+---------+-----------+
  88 //
  89 // If file has has_ttl = false, expiration field is always 0, and the blob
  90 // doesn't has expiration.
  91 //
  92 // Also note that if compression is used, value is compressed value and value
  93 // length is compressed value length.
  94 //
  95 // Header CRC is the checksum of (key_len + val_len + expiration), while
  96 // blob CRC is the checksum of (key + value).
  97 //
  98 // We could use variable length encoding (Varint64) to save more space, but it
  99 // make reader more complicated.
 100 struct BlobLogRecord {
 101   // header include fields up to blob CRC
 102   static constexpr size_t kHeaderSize = 32;
 103
 104   uint64_t key_size = 0;
 105   uint64_t value_size = 0;
 106   uint64_t expiration = 0;
 107   uint32_t header_crc = 0;
 108   uint32_t blob_crc = 0;
 109   Slice key;
 110   Slice value;
 111   std::unique_ptr<char[]> key_buf;
 112   std::unique_ptr<char[]> value_buf;
 113
 114   uint64_t record_size() const { return kHeaderSize + key_size + value_size; }
 115
 116   void EncodeHeaderTo(std::string* dst);
 117
 118   Status DecodeHeaderFrom(Slice src);
 119
 120   Status CheckBlobCRC() const;
 121 };
 122
 123 }  // namespace blob_db
 124 }  // namespace rocksdb
 125 #endif  // ROCKSDB_LITE