]>
Commit | Line | Data |
---|---|---|
11fdf7f2 TL |
1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
2 | // This source code is licensed under both the GPLv2 (found in the | |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
5 | // | |
6 | // Log format information shared by reader and writer. | |
7 | ||
8 | #pragma once | |
9 | ||
494da23a | 10 | #include <memory> |
11fdf7f2 | 11 | #include <utility> |
494da23a | 12 | |
11fdf7f2 TL |
13 | #include "rocksdb/options.h" |
14 | #include "rocksdb/slice.h" | |
15 | #include "rocksdb/status.h" | |
16 | #include "rocksdb/types.h" | |
17 | ||
f67539c2 | 18 | namespace ROCKSDB_NAMESPACE { |
11fdf7f2 TL |
19 | |
20 | constexpr uint32_t kMagicNumber = 2395959; // 0x00248f37 | |
21 | constexpr uint32_t kVersion1 = 1; | |
11fdf7f2 TL |
22 | |
23 | using ExpirationRange = std::pair<uint64_t, uint64_t>; | |
24 | ||
1e59de90 TL |
25 | // clang-format off |
26 | ||
11fdf7f2 TL |
27 | // Format of blob log file header (30 bytes): |
28 | // | |
29 | // +--------------+---------+---------+-------+-------------+-------------------+ | |
30 | // | magic number | version | cf id | flags | compression | expiration range | | |
31 | // +--------------+---------+---------+-------+-------------+-------------------+ | |
32 | // | Fixed32 | Fixed32 | Fixed32 | char | char | Fixed64 Fixed64 | | |
33 | // +--------------+---------+---------+-------+-------------+-------------------+ | |
34 | // | |
35 | // List of flags: | |
36 | // has_ttl: Whether the file contain TTL data. | |
37 | // | |
38 | // Expiration range in the header is a rough range based on | |
39 | // blob_db_options.ttl_range_secs. | |
1e59de90 TL |
40 | |
41 | // clang-format on | |
42 | ||
11fdf7f2 TL |
43 | struct BlobLogHeader { |
44 | static constexpr size_t kSize = 30; | |
45 | ||
f67539c2 TL |
46 | BlobLogHeader() = default; |
47 | BlobLogHeader(uint32_t _column_family_id, CompressionType _compression, | |
48 | bool _has_ttl, const ExpirationRange& _expiration_range) | |
49 | : column_family_id(_column_family_id), | |
50 | compression(_compression), | |
51 | has_ttl(_has_ttl), | |
52 | expiration_range(_expiration_range) {} | |
53 | ||
11fdf7f2 TL |
54 | uint32_t version = kVersion1; |
55 | uint32_t column_family_id = 0; | |
56 | CompressionType compression = kNoCompression; | |
57 | bool has_ttl = false; | |
f67539c2 | 58 | ExpirationRange expiration_range; |
11fdf7f2 TL |
59 | |
60 | void EncodeTo(std::string* dst); | |
61 | ||
62 | Status DecodeFrom(Slice slice); | |
63 | }; | |
64 | ||
1e59de90 TL |
65 | // clang-format off |
66 | ||
11fdf7f2 TL |
67 | // Format of blob log file footer (32 bytes): |
68 | // | |
69 | // +--------------+------------+-------------------+------------+ | |
70 | // | magic number | blob count | expiration range | footer CRC | | |
71 | // +--------------+------------+-------------------+------------+ | |
72 | // | Fixed32 | Fixed64 | Fixed64 + Fixed64 | Fixed32 | | |
73 | // +--------------+------------+-------------------+------------+ | |
74 | // | |
75 | // The footer will be presented only when the blob file is properly closed. | |
76 | // | |
77 | // Unlike the same field in file header, expiration range in the footer is the | |
78 | // range of smallest and largest expiration of the data in this file. | |
1e59de90 TL |
79 | |
80 | // clang-format on | |
81 | ||
11fdf7f2 TL |
82 | struct BlobLogFooter { |
83 | static constexpr size_t kSize = 32; | |
84 | ||
85 | uint64_t blob_count = 0; | |
86 | ExpirationRange expiration_range = std::make_pair(0, 0); | |
87 | uint32_t crc = 0; | |
88 | ||
89 | void EncodeTo(std::string* dst); | |
90 | ||
91 | Status DecodeFrom(Slice slice); | |
92 | }; | |
93 | ||
1e59de90 TL |
94 | // clang-format off |
95 | ||
11fdf7f2 TL |
96 | // Blob record format (32 bytes header + key + value): |
97 | // | |
98 | // +------------+--------------+------------+------------+----------+---------+-----------+ | |
99 | // | key length | value length | expiration | header CRC | blob CRC | key | value | | |
100 | // +------------+--------------+------------+------------+----------+---------+-----------+ | |
101 | // | Fixed64 | Fixed64 | Fixed64 | Fixed32 | Fixed32 | key len | value len | | |
102 | // +------------+--------------+------------+------------+----------+---------+-----------+ | |
103 | // | |
104 | // If file has has_ttl = false, expiration field is always 0, and the blob | |
105 | // doesn't has expiration. | |
106 | // | |
107 | // Also note that if compression is used, value is compressed value and value | |
108 | // length is compressed value length. | |
109 | // | |
110 | // Header CRC is the checksum of (key_len + val_len + expiration), while | |
111 | // blob CRC is the checksum of (key + value). | |
112 | // | |
113 | // We could use variable length encoding (Varint64) to save more space, but it | |
114 | // make reader more complicated. | |
1e59de90 TL |
115 | |
116 | // clang-format on | |
117 | ||
11fdf7f2 TL |
118 | struct BlobLogRecord { |
119 | // header include fields up to blob CRC | |
120 | static constexpr size_t kHeaderSize = 32; | |
121 | ||
20effc67 TL |
122 | // Note that the offset field of BlobIndex actually points to the blob value |
123 | // as opposed to the start of the blob record. The following method can | |
124 | // be used to calculate the adjustment needed to read the blob record header. | |
1e59de90 TL |
125 | static constexpr uint64_t CalculateAdjustmentForRecordHeader( |
126 | uint64_t key_size) { | |
20effc67 TL |
127 | return key_size + kHeaderSize; |
128 | } | |
129 | ||
11fdf7f2 TL |
130 | uint64_t key_size = 0; |
131 | uint64_t value_size = 0; | |
132 | uint64_t expiration = 0; | |
133 | uint32_t header_crc = 0; | |
134 | uint32_t blob_crc = 0; | |
135 | Slice key; | |
136 | Slice value; | |
494da23a TL |
137 | std::unique_ptr<char[]> key_buf; |
138 | std::unique_ptr<char[]> value_buf; | |
11fdf7f2 TL |
139 | |
140 | uint64_t record_size() const { return kHeaderSize + key_size + value_size; } | |
141 | ||
142 | void EncodeHeaderTo(std::string* dst); | |
143 | ||
144 | Status DecodeHeaderFrom(Slice src); | |
145 | ||
146 | Status CheckBlobCRC() const; | |
147 | }; | |
148 | ||
20effc67 TL |
149 | // Checks whether a blob offset is potentially valid or not. |
150 | inline bool IsValidBlobOffset(uint64_t value_offset, uint64_t key_size, | |
151 | uint64_t value_size, uint64_t file_size) { | |
152 | if (value_offset < | |
153 | BlobLogHeader::kSize + BlobLogRecord::kHeaderSize + key_size) { | |
154 | return false; | |
155 | } | |
156 | ||
157 | if (value_offset + value_size + BlobLogFooter::kSize > file_size) { | |
158 | return false; | |
159 | } | |
160 | ||
161 | return true; | |
162 | } | |
163 | ||
f67539c2 | 164 | } // namespace ROCKSDB_NAMESPACE |