]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
11fdf7f2 TL |
2 | // This source code is licensed under both the GPLv2 (found in the |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
7c673cae FG |
5 | // |
6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. | |
7 | // Use of this source code is governed by a BSD-style license that can be | |
8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. | |
9 | ||
10 | #include "db/version_edit.h" | |
11 | ||
20effc67 | 12 | #include "db/blob/blob_index.h" |
7c673cae | 13 | #include "db/version_set.h" |
f67539c2 | 14 | #include "logging/event_logger.h" |
7c673cae | 15 | #include "rocksdb/slice.h" |
1e59de90 | 16 | #include "table/unique_id_impl.h" |
f67539c2 | 17 | #include "test_util/sync_point.h" |
7c673cae | 18 | #include "util/coding.h" |
7c673cae | 19 | #include "util/string_util.h" |
7c673cae | 20 | |
f67539c2 | 21 | namespace ROCKSDB_NAMESPACE { |
20effc67 | 22 | |
1e59de90 | 23 | namespace {} // anonymous namespace |
7c673cae FG |
24 | |
25 | uint64_t PackFileNumberAndPathId(uint64_t number, uint64_t path_id) { | |
26 | assert(number <= kFileNumberMask); | |
27 | return number | (path_id * (kFileNumberMask + 1)); | |
28 | } | |
29 | ||
1e59de90 TL |
30 | Status FileMetaData::UpdateBoundaries(const Slice& key, const Slice& value, |
31 | SequenceNumber seqno, | |
32 | ValueType value_type) { | |
f67539c2 TL |
33 | if (value_type == kTypeBlobIndex) { |
34 | BlobIndex blob_index; | |
35 | const Status s = blob_index.DecodeFrom(value); | |
36 | if (!s.ok()) { | |
1e59de90 | 37 | return s; |
f67539c2 TL |
38 | } |
39 | ||
1e59de90 TL |
40 | if (!blob_index.IsInlined() && !blob_index.HasTTL()) { |
41 | if (blob_index.file_number() == kInvalidBlobFileNumber) { | |
42 | return Status::Corruption("Invalid blob file number"); | |
43 | } | |
f67539c2 | 44 | |
1e59de90 TL |
45 | if (oldest_blob_file_number == kInvalidBlobFileNumber || |
46 | oldest_blob_file_number > blob_index.file_number()) { | |
47 | oldest_blob_file_number = blob_index.file_number(); | |
48 | } | |
f67539c2 | 49 | } |
1e59de90 | 50 | } |
f67539c2 | 51 | |
1e59de90 TL |
52 | if (smallest.size() == 0) { |
53 | smallest.DecodeFrom(key); | |
f67539c2 | 54 | } |
1e59de90 TL |
55 | largest.DecodeFrom(key); |
56 | fd.smallest_seqno = std::min(fd.smallest_seqno, seqno); | |
57 | fd.largest_seqno = std::max(fd.largest_seqno, seqno); | |
58 | ||
59 | return Status::OK(); | |
f67539c2 TL |
60 | } |
61 | ||
7c673cae | 62 | void VersionEdit::Clear() { |
7c673cae | 63 | max_level_ = 0; |
f67539c2 TL |
64 | db_id_.clear(); |
65 | comparator_.clear(); | |
7c673cae FG |
66 | log_number_ = 0; |
67 | prev_log_number_ = 0; | |
7c673cae FG |
68 | next_file_number_ = 0; |
69 | max_column_family_ = 0; | |
11fdf7f2 | 70 | min_log_number_to_keep_ = 0; |
f67539c2 TL |
71 | last_sequence_ = 0; |
72 | has_db_id_ = false; | |
7c673cae FG |
73 | has_comparator_ = false; |
74 | has_log_number_ = false; | |
75 | has_prev_log_number_ = false; | |
76 | has_next_file_number_ = false; | |
7c673cae | 77 | has_max_column_family_ = false; |
11fdf7f2 | 78 | has_min_log_number_to_keep_ = false; |
f67539c2 | 79 | has_last_sequence_ = false; |
1e59de90 | 80 | compact_cursors_.clear(); |
7c673cae FG |
81 | deleted_files_.clear(); |
82 | new_files_.clear(); | |
20effc67 TL |
83 | blob_file_additions_.clear(); |
84 | blob_file_garbages_.clear(); | |
85 | wal_additions_.clear(); | |
86 | wal_deletion_.Reset(); | |
7c673cae | 87 | column_family_ = 0; |
f67539c2 TL |
88 | is_column_family_add_ = false; |
89 | is_column_family_drop_ = false; | |
7c673cae | 90 | column_family_name_.clear(); |
11fdf7f2 TL |
91 | is_in_atomic_group_ = false; |
92 | remaining_entries_ = 0; | |
1e59de90 | 93 | full_history_ts_low_.clear(); |
7c673cae FG |
94 | } |
95 | ||
96 | bool VersionEdit::EncodeTo(std::string* dst) const { | |
f67539c2 TL |
97 | if (has_db_id_) { |
98 | PutVarint32(dst, kDbId); | |
99 | PutLengthPrefixedSlice(dst, db_id_); | |
100 | } | |
7c673cae FG |
101 | if (has_comparator_) { |
102 | PutVarint32(dst, kComparator); | |
103 | PutLengthPrefixedSlice(dst, comparator_); | |
104 | } | |
105 | if (has_log_number_) { | |
106 | PutVarint32Varint64(dst, kLogNumber, log_number_); | |
107 | } | |
108 | if (has_prev_log_number_) { | |
109 | PutVarint32Varint64(dst, kPrevLogNumber, prev_log_number_); | |
110 | } | |
111 | if (has_next_file_number_) { | |
112 | PutVarint32Varint64(dst, kNextFileNumber, next_file_number_); | |
113 | } | |
7c673cae FG |
114 | if (has_max_column_family_) { |
115 | PutVarint32Varint32(dst, kMaxColumnFamily, max_column_family_); | |
116 | } | |
1e59de90 TL |
117 | if (has_min_log_number_to_keep_) { |
118 | PutVarint32Varint64(dst, kMinLogNumberToKeep, min_log_number_to_keep_); | |
119 | } | |
f67539c2 TL |
120 | if (has_last_sequence_) { |
121 | PutVarint32Varint64(dst, kLastSequence, last_sequence_); | |
122 | } | |
1e59de90 TL |
123 | for (size_t i = 0; i < compact_cursors_.size(); i++) { |
124 | if (compact_cursors_[i].second.Valid()) { | |
125 | PutVarint32(dst, kCompactCursor); | |
126 | PutVarint32(dst, compact_cursors_[i].first); // level | |
127 | PutLengthPrefixedSlice(dst, compact_cursors_[i].second.Encode()); | |
128 | } | |
129 | } | |
7c673cae FG |
130 | for (const auto& deleted : deleted_files_) { |
131 | PutVarint32Varint32Varint64(dst, kDeletedFile, deleted.first /* level */, | |
132 | deleted.second /* file number */); | |
133 | } | |
134 | ||
11fdf7f2 | 135 | bool min_log_num_written = false; |
7c673cae FG |
136 | for (size_t i = 0; i < new_files_.size(); i++) { |
137 | const FileMetaData& f = new_files_[i].second; | |
138 | if (!f.smallest.Valid() || !f.largest.Valid()) { | |
139 | return false; | |
140 | } | |
f67539c2 | 141 | PutVarint32(dst, kNewFile4); |
7c673cae | 142 | PutVarint32Varint64(dst, new_files_[i].first /* level */, f.fd.GetNumber()); |
7c673cae FG |
143 | PutVarint64(dst, f.fd.GetFileSize()); |
144 | PutLengthPrefixedSlice(dst, f.smallest.Encode()); | |
145 | PutLengthPrefixedSlice(dst, f.largest.Encode()); | |
11fdf7f2 | 146 | PutVarint64Varint64(dst, f.fd.smallest_seqno, f.fd.largest_seqno); |
f67539c2 TL |
147 | // Customized fields' format: |
148 | // +-----------------------------+ | |
149 | // | 1st field's tag (varint32) | | |
150 | // +-----------------------------+ | |
151 | // | 1st field's size (varint32) | | |
152 | // +-----------------------------+ | |
153 | // | bytes for 1st field | | |
154 | // | (based on size decoded) | | |
155 | // +-----------------------------+ | |
156 | // | | | |
157 | // | ...... | | |
158 | // | | | |
159 | // +-----------------------------+ | |
160 | // | last field's size (varint32)| | |
161 | // +-----------------------------+ | |
162 | // | bytes for last field | | |
163 | // | (based on size decoded) | | |
164 | // +-----------------------------+ | |
165 | // | terminating tag (varint32) | | |
166 | // +-----------------------------+ | |
167 | // | |
168 | // Customized encoding for fields: | |
169 | // tag kPathId: 1 byte as path_id | |
170 | // tag kNeedCompaction: | |
171 | // now only can take one char value 1 indicating need-compaction | |
172 | // | |
20effc67 | 173 | PutVarint32(dst, NewFileCustomTag::kOldestAncesterTime); |
f67539c2 TL |
174 | std::string varint_oldest_ancester_time; |
175 | PutVarint64(&varint_oldest_ancester_time, f.oldest_ancester_time); | |
176 | TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:VarintOldestAncesterTime", | |
177 | &varint_oldest_ancester_time); | |
178 | PutLengthPrefixedSlice(dst, Slice(varint_oldest_ancester_time)); | |
179 | ||
20effc67 | 180 | PutVarint32(dst, NewFileCustomTag::kFileCreationTime); |
f67539c2 TL |
181 | std::string varint_file_creation_time; |
182 | PutVarint64(&varint_file_creation_time, f.file_creation_time); | |
183 | TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:VarintFileCreationTime", | |
184 | &varint_file_creation_time); | |
185 | PutLengthPrefixedSlice(dst, Slice(varint_file_creation_time)); | |
186 | ||
20effc67 | 187 | PutVarint32(dst, NewFileCustomTag::kFileChecksum); |
f67539c2 TL |
188 | PutLengthPrefixedSlice(dst, Slice(f.file_checksum)); |
189 | ||
20effc67 | 190 | PutVarint32(dst, NewFileCustomTag::kFileChecksumFuncName); |
f67539c2 TL |
191 | PutLengthPrefixedSlice(dst, Slice(f.file_checksum_func_name)); |
192 | ||
193 | if (f.fd.GetPathId() != 0) { | |
20effc67 | 194 | PutVarint32(dst, NewFileCustomTag::kPathId); |
f67539c2 TL |
195 | char p = static_cast<char>(f.fd.GetPathId()); |
196 | PutLengthPrefixedSlice(dst, Slice(&p, 1)); | |
197 | } | |
1e59de90 TL |
198 | if (f.temperature != Temperature::kUnknown) { |
199 | PutVarint32(dst, NewFileCustomTag::kTemperature); | |
200 | char p = static_cast<char>(f.temperature); | |
201 | PutLengthPrefixedSlice(dst, Slice(&p, 1)); | |
202 | } | |
f67539c2 | 203 | if (f.marked_for_compaction) { |
20effc67 | 204 | PutVarint32(dst, NewFileCustomTag::kNeedCompaction); |
f67539c2 TL |
205 | char p = static_cast<char>(1); |
206 | PutLengthPrefixedSlice(dst, Slice(&p, 1)); | |
7c673cae | 207 | } |
f67539c2 | 208 | if (has_min_log_number_to_keep_ && !min_log_num_written) { |
20effc67 | 209 | PutVarint32(dst, NewFileCustomTag::kMinLogNumberToKeepHack); |
f67539c2 TL |
210 | std::string varint_log_number; |
211 | PutFixed64(&varint_log_number, min_log_number_to_keep_); | |
212 | PutLengthPrefixedSlice(dst, Slice(varint_log_number)); | |
213 | min_log_num_written = true; | |
214 | } | |
215 | if (f.oldest_blob_file_number != kInvalidBlobFileNumber) { | |
20effc67 | 216 | PutVarint32(dst, NewFileCustomTag::kOldestBlobFileNumber); |
f67539c2 TL |
217 | std::string oldest_blob_file_number; |
218 | PutVarint64(&oldest_blob_file_number, f.oldest_blob_file_number); | |
219 | PutLengthPrefixedSlice(dst, Slice(oldest_blob_file_number)); | |
220 | } | |
1e59de90 TL |
221 | UniqueId64x2 unique_id = f.unique_id; |
222 | TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:UniqueId", &unique_id); | |
223 | if (unique_id != kNullUniqueId64x2) { | |
224 | PutVarint32(dst, NewFileCustomTag::kUniqueId); | |
225 | std::string unique_id_str = EncodeUniqueIdBytes(&unique_id); | |
226 | PutLengthPrefixedSlice(dst, Slice(unique_id_str)); | |
227 | } | |
228 | ||
f67539c2 TL |
229 | TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:NewFile4:CustomizeFields", |
230 | dst); | |
231 | ||
20effc67 TL |
232 | PutVarint32(dst, NewFileCustomTag::kTerminate); |
233 | } | |
234 | ||
235 | for (const auto& blob_file_addition : blob_file_additions_) { | |
236 | PutVarint32(dst, kBlobFileAddition); | |
237 | blob_file_addition.EncodeTo(dst); | |
238 | } | |
239 | ||
240 | for (const auto& blob_file_garbage : blob_file_garbages_) { | |
241 | PutVarint32(dst, kBlobFileGarbage); | |
242 | blob_file_garbage.EncodeTo(dst); | |
243 | } | |
244 | ||
245 | for (const auto& wal_addition : wal_additions_) { | |
1e59de90 TL |
246 | PutVarint32(dst, kWalAddition2); |
247 | std::string encoded; | |
248 | wal_addition.EncodeTo(&encoded); | |
249 | PutLengthPrefixedSlice(dst, encoded); | |
20effc67 TL |
250 | } |
251 | ||
252 | if (!wal_deletion_.IsEmpty()) { | |
1e59de90 TL |
253 | PutVarint32(dst, kWalDeletion2); |
254 | std::string encoded; | |
255 | wal_deletion_.EncodeTo(&encoded); | |
256 | PutLengthPrefixedSlice(dst, encoded); | |
7c673cae FG |
257 | } |
258 | ||
259 | // 0 is default and does not need to be explicitly written | |
260 | if (column_family_ != 0) { | |
261 | PutVarint32Varint32(dst, kColumnFamily, column_family_); | |
262 | } | |
263 | ||
264 | if (is_column_family_add_) { | |
265 | PutVarint32(dst, kColumnFamilyAdd); | |
266 | PutLengthPrefixedSlice(dst, Slice(column_family_name_)); | |
267 | } | |
268 | ||
269 | if (is_column_family_drop_) { | |
270 | PutVarint32(dst, kColumnFamilyDrop); | |
271 | } | |
11fdf7f2 TL |
272 | |
273 | if (is_in_atomic_group_) { | |
274 | PutVarint32(dst, kInAtomicGroup); | |
275 | PutVarint32(dst, remaining_entries_); | |
276 | } | |
1e59de90 TL |
277 | |
278 | if (HasFullHistoryTsLow()) { | |
279 | PutVarint32(dst, kFullHistoryTsLow); | |
280 | PutLengthPrefixedSlice(dst, full_history_ts_low_); | |
281 | } | |
7c673cae FG |
282 | return true; |
283 | } | |
284 | ||
285 | static bool GetInternalKey(Slice* input, InternalKey* dst) { | |
286 | Slice str; | |
287 | if (GetLengthPrefixedSlice(input, &str)) { | |
288 | dst->DecodeFrom(str); | |
289 | return dst->Valid(); | |
290 | } else { | |
291 | return false; | |
292 | } | |
293 | } | |
294 | ||
11fdf7f2 | 295 | bool VersionEdit::GetLevel(Slice* input, int* level, const char** /*msg*/) { |
f67539c2 | 296 | uint32_t v = 0; |
7c673cae FG |
297 | if (GetVarint32(input, &v)) { |
298 | *level = v; | |
299 | if (max_level_ < *level) { | |
300 | max_level_ = *level; | |
301 | } | |
302 | return true; | |
303 | } else { | |
304 | return false; | |
305 | } | |
306 | } | |
307 | ||
11fdf7f2 TL |
308 | static bool is_pseudo_new_file_record_pr3488( |
309 | const int level, | |
310 | const uint64_t number, | |
311 | const uint64_t file_size, | |
312 | InternalKey& smallest, | |
313 | InternalKey& largest, | |
314 | const bool has_min_log_number_to_keep_) { | |
315 | ||
316 | if (level == 0 && number == 0 && file_size == 0 && | |
317 | has_min_log_number_to_keep_) { | |
318 | InternalKey dummy_key(Slice("dummy_key"), 0ull, ValueType::kTypeValue); | |
319 | return (*smallest.rep() == *dummy_key.rep() && | |
320 | *largest.rep() == *dummy_key.rep()); | |
321 | } else { | |
322 | return false; | |
323 | } | |
324 | } | |
325 | ||
7c673cae FG |
326 | const char* VersionEdit::DecodeNewFile4From(Slice* input) { |
327 | const char* msg = nullptr; | |
f67539c2 | 328 | int level = 0; |
7c673cae | 329 | FileMetaData f; |
f67539c2 | 330 | uint64_t number = 0; |
7c673cae | 331 | uint32_t path_id = 0; |
f67539c2 TL |
332 | uint64_t file_size = 0; |
333 | SequenceNumber smallest_seqno = 0; | |
334 | SequenceNumber largest_seqno = kMaxSequenceNumber; | |
7c673cae FG |
335 | if (GetLevel(input, &level, &msg) && GetVarint64(input, &number) && |
336 | GetVarint64(input, &file_size) && GetInternalKey(input, &f.smallest) && | |
337 | GetInternalKey(input, &f.largest) && | |
11fdf7f2 TL |
338 | GetVarint64(input, &smallest_seqno) && |
339 | GetVarint64(input, &largest_seqno)) { | |
7c673cae FG |
340 | // See comments in VersionEdit::EncodeTo() for format of customized fields |
341 | while (true) { | |
f67539c2 | 342 | uint32_t custom_tag = 0; |
7c673cae FG |
343 | Slice field; |
344 | if (!GetVarint32(input, &custom_tag)) { | |
345 | return "new-file4 custom field"; | |
346 | } | |
347 | if (custom_tag == kTerminate) { | |
348 | break; | |
349 | } | |
350 | if (!GetLengthPrefixedSlice(input, &field)) { | |
494da23a | 351 | return "new-file4 custom field length prefixed slice error"; |
7c673cae FG |
352 | } |
353 | switch (custom_tag) { | |
354 | case kPathId: | |
355 | if (field.size() != 1) { | |
356 | return "path_id field wrong size"; | |
357 | } | |
358 | path_id = field[0]; | |
359 | if (path_id > 3) { | |
360 | return "path_id wrong vaue"; | |
361 | } | |
362 | break; | |
f67539c2 TL |
363 | case kOldestAncesterTime: |
364 | if (!GetVarint64(&field, &f.oldest_ancester_time)) { | |
365 | return "invalid oldest ancester time"; | |
366 | } | |
367 | break; | |
368 | case kFileCreationTime: | |
369 | if (!GetVarint64(&field, &f.file_creation_time)) { | |
370 | return "invalid file creation time"; | |
371 | } | |
372 | break; | |
373 | case kFileChecksum: | |
374 | f.file_checksum = field.ToString(); | |
375 | break; | |
376 | case kFileChecksumFuncName: | |
377 | f.file_checksum_func_name = field.ToString(); | |
378 | break; | |
7c673cae FG |
379 | case kNeedCompaction: |
380 | if (field.size() != 1) { | |
381 | return "need_compaction field wrong size"; | |
382 | } | |
383 | f.marked_for_compaction = (field[0] == 1); | |
384 | break; | |
11fdf7f2 TL |
385 | case kMinLogNumberToKeepHack: |
386 | // This is a hack to encode kMinLogNumberToKeep in a | |
387 | // forward-compatible fashion. | |
388 | if (!GetFixed64(&field, &min_log_number_to_keep_)) { | |
389 | return "deleted log number malformatted"; | |
390 | } | |
391 | has_min_log_number_to_keep_ = true; | |
392 | break; | |
f67539c2 TL |
393 | case kOldestBlobFileNumber: |
394 | if (!GetVarint64(&field, &f.oldest_blob_file_number)) { | |
395 | return "invalid oldest blob file number"; | |
396 | } | |
397 | break; | |
1e59de90 TL |
398 | case kTemperature: |
399 | if (field.size() != 1) { | |
400 | return "temperature field wrong size"; | |
401 | } else { | |
402 | Temperature casted_field = static_cast<Temperature>(field[0]); | |
403 | if (casted_field <= Temperature::kCold) { | |
404 | f.temperature = casted_field; | |
405 | } | |
406 | } | |
407 | break; | |
408 | case kUniqueId: | |
409 | if (!DecodeUniqueIdBytes(field.ToString(), &f.unique_id).ok()) { | |
410 | f.unique_id = kNullUniqueId64x2; | |
411 | return "invalid unique id"; | |
412 | } | |
413 | break; | |
7c673cae FG |
414 | default: |
415 | if ((custom_tag & kCustomTagNonSafeIgnoreMask) != 0) { | |
416 | // Should not proceed if cannot understand it | |
417 | return "new-file4 custom field not supported"; | |
418 | } | |
419 | break; | |
420 | } | |
421 | } | |
422 | } else { | |
423 | return "new-file4 entry"; | |
424 | } | |
11fdf7f2 TL |
425 | if (is_pseudo_new_file_record_pr3488(level, number, file_size, |
426 | f.smallest, f.largest, | |
427 | has_min_log_number_to_keep_)) { | |
428 | // Since this has nothing to do with NewFile, return immediately. | |
429 | return nullptr; | |
430 | } | |
431 | f.fd = | |
432 | FileDescriptor(number, path_id, file_size, smallest_seqno, largest_seqno); | |
7c673cae FG |
433 | new_files_.push_back(std::make_pair(level, f)); |
434 | return nullptr; | |
435 | } | |
436 | ||
437 | Status VersionEdit::DecodeFrom(const Slice& src) { | |
438 | Clear(); | |
1e59de90 TL |
439 | #ifndef NDEBUG |
440 | bool ignore_ignorable_tags = false; | |
441 | TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:IgnoreIgnorableTags", | |
442 | &ignore_ignorable_tags); | |
443 | #endif | |
7c673cae FG |
444 | Slice input = src; |
445 | const char* msg = nullptr; | |
f67539c2 | 446 | uint32_t tag = 0; |
7c673cae FG |
447 | |
448 | // Temporary storage for parsing | |
f67539c2 | 449 | int level = 0; |
7c673cae FG |
450 | FileMetaData f; |
451 | Slice str; | |
452 | InternalKey key; | |
7c673cae | 453 | while (msg == nullptr && GetVarint32(&input, &tag)) { |
1e59de90 TL |
454 | #ifndef NDEBUG |
455 | if (ignore_ignorable_tags && tag > kTagSafeIgnoreMask) { | |
456 | tag = kTagSafeIgnoreMask; | |
457 | } | |
458 | #endif | |
7c673cae | 459 | switch (tag) { |
f67539c2 TL |
460 | case kDbId: |
461 | if (GetLengthPrefixedSlice(&input, &str)) { | |
462 | db_id_ = str.ToString(); | |
463 | has_db_id_ = true; | |
464 | } else { | |
465 | msg = "db id"; | |
466 | } | |
467 | break; | |
7c673cae FG |
468 | case kComparator: |
469 | if (GetLengthPrefixedSlice(&input, &str)) { | |
470 | comparator_ = str.ToString(); | |
471 | has_comparator_ = true; | |
472 | } else { | |
473 | msg = "comparator name"; | |
474 | } | |
475 | break; | |
476 | ||
477 | case kLogNumber: | |
478 | if (GetVarint64(&input, &log_number_)) { | |
479 | has_log_number_ = true; | |
480 | } else { | |
481 | msg = "log number"; | |
482 | } | |
483 | break; | |
484 | ||
485 | case kPrevLogNumber: | |
486 | if (GetVarint64(&input, &prev_log_number_)) { | |
487 | has_prev_log_number_ = true; | |
488 | } else { | |
489 | msg = "previous log number"; | |
490 | } | |
491 | break; | |
492 | ||
493 | case kNextFileNumber: | |
494 | if (GetVarint64(&input, &next_file_number_)) { | |
495 | has_next_file_number_ = true; | |
496 | } else { | |
497 | msg = "next file number"; | |
498 | } | |
499 | break; | |
500 | ||
7c673cae FG |
501 | case kMaxColumnFamily: |
502 | if (GetVarint32(&input, &max_column_family_)) { | |
503 | has_max_column_family_ = true; | |
504 | } else { | |
505 | msg = "max column family"; | |
506 | } | |
507 | break; | |
508 | ||
11fdf7f2 TL |
509 | case kMinLogNumberToKeep: |
510 | if (GetVarint64(&input, &min_log_number_to_keep_)) { | |
511 | has_min_log_number_to_keep_ = true; | |
512 | } else { | |
513 | msg = "min log number to kee"; | |
514 | } | |
515 | break; | |
516 | ||
f67539c2 TL |
517 | case kLastSequence: |
518 | if (GetVarint64(&input, &last_sequence_)) { | |
519 | has_last_sequence_ = true; | |
520 | } else { | |
521 | msg = "last sequence number"; | |
522 | } | |
523 | break; | |
524 | ||
1e59de90 TL |
525 | case kCompactCursor: |
526 | if (GetLevel(&input, &level, &msg) && GetInternalKey(&input, &key)) { | |
527 | // Here we re-use the output format of compact pointer in LevelDB | |
528 | // to persist compact_cursors_ | |
529 | compact_cursors_.push_back(std::make_pair(level, key)); | |
7c673cae FG |
530 | } else { |
531 | if (!msg) { | |
1e59de90 | 532 | msg = "compaction cursor"; |
7c673cae FG |
533 | } |
534 | } | |
535 | break; | |
536 | ||
537 | case kDeletedFile: { | |
f67539c2 | 538 | uint64_t number = 0; |
7c673cae FG |
539 | if (GetLevel(&input, &level, &msg) && GetVarint64(&input, &number)) { |
540 | deleted_files_.insert(std::make_pair(level, number)); | |
541 | } else { | |
542 | if (!msg) { | |
543 | msg = "deleted file"; | |
544 | } | |
545 | } | |
546 | break; | |
547 | } | |
548 | ||
549 | case kNewFile: { | |
f67539c2 TL |
550 | uint64_t number = 0; |
551 | uint64_t file_size = 0; | |
7c673cae FG |
552 | if (GetLevel(&input, &level, &msg) && GetVarint64(&input, &number) && |
553 | GetVarint64(&input, &file_size) && | |
554 | GetInternalKey(&input, &f.smallest) && | |
555 | GetInternalKey(&input, &f.largest)) { | |
556 | f.fd = FileDescriptor(number, 0, file_size); | |
557 | new_files_.push_back(std::make_pair(level, f)); | |
558 | } else { | |
559 | if (!msg) { | |
560 | msg = "new-file entry"; | |
561 | } | |
562 | } | |
563 | break; | |
564 | } | |
565 | case kNewFile2: { | |
f67539c2 TL |
566 | uint64_t number = 0; |
567 | uint64_t file_size = 0; | |
568 | SequenceNumber smallest_seqno = 0; | |
569 | SequenceNumber largest_seqno = kMaxSequenceNumber; | |
7c673cae FG |
570 | if (GetLevel(&input, &level, &msg) && GetVarint64(&input, &number) && |
571 | GetVarint64(&input, &file_size) && | |
572 | GetInternalKey(&input, &f.smallest) && | |
573 | GetInternalKey(&input, &f.largest) && | |
11fdf7f2 TL |
574 | GetVarint64(&input, &smallest_seqno) && |
575 | GetVarint64(&input, &largest_seqno)) { | |
576 | f.fd = FileDescriptor(number, 0, file_size, smallest_seqno, | |
577 | largest_seqno); | |
7c673cae FG |
578 | new_files_.push_back(std::make_pair(level, f)); |
579 | } else { | |
580 | if (!msg) { | |
581 | msg = "new-file2 entry"; | |
582 | } | |
583 | } | |
584 | break; | |
585 | } | |
586 | ||
587 | case kNewFile3: { | |
f67539c2 TL |
588 | uint64_t number = 0; |
589 | uint32_t path_id = 0; | |
590 | uint64_t file_size = 0; | |
591 | SequenceNumber smallest_seqno = 0; | |
592 | SequenceNumber largest_seqno = kMaxSequenceNumber; | |
7c673cae FG |
593 | if (GetLevel(&input, &level, &msg) && GetVarint64(&input, &number) && |
594 | GetVarint32(&input, &path_id) && GetVarint64(&input, &file_size) && | |
595 | GetInternalKey(&input, &f.smallest) && | |
596 | GetInternalKey(&input, &f.largest) && | |
11fdf7f2 TL |
597 | GetVarint64(&input, &smallest_seqno) && |
598 | GetVarint64(&input, &largest_seqno)) { | |
599 | f.fd = FileDescriptor(number, path_id, file_size, smallest_seqno, | |
600 | largest_seqno); | |
7c673cae FG |
601 | new_files_.push_back(std::make_pair(level, f)); |
602 | } else { | |
603 | if (!msg) { | |
604 | msg = "new-file3 entry"; | |
605 | } | |
606 | } | |
607 | break; | |
608 | } | |
609 | ||
610 | case kNewFile4: { | |
611 | msg = DecodeNewFile4From(&input); | |
612 | break; | |
613 | } | |
614 | ||
1e59de90 TL |
615 | case kBlobFileAddition: |
616 | case kBlobFileAddition_DEPRECATED: { | |
20effc67 TL |
617 | BlobFileAddition blob_file_addition; |
618 | const Status s = blob_file_addition.DecodeFrom(&input); | |
619 | if (!s.ok()) { | |
620 | return s; | |
621 | } | |
622 | ||
623 | AddBlobFile(std::move(blob_file_addition)); | |
624 | break; | |
625 | } | |
626 | ||
1e59de90 TL |
627 | case kBlobFileGarbage: |
628 | case kBlobFileGarbage_DEPRECATED: { | |
20effc67 TL |
629 | BlobFileGarbage blob_file_garbage; |
630 | const Status s = blob_file_garbage.DecodeFrom(&input); | |
631 | if (!s.ok()) { | |
632 | return s; | |
633 | } | |
634 | ||
635 | AddBlobFileGarbage(std::move(blob_file_garbage)); | |
636 | break; | |
637 | } | |
638 | ||
639 | case kWalAddition: { | |
640 | WalAddition wal_addition; | |
641 | const Status s = wal_addition.DecodeFrom(&input); | |
642 | if (!s.ok()) { | |
643 | return s; | |
644 | } | |
645 | ||
646 | wal_additions_.emplace_back(std::move(wal_addition)); | |
647 | break; | |
648 | } | |
649 | ||
1e59de90 TL |
650 | case kWalAddition2: { |
651 | Slice encoded; | |
652 | if (!GetLengthPrefixedSlice(&input, &encoded)) { | |
653 | msg = "WalAddition not prefixed by length"; | |
654 | break; | |
655 | } | |
656 | ||
657 | WalAddition wal_addition; | |
658 | const Status s = wal_addition.DecodeFrom(&encoded); | |
659 | if (!s.ok()) { | |
660 | return s; | |
661 | } | |
662 | ||
663 | wal_additions_.emplace_back(std::move(wal_addition)); | |
664 | break; | |
665 | } | |
666 | ||
20effc67 TL |
667 | case kWalDeletion: { |
668 | WalDeletion wal_deletion; | |
669 | const Status s = wal_deletion.DecodeFrom(&input); | |
670 | if (!s.ok()) { | |
671 | return s; | |
672 | } | |
673 | ||
674 | wal_deletion_ = std::move(wal_deletion); | |
675 | break; | |
676 | } | |
677 | ||
1e59de90 TL |
678 | case kWalDeletion2: { |
679 | Slice encoded; | |
680 | if (!GetLengthPrefixedSlice(&input, &encoded)) { | |
681 | msg = "WalDeletion not prefixed by length"; | |
682 | break; | |
683 | } | |
684 | ||
685 | WalDeletion wal_deletion; | |
686 | const Status s = wal_deletion.DecodeFrom(&encoded); | |
687 | if (!s.ok()) { | |
688 | return s; | |
689 | } | |
690 | ||
691 | wal_deletion_ = std::move(wal_deletion); | |
692 | break; | |
693 | } | |
694 | ||
7c673cae FG |
695 | case kColumnFamily: |
696 | if (!GetVarint32(&input, &column_family_)) { | |
697 | if (!msg) { | |
698 | msg = "set column family id"; | |
699 | } | |
700 | } | |
701 | break; | |
702 | ||
703 | case kColumnFamilyAdd: | |
704 | if (GetLengthPrefixedSlice(&input, &str)) { | |
705 | is_column_family_add_ = true; | |
706 | column_family_name_ = str.ToString(); | |
707 | } else { | |
708 | if (!msg) { | |
709 | msg = "column family add"; | |
710 | } | |
711 | } | |
712 | break; | |
713 | ||
714 | case kColumnFamilyDrop: | |
715 | is_column_family_drop_ = true; | |
716 | break; | |
717 | ||
11fdf7f2 TL |
718 | case kInAtomicGroup: |
719 | is_in_atomic_group_ = true; | |
720 | if (!GetVarint32(&input, &remaining_entries_)) { | |
721 | if (!msg) { | |
722 | msg = "remaining entries"; | |
723 | } | |
724 | } | |
725 | break; | |
726 | ||
1e59de90 TL |
727 | case kFullHistoryTsLow: |
728 | if (!GetLengthPrefixedSlice(&input, &str)) { | |
729 | msg = "full_history_ts_low"; | |
730 | } else if (str.empty()) { | |
731 | msg = "full_history_ts_low: empty"; | |
732 | } else { | |
733 | full_history_ts_low_.assign(str.data(), str.size()); | |
734 | } | |
735 | break; | |
736 | ||
7c673cae | 737 | default: |
494da23a TL |
738 | if (tag & kTagSafeIgnoreMask) { |
739 | // Tag from future which can be safely ignored. | |
740 | // The next field must be the length of the entry. | |
741 | uint32_t field_len; | |
742 | if (!GetVarint32(&input, &field_len) || | |
743 | static_cast<size_t>(field_len) > input.size()) { | |
744 | if (!msg) { | |
745 | msg = "safely ignoreable tag length error"; | |
746 | } | |
747 | } else { | |
748 | input.remove_prefix(static_cast<size_t>(field_len)); | |
749 | } | |
750 | } else { | |
751 | msg = "unknown tag"; | |
752 | } | |
7c673cae FG |
753 | break; |
754 | } | |
755 | } | |
756 | ||
757 | if (msg == nullptr && !input.empty()) { | |
758 | msg = "invalid tag"; | |
759 | } | |
760 | ||
761 | Status result; | |
762 | if (msg != nullptr) { | |
763 | result = Status::Corruption("VersionEdit", msg); | |
764 | } | |
765 | return result; | |
766 | } | |
767 | ||
768 | std::string VersionEdit::DebugString(bool hex_key) const { | |
769 | std::string r; | |
770 | r.append("VersionEdit {"); | |
f67539c2 TL |
771 | if (has_db_id_) { |
772 | r.append("\n DB ID: "); | |
773 | r.append(db_id_); | |
774 | } | |
7c673cae FG |
775 | if (has_comparator_) { |
776 | r.append("\n Comparator: "); | |
777 | r.append(comparator_); | |
778 | } | |
779 | if (has_log_number_) { | |
780 | r.append("\n LogNumber: "); | |
781 | AppendNumberTo(&r, log_number_); | |
782 | } | |
783 | if (has_prev_log_number_) { | |
784 | r.append("\n PrevLogNumber: "); | |
785 | AppendNumberTo(&r, prev_log_number_); | |
786 | } | |
787 | if (has_next_file_number_) { | |
788 | r.append("\n NextFileNumber: "); | |
789 | AppendNumberTo(&r, next_file_number_); | |
790 | } | |
f67539c2 TL |
791 | if (has_max_column_family_) { |
792 | r.append("\n MaxColumnFamily: "); | |
793 | AppendNumberTo(&r, max_column_family_); | |
794 | } | |
11fdf7f2 TL |
795 | if (has_min_log_number_to_keep_) { |
796 | r.append("\n MinLogNumberToKeep: "); | |
797 | AppendNumberTo(&r, min_log_number_to_keep_); | |
798 | } | |
7c673cae FG |
799 | if (has_last_sequence_) { |
800 | r.append("\n LastSeq: "); | |
801 | AppendNumberTo(&r, last_sequence_); | |
802 | } | |
1e59de90 TL |
803 | for (const auto& level_and_compact_cursor : compact_cursors_) { |
804 | r.append("\n CompactCursor: "); | |
805 | AppendNumberTo(&r, level_and_compact_cursor.first); | |
806 | r.append(" "); | |
807 | r.append(level_and_compact_cursor.second.DebugString(hex_key)); | |
808 | } | |
f67539c2 | 809 | for (const auto& deleted_file : deleted_files_) { |
7c673cae | 810 | r.append("\n DeleteFile: "); |
f67539c2 | 811 | AppendNumberTo(&r, deleted_file.first); |
7c673cae | 812 | r.append(" "); |
f67539c2 | 813 | AppendNumberTo(&r, deleted_file.second); |
7c673cae FG |
814 | } |
815 | for (size_t i = 0; i < new_files_.size(); i++) { | |
816 | const FileMetaData& f = new_files_[i].second; | |
817 | r.append("\n AddFile: "); | |
818 | AppendNumberTo(&r, new_files_[i].first); | |
819 | r.append(" "); | |
820 | AppendNumberTo(&r, f.fd.GetNumber()); | |
821 | r.append(" "); | |
822 | AppendNumberTo(&r, f.fd.GetFileSize()); | |
823 | r.append(" "); | |
824 | r.append(f.smallest.DebugString(hex_key)); | |
825 | r.append(" .. "); | |
826 | r.append(f.largest.DebugString(hex_key)); | |
f67539c2 TL |
827 | if (f.oldest_blob_file_number != kInvalidBlobFileNumber) { |
828 | r.append(" blob_file:"); | |
829 | AppendNumberTo(&r, f.oldest_blob_file_number); | |
830 | } | |
831 | r.append(" oldest_ancester_time:"); | |
832 | AppendNumberTo(&r, f.oldest_ancester_time); | |
833 | r.append(" file_creation_time:"); | |
834 | AppendNumberTo(&r, f.file_creation_time); | |
835 | r.append(" file_checksum:"); | |
1e59de90 | 836 | r.append(Slice(f.file_checksum).ToString(true)); |
f67539c2 TL |
837 | r.append(" file_checksum_func_name: "); |
838 | r.append(f.file_checksum_func_name); | |
1e59de90 TL |
839 | if (f.temperature != Temperature::kUnknown) { |
840 | r.append(" temperature: "); | |
841 | // Maybe change to human readable format whenthe feature becomes | |
842 | // permanent | |
843 | r.append(std::to_string(static_cast<int>(f.temperature))); | |
844 | } | |
845 | if (f.unique_id != kNullUniqueId64x2) { | |
846 | r.append(" unique_id(internal): "); | |
847 | UniqueId64x2 id = f.unique_id; | |
848 | r.append(InternalUniqueIdToHumanString(&id)); | |
849 | r.append(" public_unique_id: "); | |
850 | InternalUniqueIdToExternal(&id); | |
851 | r.append(UniqueIdToHumanString(EncodeUniqueIdBytes(&id))); | |
852 | } | |
7c673cae | 853 | } |
20effc67 TL |
854 | |
855 | for (const auto& blob_file_addition : blob_file_additions_) { | |
856 | r.append("\n BlobFileAddition: "); | |
857 | r.append(blob_file_addition.DebugString()); | |
858 | } | |
859 | ||
860 | for (const auto& blob_file_garbage : blob_file_garbages_) { | |
861 | r.append("\n BlobFileGarbage: "); | |
862 | r.append(blob_file_garbage.DebugString()); | |
863 | } | |
864 | ||
865 | for (const auto& wal_addition : wal_additions_) { | |
866 | r.append("\n WalAddition: "); | |
867 | r.append(wal_addition.DebugString()); | |
868 | } | |
869 | ||
870 | if (!wal_deletion_.IsEmpty()) { | |
871 | r.append("\n WalDeletion: "); | |
872 | r.append(wal_deletion_.DebugString()); | |
873 | } | |
874 | ||
7c673cae FG |
875 | r.append("\n ColumnFamily: "); |
876 | AppendNumberTo(&r, column_family_); | |
877 | if (is_column_family_add_) { | |
878 | r.append("\n ColumnFamilyAdd: "); | |
879 | r.append(column_family_name_); | |
880 | } | |
881 | if (is_column_family_drop_) { | |
882 | r.append("\n ColumnFamilyDrop"); | |
883 | } | |
11fdf7f2 | 884 | if (is_in_atomic_group_) { |
494da23a | 885 | r.append("\n AtomicGroup: "); |
11fdf7f2 TL |
886 | AppendNumberTo(&r, remaining_entries_); |
887 | r.append(" entries remains"); | |
888 | } | |
1e59de90 TL |
889 | if (HasFullHistoryTsLow()) { |
890 | r.append("\n FullHistoryTsLow: "); | |
891 | r.append(Slice(full_history_ts_low_).ToString(hex_key)); | |
892 | } | |
7c673cae FG |
893 | r.append("\n}\n"); |
894 | return r; | |
895 | } | |
896 | ||
897 | std::string VersionEdit::DebugJSON(int edit_num, bool hex_key) const { | |
898 | JSONWriter jw; | |
899 | jw << "EditNumber" << edit_num; | |
900 | ||
f67539c2 TL |
901 | if (has_db_id_) { |
902 | jw << "DB ID" << db_id_; | |
903 | } | |
7c673cae FG |
904 | if (has_comparator_) { |
905 | jw << "Comparator" << comparator_; | |
906 | } | |
907 | if (has_log_number_) { | |
908 | jw << "LogNumber" << log_number_; | |
909 | } | |
910 | if (has_prev_log_number_) { | |
911 | jw << "PrevLogNumber" << prev_log_number_; | |
912 | } | |
913 | if (has_next_file_number_) { | |
914 | jw << "NextFileNumber" << next_file_number_; | |
915 | } | |
f67539c2 TL |
916 | if (has_max_column_family_) { |
917 | jw << "MaxColumnFamily" << max_column_family_; | |
918 | } | |
919 | if (has_min_log_number_to_keep_) { | |
920 | jw << "MinLogNumberToKeep" << min_log_number_to_keep_; | |
921 | } | |
7c673cae FG |
922 | if (has_last_sequence_) { |
923 | jw << "LastSeq" << last_sequence_; | |
924 | } | |
925 | ||
926 | if (!deleted_files_.empty()) { | |
927 | jw << "DeletedFiles"; | |
928 | jw.StartArray(); | |
929 | ||
f67539c2 | 930 | for (const auto& deleted_file : deleted_files_) { |
7c673cae | 931 | jw.StartArrayedObject(); |
f67539c2 TL |
932 | jw << "Level" << deleted_file.first; |
933 | jw << "FileNumber" << deleted_file.second; | |
7c673cae FG |
934 | jw.EndArrayedObject(); |
935 | } | |
936 | ||
937 | jw.EndArray(); | |
938 | } | |
939 | ||
940 | if (!new_files_.empty()) { | |
941 | jw << "AddedFiles"; | |
942 | jw.StartArray(); | |
943 | ||
944 | for (size_t i = 0; i < new_files_.size(); i++) { | |
945 | jw.StartArrayedObject(); | |
946 | jw << "Level" << new_files_[i].first; | |
947 | const FileMetaData& f = new_files_[i].second; | |
948 | jw << "FileNumber" << f.fd.GetNumber(); | |
949 | jw << "FileSize" << f.fd.GetFileSize(); | |
950 | jw << "SmallestIKey" << f.smallest.DebugString(hex_key); | |
951 | jw << "LargestIKey" << f.largest.DebugString(hex_key); | |
1e59de90 TL |
952 | jw << "OldestAncesterTime" << f.oldest_ancester_time; |
953 | jw << "FileCreationTime" << f.file_creation_time; | |
954 | jw << "FileChecksum" << Slice(f.file_checksum).ToString(true); | |
955 | jw << "FileChecksumFuncName" << f.file_checksum_func_name; | |
956 | if (f.temperature != Temperature::kUnknown) { | |
957 | jw << "temperature" << std::to_string(static_cast<int>(f.temperature)); | |
958 | } | |
f67539c2 TL |
959 | if (f.oldest_blob_file_number != kInvalidBlobFileNumber) { |
960 | jw << "OldestBlobFile" << f.oldest_blob_file_number; | |
961 | } | |
1e59de90 TL |
962 | if (f.temperature != Temperature::kUnknown) { |
963 | // Maybe change to human readable format whenthe feature becomes | |
964 | // permanent | |
965 | jw << "Temperature" << static_cast<int>(f.temperature); | |
966 | } | |
7c673cae FG |
967 | jw.EndArrayedObject(); |
968 | } | |
969 | ||
970 | jw.EndArray(); | |
971 | } | |
972 | ||
20effc67 TL |
973 | if (!blob_file_additions_.empty()) { |
974 | jw << "BlobFileAdditions"; | |
975 | ||
976 | jw.StartArray(); | |
977 | ||
978 | for (const auto& blob_file_addition : blob_file_additions_) { | |
979 | jw.StartArrayedObject(); | |
980 | jw << blob_file_addition; | |
981 | jw.EndArrayedObject(); | |
982 | } | |
983 | ||
984 | jw.EndArray(); | |
985 | } | |
986 | ||
987 | if (!blob_file_garbages_.empty()) { | |
988 | jw << "BlobFileGarbages"; | |
989 | ||
990 | jw.StartArray(); | |
991 | ||
992 | for (const auto& blob_file_garbage : blob_file_garbages_) { | |
993 | jw.StartArrayedObject(); | |
994 | jw << blob_file_garbage; | |
995 | jw.EndArrayedObject(); | |
996 | } | |
997 | ||
998 | jw.EndArray(); | |
999 | } | |
1000 | ||
1001 | if (!wal_additions_.empty()) { | |
1002 | jw << "WalAdditions"; | |
1003 | ||
1004 | jw.StartArray(); | |
1005 | ||
1006 | for (const auto& wal_addition : wal_additions_) { | |
1007 | jw.StartArrayedObject(); | |
1008 | jw << wal_addition; | |
1009 | jw.EndArrayedObject(); | |
1010 | } | |
1011 | ||
1012 | jw.EndArray(); | |
1013 | } | |
1014 | ||
1015 | if (!wal_deletion_.IsEmpty()) { | |
1016 | jw << "WalDeletion"; | |
1017 | jw.StartObject(); | |
1018 | jw << wal_deletion_; | |
1019 | jw.EndObject(); | |
1020 | } | |
1021 | ||
7c673cae FG |
1022 | jw << "ColumnFamily" << column_family_; |
1023 | ||
1024 | if (is_column_family_add_) { | |
1025 | jw << "ColumnFamilyAdd" << column_family_name_; | |
1026 | } | |
1027 | if (is_column_family_drop_) { | |
1028 | jw << "ColumnFamilyDrop" << column_family_name_; | |
1029 | } | |
11fdf7f2 TL |
1030 | if (is_in_atomic_group_) { |
1031 | jw << "AtomicGroup" << remaining_entries_; | |
1032 | } | |
7c673cae | 1033 | |
1e59de90 TL |
1034 | if (HasFullHistoryTsLow()) { |
1035 | jw << "FullHistoryTsLow" << Slice(full_history_ts_low_).ToString(hex_key); | |
1036 | } | |
1037 | ||
7c673cae FG |
1038 | jw.EndObject(); |
1039 | ||
1040 | return jw.Get(); | |
1041 | } | |
1042 | ||
f67539c2 | 1043 | } // namespace ROCKSDB_NAMESPACE |