]>
Commit | Line | Data |
---|---|---|
f67539c2 TL |
1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
2 | // This source code is licensed under both the GPLv2 (found in the | |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
5 | // Copyright (c) 2013 The LevelDB Authors. All rights reserved. | |
6 | // Use of this source code is governed by a BSD-style license that can be | |
7 | // found in the LICENSE file. See the AUTHORS file for names of contributors. | |
8 | ||
9 | #pragma once | |
10 | ||
11 | #include <cassert> | |
12 | #include <map> | |
13 | #include <memory> | |
14 | #include <string> | |
15 | #include <vector> | |
16 | ||
1e59de90 | 17 | #include "rocksdb/customizable.h" |
f67539c2 TL |
18 | #include "rocksdb/status.h" |
19 | ||
20 | namespace ROCKSDB_NAMESPACE { | |
21 | ||
20effc67 TL |
22 | // The unknown file checksum. |
23 | constexpr char kUnknownFileChecksum[] = ""; | |
24 | // The unknown sst file checksum function name. | |
25 | constexpr char kUnknownFileChecksumFuncName[] = "Unknown"; | |
26 | // The standard DB file checksum function name. | |
27 | // This is the name of the checksum function returned by | |
28 | // GetFileChecksumGenCrc32cFactory(); | |
29 | constexpr char kStandardDbFileChecksumFuncName[] = "FileChecksumCrc32c"; | |
30 | ||
31 | struct FileChecksumGenContext { | |
32 | std::string file_name; | |
33 | // The name of the requested checksum generator. | |
34 | // Checksum factories may use or ignore requested_checksum_func_name, | |
35 | // and checksum factories written before this field was available are still | |
36 | // compatible. | |
37 | std::string requested_checksum_func_name; | |
38 | }; | |
39 | ||
40 | // FileChecksumGenerator is the class to generates the checksum value | |
f67539c2 | 41 | // for each file when the file is written to the file system. |
20effc67 TL |
42 | // Implementations may assume that |
43 | // * Finalize is called at most once during the life of the object | |
44 | // * All calls to Update come before Finalize | |
45 | // * All calls to GetChecksum come after Finalize | |
1e59de90 TL |
46 | // |
47 | // Exceptions MUST NOT propagate out of overridden functions into RocksDB, | |
48 | // because RocksDB is not exception-safe. This could cause undefined behavior | |
49 | // including data loss, unreported corruption, deadlocks, and more. | |
20effc67 | 50 | class FileChecksumGenerator { |
f67539c2 | 51 | public: |
20effc67 TL |
52 | virtual ~FileChecksumGenerator() {} |
53 | ||
54 | // Update the current result after process the data. For different checksum | |
55 | // functions, the temporal results may be stored and used in Update to | |
56 | // include the new data. | |
57 | virtual void Update(const char* data, size_t n) = 0; | |
f67539c2 | 58 | |
20effc67 TL |
59 | // Generate the final results if no further new data will be updated. |
60 | virtual void Finalize() = 0; | |
f67539c2 | 61 | |
20effc67 TL |
62 | // Get the checksum. The result should not be the empty string and may |
63 | // include arbitrary bytes, including non-printable characters. | |
64 | virtual std::string GetChecksum() const = 0; | |
f67539c2 TL |
65 | |
66 | // Returns a name that identifies the current file checksum function. | |
67 | virtual const char* Name() const = 0; | |
68 | }; | |
69 | ||
20effc67 | 70 | // Create the FileChecksumGenerator object for each SST file. |
1e59de90 TL |
71 | // |
72 | // Exceptions MUST NOT propagate out of overridden functions into RocksDB, | |
73 | // because RocksDB is not exception-safe. This could cause undefined behavior | |
74 | // including data loss, unreported corruption, deadlocks, and more. | |
75 | class FileChecksumGenFactory : public Customizable { | |
20effc67 | 76 | public: |
1e59de90 TL |
77 | ~FileChecksumGenFactory() override {} |
78 | static const char* Type() { return "FileChecksumGenFactory"; } | |
79 | static Status CreateFromString( | |
80 | const ConfigOptions& options, const std::string& value, | |
81 | std::shared_ptr<FileChecksumGenFactory>* result); | |
20effc67 TL |
82 | |
83 | // Create a new FileChecksumGenerator. | |
84 | virtual std::unique_ptr<FileChecksumGenerator> CreateFileChecksumGenerator( | |
85 | const FileChecksumGenContext& context) = 0; | |
86 | ||
87 | // Return the name of this FileChecksumGenFactory. | |
1e59de90 | 88 | const char* Name() const override = 0; |
20effc67 TL |
89 | }; |
90 | ||
f67539c2 | 91 | // FileChecksumList stores the checksum information of a list of files (e.g., |
1e59de90 | 92 | // SST files). The FileChecksumList can be used to store the checksum |
f67539c2 TL |
93 | // information of all SST file getting from the MANIFEST, which are |
94 | // the checksum information of all valid SST file of a DB instance. It can | |
95 | // also be used to store the checksum information of a list of SST files to | |
96 | // be ingested. | |
1e59de90 TL |
97 | // |
98 | // Exceptions MUST NOT propagate out of overridden functions into RocksDB, | |
99 | // because RocksDB is not exception-safe. This could cause undefined behavior | |
100 | // including data loss, unreported corruption, deadlocks, and more. | |
f67539c2 TL |
101 | class FileChecksumList { |
102 | public: | |
103 | virtual ~FileChecksumList() {} | |
104 | ||
105 | // Clean the previously stored file checksum information. | |
106 | virtual void reset() = 0; | |
107 | ||
108 | // Get the number of checksums in the checksum list | |
109 | virtual size_t size() const = 0; | |
110 | ||
111 | // Return all the file checksum information being stored in a unordered_map. | |
112 | // File_number is the key, the first part of the value is checksum value, | |
113 | // and the second part of the value is checksum function name. | |
114 | virtual Status GetAllFileChecksums( | |
115 | std::vector<uint64_t>* file_numbers, std::vector<std::string>* checksums, | |
116 | std::vector<std::string>* checksum_func_names) = 0; | |
117 | ||
118 | // Given the file_number, it searches if the file checksum information is | |
119 | // stored. | |
120 | virtual Status SearchOneFileChecksum(uint64_t file_number, | |
121 | std::string* checksum, | |
122 | std::string* checksum_func_name) = 0; | |
123 | ||
124 | // Insert the checksum information of one file to the FileChecksumList. | |
125 | virtual Status InsertOneFileChecksum( | |
126 | uint64_t file_number, const std::string& checksum, | |
127 | const std::string& checksum_func_name) = 0; | |
128 | ||
129 | // Remove the checksum information of one SST file. | |
130 | virtual Status RemoveOneFileChecksum(uint64_t file_number) = 0; | |
131 | }; | |
132 | ||
133 | // Create a new file checksum list. | |
134 | extern FileChecksumList* NewFileChecksumList(); | |
135 | ||
1e59de90 | 136 | // Return a shared_ptr of the builtin Crc32c based file checksum generator |
20effc67 TL |
137 | // factory object, which can be shared to create the Crc32c based checksum |
138 | // generator object. | |
139 | // Note: this implementation is compatible with many other crc32c checksum | |
140 | // implementations and uses big-endian encoding of the result, unlike most | |
141 | // other crc32c checksums in RocksDB, which alter the result with | |
142 | // crc32c::Mask and use little-endian encoding. | |
143 | extern std::shared_ptr<FileChecksumGenFactory> | |
144 | GetFileChecksumGenCrc32cFactory(); | |
f67539c2 TL |
145 | |
146 | } // namespace ROCKSDB_NAMESPACE |