]>
Commit | Line | Data |
---|---|---|
d78345bc DM |
1 | //! This module implements the proxmox backup chunked data storage |
2 | //! | |
3 | //! A chunk is simply defined as binary blob. We store them inside a | |
4 | //! `ChunkStore`, addressed by the SHA256 digest of the binary | |
5 | //! blob. This technology is also known as content-addressable | |
6 | //! storage. | |
7 | //! | |
8 | //! We store larger files by splitting them into chunks. The resulting | |
9 | //! SHA256 digest list is stored as separate index file. The | |
10 | //! `DynamicIndex*` format is able to deal with dynamic chunk sizes, | |
11 | //! whereas the `FixedIndex*` format is an optimization to store a | |
12 | //! list of equal sized chunks. | |
04652189 DM |
13 | //! |
14 | //! # ChunkStore Locking | |
15 | //! | |
16 | //! We need to be able to restart the proxmox-backup service daemons, | |
17 | //! so that we can update the software without rebooting the host. But | |
18 | //! such restarts must not abort running backup jobs, so we need to | |
19 | //! keep the old service running until those jobs are finished. This | |
c8ec450e | 20 | //! implies that we need some kind of locking for the |
04652189 DM |
21 | //! ChunkStore. Please note that it is perfectly valid to have |
22 | //! multiple parallel ChunkStore writers, even when they write the | |
23 | //! same chunk (because the chunk would have the same name and the | |
24 | //! same data). The only real problem is garbage collection, because | |
25 | //! we need to avoid deleting chunks which are still referenced. | |
26 | //! | |
27 | //! * Read Index Files: | |
28 | //! | |
29 | //! Acquire shared lock for .idx files. | |
30 | //! | |
31 | //! | |
32 | //! * Delete Index Files: | |
33 | //! | |
34 | //! Acquire exclusive lock for .idx files. This makes sure that we do | |
35 | //! not delete index files while they are still in use. | |
36 | //! | |
37 | //! | |
38 | //! * Create Index Files: | |
39 | //! | |
8a475734 | 40 | //! Acquire shared lock for ChunkStore (process wide). |
04652189 | 41 | //! |
c8ec450e DM |
42 | //! Note: When creating .idx files, we create temporary (.tmp) file, |
43 | //! then do an atomic rename ... | |
04652189 DM |
44 | //! |
45 | //! | |
46 | //! * Garbage Collect: | |
47 | //! | |
8a475734 DM |
48 | //! Acquire exclusive lock for ChunkStore (process wide). If we have |
49 | //! already an shared lock for ChunkStore, try to updraged that | |
50 | //! lock. | |
04652189 DM |
51 | //! |
52 | //! | |
53 | //! * Server Restart | |
54 | //! | |
55 | //! Try to abort running garbage collection to release exclusive | |
56 | //! ChunkStore lock asap. Start new service with existing listening | |
57 | //! socket. | |
58 | //! | |
8a475734 | 59 | //! |
c8ec450e | 60 | //! # Garbage Collection (GC) |
8a475734 DM |
61 | //! |
62 | //! Deleting backups is as easy as deleting the corresponding .idx | |
63 | //! files. Unfortunately, this does not free up any storage, because | |
64 | //! those files just contains references to chunks. | |
65 | //! | |
66 | //! To free up some storage, we run a garbage collection process at | |
67 | //! regular intervals. The collector uses an mark and sweep | |
c374f054 DM |
68 | //! approach. In the first phase, it scans all .idx files to mark used |
69 | //! chunks. The second phase then removes all unmarked chunks from the | |
8a475734 DM |
70 | //! store. |
71 | //! | |
72 | //! The above locking mechanism makes sure that we are the only | |
c8ec450e DM |
73 | //! process running GC. But we still want to be able to create backups |
74 | //! during GC, so there may be multiple backup threads/tasks | |
75 | //! running. Either started before GC started, or started while GC is | |
76 | //! running. | |
8a475734 | 77 | //! |
c8ec450e | 78 | //! ## `atime` based GC |
8a475734 | 79 | //! |
c8ec450e DM |
80 | //! The idea here is to mark chunks by updating the `atime` (access |
81 | //! timestamp) on the chunk file. This is quite simple and does not | |
c374f054 | 82 | //! need additional RAM. |
c8ec450e DM |
83 | //! |
84 | //! One minor problem is that recent Linux versions use the `relatime` | |
85 | //! mount flag by default for performance reasons (yes, we want | |
86 | //! that). When enabled, `atime` data is written to the disk only if | |
87 | //! the file has been modified since the `atime` data was last updated | |
88 | //! (`mtime`), or if the file was last accessed more than a certain | |
c374f054 DM |
89 | //! amount of time ago (by default 24h). So we may only delete chunks |
90 | //! with `atime` older than 24 hours. | |
91 | //! | |
92 | //! Another problem arise from running backups. The mark phase does | |
93 | //! not find any chunks from those backups, because there is no .idx | |
94 | //! file for them (created after the backup). Chunks created or | |
95 | //! touched by those backups may have an `atime` as old as the start | |
96 | //! time of those backup. Please not that the backup start time may | |
97 | //! predate the GC start time. Se we may only delete chunk older than | |
98 | //! the start time of those running backup jobs. | |
c8ec450e | 99 | //! |
c8ec450e DM |
100 | //! |
101 | //! ## Store `marks` in RAM using a HASH | |
102 | //! | |
103 | //! Not sure if this is better. TODO | |
cbdd8c54 | 104 | |
986bef16 DM |
105 | #[macro_export] |
106 | macro_rules! PROXMOX_BACKUP_PROTOCOL_ID_V1 { | |
107 | () => { "proxmox-backup-protocol-v1" } | |
108 | } | |
c9ec0956 | 109 | |
c20b17b4 DM |
110 | // WARNING: PLEASE DO NOT MODIFY THOSE MAGIC VALUES |
111 | ||
74792b95 DM |
112 | // openssl::sha::sha256(b"Proxmox Backup uncompressed chunk v1.0")[0..8] |
113 | pub static UNCOMPRESSED_CHUNK_MAGIC_1_0: [u8; 8] = [79, 127, 200, 4, 121, 74, 135, 239]; | |
114 | ||
115 | // openssl::sha::sha256(b"Proxmox Backup encrypted chunk v1.0")[0..8] | |
c20b17b4 DM |
116 | pub static ENCRYPTED_CHUNK_MAGIC_1_0: [u8; 8] = [8, 54, 114, 153, 70, 156, 26, 151]; |
117 | ||
118 | // openssl::sha::sha256(b"Proxmox Backup zstd compressed chunk v1.0")[0..8] | |
119 | pub static COMPRESSED_CHUNK_MAGIC_1_0: [u8; 8] = [191, 237, 46, 195, 108, 17, 228, 235]; | |
120 | ||
74792b95 DM |
121 | // openssl::sha::sha256(b"Proxmox Backup zstd compressed encrypted chunk v1.0")[0..8] |
122 | pub static ENCR_COMPR_CHUNK_MAGIC_1_0: [u8; 8] = [9, 40, 53, 200, 37, 150, 90, 196]; | |
c20b17b4 | 123 | |
3025b3a5 DM |
124 | // openssl::sha::sha256(b"Proxmox Backup uncompressed blob v1.0")[0..8] |
125 | pub static UNCOMPRESSED_BLOB_MAGIC_1_0: [u8; 8] = [66, 171, 56, 7, 190, 131, 112, 161]; | |
126 | ||
127 | //openssl::sha::sha256(b"Proxmox Backup zstd compressed blob v1.0")[0..8] | |
128 | pub static COMPRESSED_BLOB_MAGIC_1_0: [u8; 8] = [49, 185, 88, 66, 111, 182, 163, 127]; | |
129 | ||
130 | // openssl::sha::sha256(b"Proxmox Backup encrypted blob v1.0")[0..8] | |
131 | pub static ENCRYPTED_BLOB_MAGIC_1_0: [u8; 8] = [123, 103, 133, 190, 34, 45, 76, 240]; | |
132 | ||
133 | // openssl::sha::sha256(b"Proxmox Backup zstd compressed encrypted blob v1.0")[0..8] | |
134 | pub static ENCR_COMPR_BLOB_MAGIC_1_0: [u8; 8] = [230, 89, 27, 191, 11, 191, 216, 11]; | |
135 | ||
a7dd4830 DM |
136 | // openssl::sha::sha256(b"Proxmox Backup fixed sized chunk index v1.0")[0..8] |
137 | pub static FIXED_SIZED_CHUNK_INDEX_1_0: [u8; 8] = [47, 127, 65, 237, 145, 253, 15, 205]; | |
138 | ||
139 | // openssl::sha::sha256(b"Proxmox Backup dynamic sized chunk index v1.0")[0..8] | |
140 | pub static DYNAMIC_SIZED_CHUNK_INDEX_1_0: [u8; 8] = [28, 145, 78, 165, 25, 186, 179, 205]; | |
141 | ||
c38266c1 DM |
142 | mod crypt_config; |
143 | pub use crypt_config::*; | |
48b4b40b | 144 | |
826f309b DM |
145 | mod key_derivation; |
146 | pub use key_derivation::*; | |
147 | ||
b595cb9d DM |
148 | mod data_chunk; |
149 | pub use data_chunk::*; | |
150 | ||
3025b3a5 DM |
151 | mod data_blob; |
152 | pub use data_blob::*; | |
153 | ||
dafc27ae DM |
154 | mod chunk_stream; |
155 | pub use chunk_stream::*; | |
156 | ||
7e336555 DM |
157 | mod chunk_stat; |
158 | pub use chunk_stat::*; | |
159 | ||
06178f13 | 160 | pub use proxmox_protocol::Chunker; |
e5064ba6 DM |
161 | |
162 | mod chunk_store; | |
163 | pub use chunk_store::*; | |
164 | ||
7bc1d727 WB |
165 | mod index; |
166 | pub use index::*; | |
167 | ||
e5064ba6 DM |
168 | mod fixed_index; |
169 | pub use fixed_index::*; | |
170 | ||
171 | mod dynamic_index; | |
172 | pub use dynamic_index::*; | |
173 | ||
b3483782 DM |
174 | mod backup_info; |
175 | pub use backup_info::*; | |
176 | ||
e5064ba6 DM |
177 | mod datastore; |
178 | pub use datastore::*; |