]>
Commit | Line | Data |
---|---|---|
529de6c7 DM |
1 | use failure::*; |
2 | ||
e25736b4 DM |
3 | use chrono::prelude::*; |
4 | ||
3d5c11e5 | 5 | use std::path::{PathBuf, Path}; |
2c32fdde DM |
6 | use std::collections::HashMap; |
7 | use lazy_static::lazy_static; | |
8 | use std::sync::{Mutex, Arc}; | |
529de6c7 | 9 | |
e25736b4 | 10 | use crate::tools; |
529de6c7 DM |
11 | use crate::config::datastore; |
12 | use super::chunk_store::*; | |
91a905b6 | 13 | use super::fixed_index::*; |
93d5d779 | 14 | use super::dynamic_index::*; |
529de6c7 | 15 | |
ff3d3100 DM |
16 | use chrono::{Utc, TimeZone}; |
17 | ||
529de6c7 | 18 | pub struct DataStore { |
1629d2ad | 19 | chunk_store: Arc<ChunkStore>, |
64e53b28 | 20 | gc_mutex: Mutex<bool>, |
529de6c7 DM |
21 | } |
22 | ||
e25736b4 DM |
23 | #[derive(Debug)] |
24 | pub struct BackupInfo { | |
25 | pub backup_type: String, | |
26 | pub backup_id: String, | |
7ca80246 | 27 | pub backup_time: DateTime<Utc>, |
e25736b4 DM |
28 | } |
29 | ||
2c32fdde DM |
30 | lazy_static!{ |
31 | static ref datastore_map: Mutex<HashMap<String, Arc<DataStore>>> = Mutex::new(HashMap::new()); | |
32 | } | |
33 | ||
529de6c7 DM |
34 | impl DataStore { |
35 | ||
2c32fdde DM |
36 | pub fn lookup_datastore(name: &str) -> Result<Arc<DataStore>, Error> { |
37 | ||
38 | let config = datastore::config()?; | |
39 | let (_, store_config) = config.sections.get(name) | |
40 | .ok_or(format_err!("no such datastore '{}'", name))?; | |
41 | ||
42 | let path = store_config["path"].as_str().unwrap(); | |
43 | ||
44 | let mut map = datastore_map.lock().unwrap(); | |
45 | ||
46 | if let Some(datastore) = map.get(name) { | |
47 | // Compare Config - if changed, create new Datastore object! | |
a198d74f | 48 | if datastore.chunk_store.base == PathBuf::from(path) { |
2c32fdde DM |
49 | return Ok(datastore.clone()); |
50 | } | |
51 | } | |
52 | ||
53 | if let Ok(datastore) = DataStore::open(name) { | |
54 | let datastore = Arc::new(datastore); | |
55 | map.insert(name.to_string(), datastore.clone()); | |
56 | return Ok(datastore); | |
57 | } | |
58 | ||
59 | bail!("store not found"); | |
60 | } | |
61 | ||
529de6c7 DM |
62 | pub fn open(store_name: &str) -> Result<Self, Error> { |
63 | ||
64 | let config = datastore::config()?; | |
65 | let (_, store_config) = config.sections.get(store_name) | |
66 | .ok_or(format_err!("no such datastore '{}'", store_name))?; | |
67 | ||
68 | let path = store_config["path"].as_str().unwrap(); | |
69 | ||
277fc5a3 | 70 | let chunk_store = ChunkStore::open(store_name, path)?; |
529de6c7 DM |
71 | |
72 | Ok(Self { | |
1629d2ad | 73 | chunk_store: Arc::new(chunk_store), |
64e53b28 | 74 | gc_mutex: Mutex::new(false), |
529de6c7 DM |
75 | }) |
76 | } | |
77 | ||
91a905b6 | 78 | pub fn create_fixed_writer<P: AsRef<Path>>(&self, filename: P, size: usize, chunk_size: usize) -> Result<FixedIndexWriter, Error> { |
529de6c7 | 79 | |
91a905b6 | 80 | let index = FixedIndexWriter::create(self.chunk_store.clone(), filename.as_ref(), size, chunk_size)?; |
529de6c7 DM |
81 | |
82 | Ok(index) | |
83 | } | |
84 | ||
91a905b6 | 85 | pub fn open_fixed_reader<P: AsRef<Path>>(&self, filename: P) -> Result<FixedIndexReader, Error> { |
529de6c7 | 86 | |
91a905b6 | 87 | let index = FixedIndexReader::open(self.chunk_store.clone(), filename.as_ref())?; |
529de6c7 DM |
88 | |
89 | Ok(index) | |
90 | } | |
3d5c11e5 | 91 | |
93d5d779 | 92 | pub fn create_dynamic_writer<P: AsRef<Path>>( |
0433db19 DM |
93 | &self, filename: P, |
94 | chunk_size: usize | |
93d5d779 | 95 | ) -> Result<DynamicIndexWriter, Error> { |
0433db19 | 96 | |
93d5d779 | 97 | let index = DynamicIndexWriter::create( |
1629d2ad | 98 | self.chunk_store.clone(), filename.as_ref(), chunk_size)?; |
0433db19 DM |
99 | |
100 | Ok(index) | |
101 | } | |
ff3d3100 | 102 | |
93d5d779 | 103 | pub fn open_dynamic_reader<P: AsRef<Path>>(&self, filename: P) -> Result<DynamicIndexReader, Error> { |
77703d95 | 104 | |
93d5d779 | 105 | let index = DynamicIndexReader::open(self.chunk_store.clone(), filename.as_ref())?; |
77703d95 DM |
106 | |
107 | Ok(index) | |
108 | } | |
109 | ||
ff3d3100 DM |
110 | pub fn base_path(&self) -> PathBuf { |
111 | self.chunk_store.base_path() | |
112 | } | |
113 | ||
6a4c0916 DM |
114 | pub fn get_backup_dir( |
115 | &self, | |
116 | backup_type: &str, | |
117 | backup_id: &str, | |
7ca80246 | 118 | backup_time: DateTime<Utc>, |
6a4c0916 DM |
119 | ) -> PathBuf { |
120 | ||
121 | let mut relative_path = PathBuf::new(); | |
122 | ||
123 | relative_path.push(backup_type); | |
124 | ||
125 | relative_path.push(backup_id); | |
126 | ||
7ca80246 | 127 | let date_str = backup_time.format("%Y-%m-%dT%H:%M:%S").to_string(); |
6a4c0916 DM |
128 | |
129 | relative_path.push(&date_str); | |
130 | ||
131 | relative_path | |
132 | } | |
133 | ||
ff3d3100 DM |
134 | pub fn create_backup_dir( |
135 | &self, | |
136 | backup_type: &str, | |
137 | backup_id: &str, | |
138 | backup_time: i64, | |
139 | ) -> Result<PathBuf, Error> { | |
140 | let mut relative_path = PathBuf::new(); | |
141 | ||
142 | relative_path.push(backup_type); | |
143 | ||
144 | relative_path.push(backup_id); | |
145 | ||
146 | let dt = Utc.timestamp(backup_time, 0); | |
147 | let date_str = dt.format("%Y-%m-%dT%H:%M:%S").to_string(); | |
148 | ||
149 | println!("date: {}", date_str); | |
150 | ||
151 | relative_path.push(&date_str); | |
152 | ||
153 | ||
154 | let mut full_path = self.base_path(); | |
155 | full_path.push(&relative_path); | |
156 | ||
157 | std::fs::create_dir_all(&full_path)?; | |
158 | ||
159 | Ok(relative_path) | |
160 | } | |
161 | ||
e25736b4 DM |
162 | pub fn list_backups(&self) -> Result<Vec<BackupInfo>, Error> { |
163 | let path = self.base_path(); | |
164 | ||
165 | let mut list = vec![]; | |
166 | ||
167 | lazy_static! { | |
784252db DM |
168 | static ref BACKUP_TYPE_REGEX: regex::Regex = regex::Regex::new(r"^(host|vm|ct)$").unwrap(); |
169 | static ref BACKUP_ID_REGEX: regex::Regex = regex::Regex::new(r"^[A-Za-z][A-Za-z0-9_-]+$").unwrap(); | |
be0084b0 | 170 | static ref BACKUP_DATE_REGEX: regex::Regex = regex::Regex::new( |
784252db | 171 | r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$").unwrap(); |
e25736b4 DM |
172 | } |
173 | ||
174 | tools::scandir(libc::AT_FDCWD, &path, &BACKUP_TYPE_REGEX, |l0_fd, backup_type, file_type| { | |
175 | if file_type != nix::dir::Type::Directory { return Ok(()); } | |
176 | tools::scandir(l0_fd, backup_type, &BACKUP_ID_REGEX, |l1_fd, backup_id, file_type| { | |
177 | if file_type != nix::dir::Type::Directory { return Ok(()); } | |
178 | tools::scandir(l1_fd, backup_id, &BACKUP_DATE_REGEX, |_, backup_time, file_type| { | |
179 | if file_type != nix::dir::Type::Directory { return Ok(()); } | |
180 | ||
181 | let dt = Utc.datetime_from_str(backup_time, "%Y-%m-%dT%H:%M:%S")?; | |
182 | ||
183 | list.push(BackupInfo { | |
184 | backup_type: backup_type.to_owned(), | |
185 | backup_id: backup_id.to_owned(), | |
7ca80246 | 186 | backup_time: dt, |
e25736b4 DM |
187 | }); |
188 | ||
189 | Ok(()) | |
190 | }) | |
191 | }) | |
192 | })?; | |
193 | ||
194 | Ok(list) | |
195 | } | |
196 | ||
3d5c11e5 | 197 | pub fn list_images(&self) -> Result<Vec<PathBuf>, Error> { |
ff3d3100 | 198 | let base = self.base_path(); |
3d5c11e5 DM |
199 | |
200 | let mut list = vec![]; | |
201 | ||
95cea65b DM |
202 | use walkdir::WalkDir; |
203 | ||
204 | let walker = WalkDir::new(&base).same_file_system(true).into_iter(); | |
205 | ||
206 | // make sure we skip .chunks (and other hidden files to keep it simple) | |
207 | fn is_hidden(entry: &walkdir::DirEntry) -> bool { | |
208 | entry.file_name() | |
209 | .to_str() | |
210 | .map(|s| s.starts_with(".")) | |
211 | .unwrap_or(false) | |
212 | } | |
213 | ||
214 | for entry in walker.filter_entry(|e| !is_hidden(e)) { | |
215 | let path = entry?.into_path(); | |
216 | if let Some(ext) = path.extension() { | |
91a905b6 | 217 | if ext == "fidx" { |
95cea65b | 218 | list.push(path); |
93d5d779 | 219 | } else if ext == "didx" { |
95cea65b | 220 | list.push(path); |
3d5c11e5 DM |
221 | } |
222 | } | |
223 | } | |
224 | ||
225 | Ok(list) | |
226 | } | |
227 | ||
64e53b28 | 228 | fn mark_used_chunks(&self, status: &mut GarbageCollectionStatus) -> Result<(), Error> { |
3d5c11e5 DM |
229 | |
230 | let image_list = self.list_images()?; | |
231 | ||
232 | for path in image_list { | |
77703d95 | 233 | if let Some(ext) = path.extension() { |
91a905b6 DM |
234 | if ext == "fidx" { |
235 | let index = self.open_fixed_reader(&path)?; | |
77703d95 | 236 | index.mark_used_chunks(status)?; |
93d5d779 DM |
237 | } else if ext == "didx" { |
238 | let index = self.open_dynamic_reader(&path)?; | |
77703d95 DM |
239 | index.mark_used_chunks(status)?; |
240 | } | |
241 | } | |
3d5c11e5 DM |
242 | } |
243 | ||
244 | Ok(()) | |
245 | } | |
246 | ||
03e4753d | 247 | pub fn garbage_collection(&self) -> Result<(), Error> { |
3d5c11e5 | 248 | |
a198d74f | 249 | if let Ok(ref mut _mutex) = self.gc_mutex.try_lock() { |
e95950e4 | 250 | |
64e53b28 DM |
251 | let mut gc_status = GarbageCollectionStatus::default(); |
252 | gc_status.used_bytes = 0; | |
6ea3a0b7 | 253 | |
64e53b28 DM |
254 | println!("Start GC phase1 (mark chunks)"); |
255 | ||
256 | self.mark_used_chunks(&mut gc_status)?; | |
257 | ||
258 | println!("Start GC phase2 (sweep unused chunks)"); | |
77703d95 | 259 | self.chunk_store.sweep_unused_chunks(&mut gc_status)?; |
64e53b28 DM |
260 | |
261 | println!("Used bytes: {}", gc_status.used_bytes); | |
262 | println!("Used chunks: {}", gc_status.used_chunks); | |
263 | println!("Disk bytes: {}", gc_status.disk_bytes); | |
264 | println!("Disk chunks: {}", gc_status.disk_chunks); | |
265 | ||
266 | } else { | |
267 | println!("Start GC failed - (already running/locked)"); | |
268 | } | |
3d5c11e5 DM |
269 | |
270 | Ok(()) | |
271 | } | |
529de6c7 | 272 | } |