]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
11fdf7f2 TL |
2 | // This source code is licensed under both the GPLv2 (found in the |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
7c673cae FG |
5 | // |
6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. | |
7 | // Use of this source code is governed by a BSD-style license that can be | |
8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. | |
f67539c2 TL |
9 | #include "file/filename.h" |
10 | #include <cinttypes> | |
7c673cae FG |
11 | |
12 | #include <ctype.h> | |
13 | #include <stdio.h> | |
14 | #include <vector> | |
f67539c2 | 15 | #include "file/writable_file_writer.h" |
7c673cae | 16 | #include "rocksdb/env.h" |
f67539c2 | 17 | #include "test_util/sync_point.h" |
7c673cae FG |
18 | #include "util/stop_watch.h" |
19 | #include "util/string_util.h" | |
7c673cae | 20 | |
f67539c2 | 21 | namespace ROCKSDB_NAMESPACE { |
7c673cae FG |
22 | |
23 | static const std::string kRocksDbTFileExt = "sst"; | |
24 | static const std::string kLevelDbTFileExt = "ldb"; | |
25 | static const std::string kRocksDBBlobFileExt = "blob"; | |
26 | ||
27 | // Given a path, flatten the path name by replacing all chars not in | |
28 | // {[0-9,a-z,A-Z,-,_,.]} with _. And append '_LOG\0' at the end. | |
29 | // Return the number of chars stored in dest not including the trailing '\0'. | |
30 | static size_t GetInfoLogPrefix(const std::string& path, char* dest, int len) { | |
31 | const char suffix[] = "_LOG"; | |
32 | ||
33 | size_t write_idx = 0; | |
34 | size_t i = 0; | |
35 | size_t src_len = path.size(); | |
36 | ||
37 | while (i < src_len && write_idx < len - sizeof(suffix)) { | |
38 | if ((path[i] >= 'a' && path[i] <= 'z') || | |
39 | (path[i] >= '0' && path[i] <= '9') || | |
40 | (path[i] >= 'A' && path[i] <= 'Z') || | |
41 | path[i] == '-' || | |
42 | path[i] == '.' || | |
43 | path[i] == '_'){ | |
44 | dest[write_idx++] = path[i]; | |
45 | } else { | |
46 | if (i > 0) { | |
47 | dest[write_idx++] = '_'; | |
48 | } | |
49 | } | |
50 | i++; | |
51 | } | |
52 | assert(sizeof(suffix) <= len - write_idx); | |
53 | // "\0" is automatically added by snprintf | |
54 | snprintf(dest + write_idx, len - write_idx, suffix); | |
55 | write_idx += sizeof(suffix) - 1; | |
56 | return write_idx; | |
57 | } | |
58 | ||
f67539c2 TL |
59 | static std::string MakeFileName(uint64_t number, const char* suffix) { |
60 | char buf[100]; | |
61 | snprintf(buf, sizeof(buf), "%06llu.%s", | |
62 | static_cast<unsigned long long>(number), suffix); | |
63 | return buf; | |
64 | } | |
65 | ||
7c673cae FG |
66 | static std::string MakeFileName(const std::string& name, uint64_t number, |
67 | const char* suffix) { | |
f67539c2 | 68 | return name + "/" + MakeFileName(number, suffix); |
7c673cae FG |
69 | } |
70 | ||
71 | std::string LogFileName(const std::string& name, uint64_t number) { | |
72 | assert(number > 0); | |
73 | return MakeFileName(name, number, "log"); | |
74 | } | |
75 | ||
f67539c2 TL |
76 | std::string LogFileName(uint64_t number) { |
77 | assert(number > 0); | |
78 | return MakeFileName(number, "log"); | |
79 | } | |
80 | ||
20effc67 TL |
81 | std::string BlobFileName(uint64_t number) { |
82 | assert(number > 0); | |
83 | return MakeFileName(number, kRocksDBBlobFileExt.c_str()); | |
84 | } | |
85 | ||
7c673cae FG |
86 | std::string BlobFileName(const std::string& blobdirname, uint64_t number) { |
87 | assert(number > 0); | |
88 | return MakeFileName(blobdirname, number, kRocksDBBlobFileExt.c_str()); | |
89 | } | |
90 | ||
11fdf7f2 TL |
91 | std::string BlobFileName(const std::string& dbname, const std::string& blob_dir, |
92 | uint64_t number) { | |
93 | assert(number > 0); | |
94 | return MakeFileName(dbname + "/" + blob_dir, number, | |
95 | kRocksDBBlobFileExt.c_str()); | |
96 | } | |
97 | ||
7c673cae FG |
98 | std::string ArchivalDirectory(const std::string& dir) { |
99 | return dir + "/" + ARCHIVAL_DIR; | |
100 | } | |
101 | std::string ArchivedLogFileName(const std::string& name, uint64_t number) { | |
102 | assert(number > 0); | |
103 | return MakeFileName(name + "/" + ARCHIVAL_DIR, number, "log"); | |
104 | } | |
105 | ||
106 | std::string MakeTableFileName(const std::string& path, uint64_t number) { | |
107 | return MakeFileName(path, number, kRocksDbTFileExt.c_str()); | |
108 | } | |
109 | ||
f67539c2 TL |
110 | std::string MakeTableFileName(uint64_t number) { |
111 | return MakeFileName(number, kRocksDbTFileExt.c_str()); | |
112 | } | |
113 | ||
7c673cae FG |
114 | std::string Rocks2LevelTableFileName(const std::string& fullname) { |
115 | assert(fullname.size() > kRocksDbTFileExt.size() + 1); | |
116 | if (fullname.size() <= kRocksDbTFileExt.size() + 1) { | |
117 | return ""; | |
118 | } | |
119 | return fullname.substr(0, fullname.size() - kRocksDbTFileExt.size()) + | |
120 | kLevelDbTFileExt; | |
121 | } | |
122 | ||
123 | uint64_t TableFileNameToNumber(const std::string& name) { | |
124 | uint64_t number = 0; | |
125 | uint64_t base = 1; | |
126 | int pos = static_cast<int>(name.find_last_of('.')); | |
127 | while (--pos >= 0 && name[pos] >= '0' && name[pos] <= '9') { | |
128 | number += (name[pos] - '0') * base; | |
129 | base *= 10; | |
130 | } | |
131 | return number; | |
132 | } | |
133 | ||
134 | std::string TableFileName(const std::vector<DbPath>& db_paths, uint64_t number, | |
135 | uint32_t path_id) { | |
136 | assert(number > 0); | |
137 | std::string path; | |
138 | if (path_id >= db_paths.size()) { | |
139 | path = db_paths.back().path; | |
140 | } else { | |
141 | path = db_paths[path_id].path; | |
142 | } | |
143 | return MakeTableFileName(path, number); | |
144 | } | |
145 | ||
146 | void FormatFileNumber(uint64_t number, uint32_t path_id, char* out_buf, | |
147 | size_t out_buf_size) { | |
148 | if (path_id == 0) { | |
149 | snprintf(out_buf, out_buf_size, "%" PRIu64, number); | |
150 | } else { | |
151 | snprintf(out_buf, out_buf_size, "%" PRIu64 | |
152 | "(path " | |
153 | "%" PRIu32 ")", | |
154 | number, path_id); | |
155 | } | |
156 | } | |
157 | ||
158 | std::string DescriptorFileName(const std::string& dbname, uint64_t number) { | |
159 | assert(number > 0); | |
160 | char buf[100]; | |
161 | snprintf(buf, sizeof(buf), "/MANIFEST-%06llu", | |
162 | static_cast<unsigned long long>(number)); | |
163 | return dbname + buf; | |
164 | } | |
165 | ||
166 | std::string CurrentFileName(const std::string& dbname) { | |
167 | return dbname + "/CURRENT"; | |
168 | } | |
169 | ||
170 | std::string LockFileName(const std::string& dbname) { | |
171 | return dbname + "/LOCK"; | |
172 | } | |
173 | ||
174 | std::string TempFileName(const std::string& dbname, uint64_t number) { | |
175 | return MakeFileName(dbname, number, kTempFileNameSuffix.c_str()); | |
176 | } | |
177 | ||
178 | InfoLogPrefix::InfoLogPrefix(bool has_log_dir, | |
179 | const std::string& db_absolute_path) { | |
180 | if (!has_log_dir) { | |
181 | const char kInfoLogPrefix[] = "LOG"; | |
182 | // "\0" is automatically added to the end | |
183 | snprintf(buf, sizeof(buf), kInfoLogPrefix); | |
184 | prefix = Slice(buf, sizeof(kInfoLogPrefix) - 1); | |
185 | } else { | |
20effc67 TL |
186 | size_t len = |
187 | GetInfoLogPrefix(NormalizePath(db_absolute_path), buf, sizeof(buf)); | |
7c673cae FG |
188 | prefix = Slice(buf, len); |
189 | } | |
190 | } | |
191 | ||
192 | std::string InfoLogFileName(const std::string& dbname, | |
193 | const std::string& db_path, const std::string& log_dir) { | |
194 | if (log_dir.empty()) { | |
195 | return dbname + "/LOG"; | |
196 | } | |
197 | ||
198 | InfoLogPrefix info_log_prefix(true, db_path); | |
199 | return log_dir + "/" + info_log_prefix.buf; | |
200 | } | |
201 | ||
202 | // Return the name of the old info log file for "dbname". | |
203 | std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts, | |
204 | const std::string& db_path, const std::string& log_dir) { | |
205 | char buf[50]; | |
206 | snprintf(buf, sizeof(buf), "%llu", static_cast<unsigned long long>(ts)); | |
207 | ||
208 | if (log_dir.empty()) { | |
209 | return dbname + "/LOG.old." + buf; | |
210 | } | |
211 | ||
212 | InfoLogPrefix info_log_prefix(true, db_path); | |
213 | return log_dir + "/" + info_log_prefix.buf + ".old." + buf; | |
214 | } | |
215 | ||
216 | std::string OptionsFileName(const std::string& dbname, uint64_t file_num) { | |
217 | char buffer[256]; | |
218 | snprintf(buffer, sizeof(buffer), "%s%06" PRIu64, | |
219 | kOptionsFileNamePrefix.c_str(), file_num); | |
220 | return dbname + "/" + buffer; | |
221 | } | |
222 | ||
223 | std::string TempOptionsFileName(const std::string& dbname, uint64_t file_num) { | |
224 | char buffer[256]; | |
225 | snprintf(buffer, sizeof(buffer), "%s%06" PRIu64 ".%s", | |
226 | kOptionsFileNamePrefix.c_str(), file_num, | |
227 | kTempFileNameSuffix.c_str()); | |
228 | return dbname + "/" + buffer; | |
229 | } | |
230 | ||
231 | std::string MetaDatabaseName(const std::string& dbname, uint64_t number) { | |
232 | char buf[100]; | |
233 | snprintf(buf, sizeof(buf), "/METADB-%llu", | |
234 | static_cast<unsigned long long>(number)); | |
235 | return dbname + buf; | |
236 | } | |
237 | ||
238 | std::string IdentityFileName(const std::string& dbname) { | |
239 | return dbname + "/IDENTITY"; | |
240 | } | |
241 | ||
242 | // Owned filenames have the form: | |
243 | // dbname/IDENTITY | |
244 | // dbname/CURRENT | |
245 | // dbname/LOCK | |
246 | // dbname/<info_log_name_prefix> | |
247 | // dbname/<info_log_name_prefix>.old.[0-9]+ | |
248 | // dbname/MANIFEST-[0-9]+ | |
249 | // dbname/[0-9]+.(log|sst|blob) | |
250 | // dbname/METADB-[0-9]+ | |
251 | // dbname/OPTIONS-[0-9]+ | |
252 | // dbname/OPTIONS-[0-9]+.dbtmp | |
253 | // Disregards / at the beginning | |
254 | bool ParseFileName(const std::string& fname, | |
255 | uint64_t* number, | |
256 | FileType* type, | |
257 | WalFileType* log_type) { | |
258 | return ParseFileName(fname, number, "", type, log_type); | |
259 | } | |
260 | ||
261 | bool ParseFileName(const std::string& fname, uint64_t* number, | |
262 | const Slice& info_log_name_prefix, FileType* type, | |
263 | WalFileType* log_type) { | |
264 | Slice rest(fname); | |
265 | if (fname.length() > 1 && fname[0] == '/') { | |
266 | rest.remove_prefix(1); | |
267 | } | |
268 | if (rest == "IDENTITY") { | |
269 | *number = 0; | |
270 | *type = kIdentityFile; | |
271 | } else if (rest == "CURRENT") { | |
272 | *number = 0; | |
273 | *type = kCurrentFile; | |
274 | } else if (rest == "LOCK") { | |
275 | *number = 0; | |
276 | *type = kDBLockFile; | |
277 | } else if (info_log_name_prefix.size() > 0 && | |
278 | rest.starts_with(info_log_name_prefix)) { | |
279 | rest.remove_prefix(info_log_name_prefix.size()); | |
280 | if (rest == "" || rest == ".old") { | |
281 | *number = 0; | |
282 | *type = kInfoLogFile; | |
283 | } else if (rest.starts_with(".old.")) { | |
284 | uint64_t ts_suffix; | |
285 | // sizeof also counts the trailing '\0'. | |
286 | rest.remove_prefix(sizeof(".old.") - 1); | |
287 | if (!ConsumeDecimalNumber(&rest, &ts_suffix)) { | |
288 | return false; | |
289 | } | |
290 | *number = ts_suffix; | |
291 | *type = kInfoLogFile; | |
292 | } | |
293 | } else if (rest.starts_with("MANIFEST-")) { | |
294 | rest.remove_prefix(strlen("MANIFEST-")); | |
295 | uint64_t num; | |
296 | if (!ConsumeDecimalNumber(&rest, &num)) { | |
297 | return false; | |
298 | } | |
299 | if (!rest.empty()) { | |
300 | return false; | |
301 | } | |
302 | *type = kDescriptorFile; | |
303 | *number = num; | |
304 | } else if (rest.starts_with("METADB-")) { | |
305 | rest.remove_prefix(strlen("METADB-")); | |
306 | uint64_t num; | |
307 | if (!ConsumeDecimalNumber(&rest, &num)) { | |
308 | return false; | |
309 | } | |
310 | if (!rest.empty()) { | |
311 | return false; | |
312 | } | |
313 | *type = kMetaDatabase; | |
314 | *number = num; | |
315 | } else if (rest.starts_with(kOptionsFileNamePrefix)) { | |
316 | uint64_t ts_suffix; | |
317 | bool is_temp_file = false; | |
318 | rest.remove_prefix(kOptionsFileNamePrefix.size()); | |
319 | const std::string kTempFileNameSuffixWithDot = | |
320 | std::string(".") + kTempFileNameSuffix; | |
321 | if (rest.ends_with(kTempFileNameSuffixWithDot)) { | |
322 | rest.remove_suffix(kTempFileNameSuffixWithDot.size()); | |
323 | is_temp_file = true; | |
324 | } | |
325 | if (!ConsumeDecimalNumber(&rest, &ts_suffix)) { | |
326 | return false; | |
327 | } | |
328 | *number = ts_suffix; | |
329 | *type = is_temp_file ? kTempFile : kOptionsFile; | |
330 | } else { | |
331 | // Avoid strtoull() to keep filename format independent of the | |
332 | // current locale | |
333 | bool archive_dir_found = false; | |
334 | if (rest.starts_with(ARCHIVAL_DIR)) { | |
335 | if (rest.size() <= ARCHIVAL_DIR.size()) { | |
336 | return false; | |
337 | } | |
338 | rest.remove_prefix(ARCHIVAL_DIR.size() + 1); // Add 1 to remove / also | |
339 | if (log_type) { | |
340 | *log_type = kArchivedLogFile; | |
341 | } | |
342 | archive_dir_found = true; | |
343 | } | |
344 | uint64_t num; | |
345 | if (!ConsumeDecimalNumber(&rest, &num)) { | |
346 | return false; | |
347 | } | |
348 | if (rest.size() <= 1 || rest[0] != '.') { | |
349 | return false; | |
350 | } | |
351 | rest.remove_prefix(1); | |
352 | ||
353 | Slice suffix = rest; | |
354 | if (suffix == Slice("log")) { | |
20effc67 | 355 | *type = kWalFile; |
7c673cae FG |
356 | if (log_type && !archive_dir_found) { |
357 | *log_type = kAliveLogFile; | |
358 | } | |
359 | } else if (archive_dir_found) { | |
360 | return false; // Archive dir can contain only log files | |
361 | } else if (suffix == Slice(kRocksDbTFileExt) || | |
362 | suffix == Slice(kLevelDbTFileExt)) { | |
363 | *type = kTableFile; | |
364 | } else if (suffix == Slice(kRocksDBBlobFileExt)) { | |
365 | *type = kBlobFile; | |
366 | } else if (suffix == Slice(kTempFileNameSuffix)) { | |
367 | *type = kTempFile; | |
368 | } else { | |
369 | return false; | |
370 | } | |
371 | *number = num; | |
372 | } | |
373 | return true; | |
374 | } | |
375 | ||
20effc67 TL |
376 | IOStatus SetCurrentFile(FileSystem* fs, const std::string& dbname, |
377 | uint64_t descriptor_number, | |
378 | FSDirectory* directory_to_fsync) { | |
7c673cae FG |
379 | // Remove leading "dbname/" and add newline to manifest file name |
380 | std::string manifest = DescriptorFileName(dbname, descriptor_number); | |
381 | Slice contents = manifest; | |
382 | assert(contents.starts_with(dbname + "/")); | |
383 | contents.remove_prefix(dbname.size() + 1); | |
384 | std::string tmp = TempFileName(dbname, descriptor_number); | |
20effc67 | 385 | IOStatus s = WriteStringToFile(fs, contents.ToString() + "\n", tmp, true); |
7c673cae FG |
386 | if (s.ok()) { |
387 | TEST_KILL_RANDOM("SetCurrentFile:0", rocksdb_kill_odds * REDUCE_ODDS2); | |
20effc67 | 388 | s = fs->RenameFile(tmp, CurrentFileName(dbname), IOOptions(), nullptr); |
7c673cae FG |
389 | TEST_KILL_RANDOM("SetCurrentFile:1", rocksdb_kill_odds * REDUCE_ODDS2); |
390 | } | |
391 | if (s.ok()) { | |
392 | if (directory_to_fsync != nullptr) { | |
20effc67 | 393 | s = directory_to_fsync->Fsync(IOOptions(), nullptr); |
7c673cae FG |
394 | } |
395 | } else { | |
20effc67 | 396 | fs->DeleteFile(tmp, IOOptions(), nullptr); |
7c673cae FG |
397 | } |
398 | return s; | |
399 | } | |
400 | ||
f67539c2 TL |
401 | Status SetIdentityFile(Env* env, const std::string& dbname, |
402 | const std::string& db_id) { | |
403 | std::string id; | |
404 | if (db_id.empty()) { | |
405 | id = env->GenerateUniqueId(); | |
406 | } else { | |
407 | id = db_id; | |
408 | } | |
7c673cae FG |
409 | assert(!id.empty()); |
410 | // Reserve the filename dbname/000000.dbtmp for the temporary identity file | |
411 | std::string tmp = TempFileName(dbname, 0); | |
412 | Status s = WriteStringToFile(env, id, tmp, true); | |
413 | if (s.ok()) { | |
414 | s = env->RenameFile(tmp, IdentityFileName(dbname)); | |
415 | } | |
416 | if (!s.ok()) { | |
20effc67 | 417 | env->DeleteFile(tmp).PermitUncheckedError(); |
7c673cae FG |
418 | } |
419 | return s; | |
420 | } | |
421 | ||
20effc67 TL |
422 | IOStatus SyncManifest(Env* env, const ImmutableDBOptions* db_options, |
423 | WritableFileWriter* file) { | |
7c673cae FG |
424 | TEST_KILL_RANDOM("SyncManifest:0", rocksdb_kill_odds * REDUCE_ODDS2); |
425 | StopWatch sw(env, db_options->statistics.get(), MANIFEST_FILE_SYNC_MICROS); | |
426 | return file->Sync(db_options->use_fsync); | |
427 | } | |
428 | ||
f67539c2 TL |
429 | Status GetInfoLogFiles(Env* env, const std::string& db_log_dir, |
430 | const std::string& dbname, std::string* parent_dir, | |
431 | std::vector<std::string>* info_log_list) { | |
432 | assert(parent_dir != nullptr); | |
433 | assert(info_log_list != nullptr); | |
434 | uint64_t number = 0; | |
20effc67 | 435 | FileType type = kWalFile; |
f67539c2 TL |
436 | |
437 | if (!db_log_dir.empty()) { | |
438 | *parent_dir = db_log_dir; | |
439 | } else { | |
440 | *parent_dir = dbname; | |
441 | } | |
442 | ||
443 | InfoLogPrefix info_log_prefix(!db_log_dir.empty(), dbname); | |
444 | ||
445 | std::vector<std::string> file_names; | |
446 | Status s = env->GetChildren(*parent_dir, &file_names); | |
447 | ||
448 | if (!s.ok()) { | |
449 | return s; | |
450 | } | |
451 | ||
452 | for (auto& f : file_names) { | |
453 | if (ParseFileName(f, &number, info_log_prefix.prefix, &type) && | |
454 | (type == kInfoLogFile)) { | |
455 | info_log_list->push_back(f); | |
456 | } | |
457 | } | |
458 | return Status::OK(); | |
459 | } | |
460 | ||
20effc67 TL |
461 | std::string NormalizePath(const std::string& path) { |
462 | std::string dst; | |
463 | for (auto c : path) { | |
464 | if (!dst.empty() && (c == kFilePathSeparator || c == '/') && | |
465 | (dst.back() == kFilePathSeparator || dst.back() == '/')) { | |
466 | continue; | |
467 | } | |
468 | dst.push_back(c); | |
469 | } | |
470 | return dst; | |
471 | } | |
472 | ||
f67539c2 | 473 | } // namespace ROCKSDB_NAMESPACE |