]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | /* | |
4 | * Ceph - scalable distributed file system | |
5 | * | |
6 | * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net> | |
7 | * | |
8 | * This is free software; you can redistribute it and/or | |
9 | * modify it under the terms of the GNU Lesser General Public | |
10 | * License version 2.1, as published by the Free Software | |
11 | * Foundation. See file COPYING. | |
12 | * | |
13 | */ | |
14 | ||
15 | ||
16 | #ifndef OS_LFNINDEX_H | |
17 | #define OS_LFNINDEX_H | |
18 | ||
19 | #include <string> | |
20 | #include <map> | |
21 | #include <set> | |
22 | #include <vector> | |
23 | #include "include/memory.h" | |
24 | #include <exception> | |
25 | ||
26 | #include "osd/osd_types.h" | |
27 | #include "include/object.h" | |
28 | #include "common/ceph_crypto.h" | |
29 | ||
30 | #include "CollectionIndex.h" | |
31 | ||
32 | /** | |
33 | * LFNIndex also encapsulates logic for manipulating | |
34 | * subdirectories of a collection as well as the long filename | |
35 | * logic. | |
36 | * | |
37 | * The protected methods provide machinery for derived classes to | |
38 | * manipulate subdirectories and objects. | |
39 | * | |
40 | * The virtual methods are to be overridden to provide the actual | |
41 | * hashed layout. | |
42 | * | |
43 | * User must call created when an object is created. | |
44 | * | |
45 | * Syncronization: Calling code must ensure that there are no object | |
46 | * creations or deletions during the lifetime of a Path object (except | |
47 | * of an object at that path). | |
48 | * | |
49 | * Unless otherwise noted, methods which return an int return 0 on sucess | |
50 | * and a negative error code on failure. | |
51 | */ | |
52 | #define WRAP_RETRY(x) { \ | |
53 | bool failed = false; \ | |
54 | int r = 0; \ | |
55 | init_inject_failure(); \ | |
56 | while (1) { \ | |
57 | try { \ | |
58 | if (failed) { \ | |
59 | r = cleanup(); \ | |
60 | assert(r == 0); \ | |
61 | } \ | |
62 | { x } \ | |
63 | out: \ | |
64 | complete_inject_failure(); \ | |
65 | return r; \ | |
66 | } catch (RetryException) { \ | |
67 | failed = true; \ | |
68 | } catch (...) { \ | |
69 | ceph_abort(); \ | |
70 | } \ | |
71 | } \ | |
72 | return -1; \ | |
73 | } \ | |
74 | ||
75 | ||
76 | ||
77 | class LFNIndex : public CollectionIndex { | |
78 | /// Hash digest output size. | |
79 | static const int FILENAME_LFN_DIGEST_SIZE = CEPH_CRYPTO_SHA1_DIGESTSIZE; | |
80 | /// Length of filename hash. | |
81 | static const int FILENAME_HASH_LEN = FILENAME_LFN_DIGEST_SIZE; | |
82 | /// Max filename size. | |
83 | static const int FILENAME_MAX_LEN = 4096; | |
84 | /// Length of hashed filename. | |
85 | static const int FILENAME_SHORT_LEN = 255; | |
86 | /// Length of hashed filename prefix. | |
87 | static const int FILENAME_PREFIX_LEN; | |
88 | /// Length of hashed filename cookie. | |
89 | static const int FILENAME_EXTRA = 4; | |
90 | /// Lfn cookie value. | |
91 | static const string FILENAME_COOKIE; | |
92 | /// Name of LFN attribute for storing full name. | |
93 | static const string LFN_ATTR; | |
94 | /// Prefix for subdir index attributes. | |
95 | static const string PHASH_ATTR_PREFIX; | |
96 | /// Prefix for index subdirectories. | |
97 | static const string SUBDIR_PREFIX; | |
98 | ||
99 | /// Path to Index base. | |
100 | const string base_path; | |
101 | ||
102 | protected: | |
103 | const uint32_t index_version; | |
104 | ||
105 | /// true if retry injection is enabled | |
106 | struct RetryException : public exception {}; | |
107 | bool error_injection_enabled; | |
108 | bool error_injection_on; | |
109 | double error_injection_probability; | |
110 | uint64_t last_failure; | |
111 | uint64_t current_failure; | |
112 | void init_inject_failure() { | |
113 | if (error_injection_on) { | |
114 | error_injection_enabled = true; | |
115 | last_failure = current_failure = 0; | |
116 | } | |
117 | } | |
118 | void maybe_inject_failure(); | |
119 | void complete_inject_failure() { | |
120 | error_injection_enabled = false; | |
121 | } | |
122 | ||
123 | private: | |
124 | string lfn_attribute, lfn_alt_attribute; | |
125 | coll_t collection; | |
126 | ||
127 | public: | |
128 | /// Constructor | |
129 | LFNIndex( | |
130 | CephContext* cct, | |
131 | coll_t collection, | |
132 | const char *base_path, ///< [in] path to Index root | |
133 | uint32_t index_version, | |
134 | double _error_injection_probability=0) | |
135 | : CollectionIndex(cct, collection), | |
136 | base_path(base_path), | |
137 | index_version(index_version), | |
138 | error_injection_enabled(false), | |
139 | error_injection_on(_error_injection_probability != 0), | |
140 | error_injection_probability(_error_injection_probability), | |
141 | last_failure(0), current_failure(0), | |
142 | collection(collection) { | |
143 | if (index_version == HASH_INDEX_TAG) { | |
144 | lfn_attribute = LFN_ATTR; | |
145 | } else { | |
146 | char buf[100]; | |
147 | snprintf(buf, sizeof(buf), "%d", index_version); | |
148 | lfn_attribute = LFN_ATTR + string(buf); | |
149 | lfn_alt_attribute = LFN_ATTR + string(buf) + "-alt"; | |
150 | } | |
151 | } | |
152 | ||
153 | coll_t coll() const override { return collection; } | |
154 | ||
155 | /// Virtual destructor | |
156 | ~LFNIndex() override {} | |
157 | ||
158 | /// @see CollectionIndex | |
159 | int init() override; | |
160 | ||
161 | /// @see CollectionIndex | |
162 | int cleanup() override = 0; | |
163 | ||
164 | /// @see CollectionIndex | |
165 | int created( | |
166 | const ghobject_t &oid, | |
167 | const char *path | |
168 | ) override; | |
169 | ||
170 | /// @see CollectionIndex | |
171 | int unlink( | |
172 | const ghobject_t &oid | |
173 | ) override; | |
174 | ||
175 | /// @see CollectionIndex | |
176 | int lookup( | |
177 | const ghobject_t &oid, | |
178 | IndexedPath *path, | |
179 | int *hardlink | |
180 | ) override; | |
181 | ||
182 | /// @see CollectionIndex; | |
183 | int pre_hash_collection( | |
184 | uint32_t pg_num, | |
185 | uint64_t expected_num_objs | |
186 | ) override; | |
187 | ||
188 | /// @see CollectionIndex | |
189 | int collection_list_partial( | |
190 | const ghobject_t &start, | |
191 | const ghobject_t &end, | |
192 | int max_count, | |
193 | vector<ghobject_t> *ls, | |
194 | ghobject_t *next | |
195 | ) override; | |
196 | ||
197 | virtual int _split( | |
198 | uint32_t match, //< [in] value to match | |
199 | uint32_t bits, //< [in] bits to check | |
200 | CollectionIndex* dest //< [in] destination index | |
201 | ) = 0; | |
202 | ||
203 | /// @see CollectionIndex | |
204 | int split( | |
205 | uint32_t match, | |
206 | uint32_t bits, | |
207 | CollectionIndex* dest | |
208 | ) override { | |
209 | WRAP_RETRY( | |
210 | r = _split(match, bits, dest); | |
211 | goto out; | |
212 | ); | |
213 | } | |
214 | ||
215 | /** | |
216 | * Returns the length of the longest escaped name which could result | |
217 | * from any clone, shard, or rollback object of this object | |
218 | */ | |
219 | static uint64_t get_max_escaped_name_len(const hobject_t &obj); | |
220 | ||
221 | protected: | |
222 | virtual int _init() = 0; | |
223 | ||
224 | /// Will be called upon object creation | |
225 | virtual int _created( | |
226 | const vector<string> &path, ///< [in] Path to subdir. | |
227 | const ghobject_t &oid, ///< [in] Object created. | |
228 | const string &mangled_name ///< [in] Mangled filename. | |
229 | ) = 0; | |
230 | ||
231 | /// Will be called to remove an object | |
232 | virtual int _remove( | |
233 | const vector<string> &path, ///< [in] Path to subdir. | |
234 | const ghobject_t &oid, ///< [in] Object to remove. | |
235 | const string &mangled_name ///< [in] Mangled filename. | |
236 | ) = 0; | |
237 | ||
238 | /// Return the path and mangled_name for oid. | |
239 | virtual int _lookup( | |
240 | const ghobject_t &oid,///< [in] Object for lookup. | |
241 | vector<string> *path, ///< [out] Path to the object. | |
242 | string *mangled_name, ///< [out] Mangled filename. | |
243 | int *exists ///< [out] True if the object exists. | |
244 | ) = 0; | |
245 | ||
246 | /// Pre-hash the collection with the given pg number and | |
247 | /// expected number of objects in the collection. | |
248 | virtual int _pre_hash_collection( | |
249 | uint32_t pg_num, | |
250 | uint64_t expected_num_objs | |
251 | ) = 0; | |
252 | ||
253 | /// @see CollectionIndex | |
254 | virtual int _collection_list_partial( | |
255 | const ghobject_t &start, | |
256 | const ghobject_t &end, | |
257 | int max_count, | |
258 | vector<ghobject_t> *ls, | |
259 | ghobject_t *next | |
260 | ) = 0; | |
261 | ||
262 | protected: | |
263 | ||
264 | /* Non-virtual utility methods */ | |
265 | ||
266 | /// Sync a subdirectory | |
267 | int fsync_dir( | |
268 | const vector<string> &path ///< [in] Path to sync | |
269 | ); ///< @return Error Code, 0 on success | |
270 | ||
271 | /// Link an object from from into to | |
272 | int link_object( | |
273 | const vector<string> &from, ///< [in] Source subdirectory. | |
274 | const vector<string> &to, ///< [in] Dest subdirectory. | |
275 | const ghobject_t &oid, ///< [in] Object to move. | |
276 | const string &from_short_name ///< [in] Mangled filename of oid. | |
277 | ); ///< @return Error Code, 0 on success | |
278 | ||
279 | /** | |
280 | * Efficiently remove objects from a subdirectory | |
281 | * | |
282 | * remove_object invalidates mangled names in the directory requiring | |
283 | * the mangled name of each additional object to be looked up a second | |
284 | * time. remove_objects removes the need for additional lookups | |
285 | * | |
286 | * @param [in] dir Directory from which to remove. | |
287 | * @param [in] map of objects to remove to mangle names | |
288 | * @param [in,out] map of filenames to objects | |
289 | * @return Error Code, 0 on success. | |
290 | */ | |
291 | int remove_objects( | |
292 | const vector<string> &dir, | |
293 | const map<string, ghobject_t> &to_remove, | |
294 | map<string, ghobject_t> *remaining | |
295 | ); | |
296 | ||
297 | ||
298 | /** | |
299 | * Moves contents of from into to. | |
300 | * | |
301 | * Invalidates mangled names in to. If interupted, all objects will be | |
302 | * present in to before objects are removed from from. Ignores EEXIST | |
303 | * while linking into to. | |
304 | * @return Error Code, 0 on success | |
305 | */ | |
306 | int move_objects( | |
307 | const vector<string> &from, ///< [in] Source subdirectory. | |
308 | const vector<string> &to ///< [in] Dest subdirectory. | |
309 | ); | |
310 | ||
311 | /** | |
312 | * Remove an object from from. | |
313 | * | |
314 | * Invalidates mangled names in from. | |
315 | * @return Error Code, 0 on success | |
316 | */ | |
317 | int remove_object( | |
318 | const vector<string> &from, ///< [in] Directory from which to remove. | |
319 | const ghobject_t &to_remove ///< [in] Object to remove. | |
320 | ); | |
321 | ||
322 | /** | |
323 | * Gets the filename corresponding to oid in from. | |
324 | * | |
325 | * The filename may differ between subdirectories. Furthermore, | |
326 | * file creations ore removals in from may invalidate the name. | |
327 | * @return Error code on failure, 0 on success | |
328 | */ | |
329 | int get_mangled_name( | |
330 | const vector<string> &from, ///< [in] Subdirectory | |
331 | const ghobject_t &oid, ///< [in] Object | |
332 | string *mangled_name, ///< [out] Filename | |
333 | int *hardlink ///< [out] hardlink for this file, hardlink=0 mean no-exist | |
334 | ); | |
335 | ||
336 | /// do move subdir from from to dest | |
337 | static int move_subdir( | |
338 | LFNIndex &from, ///< [in] from index | |
339 | LFNIndex &dest, ///< [in] to index | |
340 | const vector<string> &path, ///< [in] path containing dir | |
341 | string dir ///< [in] dir to move | |
342 | ); | |
343 | ||
344 | /// do move object from from to dest | |
345 | static int move_object( | |
346 | LFNIndex &from, ///< [in] from index | |
347 | LFNIndex &dest, ///< [in] to index | |
348 | const vector<string> &path, ///< [in] path to split | |
349 | const pair<string, ghobject_t> &obj ///< [in] obj to move | |
350 | ); | |
351 | ||
352 | /** | |
353 | * Lists objects in to_list. | |
354 | * | |
355 | * @param [in] to_list Directory to list. | |
356 | * @param [in] max_objects Max number to list. | |
357 | * @param [in,out] handle Cookie for continuing the listing. | |
358 | * Initialize to zero to start at the beginning of the directory. | |
359 | * @param [out] out Mapping of listed object filenames to objects. | |
360 | * @return Error code on failure, 0 on success | |
361 | */ | |
362 | int list_objects( | |
363 | const vector<string> &to_list, | |
364 | int max_objects, | |
365 | long *handle, | |
366 | map<string, ghobject_t> *out | |
367 | ); | |
368 | ||
369 | /// Lists subdirectories. | |
370 | int list_subdirs( | |
371 | const vector<string> &to_list, ///< [in] Directory to list. | |
372 | vector<string> *out ///< [out] Subdirectories listed. | |
373 | ); | |
374 | ||
375 | /// Create subdirectory. | |
376 | int create_path( | |
377 | const vector<string> &to_create ///< [in] Subdirectory to create. | |
378 | ); | |
379 | ||
380 | /// Remove subdirectory. | |
381 | int remove_path( | |
382 | const vector<string> &to_remove ///< [in] Subdirectory to remove. | |
383 | ); | |
384 | ||
385 | /// Check whether to_check exists. | |
386 | int path_exists( | |
387 | const vector<string> &to_check, ///< [in] Subdirectory to check. | |
388 | int *exists ///< [out] 1 if it exists, 0 else | |
389 | ); | |
390 | ||
391 | /// Save attr_value to attr_name attribute on path. | |
392 | int add_attr_path( | |
393 | const vector<string> &path, ///< [in] Path to modify. | |
394 | const string &attr_name, ///< [in] Name of attribute. | |
395 | bufferlist &attr_value ///< [in] Value to save. | |
396 | ); | |
397 | ||
398 | /// Read into attr_value atribute attr_name on path. | |
399 | int get_attr_path( | |
400 | const vector<string> &path, ///< [in] Path to read. | |
401 | const string &attr_name, ///< [in] Attribute to read. | |
402 | bufferlist &attr_value ///< [out] Attribute value read. | |
403 | ); | |
404 | ||
405 | /// Remove attr from path | |
406 | int remove_attr_path( | |
407 | const vector<string> &path, ///< [in] path from which to remove attr | |
408 | const string &attr_name ///< [in] attr to remove | |
409 | ); ///< @return Error code, 0 on success | |
410 | ||
411 | private: | |
412 | /* lfn translation functions */ | |
413 | ||
414 | /** | |
415 | * Gets the version specific lfn attribute tag | |
416 | */ | |
417 | const string &get_lfn_attr() const { | |
418 | return lfn_attribute; | |
419 | } | |
420 | const string &get_alt_lfn_attr() const { | |
421 | return lfn_alt_attribute; | |
422 | } | |
423 | ||
424 | /** | |
425 | * Gets the filename corresponsing to oid in path. | |
426 | * | |
427 | * @param [in] path Path in which to get filename for oid. | |
428 | * @param [in] oid Object for which to get filename. | |
429 | * @param [out] mangled_name Filename for oid, pass NULL if not needed. | |
430 | * @param [out] full_path Fullpath for oid, pass NULL if not needed. | |
431 | * @param [out] hardlink of this file, 0 mean no-exist, pass NULL if | |
432 | * not needed | |
433 | * @return Error Code, 0 on success. | |
434 | */ | |
435 | int lfn_get_name( | |
436 | const vector<string> &path, | |
437 | const ghobject_t &oid, | |
438 | string *mangled_name, | |
439 | string *full_path, | |
440 | int *hardlink | |
441 | ); | |
442 | ||
443 | /// Adjusts path contents when oid is created at name mangled_name. | |
444 | int lfn_created( | |
445 | const vector<string> &path, ///< [in] Path to adjust. | |
446 | const ghobject_t &oid, ///< [in] Object created. | |
447 | const string &mangled_name ///< [in] Filename of created object. | |
448 | ); | |
449 | ||
450 | /// Removes oid from path while adjusting path contents | |
451 | int lfn_unlink( | |
452 | const vector<string> &path, ///< [in] Path containing oid. | |
453 | const ghobject_t &oid, ///< [in] Object to remove. | |
454 | const string &mangled_name ///< [in] Filename of object to remove. | |
455 | ); | |
456 | ||
457 | ///Transate a file into and ghobject_t. | |
458 | int lfn_translate( | |
459 | const vector<string> &path, ///< [in] Path containing the file. | |
460 | const string &short_name, ///< [in] Filename to translate. | |
461 | ghobject_t *out ///< [out] Object found. | |
462 | ); ///< @return Negative error code on error, 0 if not an object, 1 else | |
463 | ||
464 | /* manglers/demanglers */ | |
465 | /// Filters object filenames | |
466 | bool lfn_is_object( | |
467 | const string &short_name ///< [in] Filename to check | |
468 | ); ///< True if short_name is an object, false otherwise | |
469 | ||
470 | /// Filters subdir filenames | |
471 | bool lfn_is_subdir( | |
472 | const string &short_name, ///< [in] Filename to check. | |
473 | string *demangled_name ///< [out] Demangled subdir name. | |
474 | ); ///< @return True if short_name is a subdir, false otherwise | |
475 | ||
476 | /// Generate object name | |
477 | string lfn_generate_object_name_keyless( | |
478 | const ghobject_t &oid ///< [in] Object for which to generate. | |
479 | ); ///< @return Generated object name. | |
480 | ||
481 | /// Generate object name | |
482 | string lfn_generate_object_name_poolless( | |
483 | const ghobject_t &oid ///< [in] Object for which to generate. | |
484 | ); ///< @return Generated object name. | |
485 | ||
486 | /// Generate object name | |
487 | static string lfn_generate_object_name_current( | |
488 | const ghobject_t &oid ///< [in] Object for which to generate. | |
489 | ); ///< @return Generated object name. | |
490 | ||
491 | /// Generate object name | |
492 | string lfn_generate_object_name( | |
493 | const ghobject_t &oid ///< [in] Object for which to generate. | |
494 | ) { | |
495 | if (index_version == HASH_INDEX_TAG) | |
496 | return lfn_generate_object_name_keyless(oid); | |
497 | if (index_version == HASH_INDEX_TAG_2) | |
498 | return lfn_generate_object_name_poolless(oid); | |
499 | else | |
500 | return lfn_generate_object_name_current(oid); | |
501 | } ///< @return Generated object name. | |
502 | ||
503 | /// Parse object name | |
504 | int lfn_parse_object_name_keyless( | |
505 | const string &long_name, ///< [in] Name to parse | |
506 | ghobject_t *out ///< [out] Resulting Object | |
507 | ); ///< @return True if successfull, False otherwise. | |
508 | ||
509 | /// Parse object name | |
510 | int lfn_parse_object_name_poolless( | |
511 | const string &long_name, ///< [in] Name to parse | |
512 | ghobject_t *out ///< [out] Resulting Object | |
513 | ); ///< @return True if successfull, False otherwise. | |
514 | ||
515 | /// Parse object name | |
516 | int lfn_parse_object_name( | |
517 | const string &long_name, ///< [in] Name to parse | |
518 | ghobject_t *out ///< [out] Resulting Object | |
519 | ); ///< @return True if successfull, False otherwise. | |
520 | ||
521 | /// Checks whether short_name is a hashed filename. | |
522 | bool lfn_is_hashed_filename( | |
523 | const string &short_name ///< [in] Name to check. | |
524 | ); ///< @return True if short_name is hashed, False otherwise. | |
525 | ||
526 | /// Checks whether long_name must be hashed. | |
527 | bool lfn_must_hash( | |
528 | const string &long_name ///< [in] Name to check. | |
529 | ); ///< @return True if long_name must be hashed, False otherwise. | |
530 | ||
531 | /// Generate hashed name. | |
532 | string lfn_get_short_name( | |
533 | const ghobject_t &oid, ///< [in] Object for which to generate. | |
534 | int i ///< [in] Index of hashed name to generate. | |
535 | ); ///< @return Hashed filename. | |
536 | ||
537 | /* other common methods */ | |
538 | /// Gets the base path | |
539 | const string &get_base_path(); ///< @return Index base_path | |
540 | ||
541 | /// Get full path the subdir | |
542 | string get_full_path_subdir( | |
543 | const vector<string> &rel ///< [in] The subdir. | |
544 | ); ///< @return Full path to rel. | |
545 | ||
546 | /// Get full path to object | |
547 | string get_full_path( | |
548 | const vector<string> &rel, ///< [in] Path to object. | |
549 | const string &name ///< [in] Filename of object. | |
550 | ); ///< @return Fullpath to object at name in rel. | |
551 | ||
552 | /// Get mangled path component | |
553 | string mangle_path_component( | |
554 | const string &component ///< [in] Component to mangle | |
555 | ); /// @return Mangled component | |
556 | ||
557 | /// Demangle component | |
558 | string demangle_path_component( | |
559 | const string &component ///< [in] Subdir name to demangle | |
560 | ); ///< @return Demangled path component. | |
561 | ||
562 | /// Decompose full path into object name and filename. | |
563 | int decompose_full_path( | |
564 | const char *in, ///< [in] Full path to object. | |
565 | vector<string> *out, ///< [out] Path to object at in. | |
566 | ghobject_t *oid, ///< [out] Object at in. | |
567 | string *shortname ///< [out] Filename of object at in. | |
568 | ); ///< @return Error Code, 0 on success. | |
569 | ||
570 | /// Mangle attribute name | |
571 | string mangle_attr_name( | |
572 | const string &attr ///< [in] Attribute to mangle. | |
573 | ); ///< @return Mangled attribute name. | |
574 | ||
575 | /// checks whether long_name could hash to short_name | |
576 | bool short_name_matches( | |
577 | const char *short_name, ///< [in] name to check against | |
578 | const char *cand_long_name ///< [in] candidate long name | |
579 | ); | |
580 | ||
581 | /// Builds hashed filename | |
582 | void build_filename( | |
583 | const char *old_filename, ///< [in] Filename to convert. | |
584 | int i, ///< [in] Index of hash. | |
585 | char *filename, ///< [out] Resulting filename. | |
586 | int len ///< [in] Size of buffer for filename | |
587 | ); ///< @return Error Code, 0 on success | |
588 | ||
589 | /// Get hash of filename | |
590 | int hash_filename( | |
591 | const char *filename, ///< [in] Filename to hash. | |
592 | char *hash, ///< [out] Hash of filename. | |
593 | int len ///< [in] Size of hash buffer. | |
594 | ); ///< @return Error Code, 0 on success. | |
595 | ||
596 | friend class TestWrapLFNIndex; | |
597 | }; | |
598 | typedef LFNIndex::IndexedPath IndexedPath; | |
599 | ||
600 | #endif |