1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
23 #include "include/memory.h"
26 #include "osd/osd_types.h"
27 #include "include/object.h"
28 #include "common/ceph_crypto.h"
30 #include "CollectionIndex.h"
33 * LFNIndex also encapsulates logic for manipulating
34 * subdirectories of a collection as well as the long filename
37 * The protected methods provide machinery for derived classes to
38 * manipulate subdirectories and objects.
40 * The virtual methods are to be overridden to provide the actual
43 * User must call created when an object is created.
45 * Syncronization: Calling code must ensure that there are no object
46 * creations or deletions during the lifetime of a Path object (except
47 * of an object at that path).
49 * Unless otherwise noted, methods which return an int return 0 on sucess
50 * and a negative error code on failure.
52 #define WRAP_RETRY(x) { \
53 bool failed = false; \
55 init_inject_failure(); \
64 complete_inject_failure(); \
66 } catch (RetryException) { \
77 class LFNIndex : public CollectionIndex {
78 /// Hash digest output size.
79 static const int FILENAME_LFN_DIGEST_SIZE
= CEPH_CRYPTO_SHA1_DIGESTSIZE
;
80 /// Length of filename hash.
81 static const int FILENAME_HASH_LEN
= FILENAME_LFN_DIGEST_SIZE
;
82 /// Max filename size.
83 static const int FILENAME_MAX_LEN
= 4096;
84 /// Length of hashed filename.
85 static const int FILENAME_SHORT_LEN
= 255;
86 /// Length of hashed filename prefix.
87 static const int FILENAME_PREFIX_LEN
;
88 /// Length of hashed filename cookie.
89 static const int FILENAME_EXTRA
= 4;
91 static const string FILENAME_COOKIE
;
92 /// Name of LFN attribute for storing full name.
93 static const string LFN_ATTR
;
94 /// Prefix for subdir index attributes.
95 static const string PHASH_ATTR_PREFIX
;
96 /// Prefix for index subdirectories.
97 static const string SUBDIR_PREFIX
;
99 /// Path to Index base.
100 const string base_path
;
103 const uint32_t index_version
;
105 /// true if retry injection is enabled
106 struct RetryException
: public exception
{};
107 bool error_injection_enabled
;
108 bool error_injection_on
;
109 double error_injection_probability
;
110 uint64_t last_failure
;
111 uint64_t current_failure
;
112 void init_inject_failure() {
113 if (error_injection_on
) {
114 error_injection_enabled
= true;
115 last_failure
= current_failure
= 0;
118 void maybe_inject_failure();
119 void complete_inject_failure() {
120 error_injection_enabled
= false;
124 string lfn_attribute
, lfn_alt_attribute
;
132 const char *base_path
, ///< [in] path to Index root
133 uint32_t index_version
,
134 double _error_injection_probability
=0)
135 : CollectionIndex(cct
, collection
),
136 base_path(base_path
),
137 index_version(index_version
),
138 error_injection_enabled(false),
139 error_injection_on(_error_injection_probability
!= 0),
140 error_injection_probability(_error_injection_probability
),
141 last_failure(0), current_failure(0),
142 collection(collection
) {
143 if (index_version
== HASH_INDEX_TAG
) {
144 lfn_attribute
= LFN_ATTR
;
147 snprintf(buf
, sizeof(buf
), "%d", index_version
);
148 lfn_attribute
= LFN_ATTR
+ string(buf
);
149 lfn_alt_attribute
= LFN_ATTR
+ string(buf
) + "-alt";
153 coll_t
coll() const override
{ return collection
; }
155 /// Virtual destructor
156 ~LFNIndex() override
{}
158 /// @see CollectionIndex
161 /// @see CollectionIndex
162 int cleanup() override
= 0;
164 /// @see CollectionIndex
166 const ghobject_t
&oid
,
170 /// @see CollectionIndex
172 const ghobject_t
&oid
175 /// @see CollectionIndex
177 const ghobject_t
&oid
,
182 /// @see CollectionIndex;
183 int pre_hash_collection(
185 uint64_t expected_num_objs
188 /// @see CollectionIndex
189 int collection_list_partial(
190 const ghobject_t
&start
,
191 const ghobject_t
&end
,
193 vector
<ghobject_t
> *ls
,
198 uint32_t match
, //< [in] value to match
199 uint32_t bits
, //< [in] bits to check
200 CollectionIndex
* dest
//< [in] destination index
203 /// @see CollectionIndex
207 CollectionIndex
* dest
210 r
= _split(match
, bits
, dest
);
216 * Returns the length of the longest escaped name which could result
217 * from any clone, shard, or rollback object of this object
219 static uint64_t get_max_escaped_name_len(const hobject_t
&obj
);
222 virtual int _init() = 0;
224 /// Will be called upon object creation
225 virtual int _created(
226 const vector
<string
> &path
, ///< [in] Path to subdir.
227 const ghobject_t
&oid
, ///< [in] Object created.
228 const string
&mangled_name
///< [in] Mangled filename.
231 /// Will be called to remove an object
233 const vector
<string
> &path
, ///< [in] Path to subdir.
234 const ghobject_t
&oid
, ///< [in] Object to remove.
235 const string
&mangled_name
///< [in] Mangled filename.
238 /// Return the path and mangled_name for oid.
240 const ghobject_t
&oid
,///< [in] Object for lookup.
241 vector
<string
> *path
, ///< [out] Path to the object.
242 string
*mangled_name
, ///< [out] Mangled filename.
243 int *exists
///< [out] True if the object exists.
246 /// Pre-hash the collection with the given pg number and
247 /// expected number of objects in the collection.
248 virtual int _pre_hash_collection(
250 uint64_t expected_num_objs
253 /// @see CollectionIndex
254 virtual int _collection_list_partial(
255 const ghobject_t
&start
,
256 const ghobject_t
&end
,
258 vector
<ghobject_t
> *ls
,
264 /* Non-virtual utility methods */
266 /// Sync a subdirectory
268 const vector
<string
> &path
///< [in] Path to sync
269 ); ///< @return Error Code, 0 on success
271 /// Link an object from from into to
273 const vector
<string
> &from
, ///< [in] Source subdirectory.
274 const vector
<string
> &to
, ///< [in] Dest subdirectory.
275 const ghobject_t
&oid
, ///< [in] Object to move.
276 const string
&from_short_name
///< [in] Mangled filename of oid.
277 ); ///< @return Error Code, 0 on success
280 * Efficiently remove objects from a subdirectory
282 * remove_object invalidates mangled names in the directory requiring
283 * the mangled name of each additional object to be looked up a second
284 * time. remove_objects removes the need for additional lookups
286 * @param [in] dir Directory from which to remove.
287 * @param [in] map of objects to remove to mangle names
288 * @param [in,out] map of filenames to objects
289 * @return Error Code, 0 on success.
292 const vector
<string
> &dir
,
293 const map
<string
, ghobject_t
> &to_remove
,
294 map
<string
, ghobject_t
> *remaining
299 * Moves contents of from into to.
301 * Invalidates mangled names in to. If interupted, all objects will be
302 * present in to before objects are removed from from. Ignores EEXIST
303 * while linking into to.
304 * @return Error Code, 0 on success
307 const vector
<string
> &from
, ///< [in] Source subdirectory.
308 const vector
<string
> &to
///< [in] Dest subdirectory.
312 * Remove an object from from.
314 * Invalidates mangled names in from.
315 * @return Error Code, 0 on success
318 const vector
<string
> &from
, ///< [in] Directory from which to remove.
319 const ghobject_t
&to_remove
///< [in] Object to remove.
323 * Gets the filename corresponding to oid in from.
325 * The filename may differ between subdirectories. Furthermore,
326 * file creations ore removals in from may invalidate the name.
327 * @return Error code on failure, 0 on success
329 int get_mangled_name(
330 const vector
<string
> &from
, ///< [in] Subdirectory
331 const ghobject_t
&oid
, ///< [in] Object
332 string
*mangled_name
, ///< [out] Filename
333 int *hardlink
///< [out] hardlink for this file, hardlink=0 mean no-exist
336 /// do move subdir from from to dest
337 static int move_subdir(
338 LFNIndex
&from
, ///< [in] from index
339 LFNIndex
&dest
, ///< [in] to index
340 const vector
<string
> &path
, ///< [in] path containing dir
341 string dir
///< [in] dir to move
344 /// do move object from from to dest
345 static int move_object(
346 LFNIndex
&from
, ///< [in] from index
347 LFNIndex
&dest
, ///< [in] to index
348 const vector
<string
> &path
, ///< [in] path to split
349 const pair
<string
, ghobject_t
> &obj
///< [in] obj to move
353 * Lists objects in to_list.
355 * @param [in] to_list Directory to list.
356 * @param [in] max_objects Max number to list.
357 * @param [in,out] handle Cookie for continuing the listing.
358 * Initialize to zero to start at the beginning of the directory.
359 * @param [out] out Mapping of listed object filenames to objects.
360 * @return Error code on failure, 0 on success
363 const vector
<string
> &to_list
,
366 map
<string
, ghobject_t
> *out
369 /// Lists subdirectories.
371 const vector
<string
> &to_list
, ///< [in] Directory to list.
372 vector
<string
> *out
///< [out] Subdirectories listed.
375 /// Create subdirectory.
377 const vector
<string
> &to_create
///< [in] Subdirectory to create.
380 /// Remove subdirectory.
382 const vector
<string
> &to_remove
///< [in] Subdirectory to remove.
385 /// Check whether to_check exists.
387 const vector
<string
> &to_check
, ///< [in] Subdirectory to check.
388 int *exists
///< [out] 1 if it exists, 0 else
391 /// Save attr_value to attr_name attribute on path.
393 const vector
<string
> &path
, ///< [in] Path to modify.
394 const string
&attr_name
, ///< [in] Name of attribute.
395 bufferlist
&attr_value
///< [in] Value to save.
398 /// Read into attr_value atribute attr_name on path.
400 const vector
<string
> &path
, ///< [in] Path to read.
401 const string
&attr_name
, ///< [in] Attribute to read.
402 bufferlist
&attr_value
///< [out] Attribute value read.
405 /// Remove attr from path
406 int remove_attr_path(
407 const vector
<string
> &path
, ///< [in] path from which to remove attr
408 const string
&attr_name
///< [in] attr to remove
409 ); ///< @return Error code, 0 on success
412 /* lfn translation functions */
415 * Gets the version specific lfn attribute tag
417 const string
&get_lfn_attr() const {
418 return lfn_attribute
;
420 const string
&get_alt_lfn_attr() const {
421 return lfn_alt_attribute
;
425 * Gets the filename corresponsing to oid in path.
427 * @param [in] path Path in which to get filename for oid.
428 * @param [in] oid Object for which to get filename.
429 * @param [out] mangled_name Filename for oid, pass NULL if not needed.
430 * @param [out] full_path Fullpath for oid, pass NULL if not needed.
431 * @param [out] hardlink of this file, 0 mean no-exist, pass NULL if
433 * @return Error Code, 0 on success.
436 const vector
<string
> &path
,
437 const ghobject_t
&oid
,
438 string
*mangled_name
,
443 /// Adjusts path contents when oid is created at name mangled_name.
445 const vector
<string
> &path
, ///< [in] Path to adjust.
446 const ghobject_t
&oid
, ///< [in] Object created.
447 const string
&mangled_name
///< [in] Filename of created object.
450 /// Removes oid from path while adjusting path contents
452 const vector
<string
> &path
, ///< [in] Path containing oid.
453 const ghobject_t
&oid
, ///< [in] Object to remove.
454 const string
&mangled_name
///< [in] Filename of object to remove.
457 ///Transate a file into and ghobject_t.
459 const vector
<string
> &path
, ///< [in] Path containing the file.
460 const string
&short_name
, ///< [in] Filename to translate.
461 ghobject_t
*out
///< [out] Object found.
462 ); ///< @return Negative error code on error, 0 if not an object, 1 else
464 /* manglers/demanglers */
465 /// Filters object filenames
467 const string
&short_name
///< [in] Filename to check
468 ); ///< True if short_name is an object, false otherwise
470 /// Filters subdir filenames
472 const string
&short_name
, ///< [in] Filename to check.
473 string
*demangled_name
///< [out] Demangled subdir name.
474 ); ///< @return True if short_name is a subdir, false otherwise
476 /// Generate object name
477 string
lfn_generate_object_name_keyless(
478 const ghobject_t
&oid
///< [in] Object for which to generate.
479 ); ///< @return Generated object name.
481 /// Generate object name
482 string
lfn_generate_object_name_poolless(
483 const ghobject_t
&oid
///< [in] Object for which to generate.
484 ); ///< @return Generated object name.
486 /// Generate object name
487 static string
lfn_generate_object_name_current(
488 const ghobject_t
&oid
///< [in] Object for which to generate.
489 ); ///< @return Generated object name.
491 /// Generate object name
492 string
lfn_generate_object_name(
493 const ghobject_t
&oid
///< [in] Object for which to generate.
495 if (index_version
== HASH_INDEX_TAG
)
496 return lfn_generate_object_name_keyless(oid
);
497 if (index_version
== HASH_INDEX_TAG_2
)
498 return lfn_generate_object_name_poolless(oid
);
500 return lfn_generate_object_name_current(oid
);
501 } ///< @return Generated object name.
503 /// Parse object name
504 int lfn_parse_object_name_keyless(
505 const string
&long_name
, ///< [in] Name to parse
506 ghobject_t
*out
///< [out] Resulting Object
507 ); ///< @return True if successfull, False otherwise.
509 /// Parse object name
510 int lfn_parse_object_name_poolless(
511 const string
&long_name
, ///< [in] Name to parse
512 ghobject_t
*out
///< [out] Resulting Object
513 ); ///< @return True if successfull, False otherwise.
515 /// Parse object name
516 int lfn_parse_object_name(
517 const string
&long_name
, ///< [in] Name to parse
518 ghobject_t
*out
///< [out] Resulting Object
519 ); ///< @return True if successfull, False otherwise.
521 /// Checks whether short_name is a hashed filename.
522 bool lfn_is_hashed_filename(
523 const string
&short_name
///< [in] Name to check.
524 ); ///< @return True if short_name is hashed, False otherwise.
526 /// Checks whether long_name must be hashed.
528 const string
&long_name
///< [in] Name to check.
529 ); ///< @return True if long_name must be hashed, False otherwise.
531 /// Generate hashed name.
532 string
lfn_get_short_name(
533 const ghobject_t
&oid
, ///< [in] Object for which to generate.
534 int i
///< [in] Index of hashed name to generate.
535 ); ///< @return Hashed filename.
537 /* other common methods */
538 /// Gets the base path
539 const string
&get_base_path(); ///< @return Index base_path
541 /// Get full path the subdir
542 string
get_full_path_subdir(
543 const vector
<string
> &rel
///< [in] The subdir.
544 ); ///< @return Full path to rel.
546 /// Get full path to object
547 string
get_full_path(
548 const vector
<string
> &rel
, ///< [in] Path to object.
549 const string
&name
///< [in] Filename of object.
550 ); ///< @return Fullpath to object at name in rel.
552 /// Get mangled path component
553 string
mangle_path_component(
554 const string
&component
///< [in] Component to mangle
555 ); /// @return Mangled component
557 /// Demangle component
558 string
demangle_path_component(
559 const string
&component
///< [in] Subdir name to demangle
560 ); ///< @return Demangled path component.
562 /// Decompose full path into object name and filename.
563 int decompose_full_path(
564 const char *in
, ///< [in] Full path to object.
565 vector
<string
> *out
, ///< [out] Path to object at in.
566 ghobject_t
*oid
, ///< [out] Object at in.
567 string
*shortname
///< [out] Filename of object at in.
568 ); ///< @return Error Code, 0 on success.
570 /// Mangle attribute name
571 string
mangle_attr_name(
572 const string
&attr
///< [in] Attribute to mangle.
573 ); ///< @return Mangled attribute name.
575 /// checks whether long_name could hash to short_name
576 bool short_name_matches(
577 const char *short_name
, ///< [in] name to check against
578 const char *cand_long_name
///< [in] candidate long name
581 /// Builds hashed filename
583 const char *old_filename
, ///< [in] Filename to convert.
584 int i
, ///< [in] Index of hash.
585 char *filename
, ///< [out] Resulting filename.
586 int len
///< [in] Size of buffer for filename
587 ); ///< @return Error Code, 0 on success
589 /// Get hash of filename
591 const char *filename
, ///< [in] Filename to hash.
592 char *hash
, ///< [out] Hash of filename.
593 int len
///< [in] Size of hash buffer.
594 ); ///< @return Error Code, 0 on success.
596 friend class TestWrapLFNIndex
;
598 typedef LFNIndex::IndexedPath IndexedPath
;