8 import cephfs
as libcephfs
19 from cmd2
import __version__
as cmd2_version
20 from distutils
.version
import LooseVersion
22 if sys
.version_info
.major
< 3:
23 raise RuntimeError("cephfs-shell is only compatible with python3")
26 from cmd2
import with_argparser
28 def with_argparser(argparser
):
31 def argparser_decorator(func
):
32 @functools.wraps(func
)
33 def wrapper(thiz
, cmdline
):
34 if isinstance(cmdline
, list):
37 # do not split if it's already a list
38 arglist
= shlex
.split(cmdline
, posix
=False)
39 # in case user quotes the command args
40 arglist
= [arg
.strip('\'""') for arg
in arglist
]
42 args
= argparser
.parse_args(arglist
)
45 # argparse exits at seeing bad arguments
48 return func(thiz
, args
)
49 argparser
.prog
= func
.__name
__[3:]
50 if argparser
.description
is None and func
.__doc
__:
51 argparser
.description
= func
.__doc
__
55 return argparser_decorator
58 cephfs
= None # holds CephFS Python bindings
59 shell
= None # holds instance of class CephFSShell
61 #########################################################################
63 # Following are methods are generically useful through class CephFSShell
65 #######################################################################
67 def poutput(s
, end
='\n'):
68 shell
.poutput(s
, end
=end
)
71 def perror(msg
, **kwargs
):
72 shell
.perror(msg
, **kwargs
)
75 def mode_notation(mode
):
78 permission_bits
= {'0': '---',
91 notation
+= permission_bits
[i
]
95 def get_chunks(file_size
):
97 chunk_size
= 0x20000 # 131072 bytes, default max ssl buffer size
98 while chunk_start
+ chunk_size
< file_size
:
99 yield(chunk_start
, chunk_size
)
100 chunk_start
+= chunk_size
101 final_chunk_size
= file_size
- chunk_start
102 yield(chunk_start
, final_chunk_size
)
106 # don't convert as follows as it can lead unusable results like coverting
107 # [1, 2, 3, 4] to '[1, 2, 3, 4]' -
108 # str(param).encode('utf-8')
109 if isinstance(param
, bytes
):
111 elif isinstance(param
, str):
112 return bytes(param
, encoding
='utf-8')
113 elif isinstance(param
, list):
114 return [i
.encode('utf-8') if isinstance(i
, str) else to_bytes(i
) for \
116 elif isinstance(param
, int) or isinstance(param
, float):
117 return str(param
).encode('utf-8')
121 def ls(path
, opts
=''):
122 # opts tries to be like /bin/ls opts
123 almost_all
= 'A' in opts
125 with cephfs
.opendir(path
) as d
:
127 dent
= cephfs
.readdir(d
)
130 elif almost_all
and dent
.d_name
in (b
'.', b
'..'):
133 except libcephfs
.ObjectNotFound
as e
:
135 shell
.exit_code
= e
.get_error_code()
137 def glob(path
, pattern
):
139 parent_dir
= os
.path
.dirname(path
)
140 if parent_dir
== b
'':
142 if path
== b
'/' or is_dir_exists(os
.path
.basename(path
), parent_dir
):
143 for i
in ls(path
, opts
='A'):
144 if fnmatch
.fnmatch(i
.d_name
, pattern
):
145 paths
.append(os
.path
.join(path
, i
.d_name
))
149 def locate_file(name
, case_sensitive
=True):
150 dir_list
= sorted(set(dirwalk(cephfs
.getcwd())))
151 if not case_sensitive
:
152 return [dname
for dname
in dir_list
if name
.lower() in dname
.lower()]
154 return [dname
for dname
in dir_list
if name
in dname
]
157 def get_all_possible_paths(pattern
):
158 complete_pattern
= pattern
[:]
160 is_rel_path
= not os
.path
.isabs(pattern
)
162 dir_
= cephfs
.getcwd()
165 pattern
= pattern
[1:]
166 patterns
= pattern
.split(b
'/')
167 paths
.extend(glob(dir_
, patterns
[0]))
169 for pattern
in patterns
:
171 paths
.extend(glob(path
, pattern
))
173 complete_pattern
= os
.path
.join(cephfs
.getcwd(), complete_pattern
)
174 return [path
for path
in paths
if fnmatch
.fnmatch(path
, complete_pattern
)]
177 suffixes
= ['B', 'K', 'M', 'G', 'T', 'P']
180 def humansize(nbytes
):
182 while nbytes
>= 1024 and i
< len(suffixes
) - 1:
185 nbytes
= math
.ceil(nbytes
)
186 f
= ('%d' % nbytes
).rstrip('.')
187 return '%s%s' % (f
, suffixes
[i
])
190 def print_long(path
, is_dir
, human_readable
):
191 info
= cephfs
.stat(path
)
192 pretty
= os
.path
.basename(path
.decode('utf-8'))
194 pretty
= colorama
.Style
.BRIGHT
+ colorama
.Fore
.CYAN
+ pretty
+ '/'
195 pretty
+= colorama
.Style
.RESET_ALL
197 poutput('{}\t{:10s} {} {} {} {}'.format(
198 mode_notation(info
.st_mode
),
199 humansize(info
.st_size
), info
.st_uid
,
200 info
.st_gid
, info
.st_mtime
, pretty
, sep
='\t'))
202 poutput('{} {:12d} {} {} {} {}'.format(
203 mode_notation(info
.st_mode
), info
.st_size
, info
.st_uid
,
204 info
.st_gid
, info
.st_mtime
, pretty
, sep
='\t'))
209 Returns the word length, minus any color codes.
211 if word
[0] == '\x1b':
216 def is_dir_exists(path
, dir_
=b
''):
217 path_to_stat
= os
.path
.join(dir_
, path
)
219 return ((cephfs
.stat(path_to_stat
).st_mode
& 0o0040000) != 0)
220 except libcephfs
.Error
:
224 def is_file_exists(path
, dir_
=b
''):
226 # if its not a directory, then its a file
227 return ((cephfs
.stat(os
.path
.join(dir_
, path
)).st_mode
& 0o0040000) == 0)
228 except libcephfs
.Error
:
232 def print_list(words
, termwidth
=79):
235 words
= [word
.decode('utf-8') if isinstance(word
, bytes
) else word
for word
in words
]
236 width
= max([word_len(word
) for word
in words
]) + 2
238 ncols
= max(1, (termwidth
+ 1) // (width
+ 1))
239 nrows
= (nwords
+ ncols
- 1) // ncols
240 for row
in range(nrows
):
241 for i
in range(row
, nwords
, nrows
):
244 if word
[0] == '\x1b':
245 print_width
= print_width
+ 10
247 poutput('%-*s' % (print_width
, words
[i
]),
248 end
='\n' if i
+ nrows
>= nwords
else '')
251 def copy_from_local(local_path
, remote_path
):
255 convert_to_bytes
= False
256 if local_path
== b
'-':
258 convert_to_bytes
= True
261 file_
= open(local_path
, 'rb')
262 except PermissionError
:
264 perror('error: no permission to read local file {}'.format(
265 local_path
.decode('utf-8')))
269 fd
= cephfs
.open(remote_path
, 'w', 0o666)
270 except libcephfs
.Error
as e
:
276 data
= file_
.read(65536)
277 if not data
or len(data
) == 0:
280 data
= to_bytes(data
)
281 wrote
= cephfs
.write(fd
, data
, progress
)
291 def copy_to_local(remote_path
, local_path
):
293 if local_path
!= b
'-':
294 local_dir
= os
.path
.dirname(local_path
)
295 dir_list
= remote_path
.rsplit(b
'/', 1)
296 if not os
.path
.exists(local_dir
):
297 os
.makedirs(local_dir
)
298 if len(dir_list
) > 2 and dir_list
[1] == b
'':
300 fd
= open(local_path
, 'wb+')
301 file_
= cephfs
.open(remote_path
, 'r')
302 file_size
= cephfs
.stat(remote_path
).st_size
306 for chunk_start
, chunk_size
in get_chunks(file_size
):
307 file_chunk
= cephfs
.read(file_
, chunk_start
, chunk_size
)
308 progress
+= len(file_chunk
)
312 poutput(file_chunk
.decode('utf-8'))
320 walk a directory tree, using a generator
322 path
= os
.path
.normpath(path
)
323 for item
in ls(path
, opts
='A'):
324 fullpath
= os
.path
.join(path
, item
.d_name
)
325 src_path
= fullpath
.rsplit(b
'/', 1)[0]
327 yield os
.path
.normpath(fullpath
)
328 if is_dir_exists(item
.d_name
, src_path
):
329 for x
in dirwalk(fullpath
):
333 ##################################################################
335 # Following methods are implementation for CephFS Shell commands
337 #################################################################
339 class CephFSShell(Cmd
):
342 super().__init
__(use_ipython
=False)
343 self
.working_dir
= cephfs
.getcwd().decode('utf-8')
345 self
.interactive
= False
348 def default(self
, line
):
349 perror('Unrecognized command')
351 def set_prompt(self
):
352 self
.prompt
= ('\033[01;33mCephFS:~' + colorama
.Fore
.LIGHTCYAN_EX
353 + self
.working_dir
+ colorama
.Style
.RESET_ALL
354 + '\033[01;33m>>>\033[00m ')
356 def create_argparser(self
, command
):
358 argparse_args
= getattr(self
, 'argparse_' + command
)
359 except AttributeError:
363 self
, 'do_' + command
).__doc
__.expandtabs().splitlines()
365 blank_idx
= doc_lines
.index('')
366 usage
= doc_lines
[:blank_idx
]
367 description
= doc_lines
[blank_idx
+ 1:]
371 parser
= argparse
.ArgumentParser(
373 usage
='\n'.join(usage
),
374 description
='\n'.join(description
),
375 formatter_class
=argparse
.ArgumentDefaultsHelpFormatter
377 for args
, kwargs
in argparse_args
:
378 parser
.add_argument(*args
, **kwargs
)
381 def complete_filenames(self
, text
, line
, begidx
, endidx
):
383 completions
= [x
.d_name
.decode('utf-8') + '/' * int(x
.is_dir())
384 for x
in ls(b
".", opts
='A')]
386 if text
.count('/') > 0:
387 completions
= [text
.rsplit('/', 1)[0] + '/'
388 + x
.d_name
.decode('utf-8') + '/'
389 * int(x
.is_dir()) for x
in ls('/'
390 + text
.rsplit('/', 1)[0], opts
='A')
391 if x
.d_name
.decode('utf-8').startswith(
392 text
.rsplit('/', 1)[1])]
394 completions
= [x
.d_name
.decode('utf-8') + '/'
395 * int(x
.is_dir()) for x
in ls(b
".", opts
='A')
396 if x
.d_name
.decode('utf-8').startswith(text
)]
397 if len(completions
) == 1 and completions
[0][-1] == '/':
398 dir_
, file_
= completions
[0].rsplit('/', 1)
399 completions
.extend([dir_
+ '/' + x
.d_name
.decode('utf-8')
400 + '/' * int(x
.is_dir()) for x
in
401 ls('/' + dir_
, opts
='A')
402 if x
.d_name
.decode('utf-8').startswith(file_
)])
403 return self
.delimiter_complete(text
, line
, begidx
, endidx
, completions
, '/')
406 def onecmd(self
, line
, **kwargs
):
411 res
= Cmd
.onecmd(self
, line
, **kwargs
)
415 except ConnectionError
as e
:
416 perror('***\n{}'.format(e
))
418 except KeyboardInterrupt:
419 perror('Command aborted')
421 except (libcephfs
.Error
, Exception) as e
:
424 traceback
.print_exc(file=sys
.stdout
)
426 if hasattr(e
, 'get_error_code'):
427 self
.exit_code
= e
.get_error_code()
429 # XXX: Setting it to 3 so that exceptions not stemming from
430 # CephFS Python bindings can be distinguished.
433 class path_to_bytes(argparse
.Action
):
434 def __call__(self
, parser
, namespace
, values
, option_string
=None):
435 values
= to_bytes(values
)
436 setattr(namespace
, self
.dest
, values
)
438 # TODO: move the necessary contents from here to `class path_to_bytes`.
439 class get_list_of_bytes_path(argparse
.Action
):
440 def __call__(self
, parser
, namespace
, values
, option_string
=None):
441 values
= to_bytes(values
)
444 values
= cephfs
.getcwd()
448 values
[values
.index(i
)] = cephfs
.getcwd()
450 setattr(namespace
, self
.dest
, values
)
452 def complete_mkdir(self
, text
, line
, begidx
, endidx
):
454 auto complete of file name.
456 return self
.complete_filenames(text
, line
, begidx
, endidx
)
458 class ModeAction(argparse
.Action
):
459 def __init__(self
, option_strings
, dest
, nargs
=None, **kwargs
):
460 if nargs
is not None and nargs
!= '?':
461 raise ValueError("more than one modes not allowed")
462 super().__init
__(option_strings
, dest
, **kwargs
)
464 def __call__(self
, parser
, namespace
, values
, option_string
=None):
468 o_mode
= int(values
, base
=8)
470 res
= re
.match('((u?g?o?)|(a?))(=)(r?w?x?)', values
)
472 parser
.error("invalid mode: %s\n"
473 "mode must be a numeric octal literal\n"
474 "or ((u?g?o?)|(a?))(=)(r?w?x?)" %
477 # we are supporting only assignment of mode and not + or -
478 # as is generally available with the chmod command
480 # >>> res = re.match('((u?g?o?)|(a?))(=)(r?w?x?)', 'go=')
482 # ('go', 'go', None, '=', '')
486 parser
.error("need assignment operator between user "
487 "and mode specifiers")
489 parser
.error("invalid mode: %s\n"
490 "mode must be combination of: r | w | x" %
510 o_mode |
= (t_mode
<< 6)
512 o_mode |
= (t_mode
<< 3)
517 parser
.error("invalid mode: %s\n"
518 "mode cannot be negative" % values
)
520 parser
.error("invalid mode: %s\n"
521 "mode cannot be greater than octal 0777" % values
)
523 setattr(namespace
, self
.dest
, str(oct(o_mode
)))
525 mkdir_parser
= argparse
.ArgumentParser(
526 description
='Create the directory(ies), if they do not already exist.')
527 mkdir_parser
.add_argument('dirs', type=str,
528 action
=path_to_bytes
,
530 help='Name of new_directory.',
532 mkdir_parser
.add_argument('-m', '--mode', type=str,
534 help='Sets the access mode for the new directory.')
535 mkdir_parser
.add_argument('-p', '--parent', action
='store_true',
536 help='Create parent directories as necessary. '
537 'When this option is specified, no error is'
538 'reported if a directory already exists.')
540 @with_argparser(mkdir_parser
)
541 def do_mkdir(self
, args
):
545 for path
in args
.dirs
:
547 permission
= int(args
.mode
, 8)
551 cephfs
.mkdirs(path
, permission
)
554 cephfs
.mkdir(path
, permission
)
555 except libcephfs
.Error
as e
:
557 self
.exit_code
= e
.get_error_code()
559 def complete_put(self
, text
, line
, begidx
, endidx
):
561 auto complete of file name.
563 index_dict
= {1: self
.path_complete
}
564 return self
.index_based_complete(text
, line
, begidx
, endidx
, index_dict
)
566 put_parser
= argparse
.ArgumentParser(
567 description
='Copy a file/directory to Ceph File System from Local File System.')
568 put_parser
.add_argument('local_path', type=str, action
=path_to_bytes
,
569 help='Path of the file in the local system')
570 put_parser
.add_argument('remote_path', type=str, action
=path_to_bytes
,
571 help='Path of the file in the remote system.',
572 nargs
='?', default
='.')
573 put_parser
.add_argument('-f', '--force', action
='store_true',
574 help='Overwrites the destination if it already exists.')
576 @with_argparser(put_parser
)
577 def do_put(self
, args
):
579 Copy a local file/directory to CephFS.
581 root_src_dir
= args
.local_path
582 root_dst_dir
= args
.remote_path
583 if args
.local_path
== b
'.' or args
.local_path
== b
'./':
584 root_src_dir
= os
.getcwdb()
585 elif len(args
.local_path
.rsplit(b
'/', 1)) < 2:
586 root_src_dir
= os
.path
.join(os
.getcwdb(), args
.local_path
)
588 p
= args
.local_path
.split(b
'/')
590 root_src_dir
= os
.getcwdb()
593 root_src_dir
+= b
'/' + p
.pop(0)
595 if root_dst_dir
== b
'.':
596 if args
.local_path
!= b
'-':
597 root_dst_dir
= root_src_dir
.rsplit(b
'/', 1)[1]
598 if root_dst_dir
== b
'':
599 root_dst_dir
= root_src_dir
.rsplit(b
'/', 1)[0]
600 a
= root_dst_dir
.rsplit(b
'/', 1)
606 perror('error: no filename specified for destination')
607 # TODO: perhaps, we need a more suitable retval?
611 if root_dst_dir
[-1] != b
'/':
614 if args
.local_path
== b
'-' or os
.path
.isfile(root_src_dir
):
616 if os
.path
.isfile(root_src_dir
):
617 dst_file
= root_dst_dir
618 if is_file_exists(dst_file
):
619 perror('{}: file exists! use --force to '
620 'overwrite'.format(dst_file
.decode('utf-8')))
623 if args
.local_path
== b
'-':
625 copy_from_local(root_src_dir
, root_dst_dir
)
627 for src_dir
, dirs
, files
in os
.walk(root_src_dir
):
628 if isinstance(src_dir
, str):
629 src_dir
= to_bytes(src_dir
)
630 dst_dir
= src_dir
.replace(root_src_dir
, root_dst_dir
, 1)
631 dst_dir
= re
.sub(rb
'\/+', b
'/', cephfs
.getcwd()
633 if args
.force
and dst_dir
!= b
'/' and not is_dir_exists(
634 dst_dir
[:-1]) and not locate_file(dst_dir
):
636 cephfs
.mkdirs(dst_dir
, 0o777)
637 except libcephfs
.Error
:
639 if (not args
.force
) and dst_dir
!= b
'/' and not is_dir_exists(
640 dst_dir
) and not os
.path
.isfile(root_src_dir
):
642 cephfs
.mkdirs(dst_dir
, 0o777)
643 except libcephfs
.Error
:
644 # TODO: perhaps, set retval to 1?
648 dir_name
= os
.path
.join(dst_dir
, dir_
)
649 if not is_dir_exists(dir_name
):
651 cephfs
.mkdirs(dir_name
, 0o777)
652 except libcephfs
.Error
:
653 # TODO: perhaps, set retval to 1?
657 src_file
= os
.path
.join(src_dir
, file_
)
658 dst_file
= re
.sub(rb
'\/+', b
'/', b
'/' + dst_dir
+ b
'/' + file_
)
659 if (not args
.force
) and is_file_exists(dst_file
):
661 copy_from_local(src_file
, os
.path
.join(cephfs
.getcwd(),
664 def complete_get(self
, text
, line
, begidx
, endidx
):
666 auto complete of file name.
668 return self
.complete_filenames(text
, line
, begidx
, endidx
)
670 get_parser
= argparse
.ArgumentParser(
671 description
='Copy a file from Ceph File System from Local Directory.')
672 get_parser
.add_argument('remote_path', type=str, action
=path_to_bytes
,
673 help='Path of the file in the remote system')
674 get_parser
.add_argument('local_path', type=str, action
=path_to_bytes
,
675 help='Path of the file in the local system',
676 nargs
='?', default
='.')
677 get_parser
.add_argument('-f', '--force', action
='store_true',
678 help='Overwrites the destination if it already exists.')
680 @with_argparser(get_parser
)
681 def do_get(self
, args
):
683 Copy a file/directory from CephFS to given path.
685 root_src_dir
= args
.remote_path
686 root_dst_dir
= args
.local_path
687 fname
= root_src_dir
.rsplit(b
'/', 1)
688 if args
.local_path
== b
'.':
689 root_dst_dir
= os
.getcwdb()
690 if args
.remote_path
== b
'.':
691 root_src_dir
= cephfs
.getcwd()
692 if args
.local_path
== b
'-':
693 if args
.remote_path
== b
'.' or args
.remote_path
== b
'./':
694 perror('error: no remote file name specified')
697 copy_to_local(root_src_dir
, b
'-')
698 elif is_file_exists(args
.remote_path
):
699 copy_to_local(root_src_dir
,
700 root_dst_dir
+ b
'/' + root_src_dir
)
701 elif b
'/'in root_src_dir
and is_file_exists(fname
[1], fname
[0]):
702 copy_to_local(root_src_dir
, root_dst_dir
)
704 files
= list(reversed(sorted(dirwalk(root_src_dir
))))
707 os
.makedirs(root_dst_dir
+ b
'/' + root_src_dir
)
712 perror('{}: already exists! use --force to overwrite'.format(
713 root_src_dir
.decode('utf-8')))
714 self
.exit_code
= e
.get_error_code()
718 dst_dirpath
, dst_file
= file_
.rsplit(b
'/', 1)
719 if dst_dirpath
in files
:
720 files
.remove(dst_dirpath
)
721 dst_path
= os
.path
.join(root_dst_dir
, dst_dirpath
, dst_file
)
722 dst_path
= os
.path
.normpath(dst_path
)
723 if is_dir_exists(file_
):
725 os
.makedirs(dst_path
)
732 perror('{}: file already exists! use --force to '
733 'override'.format(file_
.decode('utf-8')))
737 copy_to_local(file_
, dst_path
)
739 copy_to_local(file_
, dst_path
)
743 def complete_ls(self
, text
, line
, begidx
, endidx
):
745 auto complete of file name.
747 return self
.complete_filenames(text
, line
, begidx
, endidx
)
749 ls_parser
= argparse
.ArgumentParser(
750 description
='Copy a file from Ceph File System from Local Directory.')
751 ls_parser
.add_argument('-l', '--long', action
='store_true',
752 help='Detailed list of items in the directory.')
753 ls_parser
.add_argument('-r', '--reverse', action
='store_true',
754 help='Reverse order of listing items in the directory.')
755 ls_parser
.add_argument('-H', action
='store_true', help='Human Readable')
756 ls_parser
.add_argument('-a', '--all', action
='store_true',
757 help='Do not Ignore entries starting with .')
758 ls_parser
.add_argument('-S', action
='store_true', help='Sort by file_size')
759 ls_parser
.add_argument('paths', help='Name of Directories',
760 action
=path_to_bytes
, nargs
='*', default
=['.'])
762 @with_argparser(ls_parser
)
763 def do_ls(self
, args
):
765 List all the files and directories in the current working directory
771 if path
.count(b
'*') > 0:
772 all_items
= get_all_possible_paths(path
)
773 if len(all_items
) == 0:
775 path
= all_items
[0].rsplit(b
'/', 1)[0]
780 for item
in ls(path
):
782 if os
.path
.basename(i
) == d_name
:
784 dirs
.append(os
.path
.join(path
, d_name
))
790 poutput(path
.decode('utf-8'), end
=':\n')
791 items
= sorted(items
, key
=lambda item
: item
.d_name
)
793 if path
!= b
'' and path
!= cephfs
.getcwd() and len(paths
) > 1:
794 poutput(path
.decode('utf-8'), end
=':\n')
795 items
= sorted(ls(path
),
796 key
=lambda item
: item
.d_name
)
798 items
= [i
for i
in items
if not i
.d_name
.startswith(b
'.')]
801 items
= sorted(items
, key
=lambda item
: cephfs
.stat(
802 path
+ b
'/' + item
.d_name
).st_size
)
805 items
= reversed(items
)
807 filepath
= item
.d_name
808 is_dir
= item
.is_dir()
810 if args
.long and args
.H
:
811 print_long(os
.path
.join(cephfs
.getcwd(), path
, filepath
),
814 print_long(os
.path
.join(cephfs
.getcwd(), path
, filepath
),
817 values
.append(colorama
.Style
.BRIGHT
819 + filepath
.decode('utf-8')
821 + colorama
.Style
.RESET_ALL
)
823 values
.append(filepath
)
825 print_list(values
, shutil
.get_terminal_size().columns
)
826 if path
!= paths
[-1]:
829 def complete_rmdir(self
, text
, line
, begidx
, endidx
):
831 auto complete of file name.
833 return self
.complete_filenames(text
, line
, begidx
, endidx
)
835 rmdir_parser
= argparse
.ArgumentParser(description
='Remove Directory.')
836 rmdir_parser
.add_argument('paths', help='Directory Path.', nargs
='+',
837 action
=path_to_bytes
)
838 rmdir_parser
.add_argument('-p', '--parent', action
='store_true',
839 help='Remove parent directories as necessary. '
840 'When this option is specified, no error '
841 'is reported if a directory has any '
842 'sub-directories, files')
844 @with_argparser(rmdir_parser
)
845 def do_rmdir(self
, args
):
846 self
.do_rmdir_helper(args
)
848 def do_rmdir_helper(self
, args
):
850 Remove a specific Directory
855 if path
.count(b
'*') > 0:
857 all_items
= get_all_possible_paths(path
)
858 if len(all_items
) > 0:
859 path
= all_items
[0].rsplit(b
'/', 1)[0]
864 for item
in ls(path
):
866 if os
.path
.basename(i
) == d_name
:
868 dirs
.append(os
.path
.join(path
, d_name
))
875 path
= os
.path
.join(cephfs
.getcwd(), path
.rsplit(b
'/')[0])
876 files
= list(sorted(set(dirwalk(path
)), reverse
=True))
879 for filepath
in files
:
881 cephfs
.rmdir(os
.path
.normpath(filepath
))
882 except libcephfs
.Error
as e
:
887 path
= os
.path
.normpath(os
.path
.join(cephfs
.getcwd(), path
))
888 if not is_pattern
and path
!= os
.path
.normpath(b
''):
891 except libcephfs
.Error
as e
:
893 self
.exit_code
= e
.get_error_code()
895 def complete_rm(self
, text
, line
, begidx
, endidx
):
897 auto complete of file name.
899 return self
.complete_filenames(text
, line
, begidx
, endidx
)
901 rm_parser
= argparse
.ArgumentParser(description
='Remove File.')
902 rm_parser
.add_argument('paths', help='File Path.', nargs
='+',
903 action
=path_to_bytes
)
905 @with_argparser(rm_parser
)
906 def do_rm(self
, args
):
908 Remove a specific file
910 file_paths
= args
.paths
911 for path
in file_paths
:
912 if path
.count(b
'*') > 0:
913 file_paths
.extend([i
for i
in get_all_possible_paths(
914 path
) if is_file_exists(i
)])
918 except libcephfs
.Error
as e
:
919 # NOTE: perhaps we need a better msg here
921 self
.exit_code
= e
.get_error_code()
923 def complete_mv(self
, text
, line
, begidx
, endidx
):
925 auto complete of file name.
927 return self
.complete_filenames(text
, line
, begidx
, endidx
)
929 mv_parser
= argparse
.ArgumentParser(description
='Move File.')
930 mv_parser
.add_argument('src_path', type=str, action
=path_to_bytes
,
931 help='Source File Path.')
932 mv_parser
.add_argument('dest_path', type=str, action
=path_to_bytes
,
933 help='Destination File Path.')
935 @with_argparser(mv_parser
)
936 def do_mv(self
, args
):
938 Rename a file or Move a file from source path to the destination
940 cephfs
.rename(args
.src_path
, args
.dest_path
)
942 def complete_cd(self
, text
, line
, begidx
, endidx
):
944 auto complete of file name.
946 return self
.complete_filenames(text
, line
, begidx
, endidx
)
948 cd_parser
= argparse
.ArgumentParser(description
='Change working directory')
949 cd_parser
.add_argument('path', type=str, help='Name of the directory.',
950 action
=path_to_bytes
, nargs
='?', default
='/')
952 @with_argparser(cd_parser
)
953 def do_cd(self
, args
):
955 Change working directory
957 cephfs
.chdir(args
.path
)
958 self
.working_dir
= cephfs
.getcwd().decode('utf-8')
961 def do_cwd(self
, arglist
):
963 Get current working directory.
965 poutput(cephfs
.getcwd().decode('utf-8'))
967 def complete_chmod(self
, text
, line
, begidx
, endidx
):
969 auto complete of file name.
971 return self
.complete_filenames(text
, line
, begidx
, endidx
)
973 chmod_parser
= argparse
.ArgumentParser(description
='Create Directory.')
974 chmod_parser
.add_argument('mode', type=str, action
=ModeAction
, help='Mode')
975 chmod_parser
.add_argument('paths', type=str, action
=path_to_bytes
,
976 help='Name of the file', nargs
='+')
978 @with_argparser(chmod_parser
)
979 def do_chmod(self
, args
):
981 Change permission of a file
983 for path
in args
.paths
:
984 mode
= int(args
.mode
, base
=8)
986 cephfs
.chmod(path
, mode
)
987 except libcephfs
.Error
as e
:
989 self
.exit_code
= e
.get_error_code()
991 def complete_cat(self
, text
, line
, begidx
, endidx
):
993 auto complete of file name.
995 return self
.complete_filenames(text
, line
, begidx
, endidx
)
997 cat_parser
= argparse
.ArgumentParser(description
='')
998 cat_parser
.add_argument('paths', help='Name of Files', action
=path_to_bytes
,
1001 @with_argparser(cat_parser
)
1002 def do_cat(self
, args
):
1004 Print contents of a file
1006 for path
in args
.paths
:
1007 if is_file_exists(path
):
1008 copy_to_local(path
, b
'-')
1010 perror('{}: no such file'.format(path
.decode('utf-8')))
1013 umask_parser
= argparse
.ArgumentParser(description
='Set umask value.')
1014 umask_parser
.add_argument('mode', help='Mode', type=str, action
=ModeAction
,
1015 nargs
='?', default
='')
1017 @with_argparser(umask_parser
)
1018 def do_umask(self
, args
):
1023 poutput(self
.umask
.zfill(4))
1025 mode
= int(args
.mode
, 8)
1026 self
.umask
= str(oct(cephfs
.umask(mode
))[2:])
1028 def complete_write(self
, text
, line
, begidx
, endidx
):
1030 auto complete of file name.
1032 return self
.complete_filenames(text
, line
, begidx
, endidx
)
1034 write_parser
= argparse
.ArgumentParser(description
='Writes data into a file')
1035 write_parser
.add_argument('path', type=str, action
=path_to_bytes
,
1036 help='Name of File')
1038 @with_argparser(write_parser
)
1039 def do_write(self
, args
):
1041 Write data into a file.
1044 copy_from_local(b
'-', args
.path
)
1046 def complete_lcd(self
, text
, line
, begidx
, endidx
):
1048 auto complete of file name.
1050 index_dict
= {1: self
.path_complete
}
1051 return self
.index_based_complete(text
, line
, begidx
, endidx
, index_dict
)
1053 lcd_parser
= argparse
.ArgumentParser(description
='')
1054 lcd_parser
.add_argument('path', type=str, action
=path_to_bytes
, help='Path')
1056 @with_argparser(lcd_parser
)
1057 def do_lcd(self
, args
):
1059 Moves into the given local directory
1062 os
.chdir(os
.path
.expanduser(args
.path
))
1063 except OSError as e
:
1064 perror("Cannot change to {}: {}".format(e
.filename
, e
.strerror
))
1065 self
.exit_code
= e
.get_error_code()
1067 def complete_lls(self
, text
, line
, begidx
, endidx
):
1069 auto complete of file name.
1071 index_dict
= {1: self
.path_complete
}
1072 return self
.index_based_complete(text
, line
, begidx
, endidx
, index_dict
)
1074 lls_parser
= argparse
.ArgumentParser(
1075 description
='List files in local system.')
1076 lls_parser
.add_argument('paths', help='Paths', action
=path_to_bytes
,
1079 @with_argparser(lls_parser
)
1080 def do_lls(self
, args
):
1082 Lists all files and folders in the current local directory
1085 print_list(os
.listdir(os
.getcwdb()))
1087 for path
in args
.paths
:
1089 items
= os
.listdir(path
)
1090 poutput("{}:".format(path
.decode('utf-8')))
1092 except OSError as e
:
1093 perror("'{}': {}".format(e
.filename
, e
.strerror
))
1094 self
.exit_code
= e
.get_error_code()
1095 # Arguments to the with_argpaser decorator function are sticky.
1096 # The items in args.path do not get overwritten in subsequent calls.
1097 # The arguments remain in args.paths after the function exits and we
1098 # neeed to clean it up to ensure the next call works as expected.
1101 def do_lpwd(self
, arglist
):
1103 Prints the absolute path of the current local directory
1105 poutput(os
.getcwd())
1107 def complete_df(self
, text
, line
, begidx
, endidx
):
1109 auto complete of file name.
1111 return self
.complete_filenames(text
, line
, begidx
, endidx
)
1113 df_parser
= argparse
.ArgumentParser(description
='Show information about\
1114 the amount of available disk space')
1115 df_parser
.add_argument('file', help='Name of the file', nargs
='*',
1116 default
=['.'], action
=path_to_bytes
)
1118 @with_argparser(df_parser
)
1119 def do_df(self
, arglist
):
1121 Display the amount of available disk space for file systems
1123 header
= True # Set to true for printing header only once
1124 if b
'.' == arglist
.file[0]:
1125 arglist
.file = ls(b
'.')
1127 for file in arglist
.file:
1128 if isinstance(file, libcephfs
.DirEntry
):
1130 if file == b
'.' or file == b
'..':
1133 statfs
= cephfs
.statfs(file)
1134 stat
= cephfs
.stat(file)
1135 block_size
= statfs
['f_blocks']*statfs
['f_bsize'] // 1024
1136 available
= block_size
- stat
.st_size
1140 use
= (stat
.st_size
*100 // block_size
)
1144 poutput('{:25s}\t{:5s}\t{:15s}{:10s}{}'.format(
1145 "1K-blocks", "Used", "Available", "Use%",
1148 poutput('{:d}\t{:18d}\t{:8d}\t{:10s} {}'.format(block_size
,
1149 stat
.st_size
, available
, str(int(use
)) + '%',
1150 file.decode('utf-8')))
1151 except libcephfs
.OSError as e
:
1152 perror("could not statfs {}: {}".format(file.decode('utf-8'),
1154 self
.exit_code
= e
.get_error_code()
1156 locate_parser
= argparse
.ArgumentParser(
1157 description
='Find file within file system')
1158 locate_parser
.add_argument('name', help='name', type=str,
1159 action
=path_to_bytes
)
1160 locate_parser
.add_argument('-c', '--count', action
='store_true',
1161 help='Count list of items located.')
1162 locate_parser
.add_argument(
1163 '-i', '--ignorecase', action
='store_true', help='Ignore case')
1165 @with_argparser(locate_parser
)
1166 def do_locate(self
, args
):
1168 Find a file within the File System
1170 if args
.name
.count(b
'*') == 1:
1171 if args
.name
[0] == b
'*':
1173 elif args
.name
[-1] == '*':
1174 args
.name
= b
'/' + args
.name
1175 args
.name
= args
.name
.replace(b
'*', b
'')
1177 locations
= locate_file(args
.name
, False)
1179 locations
= locate_file(args
.name
)
1181 poutput(len(locations
))
1183 poutput((b
'\n'.join(locations
)).decode('utf-8'))
1185 def complete_du(self
, text
, line
, begidx
, endidx
):
1187 auto complete of file name.
1189 return self
.complete_filenames(text
, line
, begidx
, endidx
)
1191 du_parser
= argparse
.ArgumentParser(
1192 description
='Disk Usage of a Directory')
1193 du_parser
.add_argument('paths', type=str, action
=get_list_of_bytes_path
,
1194 help='Name of the directory.', nargs
='*',
1196 du_parser
.add_argument('-r', action
='store_true',
1197 help='Recursive Disk usage of all directories.')
1199 @with_argparser(du_parser
)
1200 def do_du(self
, args
):
1202 Print disk usage of a given path(s).
1204 def print_disk_usage(files
):
1205 if isinstance(files
, bytes
):
1210 st
= cephfs
.lstat(f
)
1212 if stat
.S_ISDIR(st
.st_mode
):
1213 dusage
= int(cephfs
.getxattr(f
,
1214 'ceph.dir.rbytes').decode('utf-8'))
1218 # print path in local context
1219 f
= os
.path
.normpath(f
)
1220 if f
[0] is ord('/'):
1222 poutput('{:10s} {}'.format(humansize(dusage
),
1224 except libcephfs
.Error
as e
:
1226 self
.exit_code
= e
.get_error_code()
1229 for path
in args
.paths
:
1231 print_disk_usage(sorted(set(dirwalk(path
)).union({path}
)))
1233 print_disk_usage(path
)
1235 quota_parser
= argparse
.ArgumentParser(
1236 description
='Quota management for a Directory')
1237 quota_parser
.add_argument('op', choices
=['get', 'set'],
1238 help='Quota operation type.')
1239 quota_parser
.add_argument('path', type=str, action
=path_to_bytes
,
1240 help='Name of the directory.')
1241 quota_parser
.add_argument('--max_bytes', type=int, default
=-1, nargs
='?',
1242 help='Max cumulative size of the data under '
1244 quota_parser
.add_argument('--max_files', type=int, default
=-1, nargs
='?',
1245 help='Total number of files under this '
1248 @with_argparser(quota_parser
)
1249 def do_quota(self
, args
):
1253 if not is_dir_exists(args
.path
):
1254 perror('error: no such directory {}'.format(args
.path
.decode('utf-8')))
1258 if args
.op
== 'set':
1259 if (args
.max_bytes
== -1) and (args
.max_files
== -1):
1260 perror('please specify either --max_bytes or --max_files or '
1265 if args
.max_bytes
>= 0:
1266 max_bytes
= to_bytes(str(args
.max_bytes
))
1268 cephfs
.setxattr(args
.path
, 'ceph.quota.max_bytes',
1269 max_bytes
, os
.XATTR_CREATE
)
1270 poutput('max_bytes set to %d' % args
.max_bytes
)
1271 except libcephfs
.Error
as e
:
1272 cephfs
.setxattr(args
.path
, 'ceph.quota.max_bytes',
1273 max_bytes
, os
.XATTR_REPLACE
)
1274 perror('max_bytes reset to %d' % args
.max_bytes
)
1275 self
.exit_code
= e
.get_error_code()
1277 if args
.max_files
>= 0:
1278 max_files
= to_bytes(str(args
.max_files
))
1280 cephfs
.setxattr(args
.path
, 'ceph.quota.max_files',
1281 max_files
, os
.XATTR_CREATE
)
1282 poutput('max_files set to %d' % args
.max_files
)
1283 except libcephfs
.Error
as e
:
1284 cephfs
.setxattr(args
.path
, 'ceph.quota.max_files',
1285 max_files
, os
.XATTR_REPLACE
)
1286 perror('max_files reset to %d' % args
.max_files
)
1287 self
.exit_code
= e
.get_error_code()
1288 elif args
.op
== 'get':
1292 max_bytes
= cephfs
.getxattr(args
.path
, 'ceph.quota.max_bytes')
1293 poutput('max_bytes: {}'.format(max_bytes
.decode('utf-8')))
1294 except libcephfs
.Error
as e
:
1295 perror('max_bytes is not set')
1296 self
.exit_code
= e
.get_error_code()
1299 max_files
= cephfs
.getxattr(args
.path
, 'ceph.quota.max_files')
1300 poutput('max_files: {}'.format(max_files
.decode('utf-8')))
1301 except libcephfs
.Error
as e
:
1302 perror('max_files is not set')
1303 self
.exit_code
= e
.get_error_code()
1305 snap_parser
= argparse
.ArgumentParser(description
='Snapshot Management')
1306 snap_parser
.add_argument('op', type=str,
1307 help='Snapshot operation: create or delete')
1308 snap_parser
.add_argument('name', type=str, action
=path_to_bytes
,
1309 help='Name of snapshot')
1310 snap_parser
.add_argument('dir', type=str, action
=path_to_bytes
,
1311 help='Directory for which snapshot '
1312 'needs to be created or deleted')
1314 @with_argparser(snap_parser
)
1315 def do_snap(self
, args
):
1317 Snapshot management for the volume
1319 # setting self.colors to None turns off colorizing and
1320 # perror emits plain text
1324 conf_snapdir
= cephfs
.conf_get('client_snapdir')
1325 if conf_snapdir
is not None:
1326 snapdir
= conf_snapdir
1327 snapdir
= to_bytes(snapdir
)
1328 if args
.op
== 'create':
1330 if is_dir_exists(args
.dir):
1331 cephfs
.mkdir(os
.path
.join(args
.dir, snapdir
, args
.name
), 0o755)
1333 self
.perror("'{}': no such directory".format(
1334 args
.dir.decode('utf-8')))
1336 except libcephfs
.Error
as e
:
1337 self
.perror("snapshot '{}' already exists".format(
1338 args
.name
.decode('utf-8')))
1339 self
.exit_code
= e
.get_error_code()
1340 elif args
.op
== 'delete':
1341 snap_dir
= os
.path
.join(args
.dir, snapdir
, args
.name
)
1343 if is_dir_exists(snap_dir
):
1344 newargs
= argparse
.Namespace(paths
=[snap_dir
], parent
=False)
1345 self
.do_rmdir_helper(newargs
)
1347 self
.perror("'{}': no such snapshot".format(
1348 args
.name
.decode('utf-8')))
1350 except libcephfs
.Error
as e
:
1351 self
.perror("error while deleting '{}'".format(
1352 snap_dir
.decode('utf-8')))
1353 self
.exit_code
= e
.get_error_code()
1355 self
.perror("snapshot can only be created or deleted; check - "
1357 self
.exit_code
= e
.get_error_code()
1359 def do_help(self
, line
):
1361 Get details about a command.
1362 Usage: help <cmd> - for a specific command
1363 help all - for all the commands
1367 if k
.startswith('do_'):
1369 super().do_help(k
[3:])
1371 parser
= self
.create_argparser(line
)
1375 super().do_help(line
)
1377 def complete_stat(self
, text
, line
, begidx
, endidx
):
1379 auto complete of file name.
1381 return self
.complete_filenames(text
, line
, begidx
, endidx
)
1383 stat_parser
= argparse
.ArgumentParser(
1384 description
='Display file or file system status')
1385 stat_parser
.add_argument('paths', type=str, help='file paths',
1386 action
=path_to_bytes
, nargs
='+')
1388 @with_argparser(stat_parser
)
1389 def do_stat(self
, args
):
1391 Display file or file system status
1393 for path
in args
.paths
:
1395 stat
= cephfs
.stat(path
)
1396 atime
= stat
.st_atime
.isoformat(' ')
1397 mtime
= stat
.st_mtime
.isoformat(' ')
1398 ctime
= stat
.st_mtime
.isoformat(' ')
1400 poutput("File: {}\nSize: {:d}\nBlocks: {:d}\nIO Block: {:d}\n"
1401 "Device: {:d}\tInode: {:d}\tLinks: {:d}\nPermission: "
1402 "{:o}/{}\tUid: {:d}\tGid: {:d}\n\Access: {}\nModify: "
1403 "{}\nChange: {}".format(path
.decode('utf-8'),
1404 stat
.st_size
, stat
.st_blocks
, stat
.st_blksize
,
1405 stat
.st_dev
, stat
.st_ino
, stat
.st_nlink
, stat
.st_mode
,
1406 mode_notation(stat
.st_mode
), stat
.st_uid
, stat
.st_gid
,
1407 atime
, mtime
, ctime
))
1408 except libcephfs
.Error
as e
:
1410 self
.exit_code
= e
.get_error_code()
1412 setxattr_parser
= argparse
.ArgumentParser(
1413 description
='Set extended attribute for a file')
1414 setxattr_parser
.add_argument('path', type=str, action
=path_to_bytes
, help='Name of the file')
1415 setxattr_parser
.add_argument('name', type=str, help='Extended attribute name')
1416 setxattr_parser
.add_argument('value', type=str, help='Extended attribute value')
1418 @with_argparser(setxattr_parser
)
1419 def do_setxattr(self
, args
):
1421 Set extended attribute for a file
1423 val_bytes
= to_bytes(args
.value
)
1424 name_bytes
= to_bytes(args
.name
)
1426 cephfs
.setxattr(args
.path
, name_bytes
, val_bytes
, os
.XATTR_CREATE
)
1427 poutput('{} is successfully set to {}'.format(args
.name
, args
.value
))
1428 except libcephfs
.ObjectExists
:
1429 cephfs
.setxattr(args
.path
, name_bytes
, val_bytes
, os
.XATTR_REPLACE
)
1430 poutput('{} is successfully reset to {}'.format(args
.name
, args
.value
))
1431 except libcephfs
.Error
as e
:
1433 self
.exit_code
= e
.get_error_code()
1435 getxattr_parser
= argparse
.ArgumentParser(
1436 description
='Get extended attribute set for a file')
1437 getxattr_parser
.add_argument('path', type=str, action
=path_to_bytes
,
1438 help='Name of the file')
1439 getxattr_parser
.add_argument('name', type=str, help='Extended attribute name')
1441 @with_argparser(getxattr_parser
)
1442 def do_getxattr(self
, args
):
1444 Get extended attribute for a file
1447 poutput('{}'.format(cephfs
.getxattr(args
.path
,
1448 to_bytes(args
.name
)).decode('utf-8')))
1449 except libcephfs
.Error
as e
:
1451 self
.exit_code
= e
.get_error_code()
1453 listxattr_parser
= argparse
.ArgumentParser(
1454 description
='List extended attributes set for a file')
1455 listxattr_parser
.add_argument('path', type=str, action
=path_to_bytes
,
1456 help='Name of the file')
1458 @with_argparser(listxattr_parser
)
1459 def do_listxattr(self
, args
):
1461 List extended attributes for a file
1464 size
, xattr_list
= cephfs
.listxattr(args
.path
)
1466 poutput('{}'.format(xattr_list
.replace(b
'\x00', b
' ').decode('utf-8')))
1468 poutput('No extended attribute is set')
1469 except libcephfs
.Error
as e
:
1471 self
.exit_code
= e
.get_error_code()
1474 #######################################################
1476 # Following are methods that get cephfs-shell started.
1478 #####################################################
1480 def setup_cephfs(config_file
):
1486 cephfs
= libcephfs
.LibCephFS(conffile
=config_file
)
1488 except libcephfs
.ObjectNotFound
as e
:
1489 print('couldn\'t find ceph configuration not found')
1491 except libcephfs
.Error
as e
:
1495 def get_bool_vals_for_boolopts(val
):
1496 if not isinstance(val
, str):
1499 val
= val
.replace('\n', '')
1500 if val
.lower() in ['true', 'yes']:
1502 elif val
.lower() in ['false', 'no']:
1507 def read_ceph_conf(shell
, config_file
):
1509 shell
.debug
= get_bool_vals_for_boolopts(cephfs
.\
1510 conf_get('debug_shell'))
1511 shell
.allow_ansi
= get_bool_vals_for_boolopts(cephfs
.\
1512 conf_get('allow_ansi'))
1513 shell
.echo
= get_bool_vals_for_boolopts(cephfs
.conf_get('echo'))
1514 shell
.continuation_prompt
= get_bool_vals_for_boolopts(cephfs
.\
1515 conf_get('continuation_prompt'))
1516 shell
.colors
= get_bool_vals_for_boolopts(cephfs
.conf_get('colors'))
1517 shell
.editor
= get_bool_vals_for_boolopts(cephfs
.conf_get('editor'))
1518 shell
.feedback_to_output
= get_bool_vals_for_boolopts(cephfs
.\
1519 conf_get('feedback_to_output'))
1520 shell
.max_completion_items
= get_bool_vals_for_boolopts(cephfs
.\
1521 conf_get('max_completion_items'))
1522 shell
.prompt
= get_bool_vals_for_boolopts(cephfs
.conf_get('prompt'))
1523 shell
.quiet
= get_bool_vals_for_boolopts(cephfs
.conf_get('quiet'))
1524 shell
.timing
= get_bool_vals_for_boolopts(cephfs
.conf_get('timing'))
1525 except (OSError, libcephfs
.Error
) as e
:
1527 shell
.exit_code
= e
.get_error_code()
1529 if __name__
== '__main__':
1534 main_parser
= argparse
.ArgumentParser(description
='')
1535 main_parser
.add_argument('-c', '--config', action
='store',
1536 help='Configuration file_path', type=str)
1537 main_parser
.add_argument(
1538 '-b', '--batch', action
='store', help='Batch File path.', type=str)
1539 main_parser
.add_argument('-t', '--test', action
='store',
1540 help='Test against transcript(s) in FILE',
1542 main_parser
.add_argument('commands', nargs
='*',
1543 help='comma delimited commands', default
=[])
1544 args
= main_parser
.parse_args()
1546 config_file
= args
.config
1548 if LooseVersion(cmd2_version
) <= LooseVersion("0.9.13"):
1549 args
.commands
= ['load ' + args
.batch
, ',quit']
1551 args
.commands
= ['run_script ' + args
.batch
, ',quit']
1553 args
.commands
.extend(['-t,'] + [arg
+ ',' for arg
in args
.test
])
1556 sys
.argv
.append(exe
)
1557 sys
.argv
.extend([i
.strip() for i
in ' '.join(args
.commands
).split(',')])
1559 setup_cephfs(config_file
)
1560 shell
= CephFSShell()
1562 read_ceph_conf(shell
, config_file
)
1563 shell
.exit_code
= shell
.cmdloop()
1565 sys
.exit(shell
.exit_code
)