9 import cephfs
as libcephfs
18 if sys
.version_info
.major
< 3:
19 raise RuntimeError("cephfs-shell is only compatible with python3")
22 from cmd2
import with_argparser
24 def with_argparser(argparser
):
27 def argparser_decorator(func
):
28 @functools.wraps(func
)
29 def wrapper(thiz
, cmdline
):
30 if isinstance(cmdline
, list):
33 # do not split if it's already a list
34 arglist
= shlex
.split(cmdline
, posix
=False)
35 # in case user quotes the command args
36 arglist
= [arg
.strip('\'""') for arg
in arglist
]
38 args
= argparser
.parse_args(arglist
)
40 # argparse exits at seeing bad arguments
43 return func(thiz
, args
)
44 argparser
.prog
= func
.__name
__[3:]
45 if argparser
.description
is None and func
.__doc
__:
46 argparser
.description
= func
.__doc
__
50 return argparser_decorator
57 def poutput(s
, end
='\n'):
58 shell
.poutput(s
, end
=end
)
61 def setup_cephfs(config_file
):
66 cephfs
= libcephfs
.LibCephFS(conffile
=config_file
)
70 def mode_notation(mode
):
73 permission_bits
= {'0': '---',
86 notation
+= permission_bits
[i
]
90 def get_chunks(file_size
):
92 chunk_size
= 0x20000 # 131072 bytes, default max ssl buffer size
93 while chunk_start
+ chunk_size
< file_size
:
94 yield(chunk_start
, chunk_size
)
95 chunk_start
+= chunk_size
96 final_chunk_size
= file_size
- chunk_start
97 yield(chunk_start
, final_chunk_size
)
100 def to_bytes(string
):
101 return bytes(string
, encoding
='utf-8')
103 def ls(path
, opts
=''):
104 # opts tries to be like /bin/ls opts
105 almost_all
= 'A' in opts
107 with cephfs
.opendir(path
) as d
:
109 dent
= cephfs
.readdir(d
)
112 elif almost_all
and dent
.d_name
in (b
'.', b
'..'):
115 except cephfs
.ObjectNotFound
:
118 def glob(path
, pattern
):
120 parent_dir
= os
.path
.dirname(path
)
121 if parent_dir
== b
'':
123 if path
== b
'/' or is_dir_exists(os
.path
.basename(path
), parent_dir
):
124 for i
in ls(path
, opts
='A'):
125 if fnmatch
.fnmatch(i
.d_name
, pattern
):
126 paths
.append(os
.path
.join(path
, i
.d_name
))
130 def locate_file(name
, case_sensitive
=True):
131 dir_list
= sorted(set(dirwalk(cephfs
.getcwd())))
132 if not case_sensitive
:
133 return [dname
for dname
in dir_list
if name
.lower() in dname
.lower()]
135 return [dname
for dname
in dir_list
if name
in dname
]
138 def get_all_possible_paths(pattern
):
139 complete_pattern
= pattern
[:]
141 is_rel_path
= not os
.path
.isabs(pattern
)
143 dir_
= cephfs
.getcwd()
146 pattern
= pattern
[1:]
147 patterns
= pattern
.split(b
'/')
148 paths
.extend(glob(dir_
, patterns
[0]))
150 for pattern
in patterns
:
152 paths
.extend(glob(path
, pattern
))
153 return [path
for path
in paths
if fnmatch
.fnmatch(path
,
154 os
.path
.join(cephfs
.getcwd(), complete_pattern
))]
157 suffixes
= ['B', 'K', 'M', 'G', 'T', 'P']
160 def humansize(nbytes
):
162 while nbytes
>= 1024 and i
< len(suffixes
)-1:
165 nbytes
= math
.ceil(nbytes
)
166 f
= ('%d' % nbytes
).rstrip('.')
167 return '%s%s' % (f
, suffixes
[i
])
170 def print_long(path
, is_dir
, human_readable
):
171 info
= cephfs
.stat(path
)
172 pretty
= os
.path
.basename(path
.decode('utf-8'))
174 pretty
= colorama
.Style
.BRIGHT
+ colorama
.Fore
.CYAN
+ pretty
+ '/' + colorama
.Style
.RESET_ALL
176 poutput('{}\t{:10s} {} {} {} {}'.format(
177 mode_notation(info
.st_mode
),
178 humansize(info
.st_size
), info
.st_uid
,
179 info
.st_gid
, info
.st_mtime
, pretty
, sep
='\t'))
181 poutput('{} {:12d} {} {} {} {}'.format(
182 mode_notation(info
.st_mode
), info
.st_size
, info
.st_uid
,
183 info
.st_gid
, info
.st_mtime
, pretty
, sep
='\t'))
188 Returns the word length, minus any color codes.
190 if word
[0] == '\x1b':
195 def is_dir_exists(path
, dir_
=b
''):
196 path_to_stat
= os
.path
.join(dir_
, path
)
198 return ((cephfs
.stat(path_to_stat
).st_mode
& 0o0040000) != 0)
199 except libcephfs
.Error
:
203 def is_file_exists(path
, dir_
=b
''):
205 # if its not a directory, then its a file
206 return ((cephfs
.stat(os
.path
.join(dir_
, path
)).st_mode
& 0o0040000) == 0)
207 except libcephfs
.Error
:
211 def print_list(words
, termwidth
=79):
214 words
= [word
.decode('utf-8') if isinstance(word
, bytes
) else word
for word
in words
]
215 width
= max([word_len(word
) for word
in words
]) + 2
217 ncols
= max(1, (termwidth
+ 1) // (width
+ 1))
218 nrows
= (nwords
+ ncols
- 1) // ncols
219 for row
in range(nrows
):
220 for i
in range(row
, nwords
, nrows
):
223 if word
[0] == '\x1b':
224 print_width
= print_width
+ 10
226 poutput('%-*s' % (print_width
, words
[i
]),
227 end
='\n' if i
+ nrows
>= nwords
else '')
230 def copy_from_local(local_path
, remote_path
):
234 convert_to_bytes
= False
235 if local_path
== b
'-':
237 convert_to_bytes
= True
240 file_
= open(local_path
, 'rb')
241 except PermissionError
:
242 perror('error: no permission to read local file {}'.format(
243 local_path
.decode('utf-8')), end
='\n', apply_style
=True)
247 fd
= cephfs
.open(remote_path
, 'w', 0o666)
248 except libcephfs
.Error
:
249 perror('error: no permission to write remote file {}'.format(
250 remote_path
.decode('utf-8')), end
='\n', apply_style
=True)
254 data
= file_
.read(65536)
255 if not data
or len(data
) == 0:
258 data
= to_bytes(data
)
259 wrote
= cephfs
.write(fd
, data
, progress
)
269 def copy_to_local(remote_path
, local_path
):
271 if local_path
!= b
'-':
272 local_dir
= os
.path
.dirname(local_path
)
273 dir_list
= remote_path
.rsplit(b
'/', 1)
274 if not os
.path
.exists(local_dir
):
275 os
.makedirs(local_dir
)
276 if len(dir_list
) > 2 and dir_list
[1] == b
'':
278 fd
= open(local_path
, 'wb+')
279 file_
= cephfs
.open(remote_path
, 'r')
280 file_size
= cephfs
.stat(remote_path
).st_size
284 for chunk_start
, chunk_size
in get_chunks(file_size
):
285 file_chunk
= cephfs
.read(file_
, chunk_start
, chunk_size
)
286 progress
+= len(file_chunk
)
290 poutput(file_chunk
.decode('utf-8'))
298 walk a directory tree, using a generator
300 path
= os
.path
.normpath(path
)
301 for item
in ls(path
, opts
='A'):
302 fullpath
= os
.path
.join(path
, item
.d_name
)
303 src_path
= fullpath
.rsplit(b
'/', 1)[0]
305 yield os
.path
.normpath(fullpath
)
306 if is_dir_exists(item
.d_name
, src_path
):
307 for x
in dirwalk(fullpath
):
311 class CephFSShell(Cmd
):
314 super().__init
__(use_ipython
=False)
315 self
.working_dir
= cephfs
.getcwd().decode('utf-8')
317 self
.interactive
= False
320 def default(self
, line
):
321 self
.poutput('Unrecognized command')
323 def set_prompt(self
):
324 self
.prompt
= ('\033[01;33mCephFS:~' + colorama
.Fore
.LIGHTCYAN_EX
325 + self
.working_dir
+ colorama
.Style
.RESET_ALL
326 + '\033[01;33m>>>\033[00m ')
328 def create_argparser(self
, command
):
330 argparse_args
= getattr(self
, 'argparse_' + command
)
331 except AttributeError:
334 self
, 'do_' + command
).__doc
__.expandtabs().splitlines()
336 blank_idx
= doc_lines
.index('')
337 usage
= doc_lines
[:blank_idx
]
338 description
= doc_lines
[blank_idx
+ 1:]
342 parser
= argparse
.ArgumentParser(
344 usage
='\n'.join(usage
),
345 description
='\n'.join(description
),
346 formatter_class
=argparse
.ArgumentDefaultsHelpFormatter
348 for args
, kwargs
in argparse_args
:
349 parser
.add_argument(*args
, **kwargs
)
352 def complete_filenames(self
, text
, line
, begidx
, endidx
):
354 completions
= [x
.d_name
.decode('utf-8') + '/' * int(x
.is_dir())
355 for x
in ls(b
".", opts
='A')]
357 if text
.count('/') > 0:
358 completions
= [text
.rsplit('/', 1)[0] + '/'
359 + x
.d_name
.decode('utf-8') + '/'
360 * int(x
.is_dir()) for x
in ls('/'
361 + text
.rsplit('/', 1)[0], opts
='A')
362 if x
.d_name
.decode('utf-8').startswith(
363 text
.rsplit('/', 1)[1])]
365 completions
= [x
.d_name
.decode('utf-8') + '/'
366 * int(x
.is_dir()) for x
in ls(b
".", opts
='A')
367 if x
.d_name
.decode('utf-8').startswith(text
)]
368 if len(completions
) == 1 and completions
[0][-1] == '/':
369 dir_
, file_
= completions
[0].rsplit('/', 1)
370 completions
.extend([dir_
+ '/' + x
.d_name
.decode('utf-8')
371 + '/' * int(x
.is_dir()) for x
in
372 ls('/' + dir_
, opts
='A')
373 if x
.d_name
.decode('utf-8').startswith(file_
)])
374 return self
.delimiter_complete(text
, line
, begidx
, endidx
, completions
, '/')
377 def onecmd(self
, line
):
382 res
= Cmd
.onecmd(self
, line
)
386 except ConnectionError
as e
:
387 self
.poutput('***', e
)
388 except KeyboardInterrupt:
389 self
.poutput('Command aborted')
390 except Exception as e
:
392 traceback
.print_exc(file=sys
.stdout
)
394 class path_to_bytes(argparse
.Action
):
395 def __call__(self
, parser
, namespace
, values
, option_string
=None):
396 if isinstance(values
, str):
397 values
= to_bytes(values
)
398 if isinstance(values
, list):
399 values
= list(map(to_bytes
, values
))
400 setattr(namespace
, self
.dest
, values
)
402 def complete_mkdir(self
, text
, line
, begidx
, endidx
):
404 auto complete of file name.
406 return self
.complete_filenames(text
, line
, begidx
, endidx
)
408 class ModeAction(argparse
.Action
):
409 def __init__(self
, option_strings
, dest
, nargs
=None, **kwargs
):
410 if nargs
is not None and nargs
!= '?':
411 raise ValueError("more than one modes not allowed")
412 super().__init
__(option_strings
, dest
, **kwargs
)
414 def __call__(self
, parser
, namespace
, values
, option_string
=None):
418 o_mode
= int(values
, base
=8)
420 res
= re
.match('((u?g?o?)|(a?))(=)(r?w?x?)', values
)
422 parser
.error("invalid mode: %s\n"
423 "mode must be a numeric octal literal\n"
424 "or ((u?g?o?)|(a?))(=)(r?w?x?)" %
427 # we are supporting only assignment of mode and not + or -
428 # as is generally available with the chmod command
430 # >>> res = re.match('((u?g?o?)|(a?))(=)(r?w?x?)', 'go=')
432 # ('go', 'go', None, '=', '')
436 parser
.error("need assignment operator between user "
437 "and mode specifiers")
439 parser
.error("invalid mode: %s\n"
440 "mode must be combination of: r | w | x" %
460 o_mode |
= (t_mode
<< 6)
462 o_mode |
= (t_mode
<< 3)
467 parser
.error("invalid mode: %s\n"
468 "mode cannot be negative" % values
)
470 parser
.error("invalid mode: %s\n"
471 "mode cannot be greater than octal 0777" % values
)
473 setattr(namespace
, self
.dest
, str(oct(o_mode
)))
475 mkdir_parser
= argparse
.ArgumentParser(
476 description
='Create the directory(ies), if they do not already exist.')
477 mkdir_parser
.add_argument('dirs', type=str,
478 action
=path_to_bytes
,
480 help='Name of new_directory.',
482 mkdir_parser
.add_argument('-m', '--mode', type=str,
484 help='Sets the access mode for the new directory.')
485 mkdir_parser
.add_argument('-p', '--parent', action
='store_true',
486 help='Create parent directories as necessary. \
487 When this option is specified, no error is reported if a directory already \
490 @with_argparser(mkdir_parser
)
491 def do_mkdir(self
, args
):
495 for path
in args
.dirs
:
497 permission
= int(args
.mode
, 8)
501 cephfs
.mkdirs(path
, permission
)
504 cephfs
.mkdir(path
, permission
)
505 except libcephfs
.Error
:
506 self
.poutput("directory missing in the path; "
507 "you may want to pass the -p argument")
510 def complete_put(self
, text
, line
, begidx
, endidx
):
512 auto complete of file name.
514 index_dict
= {1: self
.path_complete
}
515 return self
.index_based_complete(text
, line
, begidx
, endidx
, index_dict
)
517 put_parser
= argparse
.ArgumentParser(
518 description
='Copy a file/directory to Ceph File System from Local File System.')
519 put_parser
.add_argument('local_path', type=str, action
=path_to_bytes
,
520 help='Path of the file in the local system')
521 put_parser
.add_argument('remote_path', type=str, action
=path_to_bytes
,
522 help='Path of the file in the remote system.',
523 nargs
='?', default
='.')
524 put_parser
.add_argument('-f', '--force', action
='store_true',
525 help='Overwrites the destination if it already exists.')
527 @with_argparser(put_parser
)
528 def do_put(self
, args
):
530 Copy a file to Ceph File System from Local Directory.
532 root_src_dir
= args
.local_path
533 root_dst_dir
= args
.remote_path
534 if args
.local_path
== b
'.' or args
.local_path
== b
'./':
535 root_src_dir
= os
.getcwdb()
536 elif len(args
.local_path
.rsplit(b
'/', 1)) < 2:
537 root_src_dir
= os
.path
.join(os
.getcwdb(), args
.local_path
)
539 p
= args
.local_path
.split(b
'/')
541 root_src_dir
= os
.getcwdb()
544 root_src_dir
+= b
'/' + p
.pop(0)
546 if root_dst_dir
== b
'.':
547 if args
.local_path
!= b
'-':
548 root_dst_dir
= root_src_dir
.rsplit(b
'/', 1)[1]
549 if root_dst_dir
== b
'':
550 root_dst_dir
= root_src_dir
.rsplit(b
'/', 1)[0]
551 a
= root_dst_dir
.rsplit(b
'/', 1)
557 self
.poutput("error: no filename specified for destination")
560 if root_dst_dir
[-1] != b
'/':
563 if args
.local_path
== b
'-' or os
.path
.isfile(root_src_dir
):
565 if os
.path
.isfile(root_src_dir
):
566 dst_file
= root_dst_dir
567 if is_file_exists(dst_file
):
568 self
.perror('{}: file exists! use --force to overwrite'.format(
569 dst_file
.decode('utf-8')), end
='\n',
572 if args
.local_path
== b
'-':
574 copy_from_local(root_src_dir
, root_dst_dir
)
576 for src_dir
, dirs
, files
in os
.walk(root_src_dir
):
577 if isinstance(src_dir
, str):
578 src_dir
= to_bytes(src_dir
)
579 dst_dir
= src_dir
.replace(root_src_dir
, root_dst_dir
, 1)
580 dst_dir
= re
.sub(rb
'\/+', b
'/', cephfs
.getcwd()
582 if args
.force
and dst_dir
!= b
'/' and not is_dir_exists(
583 dst_dir
[:-1]) and not locate_file(dst_dir
):
585 cephfs
.mkdirs(dst_dir
, 0o777)
586 except libcephfs
.Error
:
588 if (not args
.force
) and dst_dir
!= b
'/' and not is_dir_exists(
589 dst_dir
) and not os
.path
.isfile(root_src_dir
):
591 cephfs
.mkdirs(dst_dir
, 0o777)
592 except libcephfs
.Error
:
596 dir_name
= os
.path
.join(dst_dir
, dir_
)
597 if not is_dir_exists(dir_name
):
599 cephfs
.mkdirs(dir_name
, 0o777)
600 except libcephfs
.Error
:
604 src_file
= os
.path
.join(src_dir
, file_
)
605 dst_file
= re
.sub(rb
'\/+', b
'/', b
'/' + dst_dir
+ b
'/' + file_
)
606 if (not args
.force
) and is_file_exists(dst_file
):
608 copy_from_local(src_file
, os
.path
.join(cephfs
.getcwd(),
611 def complete_get(self
, text
, line
, begidx
, endidx
):
613 auto complete of file name.
615 return self
.complete_filenames(text
, line
, begidx
, endidx
)
617 get_parser
= argparse
.ArgumentParser(
618 description
='Copy a file from Ceph File System from Local Directory.')
619 get_parser
.add_argument('remote_path', type=str, action
=path_to_bytes
,
620 help='Path of the file in the remote system')
621 get_parser
.add_argument('local_path', type=str, action
=path_to_bytes
,
622 help='Path of the file in the local system',
623 nargs
='?', default
='.')
624 get_parser
.add_argument('-f', '--force', action
='store_true',
625 help='Overwrites the destination if it already exists.')
627 @with_argparser(get_parser
)
628 def do_get(self
, args
):
630 Copy a file/directory from Ceph File System to Local Directory.
632 root_src_dir
= args
.remote_path
633 root_dst_dir
= args
.local_path
634 fname
= root_src_dir
.rsplit(b
'/', 1)
635 if args
.local_path
== b
'.':
636 root_dst_dir
= os
.getcwdb()
637 if args
.remote_path
== b
'.':
638 root_src_dir
= cephfs
.getcwd()
639 if args
.local_path
== b
'-':
640 if args
.remote_path
== b
'.' or args
.remote_path
== b
'./':
641 self
.perror('error: no remote file name specified', end
='\n',
644 copy_to_local(root_src_dir
, b
'-')
645 elif is_file_exists(args
.remote_path
):
646 copy_to_local(root_src_dir
,
647 root_dst_dir
+ b
'/' + root_src_dir
)
648 elif b
'/'in root_src_dir
and is_file_exists(fname
[1], fname
[0]):
649 copy_to_local(root_src_dir
, root_dst_dir
)
651 files
= list(reversed(sorted(dirwalk(root_src_dir
))))
654 os
.makedirs(root_dst_dir
+ b
'/' + root_src_dir
)
659 self
.perror('{}: already exists! use --force to overwrite'.format(
660 root_src_dir
.decode('utf-8')), end
='\n',
665 dst_dirpath
, dst_file
= file_
.rsplit(b
'/', 1)
666 if dst_dirpath
in files
:
667 files
.remove(dst_dirpath
)
668 dst_path
= os
.path
.join(root_dst_dir
, dst_dirpath
, dst_file
)
669 dst_path
= os
.path
.normpath(dst_path
)
670 if is_dir_exists(file_
):
672 os
.makedirs(dst_path
)
679 self
.perror('{}: file already exists! use --force to override'.format(
680 file_
.decode('utf-8')), end
='\n',
684 copy_to_local(file_
, dst_path
)
686 copy_to_local(file_
, dst_path
)
690 def complete_ls(self
, text
, line
, begidx
, endidx
):
692 auto complete of file name.
694 return self
.complete_filenames(text
, line
, begidx
, endidx
)
696 ls_parser
= argparse
.ArgumentParser(
697 description
='Copy a file from Ceph File System from Local Directory.')
698 ls_parser
.add_argument('-l', '--long', action
='store_true',
699 help='Detailed list of items in the directory.')
700 ls_parser
.add_argument('-r', '--reverse', action
='store_true',
701 help='Reverse order of listing items in the directory.')
702 ls_parser
.add_argument('-H', action
='store_true', help='Human Readable')
703 ls_parser
.add_argument('-a', '--all', action
='store_true',
704 help='Do not Ignore entries starting with .')
705 ls_parser
.add_argument('-S', action
='store_true', help='Sort by file_size')
706 ls_parser
.add_argument('paths', help='Name of Directories',
707 action
=path_to_bytes
, nargs
='*', default
=['.'])
709 @with_argparser(ls_parser
)
710 def do_ls(self
, args
):
712 List all the files and directories in the current working directory
718 if path
.count(b
'*') > 0:
719 all_items
= get_all_possible_paths(path
)
720 if len(all_items
) == 0:
722 path
= all_items
[0].rsplit(b
'/', 1)[0]
727 for item
in ls(path
):
729 if os
.path
.basename(i
) == d_name
:
731 dirs
.append(os
.path
.join(path
, d_name
))
737 self
.poutput(path
.decode('utf-8'), end
=':\n')
738 items
= sorted(items
, key
=lambda item
: item
.d_name
)
740 if path
!= b
'' and path
!= cephfs
.getcwd() and len(paths
) > 1:
741 self
.poutput(path
.decode('utf-8'), end
=':\n')
742 items
= sorted(ls(path
),
743 key
=lambda item
: item
.d_name
)
745 items
= [i
for i
in items
if not i
.d_name
.startswith(b
'.')]
748 items
= sorted(items
, key
=lambda item
: cephfs
.stat(
749 path
+ b
'/' + item
.d_name
).st_size
)
752 items
= reversed(items
)
754 filepath
= item
.d_name
755 is_dir
= item
.is_dir()
757 if args
.long and args
.H
:
758 print_long(cephfs
.getcwd()
764 print_long(cephfs
.getcwd()
770 values
.append(colorama
.Style
.BRIGHT
772 + filepath
.decode('utf-8')
774 + colorama
.Style
.RESET_ALL
)
776 values
.append(filepath
)
778 print_list(values
, shutil
.get_terminal_size().columns
)
779 if path
!= paths
[-1]:
782 def complete_rmdir(self
, text
, line
, begidx
, endidx
):
784 auto complete of file name.
786 return self
.complete_filenames(text
, line
, begidx
, endidx
)
788 rmdir_parser
= argparse
.ArgumentParser(description
='Remove Directory.')
789 rmdir_parser
.add_argument('paths', help='Directory Path.', nargs
='+',
790 action
=path_to_bytes
)
791 rmdir_parser
.add_argument('-p', '--parent', action
='store_true',
792 help='Remove parent directories as necessary. \
793 When this option is specified, no error is reported if a directory has any \
794 sub-directories, files')
796 @with_argparser(rmdir_parser
)
797 def do_rmdir(self
, args
):
799 Remove a specific Directory
804 if path
.count(b
'*') > 0:
806 all_items
= get_all_possible_paths(path
)
807 if len(all_items
) > 0:
808 path
= all_items
[0].rsplit(b
'/', 1)[0]
813 for item
in ls(path
):
815 if os
.path
.basename(i
) == d_name
:
817 dirs
.append(os
.path
.join(path
, d_name
))
822 path
= os
.path
.normpath(os
.path
.join(cephfs
.getcwd(), path
))
824 files
= reversed(sorted(set(dirwalk(path
))))
825 for filepath
in files
:
826 filepath
= os
.path
.normpath(filepath
)
827 if filepath
[1:] != path
:
829 cephfs
.rmdir(filepath
)
830 except libcephfs
.Error
:
831 cephfs
.unlink(filepath
)
832 if not is_pattern
and path
!= os
.path
.normpath(b
''):
835 except libcephfs
.Error
:
836 self
.perror('error: no such directory {} exists'.format(
837 path
.decode('utf-8')), end
='\n',
840 def complete_rm(self
, text
, line
, begidx
, endidx
):
842 auto complete of file name.
844 return self
.complete_filenames(text
, line
, begidx
, endidx
)
846 rm_parser
= argparse
.ArgumentParser(description
='Remove File.')
847 rm_parser
.add_argument('paths', help='File Path.', nargs
='+',
848 action
=path_to_bytes
)
850 @with_argparser(rm_parser
)
851 def do_rm(self
, args
):
853 Remove a specific file
855 file_paths
= args
.paths
856 for path
in file_paths
:
857 if path
.count(b
'*') > 0:
858 file_paths
.extend([i
for i
in get_all_possible_paths(
859 path
) if is_file_exists(i
)])
863 except libcephfs
.Error
:
864 self
.perror('{}: no such file'.format(path
.decode('utf-8')),
865 end
='\n', apply_style
=True)
867 def complete_mv(self
, text
, line
, begidx
, endidx
):
869 auto complete of file name.
871 return self
.complete_filenames(text
, line
, begidx
, endidx
)
873 mv_parser
= argparse
.ArgumentParser(description
='Move File.')
874 mv_parser
.add_argument('src_path', type=str, action
=path_to_bytes
,
875 help='Source File Path.')
876 mv_parser
.add_argument('dest_path', type=str, action
=path_to_bytes
,
877 help='Destination File Path.')
879 @with_argparser(mv_parser
)
880 def do_mv(self
, args
):
882 Rename a file or Move a file from source path to the destination
885 cephfs
.rename(args
.src_path
, args
.dest_path
)
886 except libcephfs
.Error
:
887 self
.poutput("error: need a file name to move to")
889 def complete_cd(self
, text
, line
, begidx
, endidx
):
891 auto complete of file name.
893 return self
.complete_filenames(text
, line
, begidx
, endidx
)
895 cd_parser
= argparse
.ArgumentParser(description
='Change working directory')
896 cd_parser
.add_argument('path', type=str, help='Name of the directory.',
897 action
=path_to_bytes
, nargs
='?', default
='/')
899 @with_argparser(cd_parser
)
900 def do_cd(self
, args
):
902 Change working directory
905 cephfs
.chdir(args
.path
)
906 self
.working_dir
= cephfs
.getcwd().decode('utf-8')
908 except libcephfs
.Error
:
909 self
.perror('{}: no such directory'.format(args
.path
.decode('utf-8')),
910 end
='\n', apply_style
=True)
912 def do_cwd(self
, arglist
):
914 Get current working directory.
916 self
.poutput(cephfs
.getcwd().decode('utf-8'))
918 def complete_chmod(self
, text
, line
, begidx
, endidx
):
920 auto complete of file name.
922 return self
.complete_filenames(text
, line
, begidx
, endidx
)
924 chmod_parser
= argparse
.ArgumentParser(description
='Create Directory.')
925 chmod_parser
.add_argument('mode', type=str, action
=ModeAction
, help='Mode')
926 chmod_parser
.add_argument('paths', type=str, action
=path_to_bytes
,
927 help='Name of the file', nargs
='+')
929 @with_argparser(chmod_parser
)
930 def do_chmod(self
, args
):
932 Change permission of a file
934 for path
in args
.paths
:
935 mode
= int(args
.mode
, base
=8)
937 cephfs
.chmod(path
, mode
)
938 except libcephfs
.Error
:
939 self
.perror('{}: no such file or directory'.format(
940 path
.decode('utf-8')), end
='\n', apply_style
=True)
942 def complete_cat(self
, text
, line
, begidx
, endidx
):
944 auto complete of file name.
946 return self
.complete_filenames(text
, line
, begidx
, endidx
)
948 cat_parser
= argparse
.ArgumentParser(description
='')
949 cat_parser
.add_argument('paths', help='Name of Files', action
=path_to_bytes
,
952 @with_argparser(cat_parser
)
953 def do_cat(self
, args
):
955 Print contents of a file
957 for path
in args
.paths
:
958 if is_file_exists(path
):
959 copy_to_local(path
, b
'-')
961 self
.perror('{}: no such file'.format(path
.decode('utf-8')),
962 end
='\n', apply_style
=True)
964 umask_parser
= argparse
.ArgumentParser(description
='Set umask value.')
965 umask_parser
.add_argument('mode', help='Mode', type=str, action
=ModeAction
,
966 nargs
='?', default
='')
968 @with_argparser(umask_parser
)
969 def do_umask(self
, args
):
974 self
.poutput(self
.umask
.zfill(4))
976 mode
= int(args
.mode
, 8)
977 self
.umask
= str(oct(cephfs
.umask(mode
))[2:])
979 def complete_write(self
, text
, line
, begidx
, endidx
):
981 auto complete of file name.
983 return self
.complete_filenames(text
, line
, begidx
, endidx
)
985 write_parser
= argparse
.ArgumentParser(description
='Writes data into a file')
986 write_parser
.add_argument('path', type=str, action
=path_to_bytes
,
989 @with_argparser(write_parser
)
990 def do_write(self
, args
):
992 Write data into a file.
995 copy_from_local(b
'-', args
.path
)
997 def complete_lcd(self
, text
, line
, begidx
, endidx
):
999 auto complete of file name.
1001 index_dict
= {1: self
.path_complete
}
1002 return self
.index_based_complete(text
, line
, begidx
, endidx
, index_dict
)
1004 lcd_parser
= argparse
.ArgumentParser(description
='')
1005 lcd_parser
.add_argument('path', type=str, action
=path_to_bytes
, help='Path')
1007 @with_argparser(lcd_parser
)
1008 def do_lcd(self
, args
):
1010 Moves into the given local directory
1013 os
.chdir(os
.path
.expanduser(args
.path
))
1014 except OSError as e
:
1015 self
.perror("Cannot change to {}: {}".format(e
.filename
,
1018 def complete_lls(self
, text
, line
, begidx
, endidx
):
1020 auto complete of file name.
1022 index_dict
= {1: self
.path_complete
}
1023 return self
.index_based_complete(text
, line
, begidx
, endidx
, index_dict
)
1025 lls_parser
= argparse
.ArgumentParser(
1026 description
='List files in local system.')
1027 lls_parser
.add_argument('paths', help='Paths', action
=path_to_bytes
,
1030 @with_argparser(lls_parser
)
1031 def do_lls(self
, args
):
1033 Lists all files and folders in the current local directory
1036 print_list(os
.listdir(os
.getcwdb()))
1038 for path
in args
.paths
:
1040 items
= os
.listdir(path
)
1041 self
.poutput("{}:".format(path
.decode('utf-8')))
1043 except OSError as e
:
1044 self
.perror("'{}': {}".format(e
.filename
, e
.strerror
), False)
1045 # Arguments to the with_argpaser decorator function are sticky.
1046 # The items in args.path do not get overwritten in subsequent calls.
1047 # The arguments remain in args.paths after the function exits and we
1048 # neeed to clean it up to ensure the next call works as expected.
1051 def do_lpwd(self
, arglist
):
1053 Prints the absolute path of the current local directory
1055 self
.poutput(os
.getcwd())
1057 def do_df(self
, arglist
):
1059 Display the amount of available disk space for file systems
1061 for index
, i
in enumerate(ls(b
".", opts
='A')):
1063 self
.poutput('{:25s}\t{:5s}\t{:15s}{:10s}{}'.format(
1064 "1K-blocks", "Used", "Available", "Use%", "Stored on"))
1065 if not is_dir_exists(i
.d_name
):
1066 statfs
= cephfs
.statfs(i
.d_name
)
1067 stat
= cephfs
.stat(i
.d_name
)
1068 block_size
= statfs
['f_blocks']*statfs
['f_bsize'] // 1024
1069 available
= block_size
- stat
.st_size
1072 use
= (stat
.st_size
*100 // block_size
)
1073 self
.poutput('{:25d}\t{:5d}\t{:10d}\t{:5s} {}'.format(
1074 statfs
['f_fsid'], stat
.st_size
, available
,
1075 str(int(use
)) + '%', i
.d_name
.decode('utf-8')))
1077 locate_parser
= argparse
.ArgumentParser(
1078 description
='Find file within file system')
1079 locate_parser
.add_argument('name', help='name', type=str,
1080 action
=path_to_bytes
)
1081 locate_parser
.add_argument('-c', '--count', action
='store_true',
1082 help='Count list of items located.')
1083 locate_parser
.add_argument(
1084 '-i', '--ignorecase', action
='store_true', help='Ignore case')
1086 @with_argparser(locate_parser
)
1087 def do_locate(self
, args
):
1089 Find a file within the File System
1091 if args
.name
.count(b
'*') == 1:
1092 if args
.name
[0] == b
'*':
1094 elif args
.name
[-1] == '*':
1095 args
.name
= b
'/' + args
.name
1096 args
.name
= args
.name
.replace(b
'*', b
'')
1098 locations
= locate_file(args
.name
, False)
1100 locations
= locate_file(args
.name
)
1102 self
.poutput(len(locations
))
1104 self
.poutput((b
'\n'.join(locations
)).decode('utf-8'))
1106 def complete_du(self
, text
, line
, begidx
, endidx
):
1108 auto complete of file name.
1110 return self
.complete_filenames(text
, line
, begidx
, endidx
)
1112 du_parser
= argparse
.ArgumentParser(
1113 description
='Disk Usage of a Directory')
1114 du_parser
.add_argument('dirs', type=str, action
=path_to_bytes
,
1115 help='Name of the directory.', nargs
='?',
1117 du_parser
.add_argument('-r', action
='store_true',
1118 help='Recursive Disk usage of all directories.')
1120 @with_argparser(du_parser
)
1121 def do_du(self
, args
):
1123 Disk Usage of a Directory
1125 if args
.dirs
== b
'':
1126 args
.dirs
= cephfs
.getcwd()
1127 for dir_
in args
.dirs
:
1129 for i
in reversed(sorted(set(dirwalk(dir_
)))):
1130 i
= os
.path
.normpath(i
)
1132 xattr
= cephfs
.getxattr(i
, 'ceph.dir.rbytes')
1133 self
.poutput('{:10s} {}'.format(
1134 humansize(int(xattr
.decode('utf-8'))), '.'
1135 + i
.decode('utf-8')))
1136 except libcephfs
.Error
:
1139 dir_
= os
.path
.normpath(dir_
)
1140 self
.poutput('{:10s} {}'.format(humansize(int(cephfs
.getxattr(
1141 dir_
, 'ceph.dir.rbytes').decode('utf-8'))), '.'
1142 + dir_
.decode('utf-8')))
1144 quota_parser
= argparse
.ArgumentParser(
1145 description
='Quota management for a Directory')
1146 quota_parser
.add_argument('op', choices
=['get', 'set'],
1147 help='Quota operation type.')
1148 quota_parser
.add_argument('path', type=str, action
=path_to_bytes
,
1149 help='Name of the directory.')
1150 quota_parser
.add_argument('--max_bytes', type=int, default
=-1, nargs
='?',
1151 help='Max cumulative size of the data under '
1153 quota_parser
.add_argument('--max_files', type=int, default
=-1, nargs
='?',
1154 help='Total number of files under this '
1157 @with_argparser(quota_parser
)
1158 def do_quota(self
, args
):
1162 if not is_dir_exists(args
.path
):
1163 self
.perror('error: no such directory {}'.format(args
.path
.decode('utf-8')),
1164 end
='\n', apply_style
=True)
1167 if args
.op
== 'set':
1168 if (args
.max_bytes
== -1) and (args
.max_files
== -1):
1169 self
.poutput('please specify either --max_bytes or '
1170 '--max_files or both')
1173 if args
.max_bytes
>= 0:
1174 max_bytes
= to_bytes(str(args
.max_bytes
))
1176 cephfs
.setxattr(args
.path
, 'ceph.quota.max_bytes',
1177 max_bytes
, len(max_bytes
),
1179 self
.poutput('max_bytes set to %d' % args
.max_bytes
)
1180 except libcephfs
.Error
:
1181 cephfs
.setxattr(args
.path
, 'ceph.quota.max_bytes',
1182 max_bytes
, len(max_bytes
),
1184 self
.poutput('max_bytes reset to %d' % args
.max_bytes
)
1186 if args
.max_files
>= 0:
1187 max_files
= to_bytes(str(args
.max_files
))
1189 cephfs
.setxattr(args
.path
, 'ceph.quota.max_files',
1190 max_files
, len(max_files
),
1192 self
.poutput('max_files set to %d' % args
.max_files
)
1193 except libcephfs
.Error
:
1194 cephfs
.setxattr(args
.path
, 'ceph.quota.max_files',
1195 max_files
, len(max_files
),
1197 self
.poutput('max_files reset to %d' % args
.max_files
)
1198 elif args
.op
== 'get':
1202 max_bytes
= cephfs
.getxattr(args
.path
,
1203 'ceph.quota.max_bytes')
1204 self
.poutput('max_bytes: %s' % max_bytes
)
1205 except libcephfs
.Error
:
1206 self
.poutput('max_bytes is not set')
1210 max_files
= cephfs
.getxattr(args
.path
,
1211 'ceph.quota.max_files')
1212 self
.poutput('max_files: %s' % max_files
)
1213 except libcephfs
.Error
:
1214 self
.poutput('max_files is not set')
1217 def do_help(self
, line
):
1219 Get details about a command.
1220 Usage: help <cmd> - for a specific command
1221 help all - for all the commands
1225 if k
.startswith('do_'):
1226 self
.poutput('-'*80)
1227 super().do_help(k
[3:])
1229 parser
= self
.create_argparser(line
)
1233 super().do_help(line
)
1235 def complete_stat(self
, text
, line
, begidx
, endidx
):
1237 auto complete of file name.
1239 return self
.complete_filenames(text
, line
, begidx
, endidx
)
1241 stat_parser
= argparse
.ArgumentParser(
1242 description
='Display file or file system status')
1243 stat_parser
.add_argument('paths', type=str, help='file paths',
1244 action
=path_to_bytes
, nargs
='+')
1246 @with_argparser(stat_parser
)
1247 def do_stat(self
, args
):
1249 Display file or file system status
1251 for path
in args
.paths
:
1253 stat
= cephfs
.stat(path
)
1254 atime
= stat
.st_atime
.isoformat(' ')
1255 mtime
= stat
.st_mtime
.isoformat(' ')
1256 ctime
= stat
.st_mtime
.isoformat(' ')
1258 self
.poutput("File: {}\nSize: {:d}\nBlocks: {:d}\nIO Block: {:d}\n\
1259 Device: {:d}\tInode: {:d}\tLinks: {:d}\nPermission: {:o}/{}\tUid: {:d}\tGid: {:d}\n\
1260 Access: {}\nModify: {}\nChange: {}".format(path
.decode('utf-8'), stat
.st_size
,
1261 stat
.st_blocks
, stat
.st_blksize
, stat
.st_dev
,
1262 stat
.st_ino
, stat
.st_nlink
, stat
.st_mode
,
1263 mode_notation(stat
.st_mode
), stat
.st_uid
,
1264 stat
.st_gid
, atime
, mtime
, ctime
))
1265 except libcephfs
.Error
:
1266 self
.perror('{}: no such file or directory'.format(path
.decode('utf-8')),
1267 end
='\n', apply_style
=True)
1270 if __name__
== '__main__':
1273 main_parser
= argparse
.ArgumentParser(description
='')
1274 main_parser
.add_argument('-c', '--config', action
='store',
1275 help='Configuration file_path', type=str)
1276 main_parser
.add_argument(
1277 '-b', '--batch', action
='store', help='Batch File path.', type=str)
1278 main_parser
.add_argument('-t', '--test', action
='store',
1279 help='Test against transcript(s) in FILE',
1281 main_parser
.add_argument('commands', nargs
='*',
1282 help='comma delimited commands', default
=[])
1283 args
= main_parser
.parse_args()
1285 config_file
= args
.config
1287 args
.commands
= ['load ' + args
.batch
, ',quit']
1289 args
.commands
.extend(['-t,'] + [arg
+',' for arg
in args
.test
])
1291 sys
.argv
.append(exe
)
1292 sys
.argv
.extend([i
.strip() for i
in ' '.join(args
.commands
).split(',')])
1293 setup_cephfs(config_file
)
1294 shell
= CephFSShell()