ceph/src/tools/cephfs/cephfs-shell

   1 #!/usr/bin/python3
   2 # coding = utf-8
   3
   4 import argparse
   5 import os
   6 import os.path
   7 import sys
   8 from cmd2 import Cmd
   9 import cephfs as libcephfs
  10 import shutil
  11 import traceback
  12 import colorama
  13 import readline
  14 import fnmatch
  15 import math
  16 import re
  17 import shlex
  18
  19 try:
  20     from cmd2 import with_argparser
  21 except ImportError:
  22     def with_argparser(argparser):
  23         import functools
  24
  25         def argparser_decorator(func):
  26             @functools.wraps(func)
  27             def wrapper(thiz, cmdline):
  28                 if isinstance(cmdline, list):
  29                     arglist = cmdline
  30                 else:
  31                     # do not split if it's already a list
  32                     arglist = shlex.split(cmdline, posix=False)
  33                     # in case user quotes the command args
  34                     arglist = [arg.strip('\'""') for arg in arglist]
  35                 try:
  36                     args = argparser.parse_args(arglist)
  37                 except SystemExit:
  38                     # argparse exits at seeing bad arguments
  39                     return
  40                 else:
  41                     return func(thiz, args)
  42             argparser.prog = func.__name__[3:]
  43             if argparser.description is None and func.__doc__:
  44                 argparser.description = func.__doc__
  45
  46             return wrapper
  47
  48         return argparser_decorator
  49
  50
  51 cephfs = None
  52
  53
  54 def setup_cephfs(config_file):
  55     """
  56     Mouting a cephfs
  57     """
  58     global cephfs
  59     cephfs = libcephfs.LibCephFS(conffile=config_file)
  60     cephfs.mount()
  61
  62
  63 def mode_notation(mode):
  64     """
  65     """
  66     permission_bits = {'0': '---',
  67                        '1': '--x',
  68                        '2': '-w-',
  69                        '3':  '-wx',
  70                        '4': 'r--',
  71                        '5': 'r-x',
  72                        '6': 'rw-',
  73                        '7': 'rwx'}
  74     mode = str(oct(mode))
  75     notation = '-'
  76     if mode[2] == '4':
  77         notation = 'd'
  78     for i in mode[-3:]:
  79         notation += permission_bits[i]
  80     return notation
  81
  82
  83 def get_chunks(file_size):
  84     chunk_start = 0
  85     chunk_size = 0x20000  # 131072 bytes, default max ssl buffer size
  86     while chunk_start + chunk_size < file_size:
  87         yield(chunk_start, chunk_size)
  88         chunk_start += chunk_size
  89     final_chunk_size = file_size - chunk_start
  90     yield(chunk_start, final_chunk_size)
  91
  92
  93 def to_bytes(string):
  94     return bytes(string, encoding='utf-8')
  95
  96
  97 def list_items(dir_name=''):
  98     if not isinstance(dir_name, bytes):
  99         dir_name = to_bytes(dir_name)
 100     if dir_name == '':
 101         d = cephfs.opendir(cephfs.getcwd())
 102     else:
 103         d = cephfs.opendir(dir_name)
 104     dent = cephfs.readdir(d)
 105     items = []
 106     while dent:
 107         items.append(dent)
 108         dent = cephfs.readdir(d)
 109     cephfs.closedir(d)
 110     return items
 111
 112
 113 def glob(dir_name, pattern):
 114     if isinstance(dir_name, bytes):
 115         dir_name = dir_name.decode('utf-8')
 116     paths = []
 117     parent_dir = os.path.dirname(dir_name)
 118     if parent_dir == '':
 119         parent_dir = '/'
 120     if dir_name == '/' or is_dir_exists(os.path.basename(dir_name), parent_dir):
 121         for i in list_items(dir_name)[2:]:
 122             if fnmatch.fnmatch(i.d_name.decode('utf-8'), pattern):
 123                 paths.append(os.path.join(dir_name, i.d_name.decode('utf-8')))
 124     return paths
 125
 126
 127 def locate_file(name, case_sensitive=True):
 128     if not case_sensitive:
 129         return [i for i in sorted(set(dirwalk(cephfs.getcwd().decode('utf-8')))) if name.lower() in i.lower()]
 130     else:
 131         return [i for i in sorted(set(dirwalk(cephfs.getcwd().decode('utf-8')))) if name in i]
 132
 133
 134 def get_all_possible_paths(pattern):
 135     complete_pattern = pattern[:]
 136     paths = []
 137     is_rel_path = not os.path.isabs(pattern)
 138     if is_rel_path:
 139         dir_ = cephfs.getcwd()
 140     else:
 141         dir_ = '/'
 142         pattern = pattern[1:]
 143     patterns = pattern.split('/')
 144     paths.extend(glob(dir_, patterns[0]))
 145     patterns.pop(0)
 146     for pattern in patterns:
 147         for path in paths:
 148             paths.extend(glob(path, pattern))
 149     return [path for path in paths if fnmatch.fnmatch(path, os.path.join(cephfs.getcwd().decode('utf-8'), complete_pattern))]
 150
 151
 152 suffixes = ['B', 'K', 'M', 'G', 'T', 'P']
 153
 154
 155 def humansize(nbytes):
 156     i = 0
 157     while nbytes >= 1024 and i < len(suffixes)-1:
 158         nbytes /= 1024.
 159         i += 1
 160     nbytes = math.ceil(nbytes)
 161     f = ('%d' % nbytes).rstrip('.')
 162     return '%s%s' % (f, suffixes[i])
 163
 164
 165 def print_long(shell, file_name, is_dir, human_readable):
 166     if not isinstance(file_name, bytes):
 167         file_name = to_bytes(file_name)
 168     info = cephfs.stat(file_name)
 169     file_name = os.path.basename(file_name.decode('utf-8'))
 170     if is_dir:
 171         file_name = shell.colorize(file_name+'/',  'blue')
 172     if human_readable:
 173         shell.poutput('{}\t{:10s} {} {} {} {}'.format(
 174             mode_notation(info.st_mode),
 175             humansize(info.st_size), info.st_uid,
 176             info.st_gid, info.st_mtime, file_name, sep='\t'))
 177     else:
 178         shell.poutput('{} {:12d} {} {} {} {}'.format(
 179             mode_notation(info.st_mode), info.st_size, info.st_uid,
 180             info.st_gid, info.st_mtime, file_name, sep='\t'))
 181
 182
 183 def word_len(word):
 184     """
 185     Returns the word length, minus any color codes.
 186     """
 187     if word[0] == '\x1b':
 188         return len(word) - 9
 189     return len(word)
 190
 191
 192 def is_dir_exists(dir_name, dir_=''):
 193     if dir_ == '':
 194         dir_ = cephfs.getcwd()
 195     elif not isinstance(dir_, bytes):
 196         dir_ = to_bytes(dir_)
 197     if not isinstance(dir_name, bytes):
 198         dir_name = to_bytes(dir_name)
 199     return len([i for i in set(list_items(dir_)) if i.d_name == dir_name and i.is_dir()]) > 0
 200
 201
 202 def is_file_exists(file_name, dir_=''):
 203     if dir_ == '':
 204         dir_ = cephfs.getcwd()
 205     elif not isinstance(dir_, bytes):
 206         dir_ = to_bytes(dir_)
 207     if not isinstance(file_name, bytes):
 208         if file_name.count('/') > 0:
 209             file_name = to_bytes(os.path.basename(file_name))
 210         else:
 211             file_name = to_bytes(file_name)
 212     return len([i for i in set(list_items(dir_)) if i.d_name == file_name and not i.is_dir()]) > 0
 213
 214
 215 def print_list(shell, words, termwidth=79):
 216     if not words:
 217         return
 218     width = max([word_len(word) for word in words]) + 2
 219     nwords = len(words)
 220     ncols = max(1, (termwidth + 1) // (width + 1))
 221     nrows = (nwords + ncols - 1) // ncols
 222     for row in range(nrows):
 223         for i in range(row, nwords, nrows):
 224             word = words[i]
 225             if word[0] == '\x1b':
 226                 shell.poutput(
 227                     '%-*s' % (width + 10, words[i]), end='\n' if i + nrows >= nwords else '')
 228             else:
 229                 shell.poutput(
 230                     '%-*s' % (width, words[i]), end='\n' if i + nrows >= nwords else '')
 231
 232
 233 def copy_from_local(shell, local_path, remote_path):
 234     stdin = -1
 235     if local_path == '-':
 236         data = ''.join([line for line in sys.stdin])
 237         file_size = len(data)
 238     else:
 239         file_ = open(local_path, 'rb')
 240         stdin = 1
 241         file_size = os.path.getsize(local_path)
 242     fd = cephfs.open(to_bytes(remote_path), 'w', 0o666)
 243     if file_size == 0:
 244         return
 245     progress = 0
 246     while True:
 247         data = file_.read(65536)
 248         if not data:
 249             break
 250         wrote = cephfs.write(fd, data, progress)
 251         if wrote < 0:
 252             break
 253         progress += wrote
 254     cephfs.close(fd)
 255     if stdin > 0:
 256         file_.close()
 257     shell.poutput('')
 258
 259
 260 def copy_to_local(shell, remote_path, local_path):
 261     fd = None
 262     if local_path != '-':
 263         local_dir = os.path.dirname(local_path)
 264         if not os.path.exists(local_dir):
 265             os.makedirs(local_dir)
 266         if len(remote_path.rsplit('/', 1)) > 2 and remote_path.rsplit('/', 1)[1] == '':
 267             return
 268         fd = open(local_path, 'wb+')
 269     file_ = cephfs.open(to_bytes(remote_path), 'r')
 270     file_size = cephfs.stat(remote_path).st_size
 271     if file_size <= 0:
 272         return
 273     progress = 0
 274     for chunk_start, chunk_size in get_chunks(file_size):
 275         file_chunk = cephfs.read(file_, chunk_start, chunk_size)
 276         progress += len(file_chunk)
 277         if fd:
 278             fd.write(file_chunk)
 279         else:
 280             shell.poutput(file_chunk.decode('utf-8'))
 281     cephfs.close(file_)
 282     if fd:
 283         fd.close()
 284
 285
 286 def dirwalk(dir_name):
 287     """
 288     walk a directory tree, using a generator
 289     """
 290     dir_name = os.path.normpath(dir_name)
 291     for item in list_items(dir_name)[2:]:
 292         fullpath = os.path.join(dir_name, item.d_name.decode('utf-8'))
 293         yield fullpath.rsplit('/', 1)[0] + '/'
 294         if is_dir_exists(item.d_name, fullpath.rsplit('/', 1)[0]):
 295             if not len(list_items(fullpath)[2:]):
 296                 yield os.path.normpath(fullpath)
 297             else:
 298                 for x in dirwalk(fullpath):
 299                     yield x
 300         else:
 301             yield os.path.normpath(fullpath)
 302
 303
 304 class CephFSShell(Cmd):
 305
 306     def __init__(self):
 307         super().__init__(use_ipython=False)
 308         self.working_dir = cephfs.getcwd().decode('utf-8')
 309         self.set_prompt()
 310         self.interactive = False
 311         self.umask = '2'
 312
 313     def default(self, line):
 314         self.poutput('Unrecognized command:', line)
 315
 316     def set_prompt(self):
 317         self.prompt = ('\033[01;33mCephFS:~' + colorama.Fore.LIGHTCYAN_EX +
 318                        self.working_dir + colorama.Style.RESET_ALL + '\033[01;33m>>>\033[00m ')
 319
 320     def create_argparser(self, command):
 321         try:
 322             argparse_args = getattr(self, 'argparse_' + command)
 323         except AttributeError:
 324             return None
 325         doc_lines = getattr(
 326             self, 'do_' + command).__doc__.expandtabs().splitlines()
 327         if ''in doc_lines:
 328             blank_idx = doc_lines.index('')
 329             usage = doc_lines[:blank_idx]
 330             description = doc_lines[blank_idx + 1:]
 331         else:
 332             usage = doc_lines
 333             description = []
 334         parser = argparse.ArgumentParser(
 335             prog=command,
 336             usage='\n'.join(usage),
 337             description='\n'.join(description),
 338             formatter_class=argparse.ArgumentDefaultsHelpFormatter
 339         )
 340         for args, kwargs in argparse_args:
 341             parser.add_argument(*args, **kwargs)
 342         return parser
 343
 344     def complete_filenames(self, text, line, begidx, endidx):
 345         if not text:
 346             completions = [x.d_name.decode(
 347                 'utf-8') + '/' * int(x.is_dir()) for x in list_items(cephfs.getcwd())[2:]]
 348         else:
 349             if text.count('/') > 0:
 350                 completions = [text.rsplit('/', 1)[0] + '/' + x.d_name.decode('utf-8') + '/'*int(x.is_dir()) for x in list_items(
 351                     '/' + text.rsplit('/', 1)[0])[2:] if x.d_name.decode('utf-8').startswith(text.rsplit('/', 1)[1])]
 352             else:
 353                 completions = [x.d_name.decode('utf-8') + '/' * int(x.is_dir()) for x in list_items()[
 354                     2:] if x.d_name.decode('utf-8').startswith(text)]
 355             if len(completions) == 1 and completions[0][-1] == '/':
 356                 dir_, file_ = completions[0].rsplit('/', 1)
 357                 completions.extend([dir_ + '/' + x.d_name.decode('utf-8') + '/' * int(x.is_dir())
 358                                     for x in list_items('/' + dir_)[2:] if x.d_name.decode('utf-8').startswith(file_)])
 359             return self.delimiter_complete(text, line, begidx, endidx, completions, '/')
 360         return completions
 361
 362     def onecmd(self, line):
 363         """
 364         Global error catcher
 365         """
 366         try:
 367             res = Cmd.onecmd(self, line)
 368             if self.interactive:
 369                 self.set_prompt()
 370             return res
 371         except ConnectionError as e:
 372             self.poutput('***', e)
 373         except KeyboardInterrupt:
 374             self.poutput('Command aborted')
 375         except Exception as e:
 376             self.poutput(e)
 377             traceback.print_exc(file=sys.stdout)
 378
 379     def complete_mkdir(self, text, line, begidx, endidx):
 380         """
 381         auto complete of file name.
 382         """
 383         return self.complete_filenames(text, line, begidx, endidx)
 384
 385     mkdir_parser = argparse.ArgumentParser(
 386         description='Create the directory(ies), if they do not already exist.')
 387     mkdir_parser.add_argument('dirs', type=str,
 388                               metavar='DIR_NAME',
 389                               help='Name of new_directory.',
 390                               nargs='+')
 391     mkdir_parser.add_argument('-m', '--mode', type=str,
 392                               action='store',
 393                               help='Sets the access mode for the new directory.')
 394     mkdir_parser.add_argument('-p', '--parent', action='store_true',
 395                               help='Create parent directories as necessary. \
 396 When this option is specified, no error is reported if a directory already \
 397 exists.')
 398
 399     @with_argparser(mkdir_parser)
 400     def do_mkdir(self, args):
 401         """
 402         Create directory.
 403         """
 404         for dir_name in args.dirs:
 405             path = to_bytes('/' + dir_name)
 406             if args.mode:
 407                 permission = int(args.mode, 8)
 408             else:
 409                 permission = 0o777
 410             if args.parent:
 411                 cephfs.mkdirs(path, permission)
 412             else:
 413                 cephfs.mkdir(path, permission)
 414
 415     def complete_put(self, text, line, begidx, endidx):
 416         """
 417         auto complete of file name.
 418         """
 419         index_dict = {1: self.path_complete}
 420         return self.index_based_complete(text, line, begidx, endidx, index_dict)
 421
 422     put_parser = argparse.ArgumentParser(
 423         description='Copy a file/directory to Ceph File System from Local File System.')
 424     put_parser.add_argument('local_path', type=str,
 425                             help='Path of the file in the local system')
 426     put_parser.add_argument(
 427         'remote_path', type=str, help='Path of the file in the remote system.', nargs='?', default='.')
 428     put_parser.add_argument('-f', '--force', action='store_true',
 429                             help='Overwrites the destination if it already exists.')
 430
 431     @with_argparser(put_parser)
 432     def do_put(self, args):
 433         """
 434         Copy a file to Ceph File System from Local Directory.
 435         """
 436         root_src_dir = args.local_path
 437         root_dst_dir = args.remote_path
 438         if args.local_path == '.':
 439             root_src_dir = os.getcwd()
 440         if root_dst_dir == '.':
 441             root_dst_dir = root_src_dir.rsplit('/', 1)[1]
 442         elif root_dst_dir[-1] != '/':
 443             root_dst_dir += '/'
 444         if args.local_path == '-' or os.path.isfile(root_src_dir):
 445             copy_from_local(self, root_src_dir, root_dst_dir)
 446         else:
 447             for src_dir, dirs, files in os.walk(root_src_dir):
 448                 dst_dir = src_dir.replace(root_src_dir, root_dst_dir, 1)
 449                 dst_dir = re.sub('\/+', '/', cephfs.getcwd().decode('utf-8') + dst_dir)
 450                 if args.force and dst_dir != '/' and not is_dir_exists(dst_dir[:-1]) and len(locate_file(dst_dir)) == 0:
 451                     cephfs.mkdirs(to_bytes(dst_dir), 0o777)
 452                 if not args.force and dst_dir != '/' and not is_dir_exists(dst_dir) and not os.path.isfile(root_src_dir):
 453                     args.force = True
 454                     cephfs.mkdirs(to_bytes(dst_dir), 0o777)
 455                 for dir_ in dirs:
 456                     if not is_dir_exists(os.path.join(dst_dir, dir_)):
 457                         cephfs.mkdirs(
 458                             to_bytes(os.path.join(dst_dir, dir_)), 0o777)
 459                 for file_ in files:
 460                     src_file = os.path.join(src_dir, file_)
 461                     dst_file = re.sub('\/+', '/', '/' + dst_dir + '/' + file_)
 462                     if (not args.force) and is_file_exists(dst_file):
 463                         return
 464                     copy_from_local(self, src_file, os.path.join(
 465                         cephfs.getcwd().decode('utf-8'), dst_file))
 466
 467     def complete_get(self, text, line, begidx, endidx):
 468         """
 469         auto complete of file name.
 470         """
 471         return self.complete_filenames(text, line, begidx, endidx)
 472
 473     get_parser = argparse.ArgumentParser(
 474         description='Copy a file from Ceph File System from Local Directory.')
 475     get_parser.add_argument('remote_path', type=str,
 476                             help='Path of the file in the remote system')
 477     get_parser.add_argument(
 478         'local_path', type=str, help='Path of the file in the local system',  nargs='?', default='.')
 479
 480     @with_argparser(get_parser)
 481     def do_get(self, args):
 482         """
 483         Copy a file/directory  from Ceph File System to Local Directory.
 484         """
 485         root_src_dir = args.remote_path
 486         root_dst_dir = args.local_path
 487         if args.local_path == '.':
 488             root_dst_dir = os.getcwd()
 489         if args.remote_path == '.':
 490             root_src_dir = cephfs.getcwd().decode('utf-8')
 491         if args.local_path == '-':
 492             copy_to_local(self, root_src_dir, '-')
 493         elif is_file_exists(args.remote_path):
 494             copy_to_local(self, root_src_dir,
 495                           root_dst_dir + '/' + root_src_dir)
 496         elif '/'in root_src_dir and is_file_exists(root_src_dir.rsplit('/', 1)[1], root_src_dir.rsplit('/', 1)[0]):
 497             copy_to_local(self, root_src_dir, root_dst_dir)
 498         else:
 499             files = list(reversed(sorted(dirwalk(root_src_dir))))
 500             if len(files) == 0:
 501                 os.makedirs(root_dst_dir + '/' + root_src_dir)
 502             for file_ in files:
 503                 dst_dirpath, dst_file = file_.rsplit('/', 1)
 504                 if dst_dirpath in files:
 505                     files.remove(dst_dirpath)
 506                 dst_path = os.path.join(root_dst_dir, dst_dirpath, dst_file)
 507                 dst_path = os.path.normpath(dst_path)
 508                 if os.path.exists(dst_path):
 509                     continue
 510                 if is_dir_exists(file_):
 511                     os.makedirs(dst_path)
 512                 else:
 513                     copy_to_local(self, file_, dst_path)
 514         return 0
 515
 516     def complete_ls(self, text, line, begidx, endidx):
 517         """
 518         auto complete of file name.
 519         """
 520         return self.complete_filenames(text, line, begidx, endidx)
 521
 522     ls_parser = argparse.ArgumentParser(
 523         description='Copy a file from Ceph File System from Local Directory.')
 524     ls_parser.add_argument('-l', '--long', action='store_true',
 525                            help='Detailed list of items in the directory.')
 526     ls_parser.add_argument('-r', '--reverse', action='store_true',
 527                            help='Reverse order of listing items in the directory.')
 528     ls_parser.add_argument('-H', action='store_true', help='Human Readable')
 529     ls_parser.add_argument('-a', '--all', action='store_true',
 530                            help='Do not Ignore entries starting with .')
 531     ls_parser.add_argument('-S', action='store_true', help='Sort by file_size')
 532     ls_parser.add_argument(
 533         'dir_names', help='Name of Directories', nargs='*', default=[''])
 534
 535     @with_argparser(ls_parser)
 536     def do_ls(self, args):
 537         """
 538         List all the files and directories in the current working directory
 539         """
 540         directories = args.dir_names
 541         for dir_name in directories:
 542             values = []
 543             items = []
 544             if dir_name.count('*') > 0:
 545                 all_items = get_all_possible_paths(dir_name)
 546                 if len(all_items) == 0:
 547                     continue
 548                 dir_name = all_items[0].rsplit('/', 1)[0]
 549                 if dir_name == '':
 550                     dir_name = '/'
 551                 dirs = []
 552                 for i in all_items:
 553                     for item in list_items(dir_name):
 554                         d_name = item.d_name.decode('utf-8')
 555                         if os.path.basename(i) == d_name:
 556                             if item.is_dir():
 557                                 dirs.append(os.path.join(dir_name, d_name))
 558                             else:
 559                                 items.append(item)
 560                 if dirs:
 561                     directories.extend(dirs)
 562                 else:
 563                     self.poutput(dir_name, ':\n')
 564                 items = sorted(items, key=lambda item: item.d_name)
 565             else:
 566                 if dir_name != '' and dir_name != cephfs.getcwd().decode('utf-8') and len(directories) > 1:
 567                     self.poutput(dir_name, ':\n')
 568                 items = sorted(list_items(dir_name),
 569                                key=lambda item: item.d_name)
 570             if not args.all and len(items) >= 2:
 571                 items = [i for i in items if not i.d_name.decode('utf-8').startswith('.')]
 572             flag = 0
 573             if args.S:
 574                 items = sorted(items, key=lambda item: cephfs.stat(
 575                     to_bytes(dir_name + '/' + item.d_name.decode('utf-8'))).st_size)
 576             if args.reverse:
 577                 items = reversed(items)
 578             for item in items:
 579                 path = item
 580                 if not isinstance(item, str):
 581                     path = item.d_name.decode('utf-8')
 582                     if item.is_dir():
 583                         is_dir = True
 584                     else:
 585                         is_dir = False
 586                 if args.long and args.H:
 587                     print_long(self, cephfs.getcwd().decode(
 588                         'utf-8') + dir_name + '/' + path, is_dir, True)
 589                 elif args.long:
 590                     print_long(self, cephfs.getcwd().decode(
 591                         'utf-8') + dir_name + '/' + path, is_dir, False)
 592                 elif is_dir:
 593                     values.append(self.colorize(path + '/', 'blue'))
 594                 else:
 595                     values.append(path)
 596             if not args.long:
 597                 print_list(self, values, shutil.get_terminal_size().columns)
 598                 if dir_name != directories[-1]:
 599                     self.poutput('\n')
 600
 601     def complete_rmdir(self, text, line, begidx, endidx):
 602         """
 603         auto complete of file name.
 604         """
 605         return self.complete_filenames(text, line, begidx, endidx)
 606
 607     rmdir_parser = argparse.ArgumentParser(description='Remove Directory.')
 608     rmdir_parser.add_argument('dir_paths', help='Directory Path.', nargs='+')
 609     rmdir_parser.add_argument('-p', '--parent', action='store_true',
 610                               help='Remove parent directories as necessary. \
 611 When this option is specified, no error is reported if a directory has any \
 612 sub-directories, files')
 613
 614     @with_argparser(rmdir_parser)
 615     def do_rmdir(self, args):
 616         """
 617         Remove a specific Directory
 618         """
 619         is_pattern = False
 620         directories = args.dir_paths
 621         for dir_path in directories:
 622             if dir_path.count('*') > 0:
 623                 is_pattern = True
 624                 all_items = get_all_possible_paths(dir_path)
 625                 if len(all_items) > 0:
 626                     dir_path = all_items[0].rsplit('/', 1)[0]
 627                 if dir_path == '':
 628                     dir_path = '/'
 629                 dirs = []
 630                 for i in all_items:
 631                     for item in list_items(dir_path):
 632                         d_name = item.d_name.decode('utf-8')
 633                         if os.path.basename(i) == d_name:
 634                             if item.is_dir():
 635                                 dirs.append(os.path.join(dir_path, d_name))
 636                 directories.extend(dirs)
 637                 continue
 638             else:
 639                 is_pattern = False
 640             path = ''
 641             dir_path = os.path.normpath(os.path.join(
 642                 cephfs.getcwd().decode('utf-8'), dir_path))
 643             if args.parent:
 644                 files = reversed(
 645                     sorted(set(dirwalk(dir_path))))
 646                 for path in files:
 647                     path = os.path.normpath(path)
 648                     if path[1:] != dir_path:
 649                         try:
 650                             cephfs.rmdir(to_bytes(path))
 651                         except libcephfs.Error:
 652                             cephfs.unlink(to_bytes(path))
 653             if not is_pattern and dir_path != os.path.normpath(path):
 654                 cephfs.rmdir(to_bytes(dir_path))
 655
 656     def complete_rm(self, text, line, begidx, endidx):
 657         """
 658         auto complete of file name.
 659         """
 660         return self.complete_filenames(text, line, begidx, endidx)
 661
 662     rm_parser = argparse.ArgumentParser(description='Remove File.')
 663     rm_parser.add_argument('file_paths', help='File Path.', nargs='+')
 664
 665     @with_argparser(rm_parser)
 666     def do_rm(self, args):
 667         """
 668         Remove a specific file
 669         """
 670         files = args.file_paths
 671         for file_path in files:
 672             if file_path.count('*') > 0:
 673                 files.extend([i for i in get_all_possible_paths(
 674                     file_path) if is_file_exists(i)])
 675             else:
 676                 cephfs.unlink(to_bytes(file_path))
 677
 678     def complete_mv(self, text, line, begidx, endidx):
 679         """
 680          auto complete of file name.
 681         """
 682         return self.complete_filenames(text, line, begidx, endidx)
 683
 684     mv_parser = argparse.ArgumentParser(description='Move File.')
 685     mv_parser.add_argument('src_path', type=str, help='Source File Path.')
 686     mv_parser.add_argument('dest_path', type=str,
 687                            help='Destination File Path.')
 688
 689     @with_argparser(mv_parser)
 690     def do_mv(self, args):
 691         """
 692         Rename a file or Move a file from source path to the destination
 693         """
 694         cephfs.rename(to_bytes(args.src_path), to_bytes(args.dest_path))
 695
 696     def complete_cd(self, text, line, begidx, endidx):
 697         """
 698         auto complete of file name.
 699         """
 700         return self.complete_filenames(text, line, begidx, endidx)
 701
 702     cd_parser = argparse.ArgumentParser(description='Change working directory')
 703     cd_parser.add_argument('dir_name', type=str,
 704                            help='Name of the directory.', default='')
 705
 706     @with_argparser(cd_parser)
 707     def do_cd(self, args):
 708         """
 709         Change working directory
 710         """
 711         if args.dir_name == '':
 712             cephfs.chdir(b'/')
 713         if args.dir_name == '..':
 714             dir_name = cephfs.getcwd().decode('utf-8').rsplit('/', 1)[0]
 715             if dir_name != '':
 716                 cephfs.chdir(to_bytes(dir_name))
 717             else:
 718                 cephfs.chdir(b'/')
 719         else:
 720             cephfs.chdir(to_bytes(args.dir_name))
 721         self.working_dir = cephfs.getcwd().decode('utf-8')
 722         self.set_prompt()
 723
 724     def do_cwd(self, arglist):
 725         """
 726         Get current working directory.
 727         """
 728         self.poutput(cephfs.getcwd().decode('utf-8'))
 729
 730     def complete_chmod(self, text, line, begidx, endidx):
 731         """
 732         auto complete of file name.
 733         """
 734         return self.complete_filenames(text, line, begidx, endidx)
 735
 736     chmod_parser = argparse.ArgumentParser(description='Create Directory.')
 737     chmod_parser.add_argument('mode', type=int, help='Mode')
 738     chmod_parser.add_argument('file_name', type=str, help='Name of the file')
 739
 740     @with_argparser(chmod_parser)
 741     def do_chmod(self, args):
 742         """
 743         Change permission of a file
 744         """
 745         cephfs.chmod(args.file_name, args.mode)
 746
 747     def complete_cat(self, text, line, begidx, endidx):
 748         """
 749         auto complete of file name.
 750         """
 751         return self.complete_filenames(text, line, begidx, endidx)
 752
 753     cat_parser = argparse.ArgumentParser(description='')
 754     cat_parser.add_argument('file_names', help='Name of Files', nargs='+')
 755
 756     @with_argparser(cat_parser)
 757     def do_cat(self, args):
 758         """
 759         Print contents of a file
 760         """
 761         for file_name in args.file_names:
 762             self.poutput(file_name)
 763             copy_to_local(self, file_name, '-')
 764
 765     umask_parser = argparse.ArgumentParser(description='Set umask value.')
 766     umask_parser.add_argument(
 767         'mode', help='Mode', action='store', nargs='?', default='')
 768
 769     @with_argparser(umask_parser)
 770     def do_umask(self, args):
 771         """
 772         Set Umask value.
 773         """
 774         if args.mode == '':
 775             self.poutput(self.umask.zfill(4))
 776         else:
 777             mode = int(args.mode, 8)
 778             self.umask = str(oct(cephfs.umask(mode))[2:])
 779
 780     def complete_write(self, text, line, begidx, endidx):
 781         """
 782         auto complete of file name.
 783         """
 784         return self.complete_filenames(text, line, begidx, endidx)
 785
 786     write_parser = argparse.ArgumentParser(description='')
 787     write_parser.add_argument('file_name', type=str, help='Name of File')
 788
 789     @with_argparser(write_parser)
 790     def do_write(self, args):
 791         """
 792         Write data into a file.
 793         """
 794
 795         copy_from_local(self, '-', args.file_name)
 796
 797     def complete_lcd(self, text, line, begidx, endidx):
 798         """
 799         auto complete of file name.
 800         """
 801         index_dict = {1: self.path_complete}
 802         return self.index_based_complete(text, line, begidx, endidx, index_dict)
 803
 804     lcd_parser = argparse.ArgumentParser(description='')
 805     lcd_parser.add_argument('path', type=str, help='Path')
 806
 807     @with_argparser(lcd_parser)
 808     def do_lcd(self, args):
 809         """
 810         Moves into the given local directory
 811         """
 812
 813         path = os.path.expanduser(args.path)
 814         if os.path.isdir(path):
 815             os.chdir(path)
 816             # self.poutput(get_all_possible_paths(args.path))
 817
 818     def complete_lls(self, text, line, begidx, endidx):
 819         """
 820         auto complete of file name.
 821         """
 822         index_dict = {1: self.path_complete}
 823         return self.index_based_complete(text, line, begidx, endidx, index_dict)
 824
 825     lls_parser = argparse.ArgumentParser(
 826         description='List files in local system.')
 827     lls_parser.add_argument('paths', help='Paths', nargs='*', default=[''])
 828
 829     @with_argparser(lls_parser)
 830     def do_lls(self, args):
 831         """
 832         Lists all files and folders in the current local directory
 833         """
 834
 835         if len(args.paths) == 1 and args.paths[0] == '':
 836             args.paths.pop(0)
 837             args.paths.append(os.getcwd())
 838         for path in args.paths:
 839             if os.path.isabs(path):
 840                 path = os.path.relpath(os.getcwd(), '/' + path)
 841             items = os.listdir(path)
 842             print_list(self, items)
 843
 844     def do_lpwd(self, arglist):
 845         """
 846         Prints the absolute path of the current local directory
 847         """
 848         self.poutput(os.getcwd())
 849
 850     def do_df(self, arglist):
 851         """
 852         Display the amount of available disk space for file systems
 853         """
 854         for index, i in enumerate(list_items(cephfs.getcwd())[2:]):
 855             if index == 0:
 856                 self.poutput('{:25s}\t{:5s}\t{:15s}{:10s}{}'.format(
 857                     "1K-blocks", "Used", "Available", "Use%", "Stored on"))
 858             if not is_dir_exists(i.d_name):
 859                 statfs = cephfs.statfs(i.d_name)
 860                 stat = cephfs.stat(i.d_name)
 861                 block_size = statfs['f_blocks']*statfs['f_bsize'] // 1024
 862                 available = block_size - stat.st_size
 863                 use = 0
 864                 if block_size > 0:
 865                     use = (stat.st_size*100 // block_size)
 866                 self.poutput('{:25d}\t{:5d}\t{:10d}\t{:5s} {}'.format(
 867                     statfs['f_fsid'], stat.st_size, available,
 868                     str(int(use)) + '%', i.d_name.decode('utf-8')))
 869
 870     locate_parser = argparse.ArgumentParser(
 871         description='Find file within file system')
 872     locate_parser.add_argument('name', help='name', type=str)
 873     locate_parser.add_argument(
 874         '-c', '--count', action='store_true', help='Count list of items located.')
 875     locate_parser.add_argument(
 876         '-i', '--ignorecase', action='store_true', help='Ignore case')
 877
 878     @with_argparser(locate_parser)
 879     def do_locate(self, args):
 880         """
 881         Find a file within the File System
 882         """
 883         if args.name.count('*') == 1:
 884             if args.name[0] == '*':
 885                 args.name += '/'
 886             elif args.name[-1] == '*':
 887                 args.name = '/' + args.name
 888         args.name = args.name.replace('*', '')
 889         if args.ignorecase:
 890             locations = locate_file(args.name, False)
 891         else:
 892             locations = locate_file(args.name)
 893         if args.count:
 894             self.poutput(len(locations))
 895         else:
 896             self.poutput('\n'.join(locations))
 897
 898     def complete_du(self, text, line, begidx, endidx):
 899         """
 900         auto complete of file name.
 901         """
 902         return self.complete_filenames(text, line, begidx, endidx)
 903
 904     du_parser = argparse.ArgumentParser(
 905         description='Disk Usage of a Directory')
 906     du_parser.add_argument(
 907         'dirs', type=str, help='Name of the directory.', nargs='?', default='')
 908     du_parser.add_argument('-r', action='store_true',
 909                            help='Recursive Disk usage of all directories.')
 910
 911     @with_argparser(du_parser)
 912     def do_du(self, args):
 913         """
 914         Disk Usage of a Directory
 915         """
 916         if args.dirs == '':
 917             args.dirs = cephfs.getcwd().decode('utf-8')
 918         for dir_ in args.dirs:
 919             if args.r:
 920                 for i in reversed(sorted(set(dirwalk(dir_)))):
 921                     i = os.path.normpath(i)
 922                     try:
 923                         xattr = cephfs.getxattr(to_bytes(i), 'ceph.dir.rbytes')
 924                         self.poutput('{:10s} {}'.format(
 925                             humansize(int(xattr.decode('utf-8'))), '.' + i))
 926                     except libcephfs.Error:
 927                         continue
 928             else:
 929                 dir_ = os.path.normpath(dir_)
 930                 self.poutput('{:10s} {}'.format(humansize(int(cephfs.getxattr(to_bytes(
 931                     dir_), 'ceph.dir.rbytes').decode('utf-8'))), '.' + dir_))
 932
 933     def do_help(self, line):
 934         """
 935         Get details about a command.
 936             Usage: help <cmd> - for a specific command
 937                    help all - for all the commands
 938         """
 939         if line == 'all':
 940             for k in dir(self):
 941                 if k.startswith('do_'):
 942                     self.poutput('-'*80)
 943                     super().do_help(k[3:])
 944             return
 945         parser = self.create_argparser(line)
 946         if parser:
 947             parser.print_help()
 948         else:
 949             super().do_help(line)
 950
 951
 952 if __name__ == '__main__':
 953     config_file = ''
 954     exe = sys.argv[0]
 955     main_parser = argparse.ArgumentParser(description='')
 956     main_parser.add_argument(
 957         '-c', '--config', action='store', help='Configuration file_path', type=str)
 958     main_parser.add_argument(
 959         '-b', '--batch', action='store', help='Batch File path.', type=str)
 960     main_parser.add_argument('-t', '--test', action='store',
 961                              help='Test against transcript(s) in FILE', nargs='+')
 962     main_parser.add_argument('commands', nargs='*',
 963                              help='comma delimited commands', default=[])
 964     args = main_parser.parse_args()
 965     if args.config:
 966         config_file = args.config
 967     if args.batch:
 968         args.commands = ['load ' + args.batch, ',quit']
 969     if args.test:
 970         args.commands.extend(['-t,'] + [arg+',' for arg in args.test])
 971     sys.argv.clear()
 972     sys.argv.append(exe)
 973     sys.argv.extend([i.strip() for i in ' '.join(args.commands).split(',')])
 974     setup_cephfs(config_file)
 975     c = CephFSShell()
 976     c.cmdloop()