EmbeddedPkg: Extend NvVarStoreFormattedLib LIBRARY_CLASS

[mirror_edk2.git] / AppPkg / Applications / Python / Python-2.7.2 / Tools / scripts / ftpmirror.py

#! /usr/bin/env python\r
\r
"""Mirror a remote ftp subtree into a local directory tree.\r
\r
usage: ftpmirror [-v] [-q] [-i] [-m] [-n] [-r] [-s pat]\r
                 [-l username [-p passwd [-a account]]]\r
                 hostname[:port] [remotedir [localdir]]\r
-v: verbose\r
-q: quiet\r
-i: interactive mode\r
-m: macintosh server (NCSA telnet 2.4) (implies -n -s '*.o')\r
-n: don't log in\r
-r: remove local files/directories no longer pertinent\r
-l username [-p passwd [-a account]]: login info (default .netrc or anonymous)\r
-s pat: skip files matching pattern\r
hostname: remote host w/ optional port separated by ':'\r
remotedir: remote directory (default initial)\r
localdir: local directory (default current)\r
"""\r
\r
import os\r
import sys\r
import time\r
import getopt\r
import ftplib\r
import netrc\r
from fnmatch import fnmatch\r
\r
# Print usage message and exit\r
def usage(*args):\r
    sys.stdout = sys.stderr\r
    for msg in args: print msg\r
    print __doc__\r
    sys.exit(2)\r
\r
verbose = 1 # 0 for -q, 2 for -v\r
interactive = 0\r
mac = 0\r
rmok = 0\r
nologin = 0\r
skippats = ['.', '..', '.mirrorinfo']\r
\r
# Main program: parse command line and start processing\r
def main():\r
    global verbose, interactive, mac, rmok, nologin\r
    try:\r
        opts, args = getopt.getopt(sys.argv[1:], 'a:bil:mnp:qrs:v')\r
    except getopt.error, msg:\r
        usage(msg)\r
    login = ''\r
    passwd = ''\r
    account = ''\r
    if not args: usage('hostname missing')\r
    host = args[0]\r
    port = 0\r
    if ':' in host:\r
        host, port = host.split(':', 1)\r
        port = int(port)\r
    try:\r
        auth = netrc.netrc().authenticators(host)\r
        if auth is not None:\r
            login, account, passwd = auth\r
    except (netrc.NetrcParseError, IOError):\r
        pass\r
    for o, a in opts:\r
        if o == '-l': login = a\r
        if o == '-p': passwd = a\r
        if o == '-a': account = a\r
        if o == '-v': verbose = verbose + 1\r
        if o == '-q': verbose = 0\r
        if o == '-i': interactive = 1\r
        if o == '-m': mac = 1; nologin = 1; skippats.append('*.o')\r
        if o == '-n': nologin = 1\r
        if o == '-r': rmok = 1\r
        if o == '-s': skippats.append(a)\r
    remotedir = ''\r
    localdir = ''\r
    if args[1:]:\r
        remotedir = args[1]\r
        if args[2:]:\r
            localdir = args[2]\r
            if args[3:]: usage('too many arguments')\r
    #\r
    f = ftplib.FTP()\r
    if verbose: print "Connecting to '%s%s'..." % (host,\r
                                                   (port and ":%d"%port or ""))\r
    f.connect(host,port)\r
    if not nologin:\r
        if verbose:\r
            print 'Logging in as %r...' % (login or 'anonymous')\r
        f.login(login, passwd, account)\r
    if verbose: print 'OK.'\r
    pwd = f.pwd()\r
    if verbose > 1: print 'PWD =', repr(pwd)\r
    if remotedir:\r
        if verbose > 1: print 'cwd(%s)' % repr(remotedir)\r
        f.cwd(remotedir)\r
        if verbose > 1: print 'OK.'\r
        pwd = f.pwd()\r
        if verbose > 1: print 'PWD =', repr(pwd)\r
    #\r
    mirrorsubdir(f, localdir)\r
\r
# Core logic: mirror one subdirectory (recursively)\r
def mirrorsubdir(f, localdir):\r
    pwd = f.pwd()\r
    if localdir and not os.path.isdir(localdir):\r
        if verbose: print 'Creating local directory', repr(localdir)\r
        try:\r
            makedir(localdir)\r
        except os.error, msg:\r
            print "Failed to establish local directory", repr(localdir)\r
            return\r
    infofilename = os.path.join(localdir, '.mirrorinfo')\r
    try:\r
        text = open(infofilename, 'r').read()\r
    except IOError, msg:\r
        text = '{}'\r
    try:\r
        info = eval(text)\r
    except (SyntaxError, NameError):\r
        print 'Bad mirror info in', repr(infofilename)\r
        info = {}\r
    subdirs = []\r
    listing = []\r
    if verbose: print 'Listing remote directory %r...' % (pwd,)\r
    f.retrlines('LIST', listing.append)\r
    filesfound = []\r
    for line in listing:\r
        if verbose > 1: print '-->', repr(line)\r
        if mac:\r
            # Mac listing has just filenames;\r
            # trailing / means subdirectory\r
            filename = line.strip()\r
            mode = '-'\r
            if filename[-1:] == '/':\r
                filename = filename[:-1]\r
                mode = 'd'\r
            infostuff = ''\r
        else:\r
            # Parse, assuming a UNIX listing\r
            words = line.split(None, 8)\r
            if len(words) < 6:\r
                if verbose > 1: print 'Skipping short line'\r
                continue\r
            filename = words[-1].lstrip()\r
            i = filename.find(" -> ")\r
            if i >= 0:\r
                # words[0] had better start with 'l'...\r
                if verbose > 1:\r
                    print 'Found symbolic link %r' % (filename,)\r
                linkto = filename[i+4:]\r
                filename = filename[:i]\r
            infostuff = words[-5:-1]\r
            mode = words[0]\r
        skip = 0\r
        for pat in skippats:\r
            if fnmatch(filename, pat):\r
                if verbose > 1:\r
                    print 'Skip pattern', repr(pat),\r
                    print 'matches', repr(filename)\r
                skip = 1\r
                break\r
        if skip:\r
            continue\r
        if mode[0] == 'd':\r
            if verbose > 1:\r
                print 'Remembering subdirectory', repr(filename)\r
            subdirs.append(filename)\r
            continue\r
        filesfound.append(filename)\r
        if info.has_key(filename) and info[filename] == infostuff:\r
            if verbose > 1:\r
                print 'Already have this version of',repr(filename)\r
            continue\r
        fullname = os.path.join(localdir, filename)\r
        tempname = os.path.join(localdir, '@'+filename)\r
        if interactive:\r
            doit = askabout('file', filename, pwd)\r
            if not doit:\r
                if not info.has_key(filename):\r
                    info[filename] = 'Not retrieved'\r
                continue\r
        try:\r
            os.unlink(tempname)\r
        except os.error:\r
            pass\r
        if mode[0] == 'l':\r
            if verbose:\r
                print "Creating symlink %r -> %r" % (filename, linkto)\r
            try:\r
                os.symlink(linkto, tempname)\r
            except IOError, msg:\r
                print "Can't create %r: %s" % (tempname, msg)\r
                continue\r
        else:\r
            try:\r
                fp = open(tempname, 'wb')\r
            except IOError, msg:\r
                print "Can't create %r: %s" % (tempname, msg)\r
                continue\r
            if verbose:\r
                print 'Retrieving %r from %r as %r...' % (filename, pwd, fullname)\r
            if verbose:\r
                fp1 = LoggingFile(fp, 1024, sys.stdout)\r
            else:\r
                fp1 = fp\r
            t0 = time.time()\r
            try:\r
                f.retrbinary('RETR ' + filename,\r
                             fp1.write, 8*1024)\r
            except ftplib.error_perm, msg:\r
                print msg\r
            t1 = time.time()\r
            bytes = fp.tell()\r
            fp.close()\r
            if fp1 != fp:\r
                fp1.close()\r
        try:\r
            os.unlink(fullname)\r
        except os.error:\r
            pass            # Ignore the error\r
        try:\r
            os.rename(tempname, fullname)\r
        except os.error, msg:\r
            print "Can't rename %r to %r: %s" % (tempname, fullname, msg)\r
            continue\r
        info[filename] = infostuff\r
        writedict(info, infofilename)\r
        if verbose and mode[0] != 'l':\r
            dt = t1 - t0\r
            kbytes = bytes / 1024.0\r
            print int(round(kbytes)),\r
            print 'Kbytes in',\r
            print int(round(dt)),\r
            print 'seconds',\r
            if t1 > t0:\r
                print '(~%d Kbytes/sec)' % \\r
                          int(round(kbytes/dt),)\r
            print\r
    #\r
    # Remove files from info that are no longer remote\r
    deletions = 0\r
    for filename in info.keys():\r
        if filename not in filesfound:\r
            if verbose:\r
                print "Removing obsolete info entry for",\r
                print repr(filename), "in", repr(localdir or ".")\r
            del info[filename]\r
            deletions = deletions + 1\r
    if deletions:\r
        writedict(info, infofilename)\r
    #\r
    # Remove local files that are no longer in the remote directory\r
    try:\r
        if not localdir: names = os.listdir(os.curdir)\r
        else: names = os.listdir(localdir)\r
    except os.error:\r
        names = []\r
    for name in names:\r
        if name[0] == '.' or info.has_key(name) or name in subdirs:\r
            continue\r
        skip = 0\r
        for pat in skippats:\r
            if fnmatch(name, pat):\r
                if verbose > 1:\r
                    print 'Skip pattern', repr(pat),\r
                    print 'matches', repr(name)\r
                skip = 1\r
                break\r
        if skip:\r
            continue\r
        fullname = os.path.join(localdir, name)\r
        if not rmok:\r
            if verbose:\r
                print 'Local file', repr(fullname),\r
                print 'is no longer pertinent'\r
            continue\r
        if verbose: print 'Removing local file/dir', repr(fullname)\r
        remove(fullname)\r
    #\r
    # Recursively mirror subdirectories\r
    for subdir in subdirs:\r
        if interactive:\r
            doit = askabout('subdirectory', subdir, pwd)\r
            if not doit: continue\r
        if verbose: print 'Processing subdirectory', repr(subdir)\r
        localsubdir = os.path.join(localdir, subdir)\r
        pwd = f.pwd()\r
        if verbose > 1:\r
            print 'Remote directory now:', repr(pwd)\r
            print 'Remote cwd', repr(subdir)\r
        try:\r
            f.cwd(subdir)\r
        except ftplib.error_perm, msg:\r
            print "Can't chdir to", repr(subdir), ":", repr(msg)\r
        else:\r
            if verbose: print 'Mirroring as', repr(localsubdir)\r
            mirrorsubdir(f, localsubdir)\r
            if verbose > 1: print 'Remote cwd ..'\r
            f.cwd('..')\r
        newpwd = f.pwd()\r
        if newpwd != pwd:\r
            print 'Ended up in wrong directory after cd + cd ..'\r
            print 'Giving up now.'\r
            break\r
        else:\r
            if verbose > 1: print 'OK.'\r
\r
# Helper to remove a file or directory tree\r
def remove(fullname):\r
    if os.path.isdir(fullname) and not os.path.islink(fullname):\r
        try:\r
            names = os.listdir(fullname)\r
        except os.error:\r
            names = []\r
        ok = 1\r
        for name in names:\r
            if not remove(os.path.join(fullname, name)):\r
                ok = 0\r
        if not ok:\r
            return 0\r
        try:\r
            os.rmdir(fullname)\r
        except os.error, msg:\r
            print "Can't remove local directory %r: %s" % (fullname, msg)\r
            return 0\r
    else:\r
        try:\r
            os.unlink(fullname)\r
        except os.error, msg:\r
            print "Can't remove local file %r: %s" % (fullname, msg)\r
            return 0\r
    return 1\r
\r
# Wrapper around a file for writing to write a hash sign every block.\r
class LoggingFile:\r
    def __init__(self, fp, blocksize, outfp):\r
        self.fp = fp\r
        self.bytes = 0\r
        self.hashes = 0\r
        self.blocksize = blocksize\r
        self.outfp = outfp\r
    def write(self, data):\r
        self.bytes = self.bytes + len(data)\r
        hashes = int(self.bytes) / self.blocksize\r
        while hashes > self.hashes:\r
            self.outfp.write('#')\r
            self.outfp.flush()\r
            self.hashes = self.hashes + 1\r
        self.fp.write(data)\r
    def close(self):\r
        self.outfp.write('\n')\r
\r
# Ask permission to download a file.\r
def askabout(filetype, filename, pwd):\r
    prompt = 'Retrieve %s %s from %s ? [ny] ' % (filetype, filename, pwd)\r
    while 1:\r
        reply = raw_input(prompt).strip().lower()\r
        if reply in ['y', 'ye', 'yes']:\r
            return 1\r
        if reply in ['', 'n', 'no', 'nop', 'nope']:\r
            return 0\r
        print 'Please answer yes or no.'\r
\r
# Create a directory if it doesn't exist.  Recursively create the\r
# parent directory as well if needed.\r
def makedir(pathname):\r
    if os.path.isdir(pathname):\r
        return\r
    dirname = os.path.dirname(pathname)\r
    if dirname: makedir(dirname)\r
    os.mkdir(pathname, 0777)\r
\r
# Write a dictionary to a file in a way that can be read back using\r
# rval() but is still somewhat readable (i.e. not a single long line).\r
# Also creates a backup file.\r
def writedict(dict, filename):\r
    dir, fname = os.path.split(filename)\r
    tempname = os.path.join(dir, '@' + fname)\r
    backup = os.path.join(dir, fname + '~')\r
    try:\r
        os.unlink(backup)\r
    except os.error:\r
        pass\r
    fp = open(tempname, 'w')\r
    fp.write('{\n')\r
    for key, value in dict.items():\r
        fp.write('%r: %r,\n' % (key, value))\r
    fp.write('}\n')\r
    fp.close()\r
    try:\r
        os.rename(filename, backup)\r
    except os.error:\r
        pass\r
    os.rename(tempname, filename)\r
\r
\r
if __name__ == '__main__':\r
    main()\r