BaseTools: Treat BPDG.py as a python module

[mirror_edk2.git] / BaseTools / gcc / mingw-gcc-build.py

#!/usr/bin/env python\r
\r
## @file\r
#\r
# Automation of instructions from:\r
#   http://mingw-w64.svn.sourceforge.net/viewvc/mingw-w64/trunk/mingw-w64-doc/\r
#     howto-build/mingw-w64-howto-build.txt?revision=216&view=markup\r
#\r
# Copyright (c) 2008 - 2018, Intel Corporation. All rights reserved.<BR>\r
# This program and the accompanying materials\r
# are licensed and made available under the terms and conditions of the BSD License\r
# which accompanies this distribution.    The full text of the license may be found at\r
# http://opensource.org/licenses/bsd-license.php\r
#\r
# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,\r
# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.\r
#\r
\r
\r
from __future__ import print_function\r
from optparse import OptionParser\r
import os\r
import shutil\r
import subprocess\r
import sys\r
import tarfile\r
import urllib\r
import urlparse\r
try:\r
    from hashlib import md5\r
except Exception:\r
    from md5 import md5\r
\r
if sys.version_info < (2, 5):\r
    #\r
    # This script (and edk2 BaseTools) require Python 2.5 or newer\r
    #\r
    print('Python version 2.5 or later is required.')\r
    sys.exit(-1)\r
\r
#\r
# Version and Copyright\r
#\r
VersionNumber = "0.01"\r
__version__ = "%prog Version " + VersionNumber\r
__copyright__ = "Copyright (c) 2008 - 2018, Intel Corporation.  All rights reserved."\r
\r
class Config:\r
    """class Config\r
\r
    Stores the configuration options for the rest of the script.\r
\r
    Handles the command line options, and allows the code within\r
    the script to easily interact with the 'config' requested by\r
    the user.\r
    """\r
\r
    def __init__(self):\r
        self.base_dir = os.getcwd()\r
        (self.options, self.args) = self.CheckOptions()\r
        self.__init_dirs__()\r
\r
    def CheckOptions(self):\r
        Parser = \\r
            OptionParser(\r
                description=__copyright__,\r
                version=__version__,\r
                prog="mingw-gcc-build",\r
                usage="%prog [options] [target]"\r
                )\r
        Parser.add_option(\r
            "--arch",\r
            action = "store", type = "string",\r
            default = '',\r
            dest = "arch",\r
            help = "Processor architecture to build gcc for."\r
            )\r
        Parser.add_option(\r
            "--src-dir",\r
            action = "store", type = "string", dest = "src_dir",\r
            default = os.path.join(self.base_dir, 'src'),\r
            help = "Directory to download/extract binutils/gcc sources"\r
            )\r
        Parser.add_option(\r
            "--build-dir",\r
            action = "store", type = "string", dest = "build_dir",\r
            default = os.path.join(self.base_dir, 'build'),\r
            help = "Directory to download/extract binutils/gcc sources"\r
            )\r
        Parser.add_option(\r
            "--prefix",\r
            action = "store", type = "string", dest = "prefix",\r
            default = os.path.join(self.base_dir, 'install'),\r
            help = "Prefix to install binutils/gcc into"\r
            )\r
        Parser.add_option(\r
            "--skip-binutils",\r
            action = "store_true", dest = "skip_binutils",\r
            default = False,\r
            help = "Will skip building binutils"\r
            )\r
        Parser.add_option(\r
            "--skip-gcc",\r
            action = "store_true", dest = "skip_gcc",\r
            default = False,\r
            help = "Will skip building GCC"\r
            )\r
        Parser.add_option(\r
            "--symlinks",\r
            action = "store", type = "string", dest = "symlinks",\r
            default = os.path.join(self.base_dir, 'symlinks'),\r
            help = "Directory to create binutils/gcc symbolic links into."\r
            )\r
        Parser.add_option(\r
            "-v", "--verbose",\r
            action="store_true",\r
            type=None, help="Print verbose messages"\r
            )\r
\r
        (Opt, Args) = Parser.parse_args()\r
\r
        self.arch = Opt.arch.lower()\r
        allowedArchs = ('ia32', 'x64', 'ipf')\r
        if self.arch not in allowedArchs:\r
            Parser.error(\r
                'Please use --arch to specify one of: %s' %\r
                    ', '.join(allowedArchs)\r
                )\r
        self.target_arch = {'ia32': 'i686', 'x64': 'x86_64', 'ipf': 'ia64'}[self.arch]\r
        self.target_sys = {'ia32': 'pc', 'x64': 'pc', 'ipf': 'pc'}[self.arch]\r
        self.target_bin = {'ia32': 'mingw32', 'x64': 'mingw32', 'ipf': 'elf'}[self.arch]\r
        self.target_combo = '-'.join((self.target_arch, self.target_sys, self.target_bin))\r
\r
        return (Opt, Args)\r
\r
    def __init_dirs__(self):\r
        self.src_dir = os.path.realpath(os.path.expanduser(self.options.src_dir))\r
        self.build_dir = os.path.realpath(os.path.expanduser(self.options.build_dir))\r
        self.prefix = os.path.realpath(os.path.expanduser(self.options.prefix))\r
        self.symlinks = os.path.realpath(os.path.expanduser(self.options.symlinks))\r
\r
    def IsConfigOk(self):\r
\r
        building = []\r
        if not self.options.skip_binutils:\r
            building.append('binutils')\r
        if not self.options.skip_gcc:\r
            building.append('gcc')\r
        if len(building) == 0:\r
            print("Nothing will be built!")\r
            print()\r
            print("Please try using --help and then change the configuration.")\r
            return False\r
\r
        print("Current directory:")\r
        print("   ", self.base_dir)\r
        print("Sources download/extraction:", self.Relative(self.src_dir))\r
        print("Build directory            :", self.Relative(self.build_dir))\r
        print("Prefix (install) directory :", self.Relative(self.prefix))\r
        print("Create symlinks directory  :", self.Relative(self.symlinks))\r
        print("Building                   :", ', '.join(building))\r
        print()\r
        answer = raw_input("Is this configuration ok? (default = no): ")\r
        if (answer.lower() not in ('y', 'yes')):\r
            print()\r
            print("Please try using --help and then change the configuration.")\r
            return False\r
\r
        if self.arch.lower() == 'ipf':\r
            print()\r
            print('Please note that the IPF compiler built by this script has')\r
            print('not yet been validated!')\r
            print()\r
            answer = raw_input("Are you sure you want to build it? (default = no): ")\r
            if (answer.lower() not in ('y', 'yes')):\r
                print()\r
                print("Please try using --help and then change the configuration.")\r
                return False\r
\r
        print()\r
        return True\r
\r
    def Relative(self, path):\r
        if path.startswith(self.base_dir):\r
            return '.' + path[len(self.base_dir):]\r
        return path\r
\r
    def MakeDirs(self):\r
        for path in (self.src_dir, self.build_dir, self.prefix, self.symlinks):\r
            if not os.path.exists(path):\r
                os.makedirs(path)\r
\r
class SourceFiles:\r
    """class SourceFiles\r
\r
    Handles the downloading of source files used by the script.\r
    """\r
\r
    def __init__(self, config):\r
        self.config = config\r
        self.source_files = self.source_files[config.arch]\r
\r
        if config.options.skip_binutils:\r
            del self.source_files['binutils']\r
\r
        if config.options.skip_gcc:\r
            del self.source_files['gcc']\r
            del self.source_files['mingw_hdr']\r
\r
    source_files_common = {\r
        'binutils': {\r
            'url': 'http://www.kernel.org/pub/linux/devel/binutils/' + \\r
                   'binutils-$version.tar.bz2',\r
            'version': '2.20.51.0.5',\r
            'md5': '6d2de7cdf7a8389e70b124e3d73b4d37',\r
            },\r
        }\r
\r
    source_files_x64 = {\r
        'gcc': {\r
            'url': 'http://ftpmirror.gnu.org/gcc/' + \\r
                   'gcc-$version/gcc-$version.tar.bz2',\r
            'version': '4.3.0',\r
            'md5': '197ed8468b38db1d3481c3111691d85b',\r
            },\r
        }\r
\r
    source_files_ia32 = {\r
        'gcc': source_files_x64['gcc'],\r
        }\r
\r
    source_files_ipf = source_files_x64.copy()\r
    source_files_ipf['gcc']['configure-params'] = (\r
        '--with-gnu-as', '--with-gnu-ld', '--with-newlib',\r
        '--verbose', '--disable-libssp', '--disable-nls',\r
        '--enable-languages=c,c++'\r
        )\r
\r
    source_files = {\r
        'ia32': [source_files_common, source_files_ia32],\r
        'x64': [source_files_common, source_files_x64],\r
        'ipf': [source_files_common, source_files_ipf],\r
        }\r
\r
    for arch in source_files:\r
        mergedSourceFiles = {}\r
        for source_files_dict in source_files[arch]:\r
            mergedSourceFiles.update(source_files_dict)\r
        for downloadItem in mergedSourceFiles:\r
            fdata = mergedSourceFiles[downloadItem]\r
            fdata['filename'] = fdata['url'].split('/')[-1]\r
            if 'extract-dir' not in fdata:\r
                for ext in ('.tar.gz', '.tar.bz2', '.zip'):\r
                    if fdata['filename'].endswith(ext):\r
                        fdata['extract-dir'] = fdata['filename'][:-len(ext)]\r
                        break\r
            replaceables = ('extract-dir', 'filename', 'url')\r
            for replaceItem in fdata:\r
                if replaceItem in replaceables: continue\r
                if not isinstance(fdata[replaceItem], str): continue\r
                for replaceable in replaceables:\r
                    if not isinstance(fdata[replaceable], str): continue\r
                    if replaceable in fdata:\r
                        fdata[replaceable] = \\r
                            fdata[replaceable].replace(\r
                                '$' + replaceItem,\r
                                fdata[replaceItem]\r
                                )\r
        source_files[arch] = mergedSourceFiles\r
    #print 'source_files:', source_files\r
\r
    def GetAll(self):\r
\r
        def progress(received, blockSize, fileSize):\r
            if fileSize < 0: return\r
            wDots = (100 * received * blockSize) / fileSize / 10\r
            if wDots > self.dots:\r
                for i in range(wDots - self.dots):\r
                    print('.', end=' ')\r
                    sys.stdout.flush()\r
                    self.dots += 1\r
\r
        maxRetries = 1\r
        for (fname, fdata) in self.source_files.items():\r
            for retries in range(maxRetries):\r
                try:\r
                    self.dots = 0\r
                    local_file = os.path.join(self.config.src_dir, fdata['filename'])\r
                    url = fdata['url']\r
                    print('Downloading %s:' % fname, url)\r
                    if retries > 0:\r
                        print('(retry)', end=' ')\r
                    sys.stdout.flush()\r
\r
                    completed = False\r
                    if os.path.exists(local_file):\r
                        md5_pass = self.checkHash(fdata)\r
                        if md5_pass:\r
                            print('[md5 match]', end=' ')\r
                        else:\r
                            print('[md5 mismatch]', end=' ')\r
                        sys.stdout.flush()\r
                        completed = md5_pass\r
\r
                    if not completed:\r
                        urllib.urlretrieve(url, local_file, progress)\r
\r
                    #\r
                    # BUGBUG: Suggest proxy to user if download fails.\r
                    #\r
                    # export http_proxy=http://proxyservername.mycompany.com:911\r
                    # export ftp_proxy=http://proxyservername.mycompany.com:911\r
\r
                    if not completed and os.path.exists(local_file):\r
                        md5_pass = self.checkHash(fdata)\r
                        if md5_pass:\r
                            print('[md5 match]', end=' ')\r
                        else:\r
                            print('[md5 mismatch]', end=' ')\r
                        sys.stdout.flush()\r
                        completed = md5_pass\r
\r
                    if completed:\r
                        print('[done]')\r
                        break\r
                    else:\r
                        print('[failed]')\r
                        print('  Tried to retrieve', url)\r
                        print('  to', local_file)\r
                        print('Possible fixes:')\r
                        print('* If you are behind a web-proxy, try setting the', end=' ')\r
                        print('http_proxy environment variable')\r
                        print('* You can try to download this file separately', end=' ')\r
                        print('and rerun this script')\r
                        raise Exception()\r
\r
                except KeyboardInterrupt:\r
                    print('[KeyboardInterrupt]')\r
                    return False\r
\r
                except Exception as e:\r
                    print(e)\r
\r
            if not completed: return False\r
\r
        return True\r
\r
    def checkHash(self, fdata):\r
        local_file = os.path.join(self.config.src_dir, fdata['filename'])\r
        expect_md5 = fdata['md5']\r
        data = open(local_file).read()\r
        md5sum = md5()\r
        md5sum.update(data)\r
        return md5sum.hexdigest().lower() == expect_md5.lower()\r
\r
    def GetModules(self):\r
        return self.source_files.keys()\r
\r
    def GetFilenameOf(self, module):\r
        return self.source_files[module]['filename']\r
\r
    def GetMd5Of(self, module):\r
        return self.source_files[module]['md5']\r
\r
    def GetExtractDirOf(self, module):\r
        return self.source_files[module]['extract-dir']\r
\r
    def GetAdditionalParameters(self, module, step):\r
        key = step + '-params'\r
        if key in self.source_files[module]:\r
            return self.source_files[module][key]\r
        else:\r
            return tuple()\r
\r
class Extracter:\r
    """class Extracter\r
\r
    Handles the extraction of the source files from their downloaded\r
    archive files.\r
    """\r
\r
    def __init__(self, source_files, config):\r
        self.source_files = source_files\r
        self.config = config\r
\r
    def Extract(self, module):\r
        src = self.config.src_dir\r
        extractDst = os.path.join(src, self.config.arch)\r
        local_file = os.path.join(src, self.source_files.GetFilenameOf(module))\r
        moduleMd5 = self.source_files.GetMd5Of(module)\r
        extracted = os.path.join(extractDst, os.path.split(local_file)[1] + '.extracted')\r
        if not os.path.exists(extractDst):\r
            os.mkdir(extractDst)\r
\r
        extractedMd5 = None\r
        if os.path.exists(extracted):\r
            extractedMd5 = open(extracted).read()\r
\r
        if extractedMd5 != moduleMd5:\r
            print('Extracting %s:' % self.config.Relative(local_file))\r
            tar = tarfile.open(local_file)\r
            tar.extractall(extractDst)\r
            open(extracted, 'w').write(moduleMd5)\r
        else:\r
            pass\r
            #print 'Previously extracted', self.config.Relative(local_file)\r
\r
    def ExtractAll(self):\r
        for module in self.source_files.GetModules():\r
            self.Extract(module)\r
\r
class Builder:\r
    """class Builder\r
\r
    Builds and installs the GCC tool suite.\r
    """\r
\r
    def __init__(self, source_files, config):\r
        self.source_files = source_files\r
        self.config = config\r
\r
    def Build(self):\r
        if not self.config.options.skip_binutils:\r
            self.BuildModule('binutils')\r
        if not self.config.options.skip_gcc:\r
            self.BuildModule('gcc')\r
            self.MakeSymLinks()\r
\r
    def IsBuildStepComplete(self, step):\r
        return \\r
            os.path.exists(\r
                os.path.join(\r
                    self.config.build_dir, self.config.arch, step + '.completed'\r
                    )\r
                )\r
\r
    def MarkBuildStepComplete(self, step):\r
        open(\r
            os.path.join(\r
                self.config.build_dir, self.config.arch, step + '.completed'\r
                ),\r
            "w"\r
            ).close()\r
\r
\r
    def BuildModule(self, module):\r
        base_dir = os.getcwd()\r
        build_dir = os.path.join(self.config.build_dir, self.config.arch, module)\r
        module_dir = self.source_files.GetExtractDirOf(module)\r
        module_dir = os.path.realpath(os.path.join('src', self.config.arch, module_dir))\r
        configure = os.path.join(module_dir, 'configure')\r
        prefix = self.config.prefix\r
        if not os.path.exists(build_dir):\r
            os.makedirs(build_dir)\r
        os.chdir(build_dir)\r
\r
        cmd = (\r
            configure,\r
            '--target=%s' % self.config.target_combo,\r
            '--prefix=' + prefix,\r
            '--with-sysroot=' + prefix,\r
            '--disable-werror',\r
            )\r
        if os.path.exists('/opt/local/include/gmp.h'):\r
            cmd += ('--with-gmp=/opt/local',)\r
        if module == 'gcc': cmd += ('--oldincludedir=/opt/local/include',)\r
        cmd += self.source_files.GetAdditionalParameters(module, 'configure')\r
        self.RunCommand(cmd, module, 'config', skipable=True)\r
\r
        cmd = ('make',)\r
        if module == 'gcc':\r
            cmd += ('all-gcc',)\r
        self.RunCommand(cmd, module, 'build')\r
\r
        cmd = ('make',)\r
        if module == 'gcc':\r
            cmd += ('install-gcc',)\r
        else:\r
            cmd += ('install',)\r
        self.RunCommand(cmd, module, 'install')\r
\r
        os.chdir(base_dir)\r
\r
        print('%s module is now built and installed' % module)\r
\r
    def RunCommand(self, cmd, module, stage, skipable=False):\r
        if skipable:\r
            if self.IsBuildStepComplete('%s.%s' % (module, stage)):\r
                return\r
\r
        popen = lambda cmd: \\r
            subprocess.Popen(\r
                cmd,\r
                stdin=subprocess.PIPE,\r
                stdout=subprocess.PIPE,\r
                stderr=subprocess.STDOUT\r
                )\r
\r
        print('%s [%s] ...' % (module, stage), end=' ')\r
        sys.stdout.flush()\r
        p = popen(cmd)\r
        output = p.stdout.read()\r
        p.wait()\r
        if p.returncode != 0:\r
            print('[failed!]')\r
            logFile = os.path.join(self.config.build_dir, 'log.txt')\r
            f = open(logFile, "w")\r
            f.write(output)\r
            f.close()\r
            raise Exception('Failed to %s %s\n' % (stage, module) + \\r
                'See output log at %s' % self.config.Relative(logFile))\r
        else:\r
            print('[done]')\r
\r
        if skipable:\r
            self.MarkBuildStepComplete('%s.%s' % (module, stage))\r
\r
    def MakeSymLinks(self):\r
        links_dir = os.path.join(self.config.symlinks, self.config.arch)\r
        if not os.path.exists(links_dir):\r
            os.makedirs(links_dir)\r
        startPrinted = False\r
        for link in ('ar', 'ld', 'gcc'):\r
            src = os.path.join(\r
                self.config.prefix, 'bin', self.config.target_combo + '-' + link\r
                )\r
            linkdst = os.path.join(links_dir, link)\r
            if not os.path.lexists(linkdst):\r
                if not startPrinted:\r
                    print('Making symlinks in %s:' % self.config.Relative(links_dir), end=' ')\r
                    startPrinted = True\r
                print(link, end=' ')\r
                os.symlink(src, linkdst)\r
\r
        if startPrinted:\r
            print('[done]')\r
\r
class App:\r
    """class App\r
\r
    The main body of the application.\r
    """\r
\r
    def __init__(self):\r
        config = Config()\r
\r
        if not config.IsConfigOk():\r
            return\r
\r
        config.MakeDirs()\r
\r
        sources = SourceFiles(config)\r
        result = sources.GetAll()\r
        if result:\r
            print('All files have been downloaded & verified')\r
        else:\r
            print('An error occured while downloading a file')\r
            return\r
\r
        Extracter(sources, config).ExtractAll()\r
\r
        Builder(sources, config).Build()\r
\r
App()\r
\r