BaseTools/gcc/mingw-gcc-build.py

   1 #!/usr/bin/env python
   2
   3 ## @file
   4 #
   5 # Automation of instructions from:
   6 #   http://mingw-w64.svn.sourceforge.net/viewvc/mingw-w64/trunk/mingw-w64-doc/
   7 #     howto-build/mingw-w64-howto-build.txt?revision=216&view=markup
   8 #
   9 # Copyright (c) 2008 - 2018, Intel Corporation. All rights reserved.<BR>
  10 # This program and the accompanying materials
  11 # are licensed and made available under the terms and conditions of the BSD License
  12 # which accompanies this distribution.    The full text of the license may be found at
  13 # http://opensource.org/licenses/bsd-license.php
  14 #
  15 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  16 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  17 #
  18
  19
  20 from __future__ import print_function
  21 from optparse import OptionParser
  22 import os
  23 import shutil
  24 import subprocess
  25 import sys
  26 import tarfile
  27 import urllib
  28 import urlparse
  29 try:
  30     from hashlib import md5
  31 except Exception:
  32     from md5 import md5
  33
  34 if sys.version_info < (2, 5):
  35     #
  36     # This script (and edk2 BaseTools) require Python 2.5 or newer
  37     #
  38     print('Python version 2.5 or later is required.')
  39     sys.exit(-1)
  40
  41 #
  42 # Version and Copyright
  43 #
  44 VersionNumber = "0.01"
  45 __version__ = "%prog Version " + VersionNumber
  46 __copyright__ = "Copyright (c) 2008 - 2018, Intel Corporation.  All rights reserved."
  47
  48 class Config:
  49     """class Config
  50
  51     Stores the configuration options for the rest of the script.
  52
  53     Handles the command line options, and allows the code within
  54     the script to easily interact with the 'config' requested by
  55     the user.
  56     """
  57
  58     def __init__(self):
  59         self.base_dir = os.getcwd()
  60         (self.options, self.args) = self.CheckOptions()
  61         self.__init_dirs__()
  62
  63     def CheckOptions(self):
  64         Parser = \
  65             OptionParser(
  66                 description=__copyright__,
  67                 version=__version__,
  68                 prog="mingw-gcc-build",
  69                 usage="%prog [options] [target]"
  70                 )
  71         Parser.add_option(
  72             "--arch",
  73             action = "store", type = "string",
  74             default = '',
  75             dest = "arch",
  76             help = "Processor architecture to build gcc for."
  77             )
  78         Parser.add_option(
  79             "--src-dir",
  80             action = "store", type = "string", dest = "src_dir",
  81             default = os.path.join(self.base_dir, 'src'),
  82             help = "Directory to download/extract binutils/gcc sources"
  83             )
  84         Parser.add_option(
  85             "--build-dir",
  86             action = "store", type = "string", dest = "build_dir",
  87             default = os.path.join(self.base_dir, 'build'),
  88             help = "Directory to download/extract binutils/gcc sources"
  89             )
  90         Parser.add_option(
  91             "--prefix",
  92             action = "store", type = "string", dest = "prefix",
  93             default = os.path.join(self.base_dir, 'install'),
  94             help = "Prefix to install binutils/gcc into"
  95             )
  96         Parser.add_option(
  97             "--skip-binutils",
  98             action = "store_true", dest = "skip_binutils",
  99             default = False,
 100             help = "Will skip building binutils"
 101             )
 102         Parser.add_option(
 103             "--skip-gcc",
 104             action = "store_true", dest = "skip_gcc",
 105             default = False,
 106             help = "Will skip building GCC"
 107             )
 108         Parser.add_option(
 109             "--symlinks",
 110             action = "store", type = "string", dest = "symlinks",
 111             default = os.path.join(self.base_dir, 'symlinks'),
 112             help = "Directory to create binutils/gcc symbolic links into."
 113             )
 114         Parser.add_option(
 115             "-v", "--verbose",
 116             action="store_true",
 117             type=None, help="Print verbose messages"
 118             )
 119
 120         (Opt, Args) = Parser.parse_args()
 121
 122         self.arch = Opt.arch.lower()
 123         allowedArchs = ('ia32', 'x64', 'ipf')
 124         if self.arch not in allowedArchs:
 125             Parser.error(
 126                 'Please use --arch to specify one of: %s' %
 127                     ', '.join(allowedArchs)
 128                 )
 129         self.target_arch = {'ia32': 'i686', 'x64': 'x86_64', 'ipf': 'ia64'}[self.arch]
 130         self.target_sys = {'ia32': 'pc', 'x64': 'pc', 'ipf': 'pc'}[self.arch]
 131         self.target_bin = {'ia32': 'mingw32', 'x64': 'mingw32', 'ipf': 'elf'}[self.arch]
 132         self.target_combo = '-'.join((self.target_arch, self.target_sys, self.target_bin))
 133
 134         return (Opt, Args)
 135
 136     def __init_dirs__(self):
 137         self.src_dir = os.path.realpath(os.path.expanduser(self.options.src_dir))
 138         self.build_dir = os.path.realpath(os.path.expanduser(self.options.build_dir))
 139         self.prefix = os.path.realpath(os.path.expanduser(self.options.prefix))
 140         self.symlinks = os.path.realpath(os.path.expanduser(self.options.symlinks))
 141
 142     def IsConfigOk(self):
 143
 144         building = []
 145         if not self.options.skip_binutils:
 146             building.append('binutils')
 147         if not self.options.skip_gcc:
 148             building.append('gcc')
 149         if len(building) == 0:
 150             print("Nothing will be built!")
 151             print()
 152             print("Please try using --help and then change the configuration.")
 153             return False
 154
 155         print("Current directory:")
 156         print("   ", self.base_dir)
 157         print("Sources download/extraction:", self.Relative(self.src_dir))
 158         print("Build directory            :", self.Relative(self.build_dir))
 159         print("Prefix (install) directory :", self.Relative(self.prefix))
 160         print("Create symlinks directory  :", self.Relative(self.symlinks))
 161         print("Building                   :", ', '.join(building))
 162         print()
 163         answer = raw_input("Is this configuration ok? (default = no): ")
 164         if (answer.lower() not in ('y', 'yes')):
 165             print()
 166             print("Please try using --help and then change the configuration.")
 167             return False
 168
 169         if self.arch.lower() == 'ipf':
 170             print()
 171             print('Please note that the IPF compiler built by this script has')
 172             print('not yet been validated!')
 173             print()
 174             answer = raw_input("Are you sure you want to build it? (default = no): ")
 175             if (answer.lower() not in ('y', 'yes')):
 176                 print()
 177                 print("Please try using --help and then change the configuration.")
 178                 return False
 179
 180         print()
 181         return True
 182
 183     def Relative(self, path):
 184         if path.startswith(self.base_dir):
 185             return '.' + path[len(self.base_dir):]
 186         return path
 187
 188     def MakeDirs(self):
 189         for path in (self.src_dir, self.build_dir, self.prefix, self.symlinks):
 190             if not os.path.exists(path):
 191                 os.makedirs(path)
 192
 193 class SourceFiles:
 194     """class SourceFiles
 195
 196     Handles the downloading of source files used by the script.
 197     """
 198
 199     def __init__(self, config):
 200         self.config = config
 201         self.source_files = self.source_files[config.arch]
 202
 203         if config.options.skip_binutils:
 204             del self.source_files['binutils']
 205
 206         if config.options.skip_gcc:
 207             del self.source_files['gcc']
 208             del self.source_files['mingw_hdr']
 209
 210     source_files_common = {
 211         'binutils': {
 212             'url': 'http://www.kernel.org/pub/linux/devel/binutils/' + \
 213                    'binutils-$version.tar.bz2',
 214             'version': '2.20.51.0.5',
 215             'md5': '6d2de7cdf7a8389e70b124e3d73b4d37',
 216             },
 217         }
 218
 219     source_files_x64 = {
 220         'gcc': {
 221             'url': 'http://ftpmirror.gnu.org/gcc/' + \
 222                    'gcc-$version/gcc-$version.tar.bz2',
 223             'version': '4.3.0',
 224             'md5': '197ed8468b38db1d3481c3111691d85b',
 225             },
 226         }
 227
 228     source_files_ia32 = {
 229         'gcc': source_files_x64['gcc'],
 230         }
 231
 232     source_files_ipf = source_files_x64.copy()
 233     source_files_ipf['gcc']['configure-params'] = (
 234         '--with-gnu-as', '--with-gnu-ld', '--with-newlib',
 235         '--verbose', '--disable-libssp', '--disable-nls',
 236         '--enable-languages=c,c++'
 237         )
 238
 239     source_files = {
 240         'ia32': [source_files_common, source_files_ia32],
 241         'x64': [source_files_common, source_files_x64],
 242         'ipf': [source_files_common, source_files_ipf],
 243         }
 244
 245     for arch in source_files:
 246         mergedSourceFiles = {}
 247         for source_files_dict in source_files[arch]:
 248             mergedSourceFiles.update(source_files_dict)
 249         for downloadItem in mergedSourceFiles:
 250             fdata = mergedSourceFiles[downloadItem]
 251             fdata['filename'] = fdata['url'].split('/')[-1]
 252             if 'extract-dir' not in fdata:
 253                 for ext in ('.tar.gz', '.tar.bz2', '.zip'):
 254                     if fdata['filename'].endswith(ext):
 255                         fdata['extract-dir'] = fdata['filename'][:-len(ext)]
 256                         break
 257             replaceables = ('extract-dir', 'filename', 'url')
 258             for replaceItem in fdata:
 259                 if replaceItem in replaceables: continue
 260                 if not isinstance(fdata[replaceItem], str): continue
 261                 for replaceable in replaceables:
 262                     if not isinstance(fdata[replaceable], str): continue
 263                     if replaceable in fdata:
 264                         fdata[replaceable] = \
 265                             fdata[replaceable].replace(
 266                                 '$' + replaceItem,
 267                                 fdata[replaceItem]
 268                                 )
 269         source_files[arch] = mergedSourceFiles
 270     #print 'source_files:', source_files
 271
 272     def GetAll(self):
 273
 274         def progress(received, blockSize, fileSize):
 275             if fileSize < 0: return
 276             wDots = (100 * received * blockSize) / fileSize / 10
 277             if wDots > self.dots:
 278                 for i in range(wDots - self.dots):
 279                     print('.', end=' ')
 280                     sys.stdout.flush()
 281                     self.dots += 1
 282
 283         maxRetries = 1
 284         for (fname, fdata) in self.source_files.items():
 285             for retries in range(maxRetries):
 286                 try:
 287                     self.dots = 0
 288                     local_file = os.path.join(self.config.src_dir, fdata['filename'])
 289                     url = fdata['url']
 290                     print('Downloading %s:' % fname, url)
 291                     if retries > 0:
 292                         print('(retry)', end=' ')
 293                     sys.stdout.flush()
 294
 295                     completed = False
 296                     if os.path.exists(local_file):
 297                         md5_pass = self.checkHash(fdata)
 298                         if md5_pass:
 299                             print('[md5 match]', end=' ')
 300                         else:
 301                             print('[md5 mismatch]', end=' ')
 302                         sys.stdout.flush()
 303                         completed = md5_pass
 304
 305                     if not completed:
 306                         urllib.urlretrieve(url, local_file, progress)
 307
 308                     #
 309                     # BUGBUG: Suggest proxy to user if download fails.
 310                     #
 311                     # export http_proxy=http://proxyservername.mycompany.com:911
 312                     # export ftp_proxy=http://proxyservername.mycompany.com:911
 313
 314                     if not completed and os.path.exists(local_file):
 315                         md5_pass = self.checkHash(fdata)
 316                         if md5_pass:
 317                             print('[md5 match]', end=' ')
 318                         else:
 319                             print('[md5 mismatch]', end=' ')
 320                         sys.stdout.flush()
 321                         completed = md5_pass
 322
 323                     if completed:
 324                         print('[done]')
 325                         break
 326                     else:
 327                         print('[failed]')
 328                         print('  Tried to retrieve', url)
 329                         print('  to', local_file)
 330                         print('Possible fixes:')
 331                         print('* If you are behind a web-proxy, try setting the', end=' ')
 332                         print('http_proxy environment variable')
 333                         print('* You can try to download this file separately', end=' ')
 334                         print('and rerun this script')
 335                         raise Exception()
 336
 337                 except KeyboardInterrupt:
 338                     print('[KeyboardInterrupt]')
 339                     return False
 340
 341                 except Exception as e:
 342                     print(e)
 343
 344             if not completed: return False
 345
 346         return True
 347
 348     def checkHash(self, fdata):
 349         local_file = os.path.join(self.config.src_dir, fdata['filename'])
 350         expect_md5 = fdata['md5']
 351         data = open(local_file).read()
 352         md5sum = md5()
 353         md5sum.update(data)
 354         return md5sum.hexdigest().lower() == expect_md5.lower()
 355
 356     def GetModules(self):
 357         return self.source_files.keys()
 358
 359     def GetFilenameOf(self, module):
 360         return self.source_files[module]['filename']
 361
 362     def GetMd5Of(self, module):
 363         return self.source_files[module]['md5']
 364
 365     def GetExtractDirOf(self, module):
 366         return self.source_files[module]['extract-dir']
 367
 368     def GetAdditionalParameters(self, module, step):
 369         key = step + '-params'
 370         if key in self.source_files[module]:
 371             return self.source_files[module][key]
 372         else:
 373             return tuple()
 374
 375 class Extracter:
 376     """class Extracter
 377
 378     Handles the extraction of the source files from their downloaded
 379     archive files.
 380     """
 381
 382     def __init__(self, source_files, config):
 383         self.source_files = source_files
 384         self.config = config
 385
 386     def Extract(self, module):
 387         src = self.config.src_dir
 388         extractDst = os.path.join(src, self.config.arch)
 389         local_file = os.path.join(src, self.source_files.GetFilenameOf(module))
 390         moduleMd5 = self.source_files.GetMd5Of(module)
 391         extracted = os.path.join(extractDst, os.path.split(local_file)[1] + '.extracted')
 392         if not os.path.exists(extractDst):
 393             os.mkdir(extractDst)
 394
 395         extractedMd5 = None
 396         if os.path.exists(extracted):
 397             extractedMd5 = open(extracted).read()
 398
 399         if extractedMd5 != moduleMd5:
 400             print('Extracting %s:' % self.config.Relative(local_file))
 401             tar = tarfile.open(local_file)
 402             tar.extractall(extractDst)
 403             open(extracted, 'w').write(moduleMd5)
 404         else:
 405             pass
 406             #print 'Previously extracted', self.config.Relative(local_file)
 407
 408     def ExtractAll(self):
 409         for module in self.source_files.GetModules():
 410             self.Extract(module)
 411
 412 class Builder:
 413     """class Builder
 414
 415     Builds and installs the GCC tool suite.
 416     """
 417
 418     def __init__(self, source_files, config):
 419         self.source_files = source_files
 420         self.config = config
 421
 422     def Build(self):
 423         if not self.config.options.skip_binutils:
 424             self.BuildModule('binutils')
 425         if not self.config.options.skip_gcc:
 426             self.BuildModule('gcc')
 427             self.MakeSymLinks()
 428
 429     def IsBuildStepComplete(self, step):
 430         return \
 431             os.path.exists(
 432                 os.path.join(
 433                     self.config.build_dir, self.config.arch, step + '.completed'
 434                     )
 435                 )
 436
 437     def MarkBuildStepComplete(self, step):
 438         open(
 439             os.path.join(
 440                 self.config.build_dir, self.config.arch, step + '.completed'
 441                 ),
 442             "w"
 443             ).close()
 444
 445
 446     def BuildModule(self, module):
 447         base_dir = os.getcwd()
 448         build_dir = os.path.join(self.config.build_dir, self.config.arch, module)
 449         module_dir = self.source_files.GetExtractDirOf(module)
 450         module_dir = os.path.realpath(os.path.join('src', self.config.arch, module_dir))
 451         configure = os.path.join(module_dir, 'configure')
 452         prefix = self.config.prefix
 453         if not os.path.exists(build_dir):
 454             os.makedirs(build_dir)
 455         os.chdir(build_dir)
 456
 457         cmd = (
 458             configure,
 459             '--target=%s' % self.config.target_combo,
 460             '--prefix=' + prefix,
 461             '--with-sysroot=' + prefix,
 462             '--disable-werror',
 463             )
 464         if os.path.exists('/opt/local/include/gmp.h'):
 465             cmd += ('--with-gmp=/opt/local',)
 466         if module == 'gcc': cmd += ('--oldincludedir=/opt/local/include',)
 467         cmd += self.source_files.GetAdditionalParameters(module, 'configure')
 468         self.RunCommand(cmd, module, 'config', skipable=True)
 469
 470         cmd = ('make',)
 471         if module == 'gcc':
 472             cmd += ('all-gcc',)
 473         self.RunCommand(cmd, module, 'build')
 474
 475         cmd = ('make',)
 476         if module == 'gcc':
 477             cmd += ('install-gcc',)
 478         else:
 479             cmd += ('install',)
 480         self.RunCommand(cmd, module, 'install')
 481
 482         os.chdir(base_dir)
 483
 484         print('%s module is now built and installed' % module)
 485
 486     def RunCommand(self, cmd, module, stage, skipable=False):
 487         if skipable:
 488             if self.IsBuildStepComplete('%s.%s' % (module, stage)):
 489                 return
 490
 491         popen = lambda cmd: \
 492             subprocess.Popen(
 493                 cmd,
 494                 stdin=subprocess.PIPE,
 495                 stdout=subprocess.PIPE,
 496                 stderr=subprocess.STDOUT
 497                 )
 498
 499         print('%s [%s] ...' % (module, stage), end=' ')
 500         sys.stdout.flush()
 501         p = popen(cmd)
 502         output = p.stdout.read()
 503         p.wait()
 504         if p.returncode != 0:
 505             print('[failed!]')
 506             logFile = os.path.join(self.config.build_dir, 'log.txt')
 507             f = open(logFile, "w")
 508             f.write(output)
 509             f.close()
 510             raise Exception('Failed to %s %s\n' % (stage, module) + \
 511                 'See output log at %s' % self.config.Relative(logFile))
 512         else:
 513             print('[done]')
 514
 515         if skipable:
 516             self.MarkBuildStepComplete('%s.%s' % (module, stage))
 517
 518     def MakeSymLinks(self):
 519         links_dir = os.path.join(self.config.symlinks, self.config.arch)
 520         if not os.path.exists(links_dir):
 521             os.makedirs(links_dir)
 522         startPrinted = False
 523         for link in ('ar', 'ld', 'gcc'):
 524             src = os.path.join(
 525                 self.config.prefix, 'bin', self.config.target_combo + '-' + link
 526                 )
 527             linkdst = os.path.join(links_dir, link)
 528             if not os.path.lexists(linkdst):
 529                 if not startPrinted:
 530                     print('Making symlinks in %s:' % self.config.Relative(links_dir), end=' ')
 531                     startPrinted = True
 532                 print(link, end=' ')
 533                 os.symlink(src, linkdst)
 534
 535         if startPrinted:
 536             print('[done]')
 537
 538 class App:
 539     """class App
 540
 541     The main body of the application.
 542     """
 543
 544     def __init__(self):
 545         config = Config()
 546
 547         if not config.IsConfigOk():
 548             return
 549
 550         config.MakeDirs()
 551
 552         sources = SourceFiles(config)
 553         result = sources.GetAll()
 554         if result:
 555             print('All files have been downloaded & verified')
 556         else:
 557             print('An error occured while downloading a file')
 558             return
 559
 560         Extracter(sources, config).ExtractAll()
 561
 562         Builder(sources, config).Build()
 563
 564 App()
 565