BaseTools/gcc/mingw-gcc-build.py

   1 #!/usr/bin/env python
   2
   3 #
   4 # Automation of instructions from:
   5 #   http://mingw-w64.svn.sourceforge.net/viewvc/mingw-w64/trunk/mingw-w64-doc/
   6 #     howto-build/mingw-w64-howto-build.txt?revision=216&view=markup
   7 #
   8
   9 from optparse import OptionParser
  10 import os
  11 import shutil
  12 import subprocess
  13 import sys
  14 import tarfile
  15 import urllib
  16 import urlparse
  17 try:
  18     from hashlib import md5
  19 except Exception:
  20     from md5 import md5
  21
  22 if sys.version_info < (2, 5):
  23     #
  24     # This script (and edk2 BaseTools) require Python 2.5 or newer
  25     #
  26     print 'Python version 2.5 or later is required.'
  27     sys.exit(-1)
  28
  29 #
  30 # Version and Copyright
  31 #
  32 VersionNumber = "0.01"
  33 __version__ = "%prog Version " + VersionNumber
  34 __copyright__ = "Copyright (c) 2008, Intel Corporation.  All rights reserved."
  35
  36 class Config:
  37     """class Config
  38
  39     Stores the configuration options for the rest of the script.
  40
  41     Handles the command line options, and allows the code within
  42     the script to easily interact with the 'config' requested by
  43     the user.
  44     """
  45
  46     def __init__(self):
  47         self.base_dir = os.getcwd()
  48         (self.options, self.args) = self.CheckOptions()
  49         self.__init_dirs__()
  50
  51     def CheckOptions(self):
  52         Parser = \
  53             OptionParser(
  54                 description=__copyright__,
  55                 version=__version__,
  56                 prog="mingw-gcc-build",
  57                 usage="%prog [options] [target]"
  58                 )
  59         Parser.add_option(
  60             "--arch",
  61             action = "store", type = "string",
  62             default = '',
  63             dest = "arch",
  64             help = "Processor architecture to build gcc for."
  65             )
  66         Parser.add_option(
  67             "--src-dir",
  68             action = "store", type = "string", dest = "src_dir",
  69             default = os.path.join(self.base_dir, 'src'),
  70             help = "Directory to download/extract binutils/gcc sources"
  71             )
  72         Parser.add_option(
  73             "--build-dir",
  74             action = "store", type = "string", dest = "build_dir",
  75             default = os.path.join(self.base_dir, 'build'),
  76             help = "Directory to download/extract binutils/gcc sources"
  77             )
  78         Parser.add_option(
  79             "--prefix",
  80             action = "store", type = "string", dest = "prefix",
  81             default = os.path.join(self.base_dir, 'install'),
  82             help = "Prefix to install binutils/gcc into"
  83             )
  84         Parser.add_option(
  85             "--symlinks",
  86             action = "store", type = "string", dest = "symlinks",
  87             default = os.path.join(self.base_dir, 'symlinks'),
  88             help = "Directory to create binutils/gcc symbolic links into."
  89             )
  90         Parser.add_option(
  91             "-v", "--verbose",
  92             action="store_true",
  93             type=None, help="Print verbose messages"
  94             )
  95
  96         (Opt, Args) = Parser.parse_args()
  97
  98         self.arch = Opt.arch.lower()
  99         allowedArchs = ('ia32', 'x64', 'ipf')
 100         if self.arch not in allowedArchs:
 101             Parser.error(
 102                 'Please use --arch to specify one of: %s' %
 103                     ', '.join(allowedArchs)
 104                 )
 105         self.target_arch = {'ia32': 'i686', 'x64': 'x86_64', 'ipf': 'ia64'}[self.arch]
 106         self.target_sys = {'ia32': 'pc', 'x64': 'pc', 'ipf': 'pc'}[self.arch]
 107         self.target_bin = {'ia32': 'mingw32', 'x64': 'mingw32', 'ipf': 'elf'}[self.arch]
 108         self.target_combo = '-'.join((self.target_arch, self.target_sys, self.target_bin))
 109
 110         return (Opt, Args)
 111
 112     def __init_dirs__(self):
 113         self.src_dir = os.path.realpath(os.path.expanduser(self.options.src_dir))
 114         self.build_dir = os.path.realpath(os.path.expanduser(self.options.build_dir))
 115         self.prefix = os.path.realpath(os.path.expanduser(self.options.prefix))
 116         self.symlinks = os.path.realpath(os.path.expanduser(self.options.symlinks))
 117
 118     def IsConfigOk(self):
 119
 120         print "Current directory:"
 121         print "   ", self.base_dir
 122         print "Sources download/extraction:", self.Relative(self.src_dir)
 123         print "Build directory            :", self.Relative(self.build_dir)
 124         print "Prefix (install) directory :", self.Relative(self.prefix)
 125         print "Create symlinks directory  :", self.Relative(self.symlinks)
 126         print
 127         answer = raw_input("Is this configuration ok? (default = no): ")
 128         if (answer.lower() not in ('y', 'yes')):
 129             print
 130             print "Please try using --help and then change the configuration."
 131             return False
 132
 133         if self.arch.lower() == 'ipf':
 134             print
 135             print 'Please note that the IPF compiler built by this script has'
 136             print 'not yet been validated!'
 137             print
 138             answer = raw_input("Are you sure you want to build it? (default = no): ")
 139             if (answer.lower() not in ('y', 'yes')):
 140                 print
 141                 print "Please try using --help and then change the configuration."
 142                 return False
 143
 144         print
 145         return True
 146
 147     def Relative(self, path):
 148         if path.startswith(self.base_dir):
 149             return '.' + path[len(self.base_dir):]
 150         return path
 151
 152     def MakeDirs(self):
 153         for path in (self.src_dir, self.build_dir,self.prefix, self.symlinks):
 154             if not os.path.exists(path):
 155                 os.makedirs(path)
 156
 157 class SourceFiles:
 158     """class SourceFiles
 159
 160     Handles the downloading of source files used by the script.
 161     """
 162
 163     def __init__(self, config):
 164         self.config = config
 165         self.source_files = self.source_files[config.arch]
 166
 167     source_files_common = {
 168         'binutils': {
 169             'url': 'http://www.kernel.org/pub/linux/devel/binutils/' + \
 170                    'binutils-$version.tar.bz2',
 171             'version': '2.18.50.0.5',
 172             'md5': 'daee18dbbf0a6ccfc186141bee18bf62',
 173             },
 174         }
 175
 176     source_files_x64 = {
 177         'gcc': {
 178             'url': 'http://ftpmirror.gnu.org/gcc/' + \
 179                    'gcc-$version/gcc-$version.tar.bz2',
 180             'version': '4.3.0',
 181             'md5': '197ed8468b38db1d3481c3111691d85b',
 182             },
 183         'mingw_hdr': {
 184             'url': 'http://sourceforge.net/projects/' + \
 185                    'mingw-w64/files/mingw-w64/mingw-w64-snapshot/' + \
 186                    'mingw-w64-snapshot-$version.tar.bz2/download',
 187             'extract-dir': os.path.join('trunk', 'mingw-w64-headers'),
 188             'version': '20090419',
 189             'md5': '9146ecfabaf172e4cc427b88e8d218c1',
 190             },
 191         }
 192
 193     source_files_ia32 = {
 194         'gcc': source_files_x64['gcc'],
 195         'mingw_hdr': {
 196             'url': 'http://sourceforge.net/projects/' + \
 197                    'mingw/files/MinGW%20Runtime/' + \
 198                    'mingwrt-$version/' + \
 199                    'mingwrt-$version-mingw32-src.tar.gz/download',
 200             'extract-dir': 'mingwrt-$version-mingw32',
 201             'version': '3.15.2',
 202             'md5': '7bf0525f158213f3ac990ea68a5ec34d',
 203             },
 204         }
 205
 206     source_files_ipf = source_files_x64.copy()
 207     source_files_ipf['gcc']['configure-params'] = (
 208         '--with-gnu-as', '--with-gnu-ld', '--with-newlib',
 209         '--verbose', '--disable-libssp', '--disable-nls',
 210         '--enable-languages=c,c++'
 211         )
 212
 213     source_files = {
 214         'ia32': [source_files_common, source_files_ia32],
 215         'x64': [source_files_common, source_files_x64],
 216         'ipf': [source_files_common, source_files_ipf],
 217         }
 218
 219     for arch in source_files:
 220         mergedSourceFiles = {}
 221         for source_files_dict in source_files[arch]:
 222             mergedSourceFiles.update(source_files_dict)
 223         for downloadItem in mergedSourceFiles:
 224             fdata = mergedSourceFiles[downloadItem]
 225             fdata['filename'] = fdata['url'].split('/')[-1]
 226             if 'extract-dir' not in fdata:
 227                 for ext in ('.tar.gz', '.tar.bz2', '.zip'):
 228                     if fdata['filename'].endswith(ext):
 229                         fdata['extract-dir'] = fdata['filename'][:-len(ext)]
 230                         break
 231             replaceables = ('extract-dir', 'filename', 'url')
 232             for replaceItem in fdata:
 233                 if replaceItem in replaceables: continue
 234                 if type(fdata[replaceItem]) != str: continue
 235                 for replaceable in replaceables:
 236                     if type(fdata[replaceable]) != str: continue
 237                     if replaceable in fdata:
 238                         fdata[replaceable] = \
 239                             fdata[replaceable].replace(
 240                                 '$' + replaceItem,
 241                                 fdata[replaceItem]
 242                                 )
 243         source_files[arch] = mergedSourceFiles
 244     #print 'source_files:', source_files
 245
 246     def GetAll(self):
 247
 248         def progress(received, blockSize, fileSize):
 249             if fileSize < 0: return
 250             wDots = (100 * received * blockSize) / fileSize / 10
 251             if wDots > self.dots:
 252                 for i in range(wDots - self.dots):
 253                     print '.',
 254                     sys.stdout.flush()
 255                     self.dots += 1
 256
 257         maxRetries = 1
 258         for (fname, fdata) in self.source_files.items():
 259             for retries in range(maxRetries):
 260                 try:
 261                     self.dots = 0
 262                     local_file = os.path.join(self.config.src_dir, fdata['filename'])
 263                     url = fdata['url']
 264                     print 'Downloading %s:' % fname,
 265                     if retries > 0:
 266                         print '(retry)',
 267                     sys.stdout.flush()
 268
 269                     completed = False
 270                     if os.path.exists(local_file):
 271                         md5_pass = self.checkHash(fdata)
 272                         if md5_pass:
 273                             print '[md5 match]',
 274                         else:
 275                             print '[md5 mismatch]',
 276                         sys.stdout.flush()
 277                         completed = md5_pass
 278
 279                     if not completed:
 280                         urllib.urlretrieve(url, local_file, progress)
 281
 282                     #
 283                     # BUGBUG: Suggest proxy to user if download fails.
 284                     #
 285                     # export http_proxy=http://proxyservername.mycompany.com:911
 286                     # export ftp_proxy=http://proxyservername.mycompany.com:911
 287
 288                     if not completed and os.path.exists(local_file):
 289                         md5_pass = self.checkHash(fdata)
 290                         if md5_pass:
 291                             print '[md5 match]',
 292                         else:
 293                             print '[md5 mismatch]',
 294                         sys.stdout.flush()
 295                         completed = md5_pass
 296
 297                     if completed:
 298                         print '[done]'
 299                         break
 300                     else:
 301                         print '[failed]'
 302                         print '  Tried to retrieve', url
 303                         print '  to', local_file
 304                         print 'Possible fixes:'
 305                         print '* If you are behind a web-proxy, try setting the',
 306                         print 'http_proxy environment variable'
 307                         print '* You can try to download this file separately',
 308                         print 'and rerun this script'
 309                         raise Exception()
 310
 311                 except KeyboardInterrupt:
 312                     print '[KeyboardInterrupt]'
 313                     return False
 314
 315                 except:
 316                     pass
 317
 318             if not completed: return False
 319
 320         return True
 321
 322     def checkHash(self, fdata):
 323         local_file = os.path.join(self.config.src_dir, fdata['filename'])
 324         expect_md5 = fdata['md5']
 325         data = open(local_file).read()
 326         md5sum = md5()
 327         md5sum.update(data)
 328         return md5sum.hexdigest().lower() == expect_md5.lower()
 329
 330     def GetModules(self):
 331         return self.source_files.keys()
 332
 333     def GetFilenameOf(self, module):
 334         return self.source_files[module]['filename']
 335
 336     def GetMd5Of(self, module):
 337         return self.source_files[module]['md5']
 338
 339     def GetExtractDirOf(self, module):
 340         return self.source_files[module]['extract-dir']
 341
 342     def GetAdditionalParameters(self, module, step):
 343         key = step + '-params'
 344         if key in self.source_files[module]:
 345             return self.source_files[module][key]
 346         else:
 347             return tuple()
 348
 349 class Extracter:
 350     """class Extracter
 351
 352     Handles the extraction of the source files from their downloaded
 353     archive files.
 354     """
 355
 356     def __init__(self, source_files, config):
 357         self.source_files = source_files
 358         self.config = config
 359
 360     def Extract(self, module):
 361         src = self.config.src_dir
 362         extractDst = os.path.join(src, self.config.arch)
 363         local_file = os.path.join(src, self.source_files.GetFilenameOf(module))
 364         moduleMd5 = self.source_files.GetMd5Of(module)
 365         extracted = os.path.join(extractDst, os.path.split(local_file)[1] + '.extracted')
 366         if not os.path.exists(extractDst):
 367             os.mkdir(extractDst)
 368
 369         extractedMd5 = None
 370         if os.path.exists(extracted):
 371             extractedMd5 = open(extracted).read()
 372
 373         if extractedMd5 != moduleMd5:
 374             print 'Extracting %s:' % self.config.Relative(local_file)
 375             tar = tarfile.open(local_file)
 376             tar.extractall(extractDst)
 377             open(extracted, 'w').write(moduleMd5)
 378         else:
 379             pass
 380             #print 'Previously extracted', self.config.Relative(local_file)
 381
 382     def ExtractAll(self):
 383         for module in self.source_files.GetModules():
 384             self.Extract(module)
 385
 386 class Builder:
 387     """class Builder
 388
 389     Builds and installs the GCC tool suite.
 390     """
 391
 392     def __init__(self, source_files, config):
 393         self.source_files = source_files
 394         self.config = config
 395
 396     def Build(self):
 397         self.BuildModule('binutils')
 398         self.CopyIncludeDirectory()
 399         self.BuildModule('gcc')
 400         self.MakeSymLinks()
 401
 402     def IsBuildStepComplete(self, step):
 403         return \
 404             os.path.exists(
 405                 os.path.join(
 406                     self.config.build_dir, self.config.arch, step + '.completed'
 407                     )
 408                 )
 409
 410     def MarkBuildStepComplete(self, step):
 411         open(
 412             os.path.join(
 413                 self.config.build_dir, self.config.arch, step + '.completed'
 414                 ),
 415             "w"
 416             ).close()
 417
 418     def CopyIncludeDirectory(self):
 419         linkdst = os.path.join(self.config.prefix, 'mingw')
 420         src = os.path.join(
 421             self.config.src_dir,
 422             self.config.arch,
 423             self.source_files.GetExtractDirOf('mingw_hdr'),
 424             'include'
 425             )
 426         dst_parent = os.path.join(self.config.prefix, self.config.target_combo)
 427         dst = os.path.join(dst_parent, 'include')
 428         if not os.path.exists(dst):
 429             if not os.path.exists(dst_parent):
 430                 os.makedirs(dst_parent)
 431             print 'Copying headers to', self.config.Relative(dst)
 432             shutil.copytree(src, dst, True)
 433         if not os.path.lexists(linkdst):
 434             print 'Making symlink at', self.config.Relative(linkdst)
 435             os.symlink(self.config.target_combo, linkdst)
 436
 437     def BuildModule(self, module):
 438         base_dir = os.getcwd()
 439         build_dir = os.path.join(self.config.build_dir, self.config.arch, module)
 440         module_dir = self.source_files.GetExtractDirOf(module)
 441         module_dir = os.path.realpath(os.path.join('src', self.config.arch, module_dir))
 442         configure = os.path.join(module_dir, 'configure')
 443         prefix = self.config.prefix
 444         if not os.path.exists(build_dir):
 445             os.makedirs(build_dir)
 446         os.chdir(build_dir)
 447
 448         cmd = (
 449             configure,
 450             '--target=%s' % self.config.target_combo,
 451             '--prefix=' + prefix,
 452             '--with-sysroot=' + prefix,
 453             '--disable-werror',
 454             )
 455         if os.path.exists('/opt/local/include/gmp.h'):
 456             cmd += ('--with-gmp=/opt/local',)
 457         if module == 'gcc': cmd += ('--oldincludedir=/opt/local/include',)
 458         cmd += self.source_files.GetAdditionalParameters(module, 'configure')
 459         self.RunCommand(cmd, module, 'config', skipable=True)
 460
 461         cmd = ('make',)
 462         if module == 'gcc':
 463             cmd += ('all-gcc',)
 464         self.RunCommand(cmd, module, 'build')
 465
 466         cmd = ('make',)
 467         if module == 'gcc':
 468             cmd += ('install-gcc',)
 469         else:
 470             cmd += ('install',)
 471         self.RunCommand(cmd, module, 'install')
 472
 473         os.chdir(base_dir)
 474
 475         print '%s module is now built and installed' % module
 476
 477     def RunCommand(self, cmd, module, stage, skipable=False):
 478         if skipable:
 479             if self.IsBuildStepComplete('%s.%s' % (module, stage)):
 480                 return
 481
 482         popen = lambda cmd: \
 483             subprocess.Popen(
 484                 cmd,
 485                 stdin=subprocess.PIPE,
 486                 stdout=subprocess.PIPE,
 487                 stderr=subprocess.STDOUT
 488                 )
 489
 490         print '%s [%s] ...' % (module, stage),
 491         sys.stdout.flush()
 492         p = popen(cmd)
 493         output = p.stdout.read()
 494         p.wait()
 495         if p.returncode != 0:
 496             print '[failed!]'
 497             logFile = os.path.join(self.config.build_dir, 'log.txt')
 498             f = open(logFile, "w")
 499             f.write(output)
 500             f.close()
 501             raise Exception, 'Failed to %s %s\n' % (stage, module) + \
 502                 'See output log at %s' % self.config.Relative(logFile)
 503         else:
 504             print '[done]'
 505
 506         if skipable:
 507             self.MarkBuildStepComplete('%s.%s' % (module, stage))
 508
 509     def MakeSymLinks(self):
 510         links_dir = os.path.join(self.config.symlinks, self.config.arch)
 511         if not os.path.exists(links_dir):
 512             os.makedirs(links_dir)
 513         startPrinted = False
 514         for link in ('ar', 'ld', 'gcc'):
 515             src = os.path.join(
 516                 self.config.prefix, 'bin', self.config.target_combo + '-' + link
 517                 )
 518             linkdst = os.path.join(links_dir, link)
 519             if not os.path.lexists(linkdst):
 520                 if not startPrinted:
 521                     print 'Making symlinks in %s:' % self.config.Relative(links_dir),
 522                     startPrinted = True
 523                 print link,
 524                 os.symlink(src, linkdst)
 525
 526         if startPrinted:
 527             print '[done]'
 528
 529 class App:
 530     """class App
 531
 532     The main body of the application.
 533     """
 534
 535     def __init__(self):
 536         config = Config()
 537
 538         if not config.IsConfigOk():
 539             return
 540
 541         config.MakeDirs()
 542
 543         sources = SourceFiles(config)
 544         result = sources.GetAll()
 545         if result:
 546             print 'All files have been downloaded & verified'
 547         else:
 548             print 'An error occured while downloading a file'
 549             return
 550
 551         Extracter(sources, config).ExtractAll()
 552
 553         Builder(sources, config).Build()
 554
 555 App()
 556