BaseTools/gcc/mingw-gcc-build.py

   1 #!/usr/bin/env python
   2
   3 #
   4 # Automation of instructions from:
   5 #   http://mingw-w64.svn.sourceforge.net/viewvc/mingw-w64/trunk/mingw-w64-doc/
   6 #     howto-build/mingw-w64-howto-build.txt?revision=216&view=markup
   7 #
   8
   9 from optparse import OptionParser
  10 import os
  11 import shutil
  12 import subprocess
  13 import sys
  14 import tarfile
  15 import urllib
  16 import urlparse
  17 try:
  18     from hashlib import md5
  19 except Exception:
  20     from md5 import md5
  21
  22 #
  23 # Version and Copyright
  24 #
  25 VersionNumber = "0.01"
  26 __version__ = "%prog Version " + VersionNumber
  27 __copyright__ = "Copyright (c) 2008, Intel Corporation.  All rights reserved."
  28
  29 class Config:
  30     """class Config
  31
  32     Stores the configuration options for the rest of the script.
  33
  34     Handles the command line options, and allows the code within
  35     the script to easily interact with the 'config' requested by
  36     the user.
  37     """
  38
  39     def __init__(self):
  40         self.base_dir = os.getcwd()
  41         (self.options, self.args) = self.CheckOptions()
  42         self.__init_dirs__()
  43
  44     def CheckOptions(self):
  45         Parser = \
  46             OptionParser(
  47                 description=__copyright__,
  48                 version=__version__,
  49                 prog="mingw-gcc-build",
  50                 usage="%prog [options] [target]"
  51                 )
  52         Parser.add_option(
  53             "--arch",
  54             action = "store", type = "string",
  55             default = '',
  56             dest = "arch",
  57             help = "Processor architecture to build gcc for."
  58             )
  59         Parser.add_option(
  60             "--src-dir",
  61             action = "store", type = "string", dest = "src_dir",
  62             default = os.path.join(self.base_dir, 'src'),
  63             help = "Directory to download/extract binutils/gcc sources"
  64             )
  65         Parser.add_option(
  66             "--build-dir",
  67             action = "store", type = "string", dest = "build_dir",
  68             default = os.path.join(self.base_dir, 'build'),
  69             help = "Directory to download/extract binutils/gcc sources"
  70             )
  71         Parser.add_option(
  72             "--prefix",
  73             action = "store", type = "string", dest = "prefix",
  74             default = os.path.join(self.base_dir, 'install'),
  75             help = "Prefix to install binutils/gcc into"
  76             )
  77         Parser.add_option(
  78             "--symlinks",
  79             action = "store", type = "string", dest = "symlinks",
  80             default = os.path.join(self.base_dir, 'symlinks'),
  81             help = "Directory to create binutils/gcc symbolic links into."
  82             )
  83         Parser.add_option(
  84             "-v", "--verbose",
  85             action="store_true",
  86             type=None, help="Print verbose messages"
  87             )
  88
  89         (Opt, Args) = Parser.parse_args()
  90
  91         self.arch = Opt.arch.lower()
  92         allowedArchs = ('ia32', 'x64', 'ipf')
  93         if self.arch not in allowedArchs:
  94             Parser.error(
  95                 'Please use --arch to specify one of: %s' %
  96                     ', '.join(allowedArchs)
  97                 )
  98         self.target_arch = {'ia32': 'i686', 'x64': 'x86_64', 'ipf': 'ia64'}[self.arch]
  99         self.target_sys = {'ia32': 'pc', 'x64': 'pc', 'ipf': 'pc'}[self.arch]
 100         self.target_bin = {'ia32': 'mingw32', 'x64': 'mingw32', 'ipf': 'elf'}[self.arch]
 101         self.target_combo = '-'.join((self.target_arch, self.target_sys, self.target_bin))
 102
 103         return (Opt, Args)
 104
 105     def __init_dirs__(self):
 106         self.src_dir = os.path.realpath(os.path.expanduser(self.options.src_dir))
 107         self.build_dir = os.path.realpath(os.path.expanduser(self.options.build_dir))
 108         self.prefix = os.path.realpath(os.path.expanduser(self.options.prefix))
 109         self.symlinks = os.path.realpath(os.path.expanduser(self.options.symlinks))
 110
 111     def IsConfigOk(self):
 112
 113         print "Current directory:"
 114         print "   ", self.base_dir
 115         print "Sources download/extraction:", self.Relative(self.src_dir)
 116         print "Build directory            :", self.Relative(self.build_dir)
 117         print "Prefix (install) directory :", self.Relative(self.prefix)
 118         print "Create symlinks directory  :", self.Relative(self.symlinks)
 119         print
 120         answer = raw_input("Is this configuration ok? (default = no): ")
 121         if (answer.lower() not in ('y', 'yes')):
 122             print
 123             print "Please try using --help and then change the configuration."
 124             return False
 125
 126         if self.arch.lower() == 'ipf':
 127             print
 128             print 'Please note that the IPF compiler built by this script has'
 129             print 'not yet been validated!'
 130             print
 131             answer = raw_input("Are you sure you want to build it? (default = no): ")
 132             if (answer.lower() not in ('y', 'yes')):
 133                 print
 134                 print "Please try using --help and then change the configuration."
 135                 return False
 136
 137         print
 138         return True
 139
 140     def Relative(self, path):
 141         if path.startswith(self.base_dir):
 142             return '.' + path[len(self.base_dir):]
 143         return path
 144
 145     def MakeDirs(self):
 146         for path in (self.src_dir, self.build_dir,self.prefix, self.symlinks):
 147             if not os.path.exists(path):
 148                 os.makedirs(path)
 149
 150 class SourceFiles:
 151     """class SourceFiles
 152
 153     Handles the downloading of source files used by the script.
 154     """
 155
 156     def __init__(self, config):
 157         self.config = config
 158         self.source_files = self.source_files[config.arch]
 159
 160     source_files_common = {
 161         'binutils': {
 162             'url': 'http://www.kernel.org/pub/linux/devel/binutils/' + \
 163                    'binutils-$version.tar.bz2',
 164             'version': '2.18.50.0.5',
 165             'md5': 'daee18dbbf0a6ccfc186141bee18bf62',
 166             },
 167         }
 168
 169     source_files_x64 = {
 170         'gcc': {
 171             'url': 'http://gcc-ca.internet.bs/releases/' + \
 172                    'gcc-$version/gcc-$version.tar.bz2',
 173             'version': '4.3.0',
 174             'md5': '197ed8468b38db1d3481c3111691d85b',
 175             },
 176         'mingw_hdr': {
 177             'url': 'http://superb-west.dl.sourceforge.net/sourceforge/' + \
 178                    'mingw-w64/mingw-w64-snapshot-$version.tar.bz2',
 179             'extract-dir': os.path.join('trunk', 'mingw-w64-headers'),
 180             'version': '20080310',
 181             'md5': '235b2d15c2411f7d213c0c0977b2162f',
 182             },
 183         }
 184
 185     source_files_ia32 = {
 186         'gcc': {
 187             'url': 'http://superb-east.dl.sourceforge.net/sourceforge/' + \
 188                    'mingw/gcc-$version-mingw-$minor_version-src.tar.gz',
 189             'version': '4.3.0',
 190             'minor_version': 'alpha-20080403',
 191             'extract-dir': 'gcc-$version',
 192             'md5': '27961d80e304f4ef32c980833c6e8e44',
 193             'configure-params': ('--with-gnu-as', '--with-gnu-ld', '--with-newlib',
 194                                  '--verbose', '--disable-libssp', '--disable-nls',
 195                                  '--enable-languages=c,c++'
 196                                 )
 197             },
 198         'mingw_hdr': {
 199             'url': 'http://superb-west.dl.sourceforge.net/sourceforge/' + \
 200                    'mingw/mingw-runtime-$version-src.tar.gz',
 201             'extract-dir': 'mingw-runtime-$version',
 202             'version': '3.14',
 203             'md5': '7d049a8331efcfe34600c0cda6934ac6',
 204             },
 205         }
 206
 207     source_files_ipf = source_files_x64.copy()
 208     source_files_ipf['gcc']['configure-params'] = (
 209         '--with-gnu-as', '--with-gnu-ld', '--with-newlib',
 210         '--verbose', '--disable-libssp', '--disable-nls',
 211         '--enable-languages=c,c++'
 212         )
 213
 214     source_files = {
 215         'ia32': [source_files_common, source_files_ia32],
 216         'x64': [source_files_common, source_files_x64],
 217         'ipf': [source_files_common, source_files_ipf],
 218         }
 219
 220     for arch in source_files:
 221         mergedSourceFiles = {}
 222         for source_files_dict in source_files[arch]:
 223             mergedSourceFiles.update(source_files_dict)
 224         for downloadItem in mergedSourceFiles:
 225             fdata = mergedSourceFiles[downloadItem]
 226             fdata['filename'] = fdata['url'].split('/')[-1]
 227             if 'extract-dir' not in fdata:
 228                 for ext in ('.tar.gz', '.tar.bz2', '.zip'):
 229                     if fdata['filename'].endswith(ext):
 230                         fdata['extract-dir'] = fdata['filename'][:-len(ext)]
 231                         break
 232             replaceables = ('extract-dir', 'filename', 'url')
 233             for replaceItem in fdata:
 234                 if replaceItem in replaceables: continue
 235                 if type(fdata[replaceItem]) != str: continue
 236                 for replaceable in replaceables:
 237                     if type(fdata[replaceable]) != str: continue
 238                     if replaceable in fdata:
 239                         fdata[replaceable] = \
 240                             fdata[replaceable].replace(
 241                                 '$' + replaceItem,
 242                                 fdata[replaceItem]
 243                                 )
 244         source_files[arch] = mergedSourceFiles
 245     #print 'source_files:', source_files
 246
 247     def GetAll(self):
 248
 249         def progress(received, blockSize, fileSize):
 250             if fileSize < 0: return
 251             wDots = (100 * received * blockSize) / fileSize / 10
 252             if wDots > self.dots:
 253                 for i in range(wDots - self.dots):
 254                     print '.',
 255                     sys.stdout.flush()
 256                     self.dots += 1
 257
 258         maxRetries = 3
 259         for (fname, fdata) in self.source_files.items():
 260             for retries in range(maxRetries):
 261                 try:
 262                     self.dots = 0
 263                     local_file = os.path.join(self.config.src_dir, fdata['filename'])
 264                     url = fdata['url']
 265                     print 'Downloading %s:' % fname,
 266                     if retries > 0:
 267                         print '(retry)',
 268                     sys.stdout.flush()
 269
 270                     completed = False
 271                     if os.path.exists(local_file):
 272                         md5_pass = self.checkHash(fdata)
 273                         if md5_pass:
 274                             print '[md5 match]',
 275                         else:
 276                             print '[md5 mismatch]',
 277                         sys.stdout.flush()
 278                         completed = md5_pass
 279
 280                     if not completed:
 281                         urllib.urlretrieve(url, local_file, progress)
 282
 283                     #
 284                     # BUGBUG: Suggest proxy to user if download fails.
 285                     #
 286                     # export http_proxy=http://proxyservername.mycompany.com:911
 287                     # export ftp_proxy=http://proxyservername.mycompany.com:911
 288
 289                     if not completed and os.path.exists(local_file):
 290                         md5_pass = self.checkHash(fdata)
 291                         if md5_pass:
 292                             print '[md5 match]',
 293                         else:
 294                             print '[md5 mismatch]',
 295                         sys.stdout.flush()
 296                         completed = md5_pass
 297
 298                     if completed:
 299                         print '[done]'
 300                         break
 301                     else:
 302                         print '[failed]'
 303                         raise Exception()
 304
 305                 except KeyboardInterrupt:
 306                     print '[KeyboardInterrupt]'
 307                     return False
 308
 309                 except:
 310                     pass
 311
 312             if not completed: return False
 313
 314         return True
 315
 316     def checkHash(self, fdata):
 317         local_file = os.path.join(self.config.src_dir, fdata['filename'])
 318         expect_md5 = fdata['md5']
 319         data = open(local_file).read()
 320         md5sum = md5()
 321         md5sum.update(data)
 322         return md5sum.hexdigest().lower() == expect_md5.lower()
 323
 324     def GetModules(self):
 325         return self.source_files.keys()
 326
 327     def GetFilenameOf(self, module):
 328         return self.source_files[module]['filename']
 329
 330     def GetMd5Of(self, module):
 331         return self.source_files[module]['md5']
 332
 333     def GetExtractDirOf(self, module):
 334         return self.source_files[module]['extract-dir']
 335
 336     def GetAdditionalParameters(self, module, step):
 337         key = step + '-params'
 338         if key in self.source_files[module]:
 339             return self.source_files[module][key]
 340         else:
 341             return tuple()
 342
 343 class Extracter:
 344     """class Extracter
 345
 346     Handles the extraction of the source files from their downloaded
 347     archive files.
 348     """
 349
 350     def __init__(self, source_files, config):
 351         self.source_files = source_files
 352         self.config = config
 353
 354     def Extract(self, module):
 355         src = self.config.src_dir
 356         extractDst = os.path.join(src, self.config.arch)
 357         local_file = os.path.join(src, self.source_files.GetFilenameOf(module))
 358         moduleMd5 = self.source_files.GetMd5Of(module)
 359         extracted = os.path.join(extractDst, os.path.split(local_file)[1] + '.extracted')
 360         if not os.path.exists(extractDst):
 361             os.mkdir(extractDst)
 362
 363         extractedMd5 = None
 364         if os.path.exists(extracted):
 365             extractedMd5 = open(extracted).read()
 366
 367         if extractedMd5 != moduleMd5:
 368             print 'Extracting %s:' % self.config.Relative(local_file)
 369             tar = tarfile.open(local_file)
 370             tar.extractall(extractDst)
 371             open(extracted, 'w').write(moduleMd5)
 372         else:
 373             pass
 374             #print 'Previously extracted', self.config.Relative(local_file)
 375
 376     def ExtractAll(self):
 377         for module in self.source_files.GetModules():
 378             self.Extract(module)
 379
 380 class Builder:
 381     """class Builder
 382
 383     Builds and installs the GCC tool suite.
 384     """
 385
 386     def __init__(self, source_files, config):
 387         self.source_files = source_files
 388         self.config = config
 389
 390     def Build(self):
 391         self.BuildModule('binutils')
 392         self.CopyIncludeDirectory()
 393         self.BuildModule('gcc')
 394         self.MakeSymLinks()
 395
 396     def IsBuildStepComplete(self, step):
 397         return \
 398             os.path.exists(
 399                 os.path.join(
 400                     self.config.build_dir, self.config.arch, step + '.completed'
 401                     )
 402                 )
 403
 404     def MarkBuildStepComplete(self, step):
 405         open(
 406             os.path.join(
 407                 self.config.build_dir, self.config.arch, step + '.completed'
 408                 ),
 409             "w"
 410             ).close()
 411
 412     def CopyIncludeDirectory(self):
 413         linkdst = os.path.join(self.config.prefix, 'mingw')
 414         src = os.path.join(
 415             self.config.src_dir,
 416             self.config.arch,
 417             self.source_files.GetExtractDirOf('mingw_hdr'),
 418             'include'
 419             )
 420         dst_parent = os.path.join(self.config.prefix, self.config.target_combo)
 421         dst = os.path.join(dst_parent, 'include')
 422         if not os.path.exists(dst):
 423             if not os.path.exists(dst_parent):
 424                 os.makedirs(dst_parent)
 425             print 'Copying headers to', self.config.Relative(dst)
 426             shutil.copytree(src, dst, True)
 427         if not os.path.lexists(linkdst):
 428             print 'Making symlink at', self.config.Relative(linkdst)
 429             os.symlink(self.config.target_combo, linkdst)
 430
 431     def BuildModule(self, module):
 432         base_dir = os.getcwd()
 433         build_dir = os.path.join(self.config.build_dir, self.config.arch, module)
 434         module_dir = self.source_files.GetExtractDirOf(module)
 435         module_dir = os.path.realpath(os.path.join('src', self.config.arch, module_dir))
 436         configure = os.path.join(module_dir, 'configure')
 437         prefix = self.config.prefix
 438         if not os.path.exists(build_dir):
 439             os.makedirs(build_dir)
 440         os.chdir(build_dir)
 441
 442         cmd = (
 443             configure,
 444             '--target=%s' % self.config.target_combo,
 445             '--prefix=' + prefix,
 446             '--with-sysroot=' + prefix,
 447             '--disable-werror',
 448             )
 449         if os.path.exists('/opt/local/include/gmp.h'):
 450             cmd += ('--with-gmp=/opt/local',)
 451         if module == 'gcc': cmd += ('--oldincludedir=/opt/local/include',)
 452         cmd += self.source_files.GetAdditionalParameters(module, 'configure')
 453         self.RunCommand(cmd, module, 'config', skipable=True)
 454
 455         cmd = ('make',)
 456         if module == 'gcc':
 457             cmd += ('all-gcc',)
 458         self.RunCommand(cmd, module, 'build')
 459
 460         cmd = ('make',)
 461         if module == 'gcc':
 462             cmd += ('install-gcc',)
 463         else:
 464             cmd += ('install',)
 465         self.RunCommand(cmd, module, 'install')
 466
 467         os.chdir(base_dir)
 468
 469         print '%s module is now built and installed' % module
 470
 471     def RunCommand(self, cmd, module, stage, skipable=False):
 472         if skipable:
 473             if self.IsBuildStepComplete('%s.%s' % (module, stage)):
 474                 return
 475
 476         popen = lambda cmd: \
 477             subprocess.Popen(
 478                 cmd,
 479                 stdin=subprocess.PIPE,
 480                 stdout=subprocess.PIPE,
 481                 stderr=subprocess.STDOUT
 482                 )
 483
 484         print '%s [%s] ...' % (module, stage),
 485         sys.stdout.flush()
 486         p = popen(cmd)
 487         output = p.stdout.read()
 488         p.wait()
 489         if p.returncode != 0:
 490             print '[failed!]'
 491             logFile = os.path.join(self.config.build_dir, 'log.txt')
 492             f = open(logFile, "w")
 493             f.write(output)
 494             f.close()
 495             raise Exception, 'Failed to %s %s\n' % (stage, module) + \
 496                 'See output log at %s' % self.config.Relative(logFile)
 497         else:
 498             print '[done]'
 499
 500         if skipable:
 501             self.MarkBuildStepComplete('%s.%s' % (module, stage))
 502
 503     def MakeSymLinks(self):
 504         links_dir = os.path.join(self.config.symlinks, self.config.arch)
 505         if not os.path.exists(links_dir):
 506             os.makedirs(links_dir)
 507         startPrinted = False
 508         for link in ('ar', 'ld', 'gcc'):
 509             src = os.path.join(
 510                 self.config.prefix, 'bin', self.config.target_combo + '-' + link
 511                 )
 512             linkdst = os.path.join(links_dir, link)
 513             if not os.path.lexists(linkdst):
 514                 if not startPrinted:
 515                     print 'Making symlinks in %s:' % self.config.Relative(links_dir),
 516                     startPrinted = True
 517                 print link,
 518                 os.symlink(src, linkdst)
 519
 520         if startPrinted:
 521             print '[done]'
 522
 523 class App:
 524     """class App
 525
 526     The main body of the application.
 527     """
 528
 529     def __init__(self):
 530         config = Config()
 531
 532         if not config.IsConfigOk():
 533             return
 534
 535         config.MakeDirs()
 536
 537         sources = SourceFiles(config)
 538         result = sources.GetAll()
 539         if result:
 540             print 'All files have been downloaded & verified'
 541         else:
 542             print 'An error occured while downloading a file'
 543             return
 544
 545         Extracter(sources, config).ExtractAll()
 546
 547         Builder(sources, config).Build()
 548
 549 App()
 550