BaseTools: Refactor python print statements
[mirror_edk2.git] / BaseTools / gcc / mingw-gcc-build.py
1 #!/usr/bin/env python
2
3 ## @file
4 #
5 # Automation of instructions from:
6 # http://mingw-w64.svn.sourceforge.net/viewvc/mingw-w64/trunk/mingw-w64-doc/
7 # howto-build/mingw-w64-howto-build.txt?revision=216&view=markup
8 #
9 # Copyright (c) 2008 - 2010, Intel Corporation. All rights reserved.<BR>
10 # This program and the accompanying materials
11 # are licensed and made available under the terms and conditions of the BSD License
12 # which accompanies this distribution. The full text of the license may be found at
13 # http://opensource.org/licenses/bsd-license.php
14 #
15 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
16 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
17 #
18
19
20 from __future__ import print_function
21 from optparse import OptionParser
22 import os
23 import shutil
24 import subprocess
25 import sys
26 import tarfile
27 import urllib
28 import urlparse
29 try:
30 from hashlib import md5
31 except Exception:
32 from md5 import md5
33
34 if sys.version_info < (2, 5):
35 #
36 # This script (and edk2 BaseTools) require Python 2.5 or newer
37 #
38 print('Python version 2.5 or later is required.')
39 sys.exit(-1)
40
41 #
42 # Version and Copyright
43 #
44 VersionNumber = "0.01"
45 __version__ = "%prog Version " + VersionNumber
46 __copyright__ = "Copyright (c) 2008 - 2010, Intel Corporation. All rights reserved."
47
48 class Config:
49 """class Config
50
51 Stores the configuration options for the rest of the script.
52
53 Handles the command line options, and allows the code within
54 the script to easily interact with the 'config' requested by
55 the user.
56 """
57
58 def __init__(self):
59 self.base_dir = os.getcwd()
60 (self.options, self.args) = self.CheckOptions()
61 self.__init_dirs__()
62
63 def CheckOptions(self):
64 Parser = \
65 OptionParser(
66 description=__copyright__,
67 version=__version__,
68 prog="mingw-gcc-build",
69 usage="%prog [options] [target]"
70 )
71 Parser.add_option(
72 "--arch",
73 action = "store", type = "string",
74 default = '',
75 dest = "arch",
76 help = "Processor architecture to build gcc for."
77 )
78 Parser.add_option(
79 "--src-dir",
80 action = "store", type = "string", dest = "src_dir",
81 default = os.path.join(self.base_dir, 'src'),
82 help = "Directory to download/extract binutils/gcc sources"
83 )
84 Parser.add_option(
85 "--build-dir",
86 action = "store", type = "string", dest = "build_dir",
87 default = os.path.join(self.base_dir, 'build'),
88 help = "Directory to download/extract binutils/gcc sources"
89 )
90 Parser.add_option(
91 "--prefix",
92 action = "store", type = "string", dest = "prefix",
93 default = os.path.join(self.base_dir, 'install'),
94 help = "Prefix to install binutils/gcc into"
95 )
96 Parser.add_option(
97 "--skip-binutils",
98 action = "store_true", dest = "skip_binutils",
99 default = False,
100 help = "Will skip building binutils"
101 )
102 Parser.add_option(
103 "--skip-gcc",
104 action = "store_true", dest = "skip_gcc",
105 default = False,
106 help = "Will skip building GCC"
107 )
108 Parser.add_option(
109 "--symlinks",
110 action = "store", type = "string", dest = "symlinks",
111 default = os.path.join(self.base_dir, 'symlinks'),
112 help = "Directory to create binutils/gcc symbolic links into."
113 )
114 Parser.add_option(
115 "-v", "--verbose",
116 action="store_true",
117 type=None, help="Print verbose messages"
118 )
119
120 (Opt, Args) = Parser.parse_args()
121
122 self.arch = Opt.arch.lower()
123 allowedArchs = ('ia32', 'x64', 'ipf')
124 if self.arch not in allowedArchs:
125 Parser.error(
126 'Please use --arch to specify one of: %s' %
127 ', '.join(allowedArchs)
128 )
129 self.target_arch = {'ia32': 'i686', 'x64': 'x86_64', 'ipf': 'ia64'}[self.arch]
130 self.target_sys = {'ia32': 'pc', 'x64': 'pc', 'ipf': 'pc'}[self.arch]
131 self.target_bin = {'ia32': 'mingw32', 'x64': 'mingw32', 'ipf': 'elf'}[self.arch]
132 self.target_combo = '-'.join((self.target_arch, self.target_sys, self.target_bin))
133
134 return (Opt, Args)
135
136 def __init_dirs__(self):
137 self.src_dir = os.path.realpath(os.path.expanduser(self.options.src_dir))
138 self.build_dir = os.path.realpath(os.path.expanduser(self.options.build_dir))
139 self.prefix = os.path.realpath(os.path.expanduser(self.options.prefix))
140 self.symlinks = os.path.realpath(os.path.expanduser(self.options.symlinks))
141
142 def IsConfigOk(self):
143
144 building = []
145 if not self.options.skip_binutils:
146 building.append('binutils')
147 if not self.options.skip_gcc:
148 building.append('gcc')
149 if len(building) == 0:
150 print("Nothing will be built!")
151 print()
152 print("Please try using --help and then change the configuration.")
153 return False
154
155 print("Current directory:")
156 print(" ", self.base_dir)
157 print("Sources download/extraction:", self.Relative(self.src_dir))
158 print("Build directory :", self.Relative(self.build_dir))
159 print("Prefix (install) directory :", self.Relative(self.prefix))
160 print("Create symlinks directory :", self.Relative(self.symlinks))
161 print("Building :", ', '.join(building))
162 print()
163 answer = raw_input("Is this configuration ok? (default = no): ")
164 if (answer.lower() not in ('y', 'yes')):
165 print()
166 print("Please try using --help and then change the configuration.")
167 return False
168
169 if self.arch.lower() == 'ipf':
170 print()
171 print('Please note that the IPF compiler built by this script has')
172 print('not yet been validated!')
173 print()
174 answer = raw_input("Are you sure you want to build it? (default = no): ")
175 if (answer.lower() not in ('y', 'yes')):
176 print()
177 print("Please try using --help and then change the configuration.")
178 return False
179
180 print()
181 return True
182
183 def Relative(self, path):
184 if path.startswith(self.base_dir):
185 return '.' + path[len(self.base_dir):]
186 return path
187
188 def MakeDirs(self):
189 for path in (self.src_dir, self.build_dir,self.prefix, self.symlinks):
190 if not os.path.exists(path):
191 os.makedirs(path)
192
193 class SourceFiles:
194 """class SourceFiles
195
196 Handles the downloading of source files used by the script.
197 """
198
199 def __init__(self, config):
200 self.config = config
201 self.source_files = self.source_files[config.arch]
202
203 if config.options.skip_binutils:
204 del self.source_files['binutils']
205
206 if config.options.skip_gcc:
207 del self.source_files['gcc']
208 del self.source_files['mingw_hdr']
209
210 source_files_common = {
211 'binutils': {
212 'url': 'http://www.kernel.org/pub/linux/devel/binutils/' + \
213 'binutils-$version.tar.bz2',
214 'version': '2.20.51.0.5',
215 'md5': '6d2de7cdf7a8389e70b124e3d73b4d37',
216 },
217 }
218
219 source_files_x64 = {
220 'gcc': {
221 'url': 'http://ftpmirror.gnu.org/gcc/' + \
222 'gcc-$version/gcc-$version.tar.bz2',
223 'version': '4.3.0',
224 'md5': '197ed8468b38db1d3481c3111691d85b',
225 },
226 }
227
228 source_files_ia32 = {
229 'gcc': source_files_x64['gcc'],
230 }
231
232 source_files_ipf = source_files_x64.copy()
233 source_files_ipf['gcc']['configure-params'] = (
234 '--with-gnu-as', '--with-gnu-ld', '--with-newlib',
235 '--verbose', '--disable-libssp', '--disable-nls',
236 '--enable-languages=c,c++'
237 )
238
239 source_files = {
240 'ia32': [source_files_common, source_files_ia32],
241 'x64': [source_files_common, source_files_x64],
242 'ipf': [source_files_common, source_files_ipf],
243 }
244
245 for arch in source_files:
246 mergedSourceFiles = {}
247 for source_files_dict in source_files[arch]:
248 mergedSourceFiles.update(source_files_dict)
249 for downloadItem in mergedSourceFiles:
250 fdata = mergedSourceFiles[downloadItem]
251 fdata['filename'] = fdata['url'].split('/')[-1]
252 if 'extract-dir' not in fdata:
253 for ext in ('.tar.gz', '.tar.bz2', '.zip'):
254 if fdata['filename'].endswith(ext):
255 fdata['extract-dir'] = fdata['filename'][:-len(ext)]
256 break
257 replaceables = ('extract-dir', 'filename', 'url')
258 for replaceItem in fdata:
259 if replaceItem in replaceables: continue
260 if type(fdata[replaceItem]) != str: continue
261 for replaceable in replaceables:
262 if type(fdata[replaceable]) != str: continue
263 if replaceable in fdata:
264 fdata[replaceable] = \
265 fdata[replaceable].replace(
266 '$' + replaceItem,
267 fdata[replaceItem]
268 )
269 source_files[arch] = mergedSourceFiles
270 #print 'source_files:', source_files
271
272 def GetAll(self):
273
274 def progress(received, blockSize, fileSize):
275 if fileSize < 0: return
276 wDots = (100 * received * blockSize) / fileSize / 10
277 if wDots > self.dots:
278 for i in range(wDots - self.dots):
279 print('.', end=' ')
280 sys.stdout.flush()
281 self.dots += 1
282
283 maxRetries = 1
284 for (fname, fdata) in self.source_files.items():
285 for retries in range(maxRetries):
286 try:
287 self.dots = 0
288 local_file = os.path.join(self.config.src_dir, fdata['filename'])
289 url = fdata['url']
290 print('Downloading %s:' % fname, url)
291 if retries > 0:
292 print('(retry)', end=' ')
293 sys.stdout.flush()
294
295 completed = False
296 if os.path.exists(local_file):
297 md5_pass = self.checkHash(fdata)
298 if md5_pass:
299 print('[md5 match]', end=' ')
300 else:
301 print('[md5 mismatch]', end=' ')
302 sys.stdout.flush()
303 completed = md5_pass
304
305 if not completed:
306 urllib.urlretrieve(url, local_file, progress)
307
308 #
309 # BUGBUG: Suggest proxy to user if download fails.
310 #
311 # export http_proxy=http://proxyservername.mycompany.com:911
312 # export ftp_proxy=http://proxyservername.mycompany.com:911
313
314 if not completed and os.path.exists(local_file):
315 md5_pass = self.checkHash(fdata)
316 if md5_pass:
317 print('[md5 match]', end=' ')
318 else:
319 print('[md5 mismatch]', end=' ')
320 sys.stdout.flush()
321 completed = md5_pass
322
323 if completed:
324 print('[done]')
325 break
326 else:
327 print('[failed]')
328 print(' Tried to retrieve', url)
329 print(' to', local_file)
330 print('Possible fixes:')
331 print('* If you are behind a web-proxy, try setting the', end=' ')
332 print('http_proxy environment variable')
333 print('* You can try to download this file separately', end=' ')
334 print('and rerun this script')
335 raise Exception()
336
337 except KeyboardInterrupt:
338 print('[KeyboardInterrupt]')
339 return False
340
341 except Exception as e:
342 print(e)
343
344 if not completed: return False
345
346 return True
347
348 def checkHash(self, fdata):
349 local_file = os.path.join(self.config.src_dir, fdata['filename'])
350 expect_md5 = fdata['md5']
351 data = open(local_file).read()
352 md5sum = md5()
353 md5sum.update(data)
354 return md5sum.hexdigest().lower() == expect_md5.lower()
355
356 def GetModules(self):
357 return self.source_files.keys()
358
359 def GetFilenameOf(self, module):
360 return self.source_files[module]['filename']
361
362 def GetMd5Of(self, module):
363 return self.source_files[module]['md5']
364
365 def GetExtractDirOf(self, module):
366 return self.source_files[module]['extract-dir']
367
368 def GetAdditionalParameters(self, module, step):
369 key = step + '-params'
370 if key in self.source_files[module]:
371 return self.source_files[module][key]
372 else:
373 return tuple()
374
375 class Extracter:
376 """class Extracter
377
378 Handles the extraction of the source files from their downloaded
379 archive files.
380 """
381
382 def __init__(self, source_files, config):
383 self.source_files = source_files
384 self.config = config
385
386 def Extract(self, module):
387 src = self.config.src_dir
388 extractDst = os.path.join(src, self.config.arch)
389 local_file = os.path.join(src, self.source_files.GetFilenameOf(module))
390 moduleMd5 = self.source_files.GetMd5Of(module)
391 extracted = os.path.join(extractDst, os.path.split(local_file)[1] + '.extracted')
392 if not os.path.exists(extractDst):
393 os.mkdir(extractDst)
394
395 extractedMd5 = None
396 if os.path.exists(extracted):
397 extractedMd5 = open(extracted).read()
398
399 if extractedMd5 != moduleMd5:
400 print('Extracting %s:' % self.config.Relative(local_file))
401 tar = tarfile.open(local_file)
402 tar.extractall(extractDst)
403 open(extracted, 'w').write(moduleMd5)
404 else:
405 pass
406 #print 'Previously extracted', self.config.Relative(local_file)
407
408 def ExtractAll(self):
409 for module in self.source_files.GetModules():
410 self.Extract(module)
411
412 class Builder:
413 """class Builder
414
415 Builds and installs the GCC tool suite.
416 """
417
418 def __init__(self, source_files, config):
419 self.source_files = source_files
420 self.config = config
421
422 def Build(self):
423 if not self.config.options.skip_binutils:
424 self.BuildModule('binutils')
425 if not self.config.options.skip_gcc:
426 self.BuildModule('gcc')
427 self.MakeSymLinks()
428
429 def IsBuildStepComplete(self, step):
430 return \
431 os.path.exists(
432 os.path.join(
433 self.config.build_dir, self.config.arch, step + '.completed'
434 )
435 )
436
437 def MarkBuildStepComplete(self, step):
438 open(
439 os.path.join(
440 self.config.build_dir, self.config.arch, step + '.completed'
441 ),
442 "w"
443 ).close()
444
445
446 def BuildModule(self, module):
447 base_dir = os.getcwd()
448 build_dir = os.path.join(self.config.build_dir, self.config.arch, module)
449 module_dir = self.source_files.GetExtractDirOf(module)
450 module_dir = os.path.realpath(os.path.join('src', self.config.arch, module_dir))
451 configure = os.path.join(module_dir, 'configure')
452 prefix = self.config.prefix
453 if not os.path.exists(build_dir):
454 os.makedirs(build_dir)
455 os.chdir(build_dir)
456
457 cmd = (
458 configure,
459 '--target=%s' % self.config.target_combo,
460 '--prefix=' + prefix,
461 '--with-sysroot=' + prefix,
462 '--disable-werror',
463 )
464 if os.path.exists('/opt/local/include/gmp.h'):
465 cmd += ('--with-gmp=/opt/local',)
466 if module == 'gcc': cmd += ('--oldincludedir=/opt/local/include',)
467 cmd += self.source_files.GetAdditionalParameters(module, 'configure')
468 self.RunCommand(cmd, module, 'config', skipable=True)
469
470 cmd = ('make',)
471 if module == 'gcc':
472 cmd += ('all-gcc',)
473 self.RunCommand(cmd, module, 'build')
474
475 cmd = ('make',)
476 if module == 'gcc':
477 cmd += ('install-gcc',)
478 else:
479 cmd += ('install',)
480 self.RunCommand(cmd, module, 'install')
481
482 os.chdir(base_dir)
483
484 print('%s module is now built and installed' % module)
485
486 def RunCommand(self, cmd, module, stage, skipable=False):
487 if skipable:
488 if self.IsBuildStepComplete('%s.%s' % (module, stage)):
489 return
490
491 popen = lambda cmd: \
492 subprocess.Popen(
493 cmd,
494 stdin=subprocess.PIPE,
495 stdout=subprocess.PIPE,
496 stderr=subprocess.STDOUT
497 )
498
499 print('%s [%s] ...' % (module, stage), end=' ')
500 sys.stdout.flush()
501 p = popen(cmd)
502 output = p.stdout.read()
503 p.wait()
504 if p.returncode != 0:
505 print('[failed!]')
506 logFile = os.path.join(self.config.build_dir, 'log.txt')
507 f = open(logFile, "w")
508 f.write(output)
509 f.close()
510 raise Exception, 'Failed to %s %s\n' % (stage, module) + \
511 'See output log at %s' % self.config.Relative(logFile)
512 else:
513 print('[done]')
514
515 if skipable:
516 self.MarkBuildStepComplete('%s.%s' % (module, stage))
517
518 def MakeSymLinks(self):
519 links_dir = os.path.join(self.config.symlinks, self.config.arch)
520 if not os.path.exists(links_dir):
521 os.makedirs(links_dir)
522 startPrinted = False
523 for link in ('ar', 'ld', 'gcc'):
524 src = os.path.join(
525 self.config.prefix, 'bin', self.config.target_combo + '-' + link
526 )
527 linkdst = os.path.join(links_dir, link)
528 if not os.path.lexists(linkdst):
529 if not startPrinted:
530 print('Making symlinks in %s:' % self.config.Relative(links_dir), end=' ')
531 startPrinted = True
532 print(link, end=' ')
533 os.symlink(src, linkdst)
534
535 if startPrinted:
536 print('[done]')
537
538 class App:
539 """class App
540
541 The main body of the application.
542 """
543
544 def __init__(self):
545 config = Config()
546
547 if not config.IsConfigOk():
548 return
549
550 config.MakeDirs()
551
552 sources = SourceFiles(config)
553 result = sources.GetAll()
554 if result:
555 print('All files have been downloaded & verified')
556 else:
557 print('An error occured while downloading a file')
558 return
559
560 Extracter(sources, config).ExtractAll()
561
562 Builder(sources, config).Build()
563
564 App()
565