]>
Commit | Line | Data |
---|---|---|
92f5a8d4 TL |
1 | #!/usr/bin/env python |
2 | ||
3 | # Copyright Rene Rivera 2016 | |
4 | # | |
5 | # Distributed under the Boost Software License, Version 1.0. | |
6 | # (See accompanying file LICENSE_1_0.txt or copy at | |
7 | # http://www.boost.org/LICENSE_1_0.txt) | |
8 | ||
9 | import sys | |
10 | import inspect | |
11 | import optparse | |
12 | import os.path | |
13 | import string | |
14 | import time | |
15 | import subprocess | |
16 | import codecs | |
17 | import shutil | |
18 | import threading | |
19 | ||
20 | toolset_info = { | |
21 | 'clang-3.4' : { | |
22 | 'ppa' : ["ppa:h-rayflood/llvm"], | |
23 | 'package' : 'clang-3.4', | |
24 | 'command' : 'clang++-3.4', | |
25 | 'toolset' : 'clang', | |
26 | 'version' : '' | |
27 | }, | |
28 | 'clang-3.5' : { | |
29 | 'ppa' : ["ppa:h-rayflood/llvm"], | |
30 | 'package' : 'clang-3.5', | |
31 | 'command' : 'clang++-3.5', | |
32 | 'toolset' : 'clang', | |
33 | 'version' : '' | |
34 | }, | |
35 | 'clang-3.6' : { | |
36 | 'ppa' : ["ppa:h-rayflood/llvm"], | |
37 | 'package' : 'clang-3.6', | |
38 | 'command' : 'clang++-3.6', | |
39 | 'toolset' : 'clang', | |
40 | 'version' : '' | |
41 | }, | |
42 | 'clang-3.7' : { | |
43 | 'deb' : ["http://apt.llvm.org/trusty/","llvm-toolchain-trusty-3.7","main"], | |
44 | 'apt-key' : ['http://apt.llvm.org/llvm-snapshot.gpg.key'], | |
45 | 'package' : 'clang-3.7', | |
46 | 'command' : 'clang++-3.7', | |
47 | 'toolset' : 'clang', | |
48 | 'version' : '' | |
49 | }, | |
50 | 'clang-3.8' : { | |
51 | 'deb' : ["http://apt.llvm.org/trusty/","llvm-toolchain-trusty-3.8","main"], | |
52 | 'apt-key' : ['http://apt.llvm.org/llvm-snapshot.gpg.key'], | |
53 | 'package' : 'clang-3.8', | |
54 | 'command' : 'clang++-3.8', | |
55 | 'toolset' : 'clang', | |
56 | 'version' : '' | |
57 | }, | |
58 | 'clang-3.9' : { | |
59 | 'deb' : ["http://apt.llvm.org/trusty/","llvm-toolchain-trusty-3.9","main"], | |
60 | 'apt-key' : ['http://apt.llvm.org/llvm-snapshot.gpg.key'], | |
61 | 'package' : 'clang-3.9', | |
62 | 'command' : 'clang++-3.9', | |
63 | 'toolset' : 'clang', | |
64 | 'version' : '' | |
65 | }, | |
66 | 'clang-4.0' : { | |
67 | 'deb' : ["http://apt.llvm.org/trusty/","llvm-toolchain-trusty-4.0","main"], | |
68 | 'apt-key' : ['http://apt.llvm.org/llvm-snapshot.gpg.key'], | |
69 | 'package' : 'clang-4.0', | |
70 | 'command' : 'clang++-4.0', | |
71 | 'toolset' : 'clang', | |
72 | 'version' : '' | |
73 | }, | |
74 | 'clang-5.0' : { | |
75 | 'deb' : ["http://apt.llvm.org/trusty/","llvm-toolchain-trusty-5.0","main"], | |
76 | 'apt-key' : ['http://apt.llvm.org/llvm-snapshot.gpg.key'], | |
77 | 'package' : 'clang-5.0', | |
78 | 'command' : 'clang++-5.0', | |
79 | 'toolset' : 'clang', | |
80 | 'version' : '' | |
81 | }, | |
82 | 'clang-6.0' : { | |
83 | 'deb' : ["http://apt.llvm.org/trusty/","llvm-toolchain-trusty-6.0","main"], | |
84 | 'apt-key' : ['http://apt.llvm.org/llvm-snapshot.gpg.key'], | |
85 | 'package' : 'clang-6.0', | |
86 | 'command' : 'clang++-6.0', | |
87 | 'toolset' : 'clang', | |
88 | 'version' : '' | |
89 | }, | |
90 | 'gcc-4.7' : { | |
91 | 'ppa' : ["ppa:ubuntu-toolchain-r/test"], | |
92 | 'package' : 'g++-4.7', | |
93 | 'command' : 'g++-4.7', | |
94 | 'toolset' : 'gcc', | |
95 | 'version' : '' | |
96 | }, | |
97 | 'gcc-4.8' : { | |
98 | 'bin' : 'gcc-4.8', | |
99 | 'ppa' : ["ppa:ubuntu-toolchain-r/test"], | |
100 | 'package' : 'g++-4.8', | |
101 | 'command' : 'g++-4.8', | |
102 | 'toolset' : 'gcc', | |
103 | 'version' : '' | |
104 | }, | |
105 | 'gcc-4.9' : { | |
106 | 'ppa' : ["ppa:ubuntu-toolchain-r/test"], | |
107 | 'package' : 'g++-4.9', | |
108 | 'command' : 'g++-4.9', | |
109 | 'toolset' : 'gcc', | |
110 | 'version' : '' | |
111 | }, | |
112 | 'gcc-5.1' : { | |
113 | 'ppa' : ["ppa:ubuntu-toolchain-r/test"], | |
114 | 'package' : 'g++-5', | |
115 | 'command' : 'g++-5', | |
116 | 'toolset' : 'gcc', | |
117 | 'version' : '' | |
118 | }, | |
119 | 'gcc-5' : { | |
120 | 'ppa' : ["ppa:ubuntu-toolchain-r/test"], | |
121 | 'package' : 'g++-5', | |
122 | 'command' : 'g++-5', | |
123 | 'toolset' : 'gcc', | |
124 | 'version' : '' | |
125 | }, | |
126 | 'gcc-6' : { | |
127 | 'ppa' : ["ppa:ubuntu-toolchain-r/test"], | |
128 | 'package' : 'g++-6', | |
129 | 'command' : 'g++-6', | |
130 | 'toolset' : 'gcc', | |
131 | 'version' : '' | |
132 | }, | |
133 | 'gcc-7' : { | |
134 | 'ppa' : ["ppa:ubuntu-toolchain-r/test"], | |
135 | 'package' : 'g++-7', | |
136 | 'command' : 'g++-7', | |
137 | 'toolset' : 'gcc', | |
138 | 'version' : '' | |
139 | }, | |
140 | 'gcc-8' : { | |
141 | 'ppa' : ["ppa:ubuntu-toolchain-r/test"], | |
142 | 'package' : 'g++-8', | |
143 | 'command' : 'g++-8', | |
144 | 'toolset' : 'gcc', | |
145 | 'version' : '' | |
146 | }, | |
147 | 'mingw-5' : { | |
148 | 'toolset' : 'gcc', | |
149 | 'command' : 'C:\\\\MinGW\\\\bin\\\\g++.exe', | |
150 | 'version' : '' | |
151 | }, | |
152 | 'mingw64-6' : { | |
153 | 'toolset' : 'gcc', | |
154 | 'command' : 'C:\\\\mingw-w64\\\\x86_64-6.3.0-posix-seh-rt_v5-rev1\\\\mingw64\\\\bin\\\\g++.exe', | |
155 | 'version' : '' | |
156 | }, | |
157 | 'vs-2008' : { | |
158 | 'toolset' : 'msvc', | |
159 | 'command' : '', | |
160 | 'version' : '9.0' | |
161 | }, | |
162 | 'vs-2010' : { | |
163 | 'toolset' : 'msvc', | |
164 | 'command' : '', | |
165 | 'version' : '10.0' | |
166 | }, | |
167 | 'vs-2012' : { | |
168 | 'toolset' : 'msvc', | |
169 | 'command' : '', | |
170 | 'version' : '11.0' | |
171 | }, | |
172 | 'vs-2013' : { | |
173 | 'toolset' : 'msvc', | |
174 | 'command' : '', | |
175 | 'version' : '12.0' | |
176 | }, | |
177 | 'vs-2015' : { | |
178 | 'toolset' : 'msvc', | |
179 | 'command' : '', | |
180 | 'version' : '14.0' | |
181 | }, | |
182 | 'vs-2017' : { | |
183 | 'toolset' : 'msvc', | |
184 | 'command' : '', | |
185 | 'version' : '14.1' | |
186 | }, | |
187 | 'xcode-6.1' : { | |
188 | 'command' : 'clang++', | |
189 | 'toolset' : 'clang', | |
190 | 'version' : '' | |
191 | }, | |
192 | 'xcode-6.2' : { | |
193 | 'command' : 'clang++', | |
194 | 'toolset' : 'clang', | |
195 | 'version' : '' | |
196 | }, | |
197 | 'xcode-6.3' : { | |
198 | 'command' : 'clang++', | |
199 | 'toolset' : 'clang', | |
200 | 'version' : '' | |
201 | }, | |
202 | 'xcode-6.4' : { | |
203 | 'command' : 'clang++', | |
204 | 'toolset' : 'clang', | |
205 | 'version' : '' | |
206 | }, | |
207 | 'xcode-7.0' : { | |
208 | 'command' : 'clang++', | |
209 | 'toolset' : 'clang', | |
210 | 'version' : '' | |
211 | }, | |
212 | 'xcode-7.1' : { | |
213 | 'command' : 'clang++', | |
214 | 'toolset' : 'clang', | |
215 | 'version' : '' | |
216 | }, | |
217 | 'xcode-7.2' : { | |
218 | 'command' : 'clang++', | |
219 | 'toolset' : 'clang', | |
220 | 'version' : '' | |
221 | }, | |
222 | 'xcode-7.3' : { | |
223 | 'command' : 'clang++', | |
224 | 'toolset' : 'clang', | |
225 | 'version' : '' | |
226 | }, | |
227 | 'xcode-8.0' : { | |
228 | 'command' : 'clang++', | |
229 | 'toolset' : 'clang', | |
230 | 'version' : '' | |
231 | }, | |
232 | 'xcode-8.1' : { | |
233 | 'command' : 'clang++', | |
234 | 'toolset' : 'clang', | |
235 | 'version' : '' | |
236 | }, | |
237 | 'xcode-8.2' : { | |
238 | 'command' : 'clang++', | |
239 | 'toolset' : 'clang', | |
240 | 'version' : '' | |
241 | }, | |
242 | 'xcode-8.3' : { | |
243 | 'command' : 'clang++', | |
244 | 'toolset' : 'clang', | |
245 | 'version' : '' | |
246 | }, | |
247 | 'xcode-9.0' : { | |
248 | 'command' : 'clang++', | |
249 | 'toolset' : 'clang', | |
250 | 'version' : '' | |
251 | }, | |
252 | 'xcode-9.1' : { | |
253 | 'command' : 'clang++', | |
254 | 'toolset' : 'clang', | |
255 | 'version' : '' | |
256 | }, | |
257 | 'xcode-9.2' : { | |
258 | 'command' : 'clang++', | |
259 | 'toolset' : 'clang', | |
260 | 'version' : '' | |
261 | }, | |
262 | 'xcode-9.3' : { | |
263 | 'command' : 'clang++', | |
264 | 'toolset' : 'clang', | |
265 | 'version' : '' | |
266 | }, | |
267 | 'xcode-9.4' : { | |
268 | 'command' : 'clang++', | |
269 | 'toolset' : 'clang', | |
270 | 'version' : '' | |
271 | }, | |
272 | 'xcode-10.0' : { | |
273 | 'command' : 'clang++', | |
274 | 'toolset' : 'clang', | |
275 | 'version' : '' | |
276 | }, | |
277 | } | |
278 | ||
279 | class SystemCallError(Exception): | |
280 | def __init__(self, command, result): | |
281 | self.command = command | |
282 | self.result = result | |
283 | def __str__(self, *args, **kwargs): | |
284 | return "'%s' ==> %s"%("' '".join(self.command), self.result) | |
285 | ||
286 | class utils: | |
287 | ||
288 | call_stats = [] | |
289 | ||
290 | @staticmethod | |
291 | def call(*command, **kargs): | |
292 | utils.log( "%s> '%s'"%(os.getcwd(), "' '".join(command)) ) | |
293 | t = time.time() | |
294 | result = subprocess.call(command, **kargs) | |
295 | t = time.time()-t | |
296 | if result != 0: | |
297 | print "Failed: '%s' ERROR = %s"%("' '".join(command), result) | |
298 | utils.call_stats.append((t,os.getcwd(),command,result)) | |
299 | utils.log( "%s> '%s' execution time %s seconds"%(os.getcwd(), "' '".join(command), t) ) | |
300 | return result | |
301 | ||
302 | @staticmethod | |
303 | def print_call_stats(): | |
304 | utils.log("================================================================================") | |
305 | for j in sorted(utils.call_stats, reverse=True): | |
306 | utils.log("{:>12.4f}\t{}> {} ==> {}".format(*j)) | |
307 | utils.log("================================================================================") | |
308 | ||
309 | @staticmethod | |
310 | def check_call(*command, **kargs): | |
311 | cwd = os.getcwd() | |
312 | result = utils.call(*command, **kargs) | |
313 | if result != 0: | |
314 | raise(SystemCallError([cwd].extend(command), result)) | |
315 | ||
316 | @staticmethod | |
317 | def makedirs( path ): | |
318 | if not os.path.exists( path ): | |
319 | os.makedirs( path ) | |
320 | ||
321 | @staticmethod | |
322 | def log_level(): | |
323 | frames = inspect.stack() | |
324 | level = 0 | |
325 | for i in frames[ 3: ]: | |
326 | if i[0].f_locals.has_key( '__log__' ): | |
327 | level = level + i[0].f_locals[ '__log__' ] | |
328 | return level | |
329 | ||
330 | @staticmethod | |
331 | def log( message ): | |
332 | sys.stdout.flush() | |
333 | sys.stderr.flush() | |
334 | sys.stderr.write( '# ' + ' ' * utils.log_level() + message + '\n' ) | |
335 | sys.stderr.flush() | |
336 | ||
337 | @staticmethod | |
338 | def rmtree(path): | |
339 | if os.path.exists( path ): | |
340 | #~ shutil.rmtree( unicode( path ) ) | |
341 | if sys.platform == 'win32': | |
342 | os.system( 'del /f /s /q "%s" >nul 2>&1' % path ) | |
343 | shutil.rmtree( unicode( path ) ) | |
344 | else: | |
345 | os.system( 'rm -f -r "%s"' % path ) | |
346 | ||
347 | @staticmethod | |
348 | def retry( f, max_attempts=5, sleep_secs=10 ): | |
349 | for attempts in range( max_attempts, -1, -1 ): | |
350 | try: | |
351 | return f() | |
352 | except Exception, msg: | |
353 | utils.log( '%s failed with message "%s"' % ( f.__name__, msg ) ) | |
354 | if attempts == 0: | |
355 | utils.log( 'Giving up.' ) | |
356 | raise | |
357 | ||
358 | utils.log( 'Retrying (%d more attempts).' % attempts ) | |
359 | time.sleep( sleep_secs ) | |
360 | ||
361 | @staticmethod | |
362 | def web_get( source_url, destination_file, proxy = None ): | |
363 | import urllib | |
364 | ||
365 | proxies = None | |
366 | if proxy is not None: | |
367 | proxies = { | |
368 | 'https' : proxy, | |
369 | 'http' : proxy | |
370 | } | |
371 | ||
372 | src = urllib.urlopen( source_url, proxies = proxies ) | |
373 | ||
374 | f = open( destination_file, 'wb' ) | |
375 | while True: | |
376 | data = src.read( 16*1024 ) | |
377 | if len( data ) == 0: break | |
378 | f.write( data ) | |
379 | ||
380 | f.close() | |
381 | src.close() | |
382 | ||
383 | @staticmethod | |
384 | def unpack_archive( archive_path ): | |
385 | utils.log( 'Unpacking archive ("%s")...' % archive_path ) | |
386 | ||
387 | archive_name = os.path.basename( archive_path ) | |
388 | extension = archive_name[ archive_name.find( '.' ) : ] | |
389 | ||
390 | if extension in ( ".tar.gz", ".tar.bz2" ): | |
391 | import tarfile | |
392 | import stat | |
393 | ||
394 | mode = os.path.splitext( extension )[1][1:] | |
395 | tar = tarfile.open( archive_path, 'r:%s' % mode ) | |
396 | for tarinfo in tar: | |
397 | tar.extract( tarinfo ) | |
398 | if sys.platform == 'win32' and not tarinfo.isdir(): | |
399 | # workaround what appears to be a Win32-specific bug in 'tarfile' | |
400 | # (modification times for extracted files are not set properly) | |
401 | f = os.path.join( os.curdir, tarinfo.name ) | |
402 | os.chmod( f, stat.S_IWRITE ) | |
403 | os.utime( f, ( tarinfo.mtime, tarinfo.mtime ) ) | |
404 | tar.close() | |
405 | elif extension in ( ".zip" ): | |
406 | import zipfile | |
407 | ||
408 | z = zipfile.ZipFile( archive_path, 'r', zipfile.ZIP_DEFLATED ) | |
409 | for f in z.infolist(): | |
410 | destination_file_path = os.path.join( os.curdir, f.filename ) | |
411 | if destination_file_path[-1] == "/": # directory | |
412 | if not os.path.exists( destination_file_path ): | |
413 | os.makedirs( destination_file_path ) | |
414 | else: # file | |
415 | result = open( destination_file_path, 'wb' ) | |
416 | result.write( z.read( f.filename ) ) | |
417 | result.close() | |
418 | z.close() | |
419 | else: | |
420 | raise 'Do not know how to unpack archives with extension \"%s\"' % extension | |
421 | ||
422 | @staticmethod | |
423 | def make_file(filename, *text): | |
424 | text = string.join( text, '\n' ) | |
425 | with codecs.open( filename, 'w', 'utf-8' ) as f: | |
426 | f.write( text ) | |
427 | ||
428 | @staticmethod | |
429 | def append_file(filename, *text): | |
430 | with codecs.open( filename, 'a', 'utf-8' ) as f: | |
431 | f.write( string.join( text, '\n' ) ) | |
432 | ||
433 | @staticmethod | |
434 | def mem_info(): | |
435 | if sys.platform == "darwin": | |
436 | utils.call("top","-l","1","-s","0","-n","0") | |
437 | elif sys.platform.startswith("linux"): | |
438 | utils.call("free","-m","-l") | |
439 | ||
440 | @staticmethod | |
441 | def query_boost_version(boost_root): | |
442 | ''' | |
443 | Read in the Boost version from a given boost_root. | |
444 | ''' | |
445 | boost_version = None | |
446 | if os.path.exists(os.path.join(boost_root,'Jamroot')): | |
447 | with codecs.open(os.path.join(boost_root,'Jamroot'), 'r', 'utf-8') as f: | |
448 | for line in f.readlines(): | |
449 | parts = line.split() | |
450 | if len(parts) >= 5 and parts[1] == 'BOOST_VERSION': | |
451 | boost_version = parts[3] | |
452 | break | |
453 | if not boost_version: | |
454 | boost_version = 'default' | |
455 | return boost_version | |
456 | ||
457 | @staticmethod | |
458 | def git_clone(owner, repo, branch, commit = None, repo_dir = None, submodules = False, url_format = "https://github.com/%(owner)s/%(repo)s.git"): | |
459 | ''' | |
460 | This clone mimicks the way Travis-CI clones a project's repo. So far | |
461 | Travis-CI is the most limiting in the sense of only fetching partial | |
462 | history of the repo. | |
463 | ''' | |
464 | if not repo_dir: | |
465 | repo_dir = os.path.join(os.getcwd(), owner+','+repo) | |
466 | utils.makedirs(os.path.dirname(repo_dir)) | |
467 | if not os.path.exists(os.path.join(repo_dir,'.git')): | |
468 | utils.check_call("git","clone", | |
469 | "--depth=1", | |
470 | "--branch=%s"%(branch), | |
471 | url_format%{'owner':owner,'repo':repo}, | |
472 | repo_dir) | |
473 | os.chdir(repo_dir) | |
474 | else: | |
475 | os.chdir(repo_dir) | |
476 | utils.check_call("git","pull", | |
477 | # "--depth=1", # Can't do depth as we get merge errors. | |
478 | "--quiet","--no-recurse-submodules") | |
479 | if commit: | |
480 | utils.check_call("git","checkout","-qf",commit) | |
481 | if os.path.exists(os.path.join('.git','modules')): | |
482 | if sys.platform == 'win32': | |
483 | utils.check_call('dir',os.path.join('.git','modules')) | |
484 | else: | |
485 | utils.check_call('ls','-la',os.path.join('.git','modules')) | |
486 | if submodules: | |
487 | utils.check_call("git","submodule","--quiet","update", | |
488 | "--quiet","--init","--recursive", | |
489 | ) | |
490 | utils.check_call("git","submodule","--quiet","foreach","git","fetch") | |
491 | return repo_dir | |
492 | ||
493 | class parallel_call(threading.Thread): | |
494 | ''' | |
495 | Runs a synchronous command in a thread waiting for it to complete. | |
496 | ''' | |
497 | ||
498 | def __init__(self, *command, **kargs): | |
499 | super(parallel_call,self).__init__() | |
500 | self.command = command | |
501 | self.command_kargs = kargs | |
502 | self.start() | |
503 | ||
504 | def run(self): | |
505 | self.result = utils.call(*self.command, **self.command_kargs) | |
506 | ||
507 | def join(self): | |
508 | super(parallel_call,self).join() | |
509 | if self.result != 0: | |
510 | raise(SystemCallError(self.command, self.result)) | |
511 | ||
512 | def set_arg(args, k, v = None): | |
513 | if not args.get(k): | |
514 | args[k] = v | |
515 | return args[k] | |
516 | ||
517 | class script_common(object): | |
518 | ''' | |
519 | Main script to run continuous integration. | |
520 | ''' | |
521 | ||
522 | def __init__(self, ci_klass, **kargs): | |
523 | self.ci = ci_klass(self) | |
524 | ||
525 | opt = optparse.OptionParser( | |
526 | usage="%prog [options] [commands]") | |
527 | ||
528 | #~ Debug Options: | |
529 | opt.add_option( '--debug-level', | |
530 | help="debugging level; controls the amount of debugging output printed", | |
531 | type='int' ) | |
532 | opt.add_option( '-j', | |
533 | help="maximum number of parallel jobs to use for building with b2", | |
534 | type='int', dest='jobs') | |
535 | opt.add_option('--branch') | |
536 | opt.add_option('--commit') | |
537 | kargs = self.init(opt,kargs) | |
538 | kargs = self.ci.init(opt, kargs) | |
539 | set_arg(kargs,'debug_level',0) | |
540 | set_arg(kargs,'jobs',2) | |
541 | set_arg(kargs,'branch',None) | |
542 | set_arg(kargs,'commit',None) | |
543 | set_arg(kargs,'repo',None) | |
544 | set_arg(kargs,'repo_dir',None) | |
545 | set_arg(kargs,'actions',None) | |
546 | set_arg(kargs,'pull_request', None) | |
547 | ||
548 | #~ Defaults | |
549 | for (k,v) in kargs.iteritems(): | |
550 | setattr(self,k,v) | |
551 | ( _opt_, self.actions ) = opt.parse_args(None,self) | |
552 | if not self.actions or self.actions == []: | |
553 | self.actions = kargs.get('actions',None) | |
554 | if not self.actions or self.actions == []: | |
555 | self.actions = [ 'info' ] | |
556 | if not self.repo_dir: | |
557 | self.repo_dir = os.getcwd() | |
558 | self.build_dir = os.path.join(os.path.dirname(self.repo_dir), "build") | |
559 | ||
560 | # API keys. | |
561 | self.bintray_key = os.getenv('BINTRAY_KEY') | |
562 | ||
563 | try: | |
564 | self.start() | |
565 | self.command_info() | |
566 | self.main() | |
567 | utils.print_call_stats() | |
568 | except: | |
569 | utils.print_call_stats() | |
570 | raise | |
571 | ||
572 | def init(self, opt, kargs): | |
573 | return kargs | |
574 | ||
575 | def start(self): | |
576 | pass | |
577 | ||
578 | def main(self): | |
579 | for action in self.actions: | |
580 | action_m = "command_"+action.replace('-','_') | |
581 | ci_command = getattr(self.ci, action_m, None) | |
582 | ci_script = getattr(self, action_m, None) | |
583 | if ci_command or ci_script: | |
584 | utils.log( "### %s.."%(action) ) | |
585 | if os.path.exists(self.repo_dir): | |
586 | os.chdir(self.repo_dir) | |
587 | if ci_command: | |
588 | ci_command() | |
589 | elif ci_script: | |
590 | ci_script() | |
591 | ||
592 | def b2( self, *args, **kargs ): | |
593 | cmd = ['b2','--debug-configuration', '-j%s'%(self.jobs)] | |
594 | cmd.extend(args) | |
595 | ||
596 | if 'toolset' in kargs: | |
597 | cmd.append('toolset=' + kargs['toolset']) | |
598 | ||
599 | if 'parallel' in kargs: | |
600 | return parallel_call(*cmd) | |
601 | else: | |
602 | return utils.check_call(*cmd) | |
603 | ||
604 | # Common test commands in the order they should be executed.. | |
605 | ||
606 | def command_info(self): | |
607 | pass | |
608 | ||
609 | def command_install(self): | |
610 | utils.makedirs(self.build_dir) | |
611 | os.chdir(self.build_dir) | |
612 | ||
613 | def command_install_toolset(self, toolset): | |
614 | if self.ci and hasattr(self.ci,'install_toolset'): | |
615 | self.ci.install_toolset(toolset) | |
616 | ||
617 | def command_before_build(self): | |
618 | pass | |
619 | ||
620 | def command_build(self): | |
621 | pass | |
622 | ||
623 | def command_before_cache(self): | |
624 | pass | |
625 | ||
626 | def command_after_success(self): | |
627 | pass | |
628 | ||
629 | class ci_cli(object): | |
630 | ''' | |
631 | This version of the script provides a way to do manual building. It sets up | |
632 | additional environment and adds fetching of the git repos that would | |
633 | normally be done by the CI system. | |
634 | ||
635 | The common way to use this variant is to invoke something like: | |
636 | ||
637 | mkdir ci | |
638 | cd ci | |
639 | python path-to/library_test.py --branch=develop [--repo=mylib] ... | |
640 | ||
641 | Status: In working order. | |
642 | ''' | |
643 | ||
644 | def __init__(self,script): | |
645 | if sys.platform == 'darwin': | |
646 | # Requirements for running on OSX: | |
647 | # https://www.stack.nl/~dimitri/doxygen/download.html#srcbin | |
648 | # https://tug.org/mactex/morepackages.html | |
649 | doxygen_path = "/Applications/Doxygen.app/Contents/Resources" | |
650 | if os.path.isdir(doxygen_path): | |
651 | os.environ["PATH"] = doxygen_path+':'+os.environ['PATH'] | |
652 | self.script = script | |
653 | self.repo_dir = os.getcwd() | |
654 | self.exit_result = 0 | |
655 | ||
656 | def init(self, opt, kargs): | |
657 | kargs['actions'] = [ | |
658 | # 'clone', | |
659 | 'install', | |
660 | 'before_build', | |
661 | 'build', | |
662 | 'before_cache', | |
663 | 'finish' | |
664 | ] | |
665 | return kargs | |
666 | ||
667 | def finish(self, result): | |
668 | self.exit_result = result | |
669 | ||
670 | def command_finish(self): | |
671 | exit(self.exit_result) | |
672 | ||
673 | class ci_travis(object): | |
674 | ''' | |
675 | This variant build releases in the context of the Travis-CI service. | |
676 | ''' | |
677 | ||
678 | def __init__(self,script): | |
679 | self.script = script | |
680 | ||
681 | def init(self, opt, kargs): | |
682 | set_arg(kargs,'repo_dir', os.getenv("TRAVIS_BUILD_DIR")) | |
683 | set_arg(kargs,'branch', os.getenv("TRAVIS_BRANCH")) | |
684 | set_arg(kargs,'commit', os.getenv("TRAVIS_COMMIT")) | |
685 | set_arg(kargs,'repo', os.getenv("TRAVIS_REPO_SLUG").split("/")[1]) | |
686 | set_arg(kargs,'pull_request', | |
687 | os.getenv('TRAVIS_PULL_REQUEST') \ | |
688 | if os.getenv('TRAVIS_PULL_REQUEST') != 'false' else None) | |
689 | return kargs | |
690 | ||
691 | def finish(self, result): | |
692 | exit(result) | |
693 | ||
694 | def install_toolset(self, toolset): | |
695 | ''' | |
696 | Installs specific toolset on CI system. | |
697 | ''' | |
698 | info = toolset_info[toolset] | |
699 | if sys.platform.startswith('linux'): | |
700 | os.chdir(self.script.build_dir) | |
701 | if 'ppa' in info: | |
702 | for ppa in info['ppa']: | |
703 | utils.check_call( | |
704 | 'sudo','add-apt-repository','--yes',ppa) | |
705 | if 'deb' in info: | |
706 | utils.make_file('sources.list', | |
707 | "deb %s"%(' '.join(info['deb'])), | |
708 | "deb-src %s"%(' '.join(info['deb']))) | |
709 | utils.check_call('sudo','bash','-c','cat sources.list >> /etc/apt/sources.list') | |
710 | if 'apt-key' in info: | |
711 | for key in info['apt-key']: | |
712 | utils.check_call('wget',key,'-O','apt.key') | |
713 | utils.check_call('sudo','apt-key','add','apt.key') | |
714 | utils.check_call( | |
715 | 'sudo','apt-get','update','-qq') | |
716 | utils.check_call( | |
717 | 'sudo','apt-get','install','-qq',info['package']) | |
718 | if 'debugpackage' in info and info['debugpackage']: | |
719 | utils.check_call( | |
720 | 'sudo','apt-get','install','-qq',info['debugpackage']) | |
721 | ||
722 | # Travis-CI commands in the order they are executed. We need | |
723 | # these to forward to our common commands, if they are different. | |
724 | ||
725 | def command_before_install(self): | |
726 | pass | |
727 | ||
728 | def command_install(self): | |
729 | self.script.command_install() | |
730 | ||
731 | def command_before_script(self): | |
732 | self.script.command_before_build() | |
733 | ||
734 | def command_script(self): | |
735 | self.script.command_build() | |
736 | ||
737 | def command_before_cache(self): | |
738 | self.script.command_before_cache() | |
739 | ||
740 | def command_after_success(self): | |
741 | self.script.command_after_success() | |
742 | ||
743 | def command_after_failure(self): | |
744 | pass | |
745 | ||
746 | def command_before_deploy(self): | |
747 | pass | |
748 | ||
749 | def command_after_deploy(self): | |
750 | pass | |
751 | ||
752 | def command_after_script(self): | |
753 | pass | |
754 | ||
755 | class ci_circleci(object): | |
756 | ''' | |
757 | This variant build releases in the context of the CircleCI service. | |
758 | ''' | |
759 | ||
760 | def __init__(self,script): | |
761 | self.script = script | |
762 | ||
763 | def init(self, opt, kargs): | |
764 | set_arg(kargs,'repo_dir', os.path.join(os.getenv("HOME"),os.getenv("CIRCLE_PROJECT_REPONAME"))) | |
765 | set_arg(kargs,'branch', os.getenv("CIRCLE_BRANCH")) | |
766 | set_arg(kargs,'commit', os.getenv("CIRCLE_SHA1")) | |
767 | set_arg(kargs,'repo', os.getenv("CIRCLE_PROJECT_REPONAME").split("/")[1]) | |
768 | set_arg(kargs,'pull_request', os.getenv('CIRCLE_PR_NUMBER')) | |
769 | return kargs | |
770 | ||
771 | def finish(self, result): | |
772 | exit(result) | |
773 | ||
774 | def command_machine_post(self): | |
775 | # Apt update for the pckages installs we'll do later. | |
776 | utils.check_call('sudo','apt-get','-qq','update') | |
777 | # Need PyYAML to read Travis yaml in a later step. | |
778 | utils.check_call("pip","install","--user","PyYAML") | |
779 | ||
780 | def command_checkout_post(self): | |
781 | os.chdir(self.script.repo_dir) | |
782 | utils.check_call("git","submodule","update","--quiet","--init","--recursive") | |
783 | ||
784 | def command_dependencies_pre(self): | |
785 | # Read in .travis.yml for list of packages to install | |
786 | # as CircleCI doesn't have a convenient apt install method. | |
787 | import yaml | |
788 | utils.check_call('sudo','-E','apt-get','-yqq','update') | |
789 | utils.check_call('sudo','apt-get','-yqq','purge','texlive*') | |
790 | with open(os.path.join(self.script.repo_dir,'.travis.yml')) as yml: | |
791 | travis_yml = yaml.load(yml) | |
792 | utils.check_call('sudo','apt-get','-yqq', | |
793 | '--no-install-suggests','--no-install-recommends','--force-yes','install', | |
794 | *travis_yml['addons']['apt']['packages']) | |
795 | ||
796 | def command_dependencies_override(self): | |
797 | self.script.command_install() | |
798 | ||
799 | def command_dependencies_post(self): | |
800 | pass | |
801 | ||
802 | def command_database_pre(self): | |
803 | pass | |
804 | ||
805 | def command_database_override(self): | |
806 | pass | |
807 | ||
808 | def command_database_post(self): | |
809 | pass | |
810 | ||
811 | def command_test_pre(self): | |
812 | self.script.command_install() | |
813 | self.script.command_before_build() | |
814 | ||
815 | def command_test_override(self): | |
816 | # CircleCI runs all the test subsets. So in order to avoid | |
817 | # running the after_success we do it here as the build step | |
818 | # will halt accordingly. | |
819 | self.script.command_build() | |
820 | self.script.command_before_cache() | |
821 | self.script.command_after_success() | |
822 | ||
823 | def command_test_post(self): | |
824 | pass | |
825 | ||
826 | class ci_appveyor(object): | |
827 | ||
828 | def __init__(self,script): | |
829 | self.script = script | |
830 | ||
831 | def init(self, opt, kargs): | |
832 | set_arg(kargs,'repo_dir',os.getenv("APPVEYOR_BUILD_FOLDER")) | |
833 | set_arg(kargs,'branch',os.getenv("APPVEYOR_REPO_BRANCH")) | |
834 | set_arg(kargs,'commit',os.getenv("APPVEYOR_REPO_COMMIT")) | |
835 | set_arg(kargs,'repo',os.getenv("APPVEYOR_REPO_NAME").split("/")[1]) | |
836 | set_arg(kargs,'address_model',os.getenv("PLATFORM",None)) | |
837 | set_arg(kargs,'variant',os.getenv("CONFIGURATION","debug")) | |
838 | set_arg(kargs,'pull_request', os.getenv('APPVEYOR_PULL_REQUEST_NUMBER')) | |
839 | return kargs | |
840 | ||
841 | def finish(self, result): | |
842 | exit(result) | |
843 | ||
844 | # Appveyor commands in the order they are executed. We need | |
845 | # these to forward to our common commands, if they are different. | |
846 | ||
847 | def command_install(self): | |
848 | self.script.command_install() | |
849 | ||
850 | def command_before_build(self): | |
851 | os.chdir(self.script.repo_dir) | |
852 | utils.check_call("git","submodule","update","--quiet","--init","--recursive") | |
853 | self.script.command_before_build() | |
854 | ||
855 | def command_build_script(self): | |
856 | self.script.command_build() | |
857 | ||
858 | def command_after_build(self): | |
859 | self.script.command_before_cache() | |
860 | ||
861 | def command_before_test(self): | |
862 | pass | |
863 | ||
864 | def command_test_script(self): | |
865 | pass | |
866 | ||
867 | def command_after_test(self): | |
868 | pass | |
869 | ||
870 | def command_on_success(self): | |
871 | self.script.command_after_success() | |
872 | ||
873 | def command_on_failure(self): | |
874 | pass | |
875 | ||
876 | def command_on_finish(self): | |
877 | pass | |
878 | ||
879 | def main(script_klass): | |
880 | if os.getenv('TRAVIS', False): | |
881 | script_klass(ci_travis) | |
882 | elif os.getenv('CIRCLECI', False): | |
883 | script_klass(ci_circleci) | |
884 | elif os.getenv('APPVEYOR', False): | |
885 | script_klass(ci_appveyor) | |
886 | else: | |
887 | script_klass(ci_cli) |