ceph/src/boost/libs/predef/tools/ci/common.py

   1 #!/usr/bin/env python
   2
   3 # Copyright Rene Rivera 2016
   4 #
   5 # Distributed under the Boost Software License, Version 1.0.
   6 # (See accompanying file LICENSE_1_0.txt or copy at
   7 # http://www.boost.org/LICENSE_1_0.txt)
   8
   9 import sys
  10 import inspect
  11 import optparse
  12 import os.path
  13 import string
  14 import time
  15 import subprocess
  16 import codecs
  17 import shutil
  18 import threading
  19
  20 toolset_info = {
  21     'clang-3.4' : {
  22         'ppa' : ["ppa:h-rayflood/llvm"],
  23         'package' : 'clang-3.4',
  24         'command' : 'clang++-3.4',
  25         'toolset' : 'clang',
  26         'version' : ''
  27         },
  28     'clang-3.5' : {
  29         'ppa' : ["ppa:h-rayflood/llvm"],
  30         'package' : 'clang-3.5',
  31         'command' : 'clang++-3.5',
  32         'toolset' : 'clang',
  33         'version' : ''
  34         },
  35     'clang-3.6' : {
  36         'ppa' : ["ppa:h-rayflood/llvm"],
  37         'package' : 'clang-3.6',
  38         'command' : 'clang++-3.6',
  39         'toolset' : 'clang',
  40         'version' : ''
  41         },
  42     'clang-3.7' : {
  43         'deb' : ["http://apt.llvm.org/trusty/","llvm-toolchain-trusty-3.7","main"],
  44         'apt-key' : ['http://apt.llvm.org/llvm-snapshot.gpg.key'],
  45         'package' : 'clang-3.7',
  46         'command' : 'clang++-3.7',
  47         'toolset' : 'clang',
  48         'version' : ''
  49         },
  50     'clang-3.8' : {
  51         'deb' : ["http://apt.llvm.org/trusty/","llvm-toolchain-trusty-3.8","main"],
  52         'apt-key' : ['http://apt.llvm.org/llvm-snapshot.gpg.key'],
  53         'package' : 'clang-3.8',
  54         'command' : 'clang++-3.8',
  55         'toolset' : 'clang',
  56         'version' : ''
  57         },
  58     'clang-3.9' : {
  59         'deb' : ["http://apt.llvm.org/trusty/","llvm-toolchain-trusty-3.9","main"],
  60         'apt-key' : ['http://apt.llvm.org/llvm-snapshot.gpg.key'],
  61         'package' : 'clang-3.9',
  62         'command' : 'clang++-3.9',
  63         'toolset' : 'clang',
  64         'version' : ''
  65         },
  66     'clang-4.0' : {
  67         'deb' : ["http://apt.llvm.org/trusty/","llvm-toolchain-trusty-4.0","main"],
  68         'apt-key' : ['http://apt.llvm.org/llvm-snapshot.gpg.key'],
  69         'package' : 'clang-4.0',
  70         'command' : 'clang++-4.0',
  71         'toolset' : 'clang',
  72         'version' : ''
  73         },
  74     'clang-5.0' : {
  75         'deb' : ["http://apt.llvm.org/trusty/","llvm-toolchain-trusty-5.0","main"],
  76         'apt-key' : ['http://apt.llvm.org/llvm-snapshot.gpg.key'],
  77         'package' : 'clang-5.0',
  78         'command' : 'clang++-5.0',
  79         'toolset' : 'clang',
  80         'version' : ''
  81         },
  82     'clang-6.0' : {
  83         'deb' : ["http://apt.llvm.org/trusty/","llvm-toolchain-trusty-6.0","main"],
  84         'apt-key' : ['http://apt.llvm.org/llvm-snapshot.gpg.key'],
  85         'package' : 'clang-6.0',
  86         'command' : 'clang++-6.0',
  87         'toolset' : 'clang',
  88         'version' : ''
  89         },
  90     'gcc-4.7' : {
  91         'ppa' : ["ppa:ubuntu-toolchain-r/test"],
  92         'package' : 'g++-4.7',
  93         'command' : 'g++-4.7',
  94         'toolset' : 'gcc',
  95         'version' : ''
  96         },
  97     'gcc-4.8' : {
  98         'bin' : 'gcc-4.8',
  99         'ppa' : ["ppa:ubuntu-toolchain-r/test"],
 100         'package' : 'g++-4.8',
 101         'command' : 'g++-4.8',
 102         'toolset' : 'gcc',
 103         'version' : ''
 104         },
 105     'gcc-4.9' : {
 106         'ppa' : ["ppa:ubuntu-toolchain-r/test"],
 107         'package' : 'g++-4.9',
 108         'command' : 'g++-4.9',
 109         'toolset' : 'gcc',
 110         'version' : ''
 111         },
 112     'gcc-5.1' : {
 113         'ppa' : ["ppa:ubuntu-toolchain-r/test"],
 114         'package' : 'g++-5',
 115         'command' : 'g++-5',
 116         'toolset' : 'gcc',
 117         'version' : ''
 118         },
 119     'gcc-5' : {
 120         'ppa' : ["ppa:ubuntu-toolchain-r/test"],
 121         'package' : 'g++-5',
 122         'command' : 'g++-5',
 123         'toolset' : 'gcc',
 124         'version' : ''
 125         },
 126     'gcc-6' : {
 127         'ppa' : ["ppa:ubuntu-toolchain-r/test"],
 128         'package' : 'g++-6',
 129         'command' : 'g++-6',
 130         'toolset' : 'gcc',
 131         'version' : ''
 132         },
 133     'gcc-7' : {
 134         'ppa' : ["ppa:ubuntu-toolchain-r/test"],
 135         'package' : 'g++-7',
 136         'command' : 'g++-7',
 137         'toolset' : 'gcc',
 138         'version' : ''
 139         },
 140     'gcc-8' : {
 141         'ppa' : ["ppa:ubuntu-toolchain-r/test"],
 142         'package' : 'g++-8',
 143         'command' : 'g++-8',
 144         'toolset' : 'gcc',
 145         'version' : ''
 146         },
 147     'mingw-5' : {
 148         'toolset' : 'gcc',
 149         'command' : 'C:\\\\MinGW\\\\bin\\\\g++.exe',
 150         'version' : ''
 151         },
 152     'mingw64-6' : {
 153         'toolset' : 'gcc',
 154         'command' : 'C:\\\\mingw-w64\\\\x86_64-6.3.0-posix-seh-rt_v5-rev1\\\\mingw64\\\\bin\\\\g++.exe',
 155         'version' : ''
 156         },
 157     'vs-2008' : {
 158         'toolset' : 'msvc',
 159         'command' : '',
 160         'version' : '9.0'
 161         },
 162     'vs-2010' : {
 163         'toolset' : 'msvc',
 164         'command' : '',
 165         'version' : '10.0'
 166         },
 167     'vs-2012' : {
 168         'toolset' : 'msvc',
 169         'command' : '',
 170         'version' : '11.0'
 171         },
 172     'vs-2013' : {
 173         'toolset' : 'msvc',
 174         'command' : '',
 175         'version' : '12.0'
 176         },
 177     'vs-2015' : {
 178         'toolset' : 'msvc',
 179         'command' : '',
 180         'version' : '14.0'
 181         },
 182     'vs-2017' : {
 183         'toolset' : 'msvc',
 184         'command' : '',
 185         'version' : '14.1'
 186         },
 187     'xcode-6.1' : {
 188         'command' : 'clang++',
 189         'toolset' : 'clang',
 190         'version' : ''
 191         },
 192     'xcode-6.2' : {
 193         'command' : 'clang++',
 194         'toolset' : 'clang',
 195         'version' : ''
 196         },
 197     'xcode-6.3' : {
 198         'command' : 'clang++',
 199         'toolset' : 'clang',
 200         'version' : ''
 201         },
 202     'xcode-6.4' : {
 203         'command' : 'clang++',
 204         'toolset' : 'clang',
 205         'version' : ''
 206         },
 207     'xcode-7.0' : {
 208         'command' : 'clang++',
 209         'toolset' : 'clang',
 210         'version' : ''
 211         },
 212     'xcode-7.1' : {
 213         'command' : 'clang++',
 214         'toolset' : 'clang',
 215         'version' : ''
 216         },
 217     'xcode-7.2' : {
 218         'command' : 'clang++',
 219         'toolset' : 'clang',
 220         'version' : ''
 221         },
 222     'xcode-7.3' : {
 223         'command' : 'clang++',
 224         'toolset' : 'clang',
 225         'version' : ''
 226         },
 227     'xcode-8.0' : {
 228         'command' : 'clang++',
 229         'toolset' : 'clang',
 230         'version' : ''
 231         },
 232     'xcode-8.1' : {
 233         'command' : 'clang++',
 234         'toolset' : 'clang',
 235         'version' : ''
 236         },
 237     'xcode-8.2' : {
 238         'command' : 'clang++',
 239         'toolset' : 'clang',
 240         'version' : ''
 241         },
 242     'xcode-8.3' : {
 243         'command' : 'clang++',
 244         'toolset' : 'clang',
 245         'version' : ''
 246         },
 247     'xcode-9.0' : {
 248         'command' : 'clang++',
 249         'toolset' : 'clang',
 250         'version' : ''
 251         },
 252     'xcode-9.1' : {
 253         'command' : 'clang++',
 254         'toolset' : 'clang',
 255         'version' : ''
 256         },
 257     'xcode-9.2' : {
 258         'command' : 'clang++',
 259         'toolset' : 'clang',
 260         'version' : ''
 261         },
 262     'xcode-9.3' : {
 263         'command' : 'clang++',
 264         'toolset' : 'clang',
 265         'version' : ''
 266         },
 267     'xcode-9.4' : {
 268         'command' : 'clang++',
 269         'toolset' : 'clang',
 270         'version' : ''
 271         },
 272     'xcode-10.0' : {
 273         'command' : 'clang++',
 274         'toolset' : 'clang',
 275         'version' : ''
 276         },
 277     }
 278
 279 class SystemCallError(Exception):
 280     def __init__(self, command, result):
 281         self.command = command
 282         self.result = result
 283     def __str__(self, *args, **kwargs):
 284         return "'%s' ==> %s"%("' '".join(self.command), self.result)
 285
 286 class utils:
 287
 288     call_stats = []
 289
 290     @staticmethod
 291     def call(*command, **kargs):
 292         utils.log( "%s> '%s'"%(os.getcwd(), "' '".join(command)) )
 293         t = time.time()
 294         result = subprocess.call(command, **kargs)
 295         t = time.time()-t
 296         if result != 0:
 297             print "Failed: '%s' ERROR = %s"%("' '".join(command), result)
 298         utils.call_stats.append((t,os.getcwd(),command,result))
 299         utils.log( "%s> '%s' execution time %s seconds"%(os.getcwd(), "' '".join(command), t) )
 300         return result
 301
 302     @staticmethod
 303     def print_call_stats():
 304         utils.log("================================================================================")
 305         for j in sorted(utils.call_stats, reverse=True):
 306             utils.log("{:>12.4f}\t{}> {} ==> {}".format(*j))
 307         utils.log("================================================================================")
 308
 309     @staticmethod
 310     def check_call(*command, **kargs):
 311         cwd = os.getcwd()
 312         result = utils.call(*command, **kargs)
 313         if result != 0:
 314             raise(SystemCallError([cwd].extend(command), result))
 315
 316     @staticmethod
 317     def makedirs( path ):
 318         if not os.path.exists( path ):
 319             os.makedirs( path )
 320
 321     @staticmethod
 322     def log_level():
 323         frames = inspect.stack()
 324         level = 0
 325         for i in frames[ 3: ]:
 326             if i[0].f_locals.has_key( '__log__' ):
 327                 level = level + i[0].f_locals[ '__log__' ]
 328         return level
 329
 330     @staticmethod
 331     def log( message ):
 332         sys.stdout.flush()
 333         sys.stderr.flush()
 334         sys.stderr.write( '# ' + '    ' * utils.log_level() +  message + '\n' )
 335         sys.stderr.flush()
 336
 337     @staticmethod
 338     def rmtree(path):
 339         if os.path.exists( path ):
 340             #~ shutil.rmtree( unicode( path ) )
 341             if sys.platform == 'win32':
 342                 os.system( 'del /f /s /q "%s" >nul 2>&1' % path )
 343                 shutil.rmtree( unicode( path ) )
 344             else:
 345                 os.system( 'rm -f -r "%s"' % path )
 346
 347     @staticmethod
 348     def retry( f, max_attempts=5, sleep_secs=10 ):
 349         for attempts in range( max_attempts, -1, -1 ):
 350             try:
 351                 return f()
 352             except Exception, msg:
 353                 utils.log( '%s failed with message "%s"' % ( f.__name__, msg ) )
 354                 if attempts == 0:
 355                     utils.log( 'Giving up.' )
 356                     raise
 357
 358                 utils.log( 'Retrying (%d more attempts).' % attempts )
 359                 time.sleep( sleep_secs )
 360
 361     @staticmethod
 362     def web_get( source_url, destination_file, proxy = None ):
 363         import urllib
 364
 365         proxies = None
 366         if proxy is not None:
 367             proxies = {
 368                 'https' : proxy,
 369                 'http' : proxy
 370                 }
 371
 372         src = urllib.urlopen( source_url, proxies = proxies )
 373
 374         f = open( destination_file, 'wb' )
 375         while True:
 376             data = src.read( 16*1024 )
 377             if len( data ) == 0: break
 378             f.write( data )
 379
 380         f.close()
 381         src.close()
 382
 383     @staticmethod
 384     def unpack_archive( archive_path ):
 385         utils.log( 'Unpacking archive ("%s")...' % archive_path )
 386
 387         archive_name = os.path.basename( archive_path )
 388         extension = archive_name[ archive_name.find( '.' ) : ]
 389
 390         if extension in ( ".tar.gz", ".tar.bz2" ):
 391             import tarfile
 392             import stat
 393
 394             mode = os.path.splitext( extension )[1][1:]
 395             tar = tarfile.open( archive_path, 'r:%s' % mode )
 396             for tarinfo in tar:
 397                 tar.extract( tarinfo )
 398                 if sys.platform == 'win32' and not tarinfo.isdir():
 399                     # workaround what appears to be a Win32-specific bug in 'tarfile'
 400                     # (modification times for extracted files are not set properly)
 401                     f = os.path.join( os.curdir, tarinfo.name )
 402                     os.chmod( f, stat.S_IWRITE )
 403                     os.utime( f, ( tarinfo.mtime, tarinfo.mtime ) )
 404             tar.close()
 405         elif extension in ( ".zip" ):
 406             import zipfile
 407
 408             z = zipfile.ZipFile( archive_path, 'r', zipfile.ZIP_DEFLATED )
 409             for f in z.infolist():
 410                 destination_file_path = os.path.join( os.curdir, f.filename )
 411                 if destination_file_path[-1] == "/": # directory
 412                     if not os.path.exists( destination_file_path  ):
 413                         os.makedirs( destination_file_path  )
 414                 else: # file
 415                     result = open( destination_file_path, 'wb' )
 416                     result.write( z.read( f.filename ) )
 417                     result.close()
 418             z.close()
 419         else:
 420             raise 'Do not know how to unpack archives with extension \"%s\"' % extension
 421
 422     @staticmethod
 423     def make_file(filename, *text):
 424         text = string.join( text, '\n' )
 425         with codecs.open( filename, 'w', 'utf-8' ) as f:
 426             f.write( text )
 427
 428     @staticmethod
 429     def append_file(filename, *text):
 430         with codecs.open( filename, 'a', 'utf-8' ) as f:
 431             f.write( string.join( text, '\n' ) )
 432
 433     @staticmethod
 434     def mem_info():
 435         if sys.platform == "darwin":
 436             utils.call("top","-l","1","-s","0","-n","0")
 437         elif sys.platform.startswith("linux"):
 438             utils.call("free","-m","-l")
 439
 440     @staticmethod
 441     def query_boost_version(boost_root):
 442         '''
 443         Read in the Boost version from a given boost_root.
 444         '''
 445         boost_version = None
 446         if os.path.exists(os.path.join(boost_root,'Jamroot')):
 447             with codecs.open(os.path.join(boost_root,'Jamroot'), 'r', 'utf-8') as f:
 448                 for line in f.readlines():
 449                     parts = line.split()
 450                     if len(parts) >= 5 and parts[1] == 'BOOST_VERSION':
 451                         boost_version = parts[3]
 452                         break
 453         if not boost_version:
 454             boost_version = 'default'
 455         return boost_version
 456
 457     @staticmethod
 458     def git_clone(owner, repo, branch, commit = None, repo_dir = None, submodules = False, url_format = "https://github.com/%(owner)s/%(repo)s.git"):
 459         '''
 460         This clone mimicks the way Travis-CI clones a project's repo. So far
 461         Travis-CI is the most limiting in the sense of only fetching partial
 462         history of the repo.
 463         '''
 464         if not repo_dir:
 465             repo_dir = os.path.join(os.getcwd(), owner+','+repo)
 466         utils.makedirs(os.path.dirname(repo_dir))
 467         if not os.path.exists(os.path.join(repo_dir,'.git')):
 468             utils.check_call("git","clone",
 469                 "--depth=1",
 470                 "--branch=%s"%(branch),
 471                 url_format%{'owner':owner,'repo':repo},
 472                 repo_dir)
 473             os.chdir(repo_dir)
 474         else:
 475             os.chdir(repo_dir)
 476             utils.check_call("git","pull",
 477                 # "--depth=1", # Can't do depth as we get merge errors.
 478                 "--quiet","--no-recurse-submodules")
 479         if commit:
 480             utils.check_call("git","checkout","-qf",commit)
 481         if os.path.exists(os.path.join('.git','modules')):
 482             if sys.platform == 'win32':
 483                 utils.check_call('dir',os.path.join('.git','modules'))
 484             else:
 485                 utils.check_call('ls','-la',os.path.join('.git','modules'))
 486         if submodules:
 487             utils.check_call("git","submodule","--quiet","update",
 488                 "--quiet","--init","--recursive",
 489                 )
 490             utils.check_call("git","submodule","--quiet","foreach","git","fetch")
 491         return repo_dir
 492
 493 class parallel_call(threading.Thread):
 494     '''
 495     Runs a synchronous command in a thread waiting for it to complete.
 496     '''
 497
 498     def __init__(self, *command, **kargs):
 499         super(parallel_call,self).__init__()
 500         self.command = command
 501         self.command_kargs = kargs
 502         self.start()
 503
 504     def run(self):
 505         self.result = utils.call(*self.command, **self.command_kargs)
 506
 507     def join(self):
 508         super(parallel_call,self).join()
 509         if self.result != 0:
 510             raise(SystemCallError(self.command, self.result))
 511
 512 def set_arg(args, k, v = None):
 513     if not args.get(k):
 514         args[k] = v
 515     return args[k]
 516
 517 class script_common(object):
 518     '''
 519     Main script to run continuous integration.
 520     '''
 521
 522     def __init__(self, ci_klass, **kargs):
 523         self.ci = ci_klass(self)
 524
 525         opt = optparse.OptionParser(
 526             usage="%prog [options] [commands]")
 527
 528         #~ Debug Options:
 529         opt.add_option( '--debug-level',
 530             help="debugging level; controls the amount of debugging output printed",
 531             type='int' )
 532         opt.add_option( '-j',
 533             help="maximum number of parallel jobs to use for building with b2",
 534             type='int', dest='jobs')
 535         opt.add_option('--branch')
 536         opt.add_option('--commit')
 537         kargs = self.init(opt,kargs)
 538         kargs = self.ci.init(opt, kargs)
 539         set_arg(kargs,'debug_level',0)
 540         set_arg(kargs,'jobs',2)
 541         set_arg(kargs,'branch',None)
 542         set_arg(kargs,'commit',None)
 543         set_arg(kargs,'repo',None)
 544         set_arg(kargs,'repo_dir',None)
 545         set_arg(kargs,'actions',None)
 546         set_arg(kargs,'pull_request', None)
 547
 548         #~ Defaults
 549         for (k,v) in kargs.iteritems():
 550             setattr(self,k,v)
 551         ( _opt_, self.actions ) = opt.parse_args(None,self)
 552         if not self.actions or self.actions == []:
 553             self.actions = kargs.get('actions',None)
 554         if not self.actions or self.actions == []:
 555             self.actions = [ 'info' ]
 556         if not self.repo_dir:
 557             self.repo_dir = os.getcwd()
 558         self.build_dir = os.path.join(os.path.dirname(self.repo_dir), "build")
 559
 560         # API keys.
 561         self.bintray_key = os.getenv('BINTRAY_KEY')
 562
 563         try:
 564             self.start()
 565             self.command_info()
 566             self.main()
 567             utils.print_call_stats()
 568         except:
 569             utils.print_call_stats()
 570             raise
 571
 572     def init(self, opt, kargs):
 573         return kargs
 574
 575     def start(self):
 576         pass
 577
 578     def main(self):
 579         for action in self.actions:
 580             action_m = "command_"+action.replace('-','_')
 581             ci_command = getattr(self.ci, action_m, None)
 582             ci_script = getattr(self, action_m, None)
 583             if ci_command or ci_script:
 584                 utils.log( "### %s.."%(action) )
 585                 if os.path.exists(self.repo_dir):
 586                     os.chdir(self.repo_dir)
 587                 if ci_command:
 588                     ci_command()
 589                 elif ci_script:
 590                     ci_script()
 591
 592     def b2( self, *args, **kargs ):
 593         cmd = ['b2','--debug-configuration', '-j%s'%(self.jobs)]
 594         cmd.extend(args)
 595
 596         if 'toolset' in kargs:
 597             cmd.append('toolset=' + kargs['toolset'])
 598
 599         if 'parallel' in kargs:
 600             return parallel_call(*cmd)
 601         else:
 602             return utils.check_call(*cmd)
 603
 604     # Common test commands in the order they should be executed..
 605
 606     def command_info(self):
 607         pass
 608
 609     def command_install(self):
 610         utils.makedirs(self.build_dir)
 611         os.chdir(self.build_dir)
 612
 613     def command_install_toolset(self, toolset):
 614         if self.ci and hasattr(self.ci,'install_toolset'):
 615             self.ci.install_toolset(toolset)
 616
 617     def command_before_build(self):
 618         pass
 619
 620     def command_build(self):
 621         pass
 622
 623     def command_before_cache(self):
 624         pass
 625
 626     def command_after_success(self):
 627         pass
 628
 629 class ci_cli(object):
 630     '''
 631     This version of the script provides a way to do manual building. It sets up
 632     additional environment and adds fetching of the git repos that would
 633     normally be done by the CI system.
 634
 635     The common way to use this variant is to invoke something like:
 636
 637         mkdir ci
 638         cd ci
 639         python path-to/library_test.py --branch=develop [--repo=mylib] ...
 640
 641     Status: In working order.
 642     '''
 643
 644     def __init__(self,script):
 645         if sys.platform == 'darwin':
 646             # Requirements for running on OSX:
 647             # https://www.stack.nl/~dimitri/doxygen/download.html#srcbin
 648             # https://tug.org/mactex/morepackages.html
 649             doxygen_path = "/Applications/Doxygen.app/Contents/Resources"
 650             if os.path.isdir(doxygen_path):
 651                 os.environ["PATH"] = doxygen_path+':'+os.environ['PATH']
 652         self.script = script
 653         self.repo_dir = os.getcwd()
 654         self.exit_result = 0
 655
 656     def init(self, opt, kargs):
 657         kargs['actions'] = [
 658             # 'clone',
 659             'install',
 660             'before_build',
 661             'build',
 662             'before_cache',
 663             'finish'
 664             ]
 665         return kargs
 666
 667     def finish(self, result):
 668         self.exit_result = result
 669
 670     def command_finish(self):
 671         exit(self.exit_result)
 672
 673 class ci_travis(object):
 674     '''
 675     This variant build releases in the context of the Travis-CI service.
 676     '''
 677
 678     def __init__(self,script):
 679         self.script = script
 680
 681     def init(self, opt, kargs):
 682         set_arg(kargs,'repo_dir', os.getenv("TRAVIS_BUILD_DIR"))
 683         set_arg(kargs,'branch', os.getenv("TRAVIS_BRANCH"))
 684         set_arg(kargs,'commit', os.getenv("TRAVIS_COMMIT"))
 685         set_arg(kargs,'repo', os.getenv("TRAVIS_REPO_SLUG").split("/")[1])
 686         set_arg(kargs,'pull_request',
 687             os.getenv('TRAVIS_PULL_REQUEST') \
 688                 if os.getenv('TRAVIS_PULL_REQUEST') != 'false' else None)
 689         return kargs
 690
 691     def finish(self, result):
 692         exit(result)
 693
 694     def install_toolset(self, toolset):
 695         '''
 696         Installs specific toolset on CI system.
 697         '''
 698         info = toolset_info[toolset]
 699         if sys.platform.startswith('linux'):
 700             os.chdir(self.script.build_dir)
 701             if 'ppa' in info:
 702                 for ppa in info['ppa']:
 703                     utils.check_call(
 704                         'sudo','add-apt-repository','--yes',ppa)
 705             if 'deb' in info:
 706                 utils.make_file('sources.list',
 707                     "deb %s"%(' '.join(info['deb'])),
 708                     "deb-src %s"%(' '.join(info['deb'])))
 709                 utils.check_call('sudo','bash','-c','cat sources.list >> /etc/apt/sources.list')
 710             if 'apt-key' in info:
 711                 for key in info['apt-key']:
 712                     utils.check_call('wget',key,'-O','apt.key')
 713                     utils.check_call('sudo','apt-key','add','apt.key')
 714             utils.check_call(
 715                 'sudo','apt-get','update','-qq')
 716             utils.check_call(
 717                 'sudo','apt-get','install','-qq',info['package'])
 718             if 'debugpackage' in info and info['debugpackage']:
 719                 utils.check_call(
 720                     'sudo','apt-get','install','-qq',info['debugpackage'])
 721
 722     # Travis-CI commands in the order they are executed. We need
 723     # these to forward to our common commands, if they are different.
 724
 725     def command_before_install(self):
 726         pass
 727
 728     def command_install(self):
 729         self.script.command_install()
 730
 731     def command_before_script(self):
 732         self.script.command_before_build()
 733
 734     def command_script(self):
 735         self.script.command_build()
 736
 737     def command_before_cache(self):
 738         self.script.command_before_cache()
 739
 740     def command_after_success(self):
 741         self.script.command_after_success()
 742
 743     def command_after_failure(self):
 744         pass
 745
 746     def command_before_deploy(self):
 747         pass
 748
 749     def command_after_deploy(self):
 750         pass
 751
 752     def command_after_script(self):
 753         pass
 754
 755 class ci_circleci(object):
 756     '''
 757     This variant build releases in the context of the CircleCI service.
 758     '''
 759
 760     def __init__(self,script):
 761         self.script = script
 762
 763     def init(self, opt, kargs):
 764         set_arg(kargs,'repo_dir', os.path.join(os.getenv("HOME"),os.getenv("CIRCLE_PROJECT_REPONAME")))
 765         set_arg(kargs,'branch', os.getenv("CIRCLE_BRANCH"))
 766         set_arg(kargs,'commit', os.getenv("CIRCLE_SHA1"))
 767         set_arg(kargs,'repo', os.getenv("CIRCLE_PROJECT_REPONAME").split("/")[1])
 768         set_arg(kargs,'pull_request', os.getenv('CIRCLE_PR_NUMBER'))
 769         return kargs
 770
 771     def finish(self, result):
 772         exit(result)
 773
 774     def command_machine_post(self):
 775         # Apt update for the pckages installs we'll do later.
 776         utils.check_call('sudo','apt-get','-qq','update')
 777         # Need PyYAML to read Travis yaml in a later step.
 778         utils.check_call("pip","install","--user","PyYAML")
 779
 780     def command_checkout_post(self):
 781         os.chdir(self.script.repo_dir)
 782         utils.check_call("git","submodule","update","--quiet","--init","--recursive")
 783
 784     def command_dependencies_pre(self):
 785         # Read in .travis.yml for list of packages to install
 786         # as CircleCI doesn't have a convenient apt install method.
 787         import yaml
 788         utils.check_call('sudo','-E','apt-get','-yqq','update')
 789         utils.check_call('sudo','apt-get','-yqq','purge','texlive*')
 790         with open(os.path.join(self.script.repo_dir,'.travis.yml')) as yml:
 791             travis_yml = yaml.load(yml)
 792             utils.check_call('sudo','apt-get','-yqq',
 793                 '--no-install-suggests','--no-install-recommends','--force-yes','install',
 794                 *travis_yml['addons']['apt']['packages'])
 795
 796     def command_dependencies_override(self):
 797         self.script.command_install()
 798
 799     def command_dependencies_post(self):
 800         pass
 801
 802     def command_database_pre(self):
 803         pass
 804
 805     def command_database_override(self):
 806         pass
 807
 808     def command_database_post(self):
 809         pass
 810
 811     def command_test_pre(self):
 812         self.script.command_install()
 813         self.script.command_before_build()
 814
 815     def command_test_override(self):
 816         # CircleCI runs all the test subsets. So in order to avoid
 817         # running the after_success we do it here as the build step
 818         # will halt accordingly.
 819         self.script.command_build()
 820         self.script.command_before_cache()
 821         self.script.command_after_success()
 822
 823     def command_test_post(self):
 824         pass
 825
 826 class ci_appveyor(object):
 827
 828     def __init__(self,script):
 829         self.script = script
 830
 831     def init(self, opt, kargs):
 832         set_arg(kargs,'repo_dir',os.getenv("APPVEYOR_BUILD_FOLDER"))
 833         set_arg(kargs,'branch',os.getenv("APPVEYOR_REPO_BRANCH"))
 834         set_arg(kargs,'commit',os.getenv("APPVEYOR_REPO_COMMIT"))
 835         set_arg(kargs,'repo',os.getenv("APPVEYOR_REPO_NAME").split("/")[1])
 836         set_arg(kargs,'address_model',os.getenv("PLATFORM",None))
 837         set_arg(kargs,'variant',os.getenv("CONFIGURATION","debug"))
 838         set_arg(kargs,'pull_request', os.getenv('APPVEYOR_PULL_REQUEST_NUMBER'))
 839         return kargs
 840
 841     def finish(self, result):
 842         exit(result)
 843
 844     # Appveyor commands in the order they are executed. We need
 845     # these to forward to our common commands, if they are different.
 846
 847     def command_install(self):
 848         self.script.command_install()
 849
 850     def command_before_build(self):
 851         os.chdir(self.script.repo_dir)
 852         utils.check_call("git","submodule","update","--quiet","--init","--recursive")
 853         self.script.command_before_build()
 854
 855     def command_build_script(self):
 856         self.script.command_build()
 857
 858     def command_after_build(self):
 859         self.script.command_before_cache()
 860
 861     def command_before_test(self):
 862         pass
 863
 864     def command_test_script(self):
 865         pass
 866
 867     def command_after_test(self):
 868         pass
 869
 870     def command_on_success(self):
 871         self.script.command_after_success()
 872
 873     def command_on_failure(self):
 874         pass
 875
 876     def command_on_finish(self):
 877         pass
 878
 879 def main(script_klass):
 880     if os.getenv('TRAVIS', False):
 881         script_klass(ci_travis)
 882     elif os.getenv('CIRCLECI', False):
 883         script_klass(ci_circleci)
 884     elif os.getenv('APPVEYOR', False):
 885         script_klass(ci_appveyor)
 886     else:
 887         script_klass(ci_cli)