BaseTools/Source/C/Makefiles/NmakeSubdirs.py

   1 # @file NmakeSubdirs.py
   2 # This script support parallel build for nmake in windows environment.
   3 # It supports Python2.x and Python3.x both.
   4 #
   5 #  Copyright (c) 2018, Intel Corporation. All rights reserved.<BR>
   6 #
   7 #  SPDX-License-Identifier: BSD-2-Clause-Patent
   8 #
   9
  10 #
  11 # Import Modules
  12 #
  13
  14 from __future__ import print_function
  15 import argparse
  16 import threading
  17 import time
  18 import os
  19 import subprocess
  20 import multiprocessing
  21 import copy
  22 import sys
  23 __prog__        = 'NmakeSubdirs'
  24 __version__     = '%s Version %s' % (__prog__, '0.10 ')
  25 __copyright__   = 'Copyright (c) 2018, Intel Corporation. All rights reserved.'
  26 __description__ = 'Replace for NmakeSubdirs.bat in windows ,support parallel build for nmake.\n'
  27
  28 cpu_count = multiprocessing.cpu_count()
  29 output_lock = threading.Lock()
  30 def RunCommand(WorkDir=None, *Args, **kwargs):
  31     if WorkDir is None:
  32         WorkDir = os.curdir
  33     if "stderr" not in kwargs:
  34         kwargs["stderr"] = subprocess.STDOUT
  35     if "stdout" not in kwargs:
  36         kwargs["stdout"] = subprocess.PIPE
  37     p = subprocess.Popen(Args, cwd=WorkDir, stderr=kwargs["stderr"], stdout=kwargs["stdout"])
  38     stdout, stderr = p.communicate()
  39     message = ""
  40     if stdout is not None:
  41         message = stdout.decode(encoding='utf-8', errors='ignore') #for compatibility in python 2 and 3
  42
  43     if p.returncode != 0:
  44         raise RuntimeError("Error while execute command \'{0}\' in direcotry {1}\n{2}".format(" ".join(Args), WorkDir, message))
  45
  46     output_lock.acquire(True)
  47     print("execute command \"{0}\" in directory {1}".format(" ".join(Args), WorkDir))
  48     print(message)
  49     output_lock.release()
  50
  51     return p.returncode, stdout
  52
  53 class TaskUnit(object):
  54     def __init__(self, func, args, kwargs):
  55         self.func = func
  56         self.args = args
  57         self.kwargs = kwargs
  58
  59     def __eq__(self, other):
  60         return id(self).__eq__(id(other))
  61
  62     def run(self):
  63         return self.func(*self.args, **self.kwargs)
  64
  65     def __str__(self):
  66         para = list(self.args)
  67         para.extend("{0}={1}".format(k, v)for k, v in self.kwargs.items())
  68
  69         return "{0}({1})".format(self.func.__name__, ",".join(para))
  70
  71 class ThreadControl(object):
  72
  73     def __init__(self, maxthread):
  74         self._processNum = maxthread
  75         self.pending = []
  76         self.running = []
  77         self.pendingLock = threading.Lock()
  78         self.runningLock = threading.Lock()
  79         self.error = False
  80         self.errorLock = threading.Lock()
  81         self.errorMsg = "errorMsg"
  82
  83     def addTask(self, func, *args, **kwargs):
  84         self.pending.append(TaskUnit(func, args, kwargs))
  85
  86     def waitComplete(self):
  87         self._schedule.join()
  88
  89     def startSchedule(self):
  90         self._schedule = threading.Thread(target=self.Schedule)
  91         self._schedule.start()
  92
  93     def Schedule(self):
  94         for i in range(self._processNum):
  95             task = threading.Thread(target=self.startTask)
  96             task.daemon = False
  97             self.running.append(task)
  98
  99         self.runningLock.acquire(True)
 100         for thread in self.running:
 101             thread.start()
 102         self.runningLock.release()
 103
 104         while len(self.running) > 0:
 105             time.sleep(0.1)
 106         if self.error:
 107             print("subprocess not exit successfully")
 108             print(self.errorMsg)
 109
 110     def startTask(self):
 111         while True:
 112             if self.error:
 113                 break
 114             self.pendingLock.acquire(True)
 115             if len(self.pending) == 0:
 116                 self.pendingLock.release()
 117                 break
 118             task = self.pending.pop(0)
 119             self.pendingLock.release()
 120             try:
 121                 task.run()
 122             except RuntimeError as e:
 123                 if self.error: break
 124                 self.errorLock.acquire(True)
 125                 self.error = True
 126                 self.errorMsg = str(e)
 127                 time.sleep(0.1)
 128                 self.errorLock.release()
 129                 break
 130
 131         self.runningLock.acquire(True)
 132         self.running.remove(threading.currentThread())
 133         self.runningLock.release()
 134
 135 def Run():
 136     curdir = os.path.abspath(os.curdir)
 137     if len(args.subdirs) == 1:
 138         args.jobs = 1
 139     if args.jobs == 1:
 140         try:
 141             for dir in args.subdirs:
 142                 RunCommand(os.path.join(curdir, dir), "nmake", args.target, stdout=sys.stdout, stderr=subprocess.STDOUT)
 143         except RuntimeError:
 144             exit(1)
 145     else:
 146         controller = ThreadControl(args.jobs)
 147         for dir in args.subdirs:
 148             controller.addTask(RunCommand, os.path.join(curdir, dir), "nmake", args.target)
 149         controller.startSchedule()
 150         controller.waitComplete()
 151         if controller.error:
 152             exit(1)
 153
 154 if __name__ == "__main__":
 155     parser = argparse.ArgumentParser(prog=__prog__, description=__description__ + __copyright__, conflict_handler='resolve')
 156
 157     parser.add_argument("target", help="the target for nmake")
 158     parser.add_argument("subdirs", nargs="+", help="the relative dir path of makefile")
 159     parser.add_argument("--jobs", type=int, dest="jobs", default=cpu_count, help="thread number")
 160     parser.add_argument('--version', action='version', version=__version__)
 161     args = parser.parse_args()
 162     Run()
 163