BaseTools/Source/C/Makefiles/NmakeSubdirs.py

   1 # @file NmakeSubdirs.py
   2 # This script support parallel build for nmake in windows environment.
   3 # It supports Python2.x and Python3.x both.
   4 #
   5 #  Copyright (c) 2018, Intel Corporation. All rights reserved.<BR>
   6 #
   7 #  SPDX-License-Identifier: BSD-2-Clause-Patent
   8 #
   9
  10 #
  11 # Import Modules
  12 #
  13
  14 from __future__ import print_function
  15 import argparse
  16 import threading
  17 import time
  18 import os
  19 import subprocess
  20 import multiprocessing
  21 import copy
  22 import sys
  23 __prog__        = 'NmakeSubdirs'
  24 __version__     = '%s Version %s' % (__prog__, '0.10 ')
  25 __copyright__   = 'Copyright (c) 2018, Intel Corporation. All rights reserved.'
  26 __description__ = 'Replace for NmakeSubdirs.bat in windows ,support parallel build for nmake.\n'
  27
  28 cpu_count = multiprocessing.cpu_count()
  29 output_lock = threading.Lock()
  30 def RunCommand(WorkDir=None, *Args, **kwargs):
  31     if WorkDir is None:
  32         WorkDir = os.curdir
  33     if "stderr" not in kwargs:
  34         kwargs["stderr"] = subprocess.STDOUT
  35     if "stdout" not in kwargs:
  36         kwargs["stdout"] = subprocess.PIPE
  37     p = subprocess.Popen(Args, cwd=WorkDir, stderr=kwargs["stderr"], stdout=kwargs["stdout"])
  38     stdout, stderr = p.communicate()
  39     message = ""
  40     if stdout is not None:
  41         message = stdout.decode(errors='ignore') #for compatibility in python 2 and 3
  42
  43     if p.returncode != 0:
  44         raise RuntimeError("Error while execute command \'{0}\' in direcotry {1}\n{2}".format(" ".join(Args), WorkDir, message))
  45
  46     output_lock.acquire(True)
  47     print("execute command \"{0}\" in directory {1}".format(" ".join(Args), WorkDir))
  48     try:
  49         print(message)
  50     except:
  51         pass
  52     output_lock.release()
  53
  54     return p.returncode, stdout
  55
  56 class TaskUnit(object):
  57     def __init__(self, func, args, kwargs):
  58         self.func = func
  59         self.args = args
  60         self.kwargs = kwargs
  61
  62     def __eq__(self, other):
  63         return id(self).__eq__(id(other))
  64
  65     def run(self):
  66         return self.func(*self.args, **self.kwargs)
  67
  68     def __str__(self):
  69         para = list(self.args)
  70         para.extend("{0}={1}".format(k, v)for k, v in self.kwargs.items())
  71
  72         return "{0}({1})".format(self.func.__name__, ",".join(para))
  73
  74 class ThreadControl(object):
  75
  76     def __init__(self, maxthread):
  77         self._processNum = maxthread
  78         self.pending = []
  79         self.running = []
  80         self.pendingLock = threading.Lock()
  81         self.runningLock = threading.Lock()
  82         self.error = False
  83         self.errorLock = threading.Lock()
  84         self.errorMsg = "errorMsg"
  85
  86     def addTask(self, func, *args, **kwargs):
  87         self.pending.append(TaskUnit(func, args, kwargs))
  88
  89     def waitComplete(self):
  90         self._schedule.join()
  91
  92     def startSchedule(self):
  93         self._schedule = threading.Thread(target=self.Schedule)
  94         self._schedule.start()
  95
  96     def Schedule(self):
  97         for i in range(self._processNum):
  98             task = threading.Thread(target=self.startTask)
  99             task.daemon = False
 100             self.running.append(task)
 101
 102         self.runningLock.acquire(True)
 103         for thread in self.running:
 104             thread.start()
 105         self.runningLock.release()
 106
 107         while len(self.running) > 0:
 108             time.sleep(0.1)
 109         if self.error:
 110             print("subprocess not exit successfully")
 111             print(self.errorMsg)
 112
 113     def startTask(self):
 114         while True:
 115             if self.error:
 116                 break
 117             self.pendingLock.acquire(True)
 118             if len(self.pending) == 0:
 119                 self.pendingLock.release()
 120                 break
 121             task = self.pending.pop(0)
 122             self.pendingLock.release()
 123             try:
 124                 task.run()
 125             except RuntimeError as e:
 126                 if self.error: break
 127                 self.errorLock.acquire(True)
 128                 self.error = True
 129                 self.errorMsg = str(e)
 130                 time.sleep(0.1)
 131                 self.errorLock.release()
 132                 break
 133
 134         self.runningLock.acquire(True)
 135         self.running.remove(threading.currentThread())
 136         self.runningLock.release()
 137
 138 def Run():
 139     curdir = os.path.abspath(os.curdir)
 140     if len(args.subdirs) == 1:
 141         args.jobs = 1
 142     if args.jobs == 1:
 143         try:
 144             for dir in args.subdirs:
 145                 RunCommand(os.path.join(curdir, dir), "nmake", args.target, stdout=sys.stdout, stderr=subprocess.STDOUT)
 146         except RuntimeError:
 147             exit(1)
 148     else:
 149         controller = ThreadControl(args.jobs)
 150         for dir in args.subdirs:
 151             controller.addTask(RunCommand, os.path.join(curdir, dir), "nmake", args.target)
 152         controller.startSchedule()
 153         controller.waitComplete()
 154         if controller.error:
 155             exit(1)
 156
 157 if __name__ == "__main__":
 158     parser = argparse.ArgumentParser(prog=__prog__, description=__description__ + __copyright__, conflict_handler='resolve')
 159
 160     parser.add_argument("target", help="the target for nmake")
 161     parser.add_argument("subdirs", nargs="+", help="the relative dir path of makefile")
 162     parser.add_argument("--jobs", type=int, dest="jobs", default=cpu_count, help="thread number")
 163     parser.add_argument('--version', action='version', version=__version__)
 164     args = parser.parse_args()
 165     Run()
 166