BaseTools/Source/C/Makefiles/NmakeSubdirs.py

   1 # @file NmakeSubdirs.py
   2 # This script support parallel build for nmake in windows environment.
   3 # It supports Python2.x and Python3.x both.
   4 #
   5 #  Copyright (c) 2018, Intel Corporation. All rights reserved.<BR>
   6 #
   7 #  This program and the accompanying materials
   8 #  are licensed and made available under the terms and conditions of the BSD License
   9 #  which accompanies this distribution.  The full text of the license may be found at
  10 #  http://opensource.org/licenses/bsd-license.php
  11 #
  12 #  THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  13 #  WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  14 #
  15
  16 #
  17 # Import Modules
  18 #
  19
  20 from __future__ import print_function
  21 import argparse
  22 import threading
  23 import time
  24 import os
  25 import subprocess
  26 import multiprocessing
  27 import copy
  28 import sys
  29 __prog__        = 'NmakeSubdirs'
  30 __version__     = '%s Version %s' % (__prog__, '0.10 ')
  31 __copyright__   = 'Copyright (c) 2018, Intel Corporation. All rights reserved.'
  32 __description__ = 'Replace for NmakeSubdirs.bat in windows ,support parallel build for nmake.\n'
  33
  34 cpu_count = multiprocessing.cpu_count()
  35 output_lock = threading.Lock()
  36 def RunCommand(WorkDir=None, *Args, **kwargs):
  37     if WorkDir is None:
  38         WorkDir = os.curdir
  39     if "stderr" not in kwargs:
  40         kwargs["stderr"] = subprocess.STDOUT
  41     if "stdout" not in kwargs:
  42         kwargs["stdout"] = subprocess.PIPE
  43     p = subprocess.Popen(Args, cwd=WorkDir, stderr=kwargs["stderr"], stdout=kwargs["stdout"])
  44     stdout, stderr = p.communicate()
  45     message = ""
  46     if stdout is not None:
  47         message = stdout.decode(encoding='utf-8', errors='ignore') #for compatibility in python 2 and 3
  48
  49     if p.returncode != 0:
  50         raise RuntimeError("Error while execute command \'{0}\' in direcotry {1}\n{2}".format(" ".join(Args), WorkDir, message))
  51
  52     output_lock.acquire(True)
  53     print("execute command \"{0}\" in directory {1}".format(" ".join(Args), WorkDir))
  54     print(message)
  55     output_lock.release()
  56
  57     return p.returncode, stdout
  58
  59 class TaskUnit(object):
  60     def __init__(self, func, args, kwargs):
  61         self.func = func
  62         self.args = args
  63         self.kwargs = kwargs
  64
  65     def __eq__(self, other):
  66         return id(self).__eq__(id(other))
  67
  68     def run(self):
  69         return self.func(*self.args, **self.kwargs)
  70
  71     def __str__(self):
  72         para = list(self.args)
  73         para.extend("{0}={1}".format(k, v)for k, v in self.kwargs.items())
  74
  75         return "{0}({1})".format(self.func.__name__, ",".join(para))
  76
  77 class ThreadControl(object):
  78
  79     def __init__(self, maxthread):
  80         self._processNum = maxthread
  81         self.pending = []
  82         self.running = []
  83         self.pendingLock = threading.Lock()
  84         self.runningLock = threading.Lock()
  85         self.error = False
  86         self.errorLock = threading.Lock()
  87         self.errorMsg = "errorMsg"
  88
  89     def addTask(self, func, *args, **kwargs):
  90         self.pending.append(TaskUnit(func, args, kwargs))
  91
  92     def waitComplete(self):
  93         self._schedule.join()
  94
  95     def startSchedule(self):
  96         self._schedule = threading.Thread(target=self.Schedule)
  97         self._schedule.start()
  98
  99     def Schedule(self):
 100         for i in range(self._processNum):
 101             task = threading.Thread(target=self.startTask)
 102             task.daemon = False
 103             self.running.append(task)
 104
 105         self.runningLock.acquire(True)
 106         for thread in self.running:
 107             thread.start()
 108         self.runningLock.release()
 109
 110         while len(self.running) > 0:
 111             time.sleep(0.1)
 112         if self.error:
 113             print("subprocess not exit sucessfully")
 114             print(self.errorMsg)
 115
 116     def startTask(self):
 117         while True:
 118             if self.error:
 119                 break
 120             self.pendingLock.acquire(True)
 121             if len(self.pending) == 0:
 122                 self.pendingLock.release()
 123                 break
 124             task = self.pending.pop(0)
 125             self.pendingLock.release()
 126             try:
 127                 task.run()
 128             except RuntimeError as e:
 129                 if self.error: break
 130                 self.errorLock.acquire(True)
 131                 self.error = True
 132                 self.errorMsg = str(e)
 133                 time.sleep(0.1)
 134                 self.errorLock.release()
 135                 break
 136
 137         self.runningLock.acquire(True)
 138         self.running.remove(threading.currentThread())
 139         self.runningLock.release()
 140
 141 def Run():
 142     curdir = os.path.abspath(os.curdir)
 143     if len(args.subdirs) == 1:
 144         args.jobs = 1
 145     if args.jobs == 1:
 146         try:
 147             for dir in args.subdirs:
 148                 RunCommand(os.path.join(curdir, dir), "nmake", args.target, stdout=sys.stdout, stderr=subprocess.STDOUT)
 149         except RuntimeError:
 150             exit(1)
 151     else:
 152         controller = ThreadControl(args.jobs)
 153         for dir in args.subdirs:
 154             controller.addTask(RunCommand, os.path.join(curdir, dir), "nmake", args.target)
 155         controller.startSchedule()
 156         controller.waitComplete()
 157         if controller.error:
 158             exit(1)
 159
 160 if __name__ == "__main__":
 161     parser = argparse.ArgumentParser(prog=__prog__, description=__description__ + __copyright__, conflict_handler='resolve')
 162
 163     parser.add_argument("target", help="the target for nmake")
 164     parser.add_argument("subdirs", nargs="+", help="the relative dir path of makefile")
 165     parser.add_argument("--jobs", type=int, dest="jobs", default=cpu_count, help="thread number")
 166     parser.add_argument('--version', action='version', version=__version__)
 167     args = parser.parse_args()
 168     Run()
 169