ceph/src/boost/tools/build/src/build/project.py

   1 # Status: ported.
   2 # Base revision: 64488
   3
   4 # Copyright 2002, 2003 Dave Abrahams
   5 # Copyright 2002, 2005, 2006 Rene Rivera
   6 # Copyright 2002, 2003, 2004, 2005, 2006 Vladimir Prus
   7 # Distributed under the Boost Software License, Version 1.0.
   8 # (See accompanying file LICENSE_1_0.txt or copy at
   9 # http://www.boost.org/LICENSE_1_0.txt)
  10
  11 # Implements project representation and loading. Each project is represented
  12 # by:
  13 #  - a module where all the Jamfile content live.
  14 #  - an instance of 'project-attributes' class.
  15 #    (given a module name, can be obtained using the 'attributes' rule)
  16 #  - an instance of 'project-target' class (from targets.jam)
  17 #    (given a module name, can be obtained using the 'target' rule)
  18 #
  19 # Typically, projects are created as result of loading a Jamfile, which is done
  20 # by rules 'load' and 'initialize', below. First, module for Jamfile is loaded
  21 # and new project-attributes instance is created. Some rules necessary for
  22 # project are added to the module (see 'project-rules' module) at the bottom of
  23 # this file. Default project attributes are set (inheriting attributes of
  24 # parent project, if it exists). After that the Jamfile is read. It can declare
  25 # its own attributes using the 'project' rule which will be combined with any
  26 # already set attributes.
  27 #
  28 # The 'project' rule can also declare a project id which will be associated
  29 # with the project module.
  30 #
  31 # There can also be 'standalone' projects. They are created by calling
  32 # 'initialize' on an arbitrary module and not specifying their location. After
  33 # the call, the module can call the 'project' rule, declare main targets and
  34 # behave as a regular project except that, since it is not associated with any
  35 # location, it should only declare prebuilt targets.
  36 #
  37 # The list of all loaded Jamfiles is stored in the .project-locations variable.
  38 # It is possible to obtain a module name for a location using the 'module-name'
  39 # rule. Standalone projects are not recorded and can only be references using
  40 # their project id.
  41
  42 import b2.util.path
  43 import b2.build.targets
  44 from b2.build import property_set, property
  45 from b2.build.errors import ExceptionWithUserContext
  46 from b2.manager import get_manager
  47
  48 import bjam
  49 import b2
  50
  51 import re
  52 import sys
  53 import pkgutil
  54 import os
  55 import string
  56 import imp
  57 import traceback
  58 import b2.util.option as option
  59
  60 from b2.util import (
  61     record_jam_to_value_mapping, qualify_jam_action, is_iterable_typed, bjam_signature,
  62     is_iterable)
  63
  64
  65 class ProjectRegistry:
  66
  67     def __init__(self, manager, global_build_dir):
  68         self.manager = manager
  69         self.global_build_dir = global_build_dir
  70         self.project_rules_ = ProjectRules(self)
  71
  72         # The target corresponding to the project being loaded now
  73         self.current_project = None
  74
  75         # The set of names of loaded project modules
  76         self.jamfile_modules = {}
  77
  78         # Mapping from location to module name
  79         self.location2module = {}
  80
  81         # Mapping from project id to project module
  82         self.id2module = {}
  83
  84         # Map from Jamfile directory to parent Jamfile/Jamroot
  85         # location.
  86         self.dir2parent_jamfile = {}
  87
  88         # Map from directory to the name of Jamfile in
  89         # that directory (or None).
  90         self.dir2jamfile = {}
  91
  92         # Map from project module to attributes object.
  93         self.module2attributes = {}
  94
  95         # Map from project module to target for the project
  96         self.module2target = {}
  97
  98         # Map from names to Python modules, for modules loaded
  99         # via 'using' and 'import' rules in Jamfiles.
 100         self.loaded_tool_modules_ = {}
 101
 102         self.loaded_tool_module_path_ = {}
 103
 104         # Map from project target to the list of
 105         # (id,location) pairs corresponding to all 'use-project'
 106         # invocations.
 107         # TODO: should not have a global map, keep this
 108         # in ProjectTarget.
 109         self.used_projects = {}
 110
 111         self.saved_current_project = []
 112
 113         self.JAMROOT = self.manager.getenv("JAMROOT");
 114
 115         # Note the use of character groups, as opposed to listing
 116         # 'Jamroot' and 'jamroot'. With the latter, we'd get duplicate
 117         # matches on windows and would have to eliminate duplicates.
 118         if not self.JAMROOT:
 119             self.JAMROOT = ["project-root.jam", "[Jj]amroot", "[Jj]amroot.jam"]
 120
 121         # Default patterns to search for the Jamfiles to use for build
 122         # declarations.
 123         self.JAMFILE = self.manager.getenv("JAMFILE")
 124
 125         if not self.JAMFILE:
 126             self.JAMFILE = ["[Bb]uild.jam", "[Jj]amfile.v2", "[Jj]amfile",
 127                             "[Jj]amfile.jam"]
 128
 129         self.__python_module_cache = {}
 130
 131
 132     def load (self, jamfile_location):
 133         """Loads jamfile at the given location. After loading, project global
 134         file and jamfile needed by the loaded one will be loaded recursively.
 135         If the jamfile at that location is loaded already, does nothing.
 136         Returns the project module for the Jamfile."""
 137         assert isinstance(jamfile_location, basestring)
 138
 139         absolute = os.path.join(os.getcwd(), jamfile_location)
 140         absolute = os.path.normpath(absolute)
 141         jamfile_location = b2.util.path.relpath(os.getcwd(), absolute)
 142
 143         mname = self.module_name(jamfile_location)
 144         # If Jamfile is already loaded, do not try again.
 145         if not mname in self.jamfile_modules:
 146
 147             if "--debug-loading" in self.manager.argv():
 148                 print "Loading Jamfile at '%s'" % jamfile_location
 149
 150             self.load_jamfile(jamfile_location, mname)
 151
 152             # We want to make sure that child project are loaded only
 153             # after parent projects. In particular, because parent projects
 154             # define attributes which are inherited by children, and we do not
 155             # want children to be loaded before parents has defined everything.
 156             #
 157             # While "build-project" and "use-project" can potentially refer
 158             # to child projects from parent projects, we do not immediately
 159             # load child projects when seing those attributes. Instead,
 160             # we record the minimal information that will be used only later.
 161
 162             self.load_used_projects(mname)
 163
 164         return mname
 165
 166     def load_used_projects(self, module_name):
 167         assert isinstance(module_name, basestring)
 168         # local used = [ modules.peek $(module-name) : .used-projects ] ;
 169         used = self.used_projects[module_name]
 170
 171         location = self.attribute(module_name, "location")
 172         for u in used:
 173             id = u[0]
 174             where = u[1]
 175
 176             self.use(id, os.path.join(location, where))
 177
 178     def load_parent(self, location):
 179         """Loads parent of Jamfile at 'location'.
 180         Issues an error if nothing is found."""
 181         assert isinstance(location, basestring)
 182         found = b2.util.path.glob_in_parents(
 183             location, self.JAMROOT + self.JAMFILE)
 184
 185         if not found:
 186             print "error: Could not find parent for project at '%s'" % location
 187             print "error: Did not find Jamfile.jam or Jamroot.jam in any parent directory."
 188             sys.exit(1)
 189
 190         return self.load(os.path.dirname(found[0]))
 191
 192     def find(self, name, current_location):
 193         """Given 'name' which can be project-id or plain directory name,
 194         return project module corresponding to that id or directory.
 195         Returns nothing of project is not found."""
 196         assert isinstance(name, basestring)
 197         assert isinstance(current_location, basestring)
 198
 199         project_module = None
 200
 201         # Try interpreting name as project id.
 202         if name[0] == '/':
 203             project_module = self.id2module.get(name)
 204
 205         if not project_module:
 206             location = os.path.join(current_location, name)
 207             # If no project is registered for the given location, try to
 208             # load it. First see if we have Jamfile. If not we might have project
 209             # root, willing to act as Jamfile. In that case, project-root
 210             # must be placed in the directory referred by id.
 211
 212             project_module = self.module_name(location)
 213             if not project_module in self.jamfile_modules:
 214                 if b2.util.path.glob([location], self.JAMROOT + self.JAMFILE):
 215                     project_module = self.load(location)
 216                 else:
 217                     project_module = None
 218
 219         return project_module
 220
 221     def module_name(self, jamfile_location):
 222         """Returns the name of module corresponding to 'jamfile-location'.
 223         If no module corresponds to location yet, associates default
 224         module name with that location."""
 225         assert isinstance(jamfile_location, basestring)
 226         module = self.location2module.get(jamfile_location)
 227         if not module:
 228             # Root the path, so that locations are always umbiguious.
 229             # Without this, we can't decide if '../../exe/program1' and '.'
 230             # are the same paths, or not.
 231             jamfile_location = os.path.realpath(
 232                 os.path.join(os.getcwd(), jamfile_location))
 233             module = "Jamfile<%s>" % jamfile_location
 234             self.location2module[jamfile_location] = module
 235         return module
 236
 237     def find_jamfile (self, dir, parent_root=0, no_errors=0):
 238         """Find the Jamfile at the given location. This returns the
 239         exact names of all the Jamfiles in the given directory. The optional
 240         parent-root argument causes this to search not the given directory
 241         but the ones above it up to the directory given in it."""
 242         assert isinstance(dir, basestring)
 243         assert isinstance(parent_root, (int, bool))
 244         assert isinstance(no_errors, (int, bool))
 245
 246         # Glob for all the possible Jamfiles according to the match pattern.
 247         #
 248         jamfile_glob = None
 249         if parent_root:
 250             parent = self.dir2parent_jamfile.get(dir)
 251             if not parent:
 252                 parent = b2.util.path.glob_in_parents(dir,
 253                                                                self.JAMFILE)
 254                 self.dir2parent_jamfile[dir] = parent
 255             jamfile_glob = parent
 256         else:
 257             jamfile = self.dir2jamfile.get(dir)
 258             if not jamfile:
 259                 jamfile = b2.util.path.glob([dir], self.JAMFILE)
 260                 self.dir2jamfile[dir] = jamfile
 261             jamfile_glob = jamfile
 262
 263         if len(jamfile_glob) > 1:
 264             # Multiple Jamfiles found in the same place. Warn about this.
 265             # And ensure we use only one of them.
 266             # As a temporary convenience measure, if there's Jamfile.v2 amount
 267             # found files, suppress the warning and use it.
 268             #
 269             pattern = "(.*[Jj]amfile\\.v2)|(.*[Bb]uild\\.jam)"
 270             v2_jamfiles = [x for x in jamfile_glob if re.match(pattern, x)]
 271             if len(v2_jamfiles) == 1:
 272                 jamfile_glob = v2_jamfiles
 273             else:
 274                 print """warning: Found multiple Jamfiles at '%s'!""" % (dir)
 275                 for j in jamfile_glob:
 276                     print "    -", j
 277                 print "Loading the first one"
 278
 279         # Could not find it, error.
 280         if not no_errors and not jamfile_glob:
 281             self.manager.errors()(
 282                 """Unable to load Jamfile.
 283 Could not find a Jamfile in directory '%s'
 284 Attempted to find it with pattern '%s'.
 285 Please consult the documentation at 'http://boost.org/boost-build2'."""
 286                 % (dir, string.join(self.JAMFILE)))
 287
 288         if jamfile_glob:
 289             return jamfile_glob[0]
 290
 291     def load_jamfile(self, dir, jamfile_module):
 292         """Load a Jamfile at the given directory. Returns nothing.
 293         Will attempt to load the file as indicated by the JAMFILE patterns.
 294         Effect of calling this rule twice with the same 'dir' is underfined."""
 295         assert isinstance(dir, basestring)
 296         assert isinstance(jamfile_module, basestring)
 297
 298         # See if the Jamfile is where it should be.
 299         is_jamroot = False
 300         jamfile_to_load = b2.util.path.glob([dir], self.JAMROOT)
 301         if jamfile_to_load:
 302             if len(jamfile_to_load) > 1:
 303                 get_manager().errors()(
 304                     "Multiple Jamfiles found at '{}'\n"
 305                     "Filenames are: {}"
 306                     .format(dir, ' '.join(os.path.basename(j) for j in jamfile_to_load))
 307                 )
 308             is_jamroot = True
 309             jamfile_to_load = jamfile_to_load[0]
 310         else:
 311             jamfile_to_load = self.find_jamfile(dir)
 312
 313         dir = os.path.dirname(jamfile_to_load)
 314         if not dir:
 315             dir = "."
 316
 317         self.used_projects[jamfile_module] = []
 318
 319         # Now load the Jamfile in it's own context.
 320         # The call to 'initialize' may load parent Jamfile, which might have
 321         # 'use-project' statement that causes a second attempt to load the
 322         # same project we're loading now.  Checking inside .jamfile-modules
 323         # prevents that second attempt from messing up.
 324         if not jamfile_module in self.jamfile_modules:
 325             previous_project = self.current_project
 326             # Initialize the jamfile module before loading.
 327             self.initialize(jamfile_module, dir, os.path.basename(jamfile_to_load))
 328
 329             if not jamfile_module in self.jamfile_modules:
 330                 saved_project = self.current_project
 331                 self.jamfile_modules[jamfile_module] = True
 332
 333                 bjam.call("load", jamfile_module, jamfile_to_load)
 334
 335                 if is_jamroot:
 336                     jamfile = self.find_jamfile(dir, no_errors=True)
 337                     if jamfile:
 338                         bjam.call("load", jamfile_module, jamfile)
 339
 340                 # Now do some checks
 341                 if self.current_project != saved_project:
 342                     from textwrap import dedent
 343                     self.manager.errors()(dedent(
 344                         """
 345                         The value of the .current-project variable has magically changed
 346                         after loading a Jamfile. This means some of the targets might be
 347                         defined a the wrong project.
 348                         after loading %s
 349                         expected value %s
 350                         actual value %s
 351                         """
 352                         % (jamfile_module, saved_project, self.current_project)
 353                     ))
 354
 355                 self.end_load(previous_project)
 356
 357                 if self.global_build_dir:
 358                     id = self.attributeDefault(jamfile_module, "id", None)
 359                     project_root = self.attribute(jamfile_module, "project-root")
 360                     location = self.attribute(jamfile_module, "location")
 361
 362                     if location and project_root == dir:
 363                         # This is Jamroot
 364                         if not id:
 365                             # FIXME: go via errors module, so that contexts are
 366                             # shown?
 367                             print "warning: the --build-dir option was specified"
 368                             print "warning: but Jamroot at '%s'" % dir
 369                             print "warning: specified no project id"
 370                             print "warning: the --build-dir option will be ignored"
 371
 372     def end_load(self, previous_project=None):
 373         if not self.current_project:
 374             self.manager.errors()(
 375                 'Ending project loading requested when there was no project currently '
 376                 'being loaded.'
 377             )
 378
 379         if not previous_project and self.saved_current_project:
 380             self.manager.errors()(
 381                 'Ending project loading requested with no "previous project" when there '
 382                 'other projects still being loaded recursively.'
 383             )
 384
 385         self.current_project = previous_project
 386
 387     def load_standalone(self, jamfile_module, file):
 388         """Loads 'file' as standalone project that has no location
 389         associated with it.  This is mostly useful for user-config.jam,
 390         which should be able to define targets, but although it has
 391         some location in filesystem, we do not want any build to
 392         happen in user's HOME, for example.
 393
 394         The caller is required to never call this method twice on
 395         the same file.
 396         """
 397         assert isinstance(jamfile_module, basestring)
 398         assert isinstance(file, basestring)
 399
 400         self.used_projects[jamfile_module] = []
 401         bjam.call("load", jamfile_module, file)
 402         self.load_used_projects(jamfile_module)
 403
 404     def is_jamroot(self, basename):
 405         assert isinstance(basename, basestring)
 406         match = [ pat for pat in self.JAMROOT if re.match(pat, basename)]
 407         if match:
 408             return 1
 409         else:
 410             return 0
 411
 412     def initialize(self, module_name, location=None, basename=None, standalone_path=''):
 413         """Initialize the module for a project.
 414
 415         module-name is the name of the project module.
 416         location is the location (directory) of the project to initialize.
 417                  If not specified, standalone project will be initialized
 418         standalone_path is the path to the source-location.
 419                         this should only be called from the python side.
 420         """
 421         assert isinstance(module_name, basestring)
 422         assert isinstance(location, basestring) or location is None
 423         assert isinstance(basename, basestring) or basename is None
 424         jamroot = False
 425         parent_module = None
 426         if module_name == "test-config":
 427             # No parent
 428             pass
 429         elif module_name == "site-config":
 430             parent_module = "test-config"
 431         elif module_name == "user-config":
 432             parent_module = "site-config"
 433         elif module_name == "project-config":
 434             parent_module = "user-config"
 435         elif location and not self.is_jamroot(basename):
 436             # We search for parent/project-root only if jamfile was specified
 437             # --- i.e
 438             # if the project is not standalone.
 439             parent_module = self.load_parent(location)
 440         elif location:
 441             # It's either jamroot, or standalone project.
 442             # If it's jamroot, inherit from user-config.
 443             # If project-config module exist, inherit from it.
 444             parent_module = 'user-config'
 445             if 'project-config' in self.module2attributes:
 446                 parent_module = 'project-config'
 447             jamroot = True
 448
 449         # TODO: need to consider if standalone projects can do anything but defining
 450         # prebuilt targets. If so, we need to give more sensible "location", so that
 451         # source paths are correct.
 452         if not location:
 453             location = ""
 454
 455         # the call to load_parent() above can end up loading this module again
 456         # make sure we don't reinitialize the module's attributes
 457         if module_name not in self.module2attributes:
 458             if "--debug-loading" in self.manager.argv():
 459                 print "Initializing project '%s'" % module_name
 460             attributes = ProjectAttributes(self.manager, location, module_name)
 461             self.module2attributes[module_name] = attributes
 462
 463             python_standalone = False
 464             if location:
 465                 attributes.set("source-location", [location], exact=1)
 466             elif not module_name in ["test-config", "site-config", "user-config", "project-config"]:
 467                 # This is a standalone project with known location. Set source location
 468                 # so that it can declare targets. This is intended so that you can put
 469                 # a .jam file in your sources and use it via 'using'. Standard modules
 470                 # (in 'tools' subdir) may not assume source dir is set.
 471                 source_location = standalone_path
 472                 if not source_location:
 473                     source_location = self.loaded_tool_module_path_.get(module_name)
 474                 if not source_location:
 475                     self.manager.errors()('Standalone module path not found for "{}"'
 476                                           .format(module_name))
 477                 attributes.set("source-location", [source_location], exact=1)
 478                 python_standalone = True
 479
 480             attributes.set("requirements", property_set.empty(), exact=True)
 481             attributes.set("usage-requirements", property_set.empty(), exact=True)
 482             attributes.set("default-build", property_set.empty(), exact=True)
 483             attributes.set("projects-to-build", [], exact=True)
 484             attributes.set("project-root", None, exact=True)
 485             attributes.set("build-dir", None, exact=True)
 486
 487             self.project_rules_.init_project(module_name, python_standalone)
 488
 489             if parent_module:
 490                 self.inherit_attributes(module_name, parent_module)
 491                 attributes.set("parent-module", parent_module, exact=1)
 492
 493             if jamroot:
 494                 attributes.set("project-root", location, exact=1)
 495
 496             parent = None
 497             if parent_module:
 498                 parent = self.target(parent_module)
 499
 500             if module_name not in self.module2target:
 501                 target = b2.build.targets.ProjectTarget(self.manager,
 502                     module_name, module_name, parent,
 503                     self.attribute(module_name, "requirements"),
 504                     # FIXME: why we need to pass this? It's not
 505                     # passed in jam code.
 506                     self.attribute(module_name, "default-build"))
 507                 self.module2target[module_name] = target
 508
 509         self.current_project = self.target(module_name)
 510
 511     def inherit_attributes(self, project_module, parent_module):
 512         """Make 'project-module' inherit attributes of project
 513         root and parent module."""
 514         assert isinstance(project_module, basestring)
 515         assert isinstance(parent_module, basestring)
 516
 517         attributes = self.module2attributes[project_module]
 518         pattributes = self.module2attributes[parent_module]
 519
 520         # Parent module might be locationless user-config.
 521         # FIXME:
 522         #if [ modules.binding $(parent-module) ]
 523         #{
 524         #    $(attributes).set parent : [ path.parent
 525         #                                 [ path.make [ modules.binding $(parent-module) ] ] ] ;
 526         #    }
 527
 528         attributes.set("project-root", pattributes.get("project-root"), exact=True)
 529         attributes.set("default-build", pattributes.get("default-build"), exact=True)
 530         attributes.set("requirements", pattributes.get("requirements"), exact=True)
 531         attributes.set("usage-requirements",
 532                        pattributes.get("usage-requirements"), exact=1)
 533
 534         parent_build_dir = pattributes.get("build-dir")
 535
 536         if parent_build_dir:
 537         # Have to compute relative path from parent dir to our dir
 538         # Convert both paths to absolute, since we cannot
 539         # find relative path from ".." to "."
 540
 541              location = attributes.get("location")
 542              parent_location = pattributes.get("location")
 543
 544              our_dir = os.path.join(os.getcwd(), location)
 545              parent_dir = os.path.join(os.getcwd(), parent_location)
 546
 547              build_dir = os.path.join(parent_build_dir,
 548                                       os.path.relpath(our_dir, parent_dir))
 549              attributes.set("build-dir", build_dir, exact=True)
 550
 551     def register_id(self, id, module):
 552         """Associate the given id with the given project module."""
 553         assert isinstance(id, basestring)
 554         assert isinstance(module, basestring)
 555         self.id2module[id] = module
 556
 557     def current(self):
 558         """Returns the project which is currently being loaded."""
 559         if not self.current_project:
 560             get_manager().errors()(
 561                 'Reference to the project currently being loaded requested '
 562                 'when there was no project module being loaded.'
 563             )
 564         return self.current_project
 565
 566     def set_current(self, c):
 567         if __debug__:
 568             from .targets import ProjectTarget
 569             assert isinstance(c, ProjectTarget)
 570         self.current_project = c
 571
 572     def push_current(self, project):
 573         """Temporary changes the current project to 'project'. Should
 574         be followed by 'pop-current'."""
 575         if __debug__:
 576             from .targets import ProjectTarget
 577             assert isinstance(project, ProjectTarget)
 578         self.saved_current_project.append(self.current_project)
 579         self.current_project = project
 580
 581     def pop_current(self):
 582         if self.saved_current_project:
 583             self.current_project = self.saved_current_project.pop()
 584         else:
 585             self.current_project = None
 586
 587     def attributes(self, project):
 588         """Returns the project-attribute instance for the
 589         specified jamfile module."""
 590         assert isinstance(project, basestring)
 591         return self.module2attributes[project]
 592
 593     def attribute(self, project, attribute):
 594         """Returns the value of the specified attribute in the
 595         specified jamfile module."""
 596         assert isinstance(project, basestring)
 597         assert isinstance(attribute, basestring)
 598         try:
 599             return self.module2attributes[project].get(attribute)
 600         except:
 601             raise BaseException("No attribute '%s' for project %s" % (attribute, project))
 602
 603     def attributeDefault(self, project, attribute, default):
 604         """Returns the value of the specified attribute in the
 605         specified jamfile module."""
 606         assert isinstance(project, basestring)
 607         assert isinstance(attribute, basestring)
 608         assert isinstance(default, basestring) or default is None
 609         return self.module2attributes[project].getDefault(attribute, default)
 610
 611     def target(self, project_module):
 612         """Returns the project target corresponding to the 'project-module'."""
 613         assert isinstance(project_module, basestring)
 614         if project_module not in self.module2target:
 615             self.module2target[project_module] = \
 616                 b2.build.targets.ProjectTarget(project_module, project_module,
 617                               self.attribute(project_module, "requirements"))
 618
 619         return self.module2target[project_module]
 620
 621     def use(self, id, location):
 622         # Use/load a project.
 623         assert isinstance(id, basestring)
 624         assert isinstance(location, basestring)
 625         saved_project = self.current_project
 626         project_module = self.load(location)
 627         declared_id = self.attributeDefault(project_module, "id", "")
 628
 629         if not declared_id or declared_id != id:
 630             # The project at 'location' either have no id or
 631             # that id is not equal to the 'id' parameter.
 632             if id in self.id2module and self.id2module[id] != project_module:
 633                 self.manager.errors()(
 634 """Attempt to redeclare already existing project id '%s' at location '%s'""" % (id, location))
 635             self.id2module[id] = project_module
 636
 637         self.current_project = saved_project
 638
 639     def add_rule(self, name, callable_):
 640         """Makes rule 'name' available to all subsequently loaded Jamfiles.
 641
 642         Calling that rule wil relay to 'callable'."""
 643         assert isinstance(name, basestring)
 644         assert callable(callable_)
 645         self.project_rules_.add_rule(name, callable_)
 646
 647     def project_rules(self):
 648         return self.project_rules_
 649
 650     def glob_internal(self, project, wildcards, excludes, rule_name):
 651         if __debug__:
 652             from .targets import ProjectTarget
 653             assert isinstance(project, ProjectTarget)
 654             assert is_iterable_typed(wildcards, basestring)
 655             assert is_iterable_typed(excludes, basestring) or excludes is None
 656             assert isinstance(rule_name, basestring)
 657         location = project.get("source-location")[0]
 658
 659         result = []
 660         callable = b2.util.path.__dict__[rule_name]
 661
 662         paths = callable([location], wildcards, excludes)
 663         has_dir = 0
 664         for w in wildcards:
 665             if os.path.dirname(w):
 666                 has_dir = 1
 667                 break
 668
 669         if has_dir or rule_name != "glob":
 670             result = []
 671             # The paths we've found are relative to current directory,
 672             # but the names specified in sources list are assumed to
 673             # be relative to source directory of the corresponding
 674             # prject. Either translate them or make absolute.
 675
 676             for p in paths:
 677                 rel = os.path.relpath(p, location)
 678                 # If the path is below source location, use relative path.
 679                 if not ".." in rel:
 680                     result.append(rel)
 681                 else:
 682                     # Otherwise, use full path just to avoid any ambiguities.
 683                     result.append(os.path.abspath(p))
 684
 685         else:
 686             # There were not directory in wildcard, so the files are all
 687             # in the source directory of the project. Just drop the
 688             # directory, instead of making paths absolute.
 689             result = [os.path.basename(p) for p in paths]
 690
 691         return result
 692
 693     def __build_python_module_cache(self):
 694         """Recursively walks through the b2/src subdirectories and
 695         creates an index of base module name to package name. The
 696         index is stored within self.__python_module_cache and allows
 697         for an O(1) module lookup.
 698
 699         For example, given the base module name `toolset`,
 700         self.__python_module_cache['toolset'] will return
 701         'b2.build.toolset'
 702
 703         pkgutil.walk_packages() will find any python package
 704         provided a directory contains an __init__.py. This has the
 705         added benefit of allowing libraries to be installed and
 706         automatically avaiable within the contrib directory.
 707
 708         *Note*: pkgutil.walk_packages() will import any subpackage
 709         in order to access its __path__variable. Meaning:
 710         any initialization code will be run if the package hasn't
 711         already been imported.
 712         """
 713         cache = {}
 714         for importer, mname, ispkg in pkgutil.walk_packages(b2.__path__, prefix='b2.'):
 715             basename = mname.split('.')[-1]
 716             # since the jam code is only going to have "import toolset ;"
 717             # it doesn't matter if there are separately named "b2.build.toolset" and
 718             # "b2.contrib.toolset" as it is impossible to know which the user is
 719             # referring to.
 720             if basename in cache:
 721                 self.manager.errors()('duplicate module name "{0}" '
 722                                       'found in boost-build path'.format(basename))
 723             cache[basename] = mname
 724         self.__python_module_cache = cache
 725
 726     def load_module(self, name, extra_path=None):
 727         """Load a Python module that should be useable from Jamfiles.
 728
 729         There are generally two types of modules Jamfiles might want to
 730         use:
 731         - Core Boost.Build. Those are imported using plain names, e.g.
 732         'toolset', so this function checks if we have module named
 733         b2.package.module already.
 734         - Python modules in the same directory as Jamfile. We don't
 735         want to even temporary add Jamfile's directory to sys.path,
 736         since then we might get naming conflicts between standard
 737         Python modules and those.
 738         """
 739         assert isinstance(name, basestring)
 740         assert is_iterable_typed(extra_path, basestring) or extra_path is None
 741         # See if we loaded module of this name already
 742         existing = self.loaded_tool_modules_.get(name)
 743         if existing:
 744             return existing
 745
 746         # check the extra path as well as any paths outside
 747         # of the b2 package and import the  module if it exists
 748         b2_path = os.path.normpath(b2.__path__[0])
 749         # normalize the pathing in the BOOST_BUILD_PATH.
 750         # this allows for using startswith() to determine
 751         # if a path is a subdirectory of the b2 root_path
 752         paths = [os.path.normpath(p) for p in self.manager.boost_build_path()]
 753         # remove all paths that start with b2's root_path
 754         paths = [p for p in paths if not p.startswith(b2_path)]
 755         # add any extra paths
 756         paths.extend(extra_path)
 757
 758         try:
 759             # find_module is used so that the pyc's can be used.
 760             # an ImportError is raised if not found
 761             f, location, description = imp.find_module(name, paths)
 762         except ImportError:
 763             # if the module is not found in the b2 package,
 764             # this error will be handled later
 765             pass
 766         else:
 767             # we've found the module, now let's try loading it.
 768             # it's possible that the module itself contains an ImportError
 769             # which is why we're loading it in this else clause so that the
 770             # proper error message is shown to the end user.
 771             # TODO: does this module name really need to be mangled like this?
 772             mname = name + "__for_jamfile"
 773             self.loaded_tool_module_path_[mname] = location
 774             module = imp.load_module(mname, f, location, description)
 775             self.loaded_tool_modules_[name] = module
 776             return module
 777
 778         # the cache is created here due to possibly importing packages
 779         # that end up calling get_manager() which might fail
 780         if not self.__python_module_cache:
 781             self.__build_python_module_cache()
 782
 783         underscore_name = name.replace('-', '_')
 784         # check to see if the module is within the b2 package
 785         # and already loaded
 786         mname = self.__python_module_cache.get(underscore_name)
 787         if mname in sys.modules:
 788             return sys.modules[mname]
 789         # otherwise, if the module name is within the cache,
 790         # the module exists within the BOOST_BUILD_PATH,
 791         # load it.
 792         elif mname:
 793             # in some cases, self.loaded_tool_module_path_ needs to
 794             # have the path to the file during the import
 795             # (project.initialize() for example),
 796             # so the path needs to be set *before* importing the module.
 797             path = os.path.join(b2.__path__[0], *mname.split('.')[1:])
 798             self.loaded_tool_module_path_[mname] = path
 799             # mname is guaranteed to be importable since it was
 800             # found within the cache
 801             __import__(mname)
 802             module = sys.modules[mname]
 803             self.loaded_tool_modules_[name] = module
 804             return module
 805
 806         self.manager.errors()("Cannot find module '%s'" % name)
 807
 808
 809
 810 # FIXME:
 811 # Defines a Boost.Build extension project. Such extensions usually
 812 # contain library targets and features that can be used by many people.
 813 # Even though extensions are really projects, they can be initialize as
 814 # a module would be with the "using" (project.project-rules.using)
 815 # mechanism.
 816 #rule extension ( id : options * : * )
 817 #{
 818 #    # The caller is a standalone module for the extension.
 819 #    local mod = [ CALLER_MODULE ] ;
 820 #
 821 #    # We need to do the rest within the extension module.
 822 #    module $(mod)
 823 #    {
 824 #        import path ;
 825 #
 826 #        # Find the root project.
 827 #        local root-project = [ project.current ] ;
 828 #        root-project = [ $(root-project).project-module ] ;
 829 #        while
 830 #            [ project.attribute $(root-project) parent-module ] &&
 831 #            [ project.attribute $(root-project) parent-module ] != user-config
 832 #        {
 833 #            root-project = [ project.attribute $(root-project) parent-module ] ;
 834 #        }
 835 #
 836 #        # Create the project data, and bring in the project rules
 837 #        # into the module.
 838 #        project.initialize $(__name__) :
 839 #            [ path.join [ project.attribute $(root-project) location ] ext $(1:L) ] ;
 840 #
 841 #        # Create the project itself, i.e. the attributes.
 842 #        # All extensions are created in the "/ext" project space.
 843 #        project /ext/$(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ;
 844 #        local attributes = [ project.attributes $(__name__) ] ;
 845 #
 846 #        # Inherit from the root project of whomever is defining us.
 847 #        project.inherit-attributes $(__name__) : $(root-project) ;
 848 #        $(attributes).set parent-module : $(root-project) : exact ;
 849 #    }
 850 #}
 851
 852
 853 class ProjectAttributes:
 854     """Class keeping all the attributes of a project.
 855
 856     The standard attributes are 'id', "location", "project-root", "parent"
 857     "requirements", "default-build", "source-location" and "projects-to-build".
 858     """
 859
 860     def __init__(self, manager, location, project_module):
 861         self.manager = manager
 862         self.location = location
 863         self.project_module = project_module
 864         self.attributes = {}
 865         self.usage_requirements = None
 866
 867     def set(self, attribute, specification, exact=False):
 868         """Set the named attribute from the specification given by the user.
 869         The value actually set may be different."""
 870         assert isinstance(attribute, basestring)
 871         assert isinstance(exact, (int, bool))
 872         if __debug__ and not exact:
 873             if attribute == 'requirements':
 874                 assert (isinstance(specification, property_set.PropertySet)
 875                         or all(isinstance(s, basestring) for s in specification))
 876             elif attribute in (
 877             'usage-requirements', 'default-build', 'source-location', 'build-dir', 'id'):
 878                 assert is_iterable_typed(specification, basestring)
 879         elif __debug__:
 880             assert (
 881                 isinstance(specification, (property_set.PropertySet, type(None), basestring))
 882                     or all(isinstance(s, basestring) for s in specification)
 883             )
 884         if exact:
 885             self.__dict__[attribute] = specification
 886
 887         elif attribute == "requirements":
 888             self.requirements = property_set.refine_from_user_input(
 889                 self.requirements, specification,
 890                 self.project_module, self.location)
 891
 892         elif attribute == "usage-requirements":
 893             unconditional = []
 894             for p in specification:
 895                 split = property.split_conditional(p)
 896                 if split:
 897                     unconditional.append(split[1])
 898                 else:
 899                     unconditional.append(p)
 900
 901             non_free = property.remove("free", unconditional)
 902             if non_free:
 903                 get_manager().errors()("usage-requirements %s have non-free properties %s" \
 904                                        % (specification, non_free))
 905
 906             t = property.translate_paths(
 907                     property.create_from_strings(specification, allow_condition=True),
 908                     self.location)
 909
 910             existing = self.__dict__.get("usage-requirements")
 911             if existing:
 912                 new = property_set.create(existing.all() +  t)
 913             else:
 914                 new = property_set.create(t)
 915             self.__dict__["usage-requirements"] = new
 916
 917
 918         elif attribute == "default-build":
 919             self.__dict__["default-build"] = property_set.create(specification)
 920
 921         elif attribute == "source-location":
 922             source_location = []
 923             for path in specification:
 924                 source_location.append(os.path.join(self.location, path))
 925             self.__dict__["source-location"] = source_location
 926
 927         elif attribute == "build-dir":
 928             self.__dict__["build-dir"] = os.path.join(self.location, specification[0])
 929
 930         elif attribute == "id":
 931             id = specification[0]
 932             if id[0] != '/':
 933                 id = "/" + id
 934             self.manager.projects().register_id(id, self.project_module)
 935             self.__dict__["id"] = id
 936
 937         elif not attribute in ["default-build", "location",
 938                                "source-location", "parent",
 939                                "projects-to-build", "project-root"]:
 940             self.manager.errors()(
 941 """Invalid project attribute '%s' specified
 942 for project at '%s'""" % (attribute, self.location))
 943         else:
 944             self.__dict__[attribute] = specification
 945
 946     def get(self, attribute):
 947         assert isinstance(attribute, basestring)
 948         return self.__dict__[attribute]
 949
 950     def getDefault(self, attribute, default):
 951         assert isinstance(attribute, basestring)
 952         return self.__dict__.get(attribute, default)
 953
 954     def dump(self):
 955         """Prints the project attributes."""
 956         id = self.get("id")
 957         if not id:
 958             id = "(none)"
 959         else:
 960             id = id[0]
 961
 962         parent = self.get("parent")
 963         if not parent:
 964             parent = "(none)"
 965         else:
 966             parent = parent[0]
 967
 968         print "'%s'" % id
 969         print "Parent project:%s", parent
 970         print "Requirements:%s", self.get("requirements")
 971         print "Default build:%s", string.join(self.get("debuild-build"))
 972         print "Source location:%s", string.join(self.get("source-location"))
 973         print "Projects to build:%s", string.join(self.get("projects-to-build").sort());
 974
 975 class ProjectRules:
 976     """Class keeping all rules that are made available to Jamfile."""
 977
 978     def __init__(self, registry):
 979         self.registry = registry
 980         self.manager_ = registry.manager
 981         self.rules = {}
 982         self.local_names = [x for x in self.__class__.__dict__
 983                             if x not in ["__init__", "init_project", "add_rule",
 984                                          "error_reporting_wrapper", "add_rule_for_type", "reverse"]]
 985         self.all_names_ = [x for x in self.local_names]
 986
 987     def _import_rule(self, bjam_module, name, callable_):
 988         assert isinstance(bjam_module, basestring)
 989         assert isinstance(name, basestring)
 990         assert callable(callable_)
 991         if hasattr(callable_, "bjam_signature"):
 992             bjam.import_rule(bjam_module, name, self.make_wrapper(callable_), callable_.bjam_signature)
 993         else:
 994             bjam.import_rule(bjam_module, name, self.make_wrapper(callable_))
 995
 996
 997     def add_rule_for_type(self, type):
 998         assert isinstance(type, basestring)
 999         rule_name = type.lower().replace("_", "-")
1000
1001         @bjam_signature([['name'], ['sources', '*'], ['requirements', '*'],
1002                          ['default_build', '*'], ['usage_requirements', '*']])
1003         def xpto (name, sources=[], requirements=[], default_build=[], usage_requirements=[]):
1004
1005             return self.manager_.targets().create_typed_target(
1006                 type, self.registry.current(), name, sources,
1007                 requirements, default_build, usage_requirements)
1008
1009         self.add_rule(rule_name, xpto)
1010
1011     def add_rule(self, name, callable_):
1012         assert isinstance(name, basestring)
1013         assert callable(callable_)
1014         self.rules[name] = callable_
1015         self.all_names_.append(name)
1016
1017         # Add new rule at global bjam scope. This might not be ideal,
1018         # added because if a jamroot does 'import foo' where foo calls
1019         # add_rule, we need to import new rule to jamroot scope, and
1020         # I'm lazy to do this now.
1021         self._import_rule("", name, callable_)
1022
1023     def all_names(self):
1024         return self.all_names_
1025
1026     def call_and_report_errors(self, callable_, *args, **kw):
1027         assert callable(callable_)
1028         result = None
1029         try:
1030             self.manager_.errors().push_jamfile_context()
1031             result = callable_(*args, **kw)
1032         except ExceptionWithUserContext, e:
1033             e.report()
1034         except Exception, e:
1035             try:
1036                 self.manager_.errors().handle_stray_exception (e)
1037             except ExceptionWithUserContext, e:
1038                 e.report()
1039         finally:
1040             self.manager_.errors().pop_jamfile_context()
1041
1042         return result
1043
1044     def make_wrapper(self, callable_):
1045         """Given a free-standing function 'callable', return a new
1046         callable that will call 'callable' and report all exceptins,
1047         using 'call_and_report_errors'."""
1048         assert callable(callable_)
1049         def wrapper(*args, **kw):
1050             return self.call_and_report_errors(callable_, *args, **kw)
1051         return wrapper
1052
1053     def init_project(self, project_module, python_standalone=False):
1054         assert isinstance(project_module, basestring)
1055         assert isinstance(python_standalone, bool)
1056         if python_standalone:
1057             m = sys.modules[project_module]
1058
1059             for n in self.local_names:
1060                 if n != "import_":
1061                     setattr(m, n, getattr(self, n))
1062
1063             for n in self.rules:
1064                 setattr(m, n, self.rules[n])
1065
1066             return
1067
1068         for n in self.local_names:
1069             # Using 'getattr' here gives us a bound method,
1070             # while using self.__dict__[r] would give unbound one.
1071             v = getattr(self, n)
1072             if callable(v):
1073                 if n == "import_":
1074                     n = "import"
1075                 else:
1076                     n = string.replace(n, "_", "-")
1077
1078                 self._import_rule(project_module, n, v)
1079
1080         for n in self.rules:
1081             self._import_rule(project_module, n, self.rules[n])
1082
1083     def project(self, *args):
1084         assert is_iterable(args) and all(is_iterable(arg) for arg in args)
1085         jamfile_module = self.registry.current().project_module()
1086         attributes = self.registry.attributes(jamfile_module)
1087
1088         id = None
1089         if args and args[0]:
1090             id = args[0][0]
1091             args = args[1:]
1092
1093         if id:
1094             attributes.set('id', [id])
1095
1096         explicit_build_dir = None
1097         for a in args:
1098             if a:
1099                 attributes.set(a[0], a[1:], exact=0)
1100                 if a[0] == "build-dir":
1101                     explicit_build_dir = a[1]
1102
1103         # If '--build-dir' is specified, change the build dir for the project.
1104         if self.registry.global_build_dir:
1105
1106             location = attributes.get("location")
1107             # Project with empty location is 'standalone' project, like
1108             # user-config, or qt.  It has no build dir.
1109             # If we try to set build dir for user-config, we'll then
1110             # try to inherit it, with either weird, or wrong consequences.
1111             if location and location == attributes.get("project-root"):
1112                 # Re-read the project id, since it might have been changed in
1113                 # the project's attributes.
1114                 id = attributes.get('id')
1115
1116                 # This is Jamroot.
1117                 if id:
1118                     if explicit_build_dir and os.path.isabs(explicit_build_dir):
1119                         self.registry.manager.errors()(
1120 """Absolute directory specified via 'build-dir' project attribute
1121 Don't know how to combine that with the --build-dir option.""")
1122
1123                     rid = id
1124                     if rid[0] == '/':
1125                         rid = rid[1:]
1126
1127                     p = os.path.join(self.registry.global_build_dir, rid)
1128                     if explicit_build_dir:
1129                         p = os.path.join(p, explicit_build_dir)
1130                     attributes.set("build-dir", p, exact=1)
1131             elif explicit_build_dir:
1132                 self.registry.manager.errors()(
1133 """When --build-dir is specified, the 'build-dir'
1134 attribute is allowed only for top-level 'project' invocations""")
1135
1136     def constant(self, name, value):
1137         """Declare and set a project global constant.
1138         Project global constants are normal variables but should
1139         not be changed. They are applied to every child Jamfile."""
1140         assert is_iterable_typed(name, basestring)
1141         assert is_iterable_typed(value, basestring)
1142         self.registry.current().add_constant(name[0], value)
1143
1144     def path_constant(self, name, value):
1145         """Declare and set a project global constant, whose value is a path. The
1146         path is adjusted to be relative to the invocation directory. The given
1147         value path is taken to be either absolute, or relative to this project
1148         root."""
1149         assert is_iterable_typed(name, basestring)
1150         assert is_iterable_typed(value, basestring)
1151         if len(value) > 1:
1152             self.registry.manager.errors()("path constant should have one element")
1153         self.registry.current().add_constant(name[0], value, path=1)
1154
1155     def use_project(self, id, where):
1156         # See comment in 'load' for explanation why we record the
1157         # parameters as opposed to loading the project now.
1158         assert is_iterable_typed(id, basestring)
1159         assert is_iterable_typed(where, basestring)
1160         m = self.registry.current().project_module()
1161         self.registry.used_projects[m].append((id[0], where[0]))
1162
1163     def build_project(self, dir):
1164         assert is_iterable_typed(dir, basestring)
1165         jamfile_module = self.registry.current().project_module()
1166         attributes = self.registry.attributes(jamfile_module)
1167         now = attributes.get("projects-to-build")
1168         attributes.set("projects-to-build", now + dir, exact=True)
1169
1170     def explicit(self, target_names):
1171         assert is_iterable_typed(target_names, basestring)
1172         self.registry.current().mark_targets_as_explicit(target_names)
1173
1174     def always(self, target_names):
1175         assert is_iterable_typed(target_names, basestring)
1176         self.registry.current().mark_targets_as_always(target_names)
1177
1178     def glob(self, wildcards, excludes=None):
1179         assert is_iterable_typed(wildcards, basestring)
1180         assert is_iterable_typed(excludes, basestring)or excludes is None
1181         return self.registry.glob_internal(self.registry.current(),
1182                                            wildcards, excludes, "glob")
1183
1184     def glob_tree(self, wildcards, excludes=None):
1185         assert is_iterable_typed(wildcards, basestring)
1186         assert is_iterable_typed(excludes, basestring) or excludes is None
1187         bad = 0
1188         for p in wildcards:
1189             if os.path.dirname(p):
1190                 bad = 1
1191
1192         if excludes:
1193             for p in excludes:
1194                 if os.path.dirname(p):
1195                     bad = 1
1196
1197         if bad:
1198             self.registry.manager.errors()(
1199 "The patterns to 'glob-tree' may not include directory")
1200         return self.registry.glob_internal(self.registry.current(),
1201                                            wildcards, excludes, "glob_tree")
1202
1203
1204     def using(self, toolset, *args):
1205         # The module referred by 'using' can be placed in
1206         # the same directory as Jamfile, and the user
1207         # will expect the module to be found even though
1208         # the directory is not in BOOST_BUILD_PATH.
1209         # So temporary change the search path.
1210         assert is_iterable_typed(toolset, basestring)
1211         current = self.registry.current()
1212         location = current.get('location')
1213
1214         m = self.registry.load_module(toolset[0], [location])
1215         if "init" not in m.__dict__:
1216             self.registry.manager.errors()(
1217                 "Tool module '%s' does not define the 'init' method" % toolset[0])
1218         m.init(*args)
1219
1220         # The above might have clobbered .current-project. Restore the correct
1221         # value.
1222         self.registry.set_current(current)
1223
1224     def import_(self, name, names_to_import=None, local_names=None):
1225         assert is_iterable_typed(name, basestring)
1226         assert is_iterable_typed(names_to_import, basestring) or names_to_import is None
1227         assert is_iterable_typed(local_names, basestring)or local_names is None
1228         name = name[0]
1229         py_name = name
1230         if py_name == "os":
1231             py_name = "os_j"
1232         jamfile_module = self.registry.current().project_module()
1233         attributes = self.registry.attributes(jamfile_module)
1234         location = attributes.get("location")
1235
1236         saved = self.registry.current()
1237
1238         m = self.registry.load_module(py_name, [location])
1239
1240         for f in m.__dict__:
1241             v = m.__dict__[f]
1242             f = f.replace("_", "-")
1243             if callable(v):
1244                 qn = name + "." + f
1245                 self._import_rule(jamfile_module, qn, v)
1246                 record_jam_to_value_mapping(qualify_jam_action(qn, jamfile_module), v)
1247
1248
1249         if names_to_import:
1250             if not local_names:
1251                 local_names = names_to_import
1252
1253             if len(names_to_import) != len(local_names):
1254                 self.registry.manager.errors()(
1255 """The number of names to import and local names do not match.""")
1256
1257             for n, l in zip(names_to_import, local_names):
1258                 self._import_rule(jamfile_module, l, m.__dict__[n])
1259
1260         self.registry.set_current(saved)
1261
1262     def conditional(self, condition, requirements):
1263         """Calculates conditional requirements for multiple requirements
1264         at once. This is a shorthand to be reduce duplication and to
1265         keep an inline declarative syntax. For example:
1266
1267             lib x : x.cpp : [ conditional <toolset>gcc <variant>debug :
1268                 <define>DEBUG_EXCEPTION <define>DEBUG_TRACE ] ;
1269         """
1270         assert is_iterable_typed(condition, basestring)
1271         assert is_iterable_typed(requirements, basestring)
1272         c = string.join(condition, ",")
1273         if c.find(":") != -1:
1274             return [c + r for r in requirements]
1275         else:
1276             return [c + ":" + r for r in requirements]
1277
1278     def option(self, name, value):
1279         assert is_iterable(name) and isinstance(name[0], basestring)
1280         assert is_iterable(value) and isinstance(value[0], basestring)
1281         name = name[0]
1282         if not name in ["site-config", "user-config", "project-config"]:
1283             get_manager().errors()("The 'option' rule may be used only in site-config or user-config")
1284
1285         option.set(name, value[0])