From 1dc016b03e009a92ddec89400c29c11c0f9d7b8a Mon Sep 17 00:00:00 2001 From: Aaron Boman Date: Wed, 3 Sep 2014 13:48:42 -0500 Subject: [PATCH 1/2] Python Port: Make importing a little more robust. - create a b2. index for O(1) package/module searching. - make the .pyc's sit next to their respective files; preventing pollution of workspace - restore the __file__ variable for imported files - prevent importing the same module twice under two separate names; also preventing running initialization code twice (registering types and features) - create an __init__.py for the contrib directory - requires Python 2.3 or newer for pkgutil --- src/build/project.py | 100 +++++++++++++++++++++++++++------------- src/contrib/__init__.py | 0 2 files changed, 69 insertions(+), 31 deletions(-) create mode 100644 src/contrib/__init__.py diff --git a/src/build/project.py b/src/build/project.py index 1cbc0e8c0..c8f11f1d4 100644 --- a/src/build/project.py +++ b/src/build/project.py @@ -45,9 +45,11 @@ from b2.build.errors import ExceptionWithUserContext import b2.build.targets import bjam +import b2 import re import sys +import pkgutil import os import string import imp @@ -120,6 +122,8 @@ class ProjectRegistry: self.JAMFILE = ["[Bb]uild.jam", "[Jj]amfile.v2", "[Jj]amfile", "[Jj]amfile.jam"] + self.__python_module_cache = {} + def load (self, jamfile_location): """Loads jamfile at the given location. After loading, project global @@ -607,6 +611,39 @@ actual value %s""" % (jamfile_module, saved_project, self.current_project)) return result + def __build_python_module_cache(self): + """Recursively walks through the b2/src subdirectories and + creates an index of base module name to package name. The + index is stored within self.__python_module_cache and allows + for an O(1) module lookup. + + For example, given the base module name `toolset`, + self.__python_module_cache['toolset'] will return + 'b2.build.toolset' + + pkgutil.walk_packages() will find any python package + provided a directory contains an __init__.py. This has the + added benefit of allowing libraries to be installed and + automatically avaiable within the contrib directory. + + *Note*: pkgutil.walk_packages() will import any subpackage + in order to access its __path__variable. Meaning: + any initialization code will be run if the package hasn't + already been imported. + """ + cache = {} + for importer, mname, ispkg in pkgutil.walk_packages(b2.__path__, prefix='b2.'): + basename = mname.split('.')[-1] + # since the jam code is only going to have "import toolset ;" + # it doesn't matter if there are separately named "b2.build.toolset" and + # "b2.contrib.toolset" as it is impossible to know which the user is + # referring to. + if basename in cache: + self.manager.errors()('duplicate module name "{0}" ' + 'found in boost-build path'.format(basename)) + cache[basename] = mname + self.__python_module_cache = cache + def load_module(self, name, extra_path=None): """Load a Python module that should be useable from Jamfiles. @@ -620,50 +657,51 @@ actual value %s""" % (jamfile_module, saved_project, self.current_project)) since then we might get naming conflicts between standard Python modules and those. """ - # See if we loaded module of this name already existing = self.loaded_tool_modules_.get(name) if existing: return existing - # See if we have a module b2.whatever., where - # is what is passed to this function - modules = sys.modules - for class_name in modules: - parts = class_name.split('.') - if name is class_name or parts[0] == "b2" \ - and parts[-1] == name.replace("-", "_"): - module = modules[class_name] - self.loaded_tool_modules_[name] = module - return module - - # Lookup a module in BOOST_BUILD_PATH - path = extra_path - if not path: - path = [] - path.extend(self.manager.boost_build_path()) + # check the extra path first and load the + # module if it exists location = None - for p in path: + for p in extra_path: l = os.path.join(p, name + ".py") if os.path.exists(l): location = l break - - if not location: - self.manager.errors()("Cannot find module '%s'" % name) - mname = name + "__for_jamfile" - file = open(location) - try: - # TODO: this means we'll never make use of .pyc module, - # which might be a problem, or not. - self.loaded_tool_module_path_[mname] = location - module = imp.load_module(mname, file, os.path.basename(location), - (".py", "r", imp.PY_SOURCE)) + if location: + with open(location) as f: + self.loaded_tool_module_path_[mname] = location + module = imp.load_module(mname, f, location, + (".py", "r", imp.PY_SOURCE)) + self.loaded_tool_modules_[name] = module + return module + + # the cache is created here due to possibly importing packages + # that end up calling get_manager() which might fail + if not self.__python_module_cache: + self.__build_python_module_cache() + + underscore_name = name.replace('-', '_') + # check to see if the module is within the BOOST_BUILD_PATH + # and already loaded + mname = self.__python_module_cache.get(underscore_name) + if mname in sys.modules: + return sys.modules[mname] + # otherwise, if the module name is within the cache, + # the module exists within the BOOST_BUILD_PATH, + # load it. + elif mname: + # __import__ can be used here since the module + # is guaranteed to be found under the `b2` namespace. + __import__(mname) + module = sys.modules[mname] self.loaded_tool_modules_[name] = module return module - finally: - file.close() + + self.manager.errors()("Cannot find module '%s'" % name) diff --git a/src/contrib/__init__.py b/src/contrib/__init__.py new file mode 100644 index 000000000..e69de29bb From 60130a7a0a7e7c623b66a1a9e875b88596e9b8c6 Mon Sep 17 00:00:00 2001 From: Aaron Boman Date: Wed, 10 Sep 2014 11:21:32 -0500 Subject: [PATCH 2/2] Make sure all of BOOST_BUILD_PATH is checked for module importing. --- src/build/project.py | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/src/build/project.py b/src/build/project.py index c8f11f1d4..414aecceb 100644 --- a/src/build/project.py +++ b/src/build/project.py @@ -662,22 +662,31 @@ actual value %s""" % (jamfile_module, saved_project, self.current_project)) if existing: return existing - # check the extra path first and load the - # module if it exists - location = None - for p in extra_path: - l = os.path.join(p, name + ".py") - if os.path.exists(l): - location = l - break - mname = name + "__for_jamfile" - if location: - with open(location) as f: - self.loaded_tool_module_path_[mname] = location - module = imp.load_module(mname, f, location, - (".py", "r", imp.PY_SOURCE)) - self.loaded_tool_modules_[name] = module - return module + # check the extra path as well as any paths outside + # of the b2 package and import the module if it exists + b2_path = os.path.normpath(b2.__path__[0]) + # normalize the pathing in the BOOST_BUILD_PATH. + # this allows for using startswith() to determine + # if a path is a subdirectory of the b2 root_path + paths = [os.path.normpath(p) for p in self.manager.boost_build_path()] + # remove all paths that start with b2's root_path + paths = [p for p in paths if not p.startswith(b2_path)] + # add any extra paths + paths.extend(extra_path) + + try: + # find_module is used so that the pyc's can be used. + # an ImportError is raised if not found + f, location, description = imp.find_module(name, paths) + mname = name + "__for_jamfile" + self.loaded_tool_module_path_[mname] = location + module = imp.load_module(mname, f, location, description) + self.loaded_tool_modules_[name] = module + return module + except ImportError: + # if the module is not found in the b2 package, + # this error will be handled later + pass # the cache is created here due to possibly importing packages # that end up calling get_manager() which might fail @@ -685,7 +694,7 @@ actual value %s""" % (jamfile_module, saved_project, self.current_project)) self.__build_python_module_cache() underscore_name = name.replace('-', '_') - # check to see if the module is within the BOOST_BUILD_PATH + # check to see if the module is within the b2 package # and already loaded mname = self.__python_module_cache.get(underscore_name) if mname in sys.modules: