diff --git a/pyste/NEWS b/pyste/NEWS index 43a1010c..9bb1d4c7 100644 --- a/pyste/NEWS +++ b/pyste/NEWS @@ -1,3 +1,6 @@ +4 June 2003 +Major improvements in memory usage. + 3 June 2003 Appliced a patch from Giulio Eulisse that allows unnamed enumerations to be exported with an AllFromHeader construct. Thanks a lot Giulio! diff --git a/pyste/src/ClassExporter.py b/pyste/src/ClassExporter.py index bfb38cee..a1a82d14 100644 --- a/pyste/src/ClassExporter.py +++ b/pyste/src/ClassExporter.py @@ -15,8 +15,7 @@ import re #============================================================================== class ClassExporter(Exporter): 'Generates boost.python code to export a class declaration' - - + def __init__(self, info, parser_tail=None): Exporter.__init__(self, info, parser_tail) # sections of code @@ -566,7 +565,8 @@ class ClassExporter(Exporter): nested_info.include = self.info.include nested_info.name = nested_class.FullName() exporter = ClassExporter(nested_info) - exporter.SetDeclarations(self.declarations + [nested_class]) + self.declarations.append(nested_class) + exporter.SetDeclarations(self.declarations) codeunit = SingleCodeUnit(None, None) exporter.Export(codeunit, exported_names) self.nested_codeunits.append(codeunit) @@ -579,7 +579,8 @@ class ClassExporter(Exporter): enum_info.include = self.info.include enum_info.name = enum.FullName() exporter = EnumExporter(enum_info) - exporter.SetDeclarations(self.declarations + [enum]) + self.declarations.append(enum) + exporter.SetDeclarations(self.declarations) codeunit = SingleCodeUnit(None, None) exporter.Export(codeunit, None) self.nested_codeunits.append(codeunit) diff --git a/pyste/src/CppParser.py b/pyste/src/CppParser.py index 6412082e..716779ec 100644 --- a/pyste/src/CppParser.py +++ b/pyste/src/CppParser.py @@ -19,8 +19,6 @@ class CppParser: defines = [] self.includes = includes self.defines = defines - self._cache = [] - self._CACHE_SIZE = 5 def _includeparams(self, filename): @@ -37,21 +35,6 @@ class CppParser: defines = ['-D "%s"' % x for x in self.defines] return ' '.join(defines) - - def UpdateCache(self, include, tail, decl_name, declarations, header): - self._cache.append((include, tail, decl_name, declarations, header)) - if len(self._cache) > self._CACHE_SIZE: - self._cache.pop(0) - - - def Cache(self, include, tail, decl_name): - for cache_include, cache_tail, cache_decl, declarations, header in self._cache: - if cache_include == include \ - and cache_tail == tail \ - and cache_decl == decl_name: - return declarations, header - return None - def FindFileName(self, include): if os.path.isfile(include): @@ -71,10 +54,6 @@ class CppParser: filename and the tail code is appended to it before being passed on to gcc. This temp filename is then returned. ''' - # check if this header was already parsed - cached = self.Cache(include, tail, decl_name) - if cached: - return cached filename = self.FindFileName(include) # copy file to temp folder, if needed if tail: @@ -101,8 +80,6 @@ class CppParser: raise CppParserError, 'Error executing gccxml' # parse the resulting xml declarations = ParseDeclarations(xmlfile) - # cache the results - self.UpdateCache(include, tail, decl_name, declarations, infilename) # return the declarations return declarations, infilename finally: diff --git a/pyste/src/Exporter.py b/pyste/src/Exporter.py index c47a05c4..e72c3b39 100644 --- a/pyste/src/Exporter.py +++ b/pyste/src/Exporter.py @@ -16,6 +16,10 @@ class Exporter: def Name(self): return self.info.name + + def Tail(self): + return self.parser_tail + def Parse(self, parser): self.parser = parser @@ -26,6 +30,10 @@ class Exporter: self.SetDeclarations(declarations) + def SetParsedHeader(self, parsed_header): + self.parser_header = parsed_header + + def SetDeclarations(self, declarations): self.declarations = declarations @@ -44,12 +52,11 @@ class Exporter: pass - def Unit(self): - raise NotImplementedError - - def GetDeclarations(self, fullname): - decls = [x for x in self.declarations if x.FullName() == fullname] + decls = [] + for decl in self.declarations: + if decl.FullName() == fullname: + decls.append(decl) if not decls: raise RuntimeError, 'no %s declaration found!' % fullname return decls @@ -70,3 +77,7 @@ class Exporter: def Unit(self): return self.info.include + + + def Header(self): + return self.info.include diff --git a/pyste/src/FunctionExporter.py b/pyste/src/FunctionExporter.py index a3c9a361..927de474 100644 --- a/pyste/src/FunctionExporter.py +++ b/pyste/src/FunctionExporter.py @@ -16,7 +16,7 @@ class FunctionExporter(Exporter): Exporter.__init__(self, info, tail) self._exported_opaque_pointers = {} - + def Export(self, codeunit, exported_names): if not self.info.exclude: decls = self.GetDeclarations(self.info.name) diff --git a/pyste/src/HeaderExporter.py b/pyste/src/HeaderExporter.py index 141f4da4..4449508b 100644 --- a/pyste/src/HeaderExporter.py +++ b/pyste/src/HeaderExporter.py @@ -22,24 +22,23 @@ class HeaderExporter(Exporter): pass - def SetDeclarations(self, declarations): - def IsInternalName(name): - '''Returns true if the given name looks like a internal compiler - structure''' - return name.startswith('_') + def IsInternalName(self, name): + '''Returns true if the given name looks like a internal compiler + structure''' + return name.startswith('_') - Exporter.SetDeclarations(self, declarations) + + def Export(self, codeunit, exported_names): header = os.path.normpath(self.parser_header) - for decl in declarations: + for decl in self.declarations: # check if this declaration is in the header location = os.path.normpath(decl.location[0]) - if location != header or IsInternalName(decl.name): - continue - # ok, check the type of the declaration and export it accordingly - self.HandleDeclaration(decl) + if location == header and not self.IsInternalName(decl.name): + # ok, check the type of the declaration and export it accordingly + self.HandleDeclaration(decl, codeunit, exported_names) - def HandleDeclaration(self, decl): + def HandleDeclaration(self, decl, codeunit, exported_names): '''Dispatch the declaration to the appropriate method, that must create a suitable info object for a Exporter, create a Exporter, set its declarations and append it to the list of exporters. @@ -53,10 +52,10 @@ class HeaderExporter(Exporter): exporter_class = dispatch_table.get(type(decl)) if exporter_class is not None: - self.HandleExporter(decl, exporter_class) + self.HandleExporter(decl, exporter_class, codeunit, exported_names) - def HandleExporter(self, decl, exporter_type): + def HandleExporter(self, decl, exporter_type, codeunit, exported_names): # only export complete declarations if not getattr(decl, "incomplete", False): info = self.info[decl.name] @@ -64,7 +63,9 @@ class HeaderExporter(Exporter): info.include = self.info.include exporter = exporter_type(info) exporter.SetDeclarations(self.declarations) - exporters.exporters.append(exporter) + exporter.SetParsedHeader(self.parser_header) + codeunit.SetCurrent(exporter.Unit()) + exporter.GenerateCode(codeunit, exported_names) def Unit(self): diff --git a/pyste/src/IncludeExporter.py b/pyste/src/IncludeExporter.py index 7784d577..c60a2848 100644 --- a/pyste/src/IncludeExporter.py +++ b/pyste/src/IncludeExporter.py @@ -22,3 +22,6 @@ class IncludeExporter(Exporter): def Unit(self): return '__all__' # include it in all generated cpps (multiple mode) + + def Header(self): + return None # means "don't try to parse me!" diff --git a/pyste/src/SingleCodeUnit.py b/pyste/src/SingleCodeUnit.py index 1ac590f9..6f3fc2f6 100644 --- a/pyste/src/SingleCodeUnit.py +++ b/pyste/src/SingleCodeUnit.py @@ -49,6 +49,14 @@ class SingleCodeUnit: def Section(self, section): return self.code[section] + + def SetCurrent(self, current): + pass + + + def Current(self): + pass + def Save(self): 'Writes this code unit to the filename' diff --git a/pyste/src/pyste.py b/pyste/src/pyste.py index 1314f10f..39938caf 100644 --- a/pyste/src/pyste.py +++ b/pyste/src/pyste.py @@ -33,11 +33,13 @@ import MultipleCodeUnit import infos import exporterutils import settings +import gc +import sys from policies import * from CppParser import CppParser, CppParserError import time -__VERSION__ = '0.9.2' +__VERSION__ = '0.9.3' def RecursiveIncludes(include): 'Return a list containg the include dir and all its subdirectories' @@ -157,32 +159,63 @@ def Main(): for interface in interfaces: context = CreateContext() execfile(interface, context) - # parse all the C++ code + # create the parser parser = CppParser(includes, defines) - exports = exporters.exporters[:] - for export in exports: - try: - export.Parse(parser) - except CppParserError, e: - print '\n' - print '***', e, ': exitting' - return 2 - print - # sort the exporters by its order - exports = [(x.Order(), x) for x in exporters.exporters] - exports.sort() - exports = [x for _, x in exports] - # now generate the wrapper code + # prepare to generate the wrapper code if multiple: codeunit = MultipleCodeUnit.MultipleCodeUnit(module, out) else: - codeunit = SingleCodeUnit.SingleCodeUnit(module, out) - exported_names = [] - for export in exports: - if multiple: - codeunit.SetCurrent(export.Unit()) - export.GenerateCode(codeunit, exported_names) - exported_names.append(export.Name()) + codeunit = SingleCodeUnit.SingleCodeUnit(module, out) + # group exporters by header files + groups = {} + for export in exporters.exporters: + header = export.Header() + if header in groups: + groups[header].append(export) + else: + groups[header] = [export] + # stop referencing the exporters here + exporters.exporters = None + # export all the exporters in each group, releasing memory as doing so + while len(groups) > 0: + # get the first group + header = groups.keys()[0] + exports = groups[header] + del groups[header] + # gather all tails into one + all_tails = [] + for export in exports: + if export.Tail(): + all_tails.append(export.Tail()) + all_tails = '\n'.join(all_tails) + # parse header (if there's one) + if header: + try: + declarations, parsed_header = parser.parse(header, all_tails) + except CppParserError, e: + print>>sys.stderr, '\n\n***', e, ': exitting' + return 2 + else: + declarations = [] + parsed_header = None + # first set the declarations and parsed_header for all the exporters + for export in exports: + export.SetDeclarations(declarations) + export.SetParsedHeader(parsed_header) + # sort the exporters by their order + exports = [(x.Order(), x) for x in exports] + exports.sort() + exports = [x for _, x in exports] + # maintain a dict of exported_names for this group + exported_names = {} + for export in exports: + if multiple: + codeunit.SetCurrent(export.Unit()) + export.GenerateCode(codeunit, exported_names) + exported_names[export.Name()] = 1 + # force collect of cyclic references + gc.collect() + # finally save the code unit codeunit.Save() print 'Module %s generated' % module return 0 diff --git a/pyste/tests/test_all.py b/pyste/tests/test_all.py index f2dbee66..3ae2f911 100644 --- a/pyste/tests/test_all.py +++ b/pyste/tests/test_all.py @@ -176,7 +176,7 @@ if __name__ == '__main__': else: module = None try: - main('--multiple', module) + #main('--multiple', module) main('', module) except RuntimeError, e: print e