diff --git a/MakeBoostDistro.py b/MakeBoostDistro.py index a52ef6a..391025f 100755 --- a/MakeBoostDistro.py +++ b/MakeBoostDistro.py @@ -10,9 +10,12 @@ # # Usage: %0 source dest +from __future__ import print_function + import os, sys import shutil import stat +import six IgnoreFiles = shutil.ignore_patterns( '[.]*', @@ -51,7 +54,7 @@ def MergeTree(src, dst, symlinks = False): MergeTree(s, d, symlinks) else: if os.path.exists(d): - print "## Overwriting file %s with %s" % (d, s) + print("## Overwriting file %s with %s" % (d, s)) shutil.copy2(s, d) @@ -83,7 +86,7 @@ def CopyInclude(src, dst): MergeTree(s, d, symlinks=False) else: if os.path.exists(d): - print "## Overwriting file %s with %s" % (d, s) + print("## Overwriting file %s with %s" % (d, s)) CopyFile(src, dst, item) @@ -138,15 +141,15 @@ BoostSpecialFolders = [ "doc", "more", "status", "tools" ] SourceRoot = sys.argv[1] DestRoot = sys.argv[2] -print "Source = %s" % SourceRoot -print "Dest = %s" % DestRoot +print("Source = %s" % SourceRoot) +print("Dest = %s" % DestRoot) if not os.path.exists(SourceRoot): - print "## Error: %s does not exist" % SourceRoot + print("## Error: %s does not exist" % SourceRoot) exit(1) if not os.path.exists(DestRoot): - print "Creating destination directory %s" % DestRoot + print("Creating destination directory %s" % DestRoot) os.makedirs(DestRoot) DestHeaders = os.path.join(DestRoot, BoostHeaders) @@ -187,7 +190,7 @@ for f in os.listdir(SourceLibs): BoostSubProjects.add((f,s)) for p in BoostSubProjects: - if isinstance(p, basestring): + if isinstance(p, six.string_types): CopySubProject(SourceLibs, DestLibs, DestHeaders, p) else: NestedSource = os.path.join(SourceRoot,"libs",p[0]) diff --git a/ci_boost_build.py b/ci_boost_build.py index c1f4d8c..541a715 100644 --- a/ci_boost_build.py +++ b/ci_boost_build.py @@ -12,6 +12,14 @@ import os.path from ci_boost_common import main, utils, script_common +# Check python version +if sys.version_info[0] == 2 : + pythonversion="2" + pythonbinary="python2" +else: + pythonversion="3" + pythonbinary="python3" + class script(script_common): def __init__(self, ci_klass, **kargs): @@ -43,7 +51,7 @@ class script(script_common): utils.check_call('chmod','+x','MakeBoostDistro.py') os.chdir(self.root_dir) - utils.check_call('python',os.path.join(self.build_dir,'MakeBoostDistro.py'), + utils.check_call(pythonbinary,os.path.join(self.build_dir,'MakeBoostDistro.py'), self.root_dir, 'release') self.root_dir = os.path.join( self.root_dir, 'release' ) diff --git a/ci_boost_common.py b/ci_boost_common.py index 56676c9..458f5c1 100644 --- a/ci_boost_common.py +++ b/ci_boost_common.py @@ -6,6 +6,8 @@ # (See accompanying file LICENSE_1_0.txt or copy at # http://www.boost.org/LICENSE_1_0.txt) +from __future__ import print_function + import sys import inspect import optparse @@ -18,6 +20,12 @@ import shutil import threading import distutils.dir_util +# For urllib +from future.standard_library import install_aliases +install_aliases() + +from builtins import str + class SystemCallError(Exception): def __init__(self, command, result): self.command = command @@ -36,7 +44,7 @@ class utils: result = subprocess.call(command, **kargs) t = time.time()-t if result != 0: - print "Failed: '%s' ERROR = %s"%("' '".join(command), result) + print("Failed: '%s' ERROR = %s"%("' '".join(command), result)) utils.call_stats.append((t,os.getcwd(),command,result)) utils.log( "%s> '%s' execution time %s seconds"%(os.getcwd(), "' '".join(command), t) ) return result @@ -53,7 +61,7 @@ class utils: cwd = os.getcwd() result = utils.call(*command, **kargs) if result != 0: - raise(SystemCallError([cwd].extend(command), result)) + raise SystemCallError([cwd].extend(command), result) @staticmethod def makedirs( path ): @@ -65,7 +73,7 @@ class utils: frames = inspect.stack() level = 0 for i in frames[ 3: ]: - if i[0].f_locals.has_key( '__log__' ): + if '__log__' in i[0].f_locals: level = level + i[0].f_locals[ '__log__' ] return level @@ -82,6 +90,11 @@ class utils: #~ shutil.rmtree( unicode( path ) ) if sys.platform == 'win32': os.system( 'del /f /s /q "%s" >nul 2>&1' % path ) + # Python 3 compatibility hack + try: + unicode('') + except NameError: + unicode = str shutil.rmtree( unicode( path ) ) else: os.system( 'rm -f -r "%s"' % path ) @@ -91,7 +104,7 @@ class utils: for attempts in range( max_attempts, -1, -1 ): try: return f() - except Exception, msg: + except Exception as msg: utils.log( '%s failed with message "%s"' % ( f.__name__, msg ) ) if attempts == 0: utils.log( 'Giving up.' ) @@ -102,7 +115,7 @@ class utils: @staticmethod def web_get( source_url, destination_file, proxy = None ): - import urllib + import urllib.request, urllib.parse, urllib.error proxies = None if proxy is not None: @@ -111,7 +124,7 @@ class utils: 'http' : proxy } - src = urllib.urlopen( source_url, proxies = proxies ) + src = urllib.request.urlopen( source_url, proxies = proxies ) f = open( destination_file, 'wb' ) while True: @@ -164,7 +177,7 @@ class utils: @staticmethod def make_file(filename, *text): f = codecs.open( filename, 'w', 'utf-8' ) - f.write( string.join( text, '\n' ) ) + f.write( '\n'.join(text) ) f.close() @staticmethod @@ -188,7 +201,7 @@ class parallel_call(threading.Thread): def join(self): super(parallel_call,self).join() if self.result != 0: - raise(SystemCallError(self.command, self.result)) + raise SystemCallError(self.command, self.result) class script_common(object): ''' diff --git a/ci_boost_release.py b/ci_boost_release.py index a7a9165..1feb0dc 100755 --- a/ci_boost_release.py +++ b/ci_boost_release.py @@ -6,7 +6,10 @@ # (See accompanying file LICENSE_1_0.txt or copy at # http://www.boost.org/LICENSE_1_0.txt) +from __future__ import print_function + import os.path +import sys import time import shutil import site @@ -15,6 +18,14 @@ import subprocess from ci_boost_common import main, utils, script_common, parallel_call +# Check python version +if sys.version_info[0] == 2 : + pythonversion="2" + pythonbinary="python2" +else: + pythonversion="3" + pythonbinary="python3" + class script(script_common): ''' Main script to build/test Boost C++ Libraries continuous releases. This base @@ -98,7 +109,10 @@ class script(script_common): utils.check_call("gem","install","asciidoctor", "--version", "1.5.8") utils.check_call("asciidoctor","--version") utils.check_call("gem","install","rouge") - utils.check_call("gem","install","pygments.rb", "--version", "1.2.1") + if pythonversion=="2": + utils.check_call("gem","install","pygments.rb", "--version", "1.2.1") + else: + utils.check_call("gem","install","pygments.rb", "--version", "2.1.0") utils.check_call("pip","install","--user","Pygments==2.1") utils.check_call("pip","install","--user","https://github.com/bfgroup/jam_pygments/archive/master.zip") os.chdir(self.root_dir) @@ -220,10 +234,10 @@ class script(script_common): # Make the real distribution tree from the base tree. os.chdir(os.path.join(self.build_dir)) - utils.check_call('wget','https://raw.githubusercontent.com/boostorg/release-tools/develop/MakeBoostDistro.py') + utils.check_call('wget','https://raw.githubusercontent.com/sdarwin/release-tools/python3/MakeBoostDistro.py','-O','MakeBoostDistro.py') utils.check_call('chmod','+x','MakeBoostDistro.py') os.chdir(os.path.dirname(self.root_dir)) - utils.check_call('python',os.path.join(self.build_dir,'MakeBoostDistro.py'), + utils.check_call(pythonbinary,os.path.join(self.build_dir,'MakeBoostDistro.py'), self.root_dir,self.boost_release_name) packages = [] @@ -269,7 +283,7 @@ class script(script_common): # Create archive info data files. for archive_file in archive_files: - sha256_sum = hashlib.sha256(open(archive_file).read()).hexdigest() + sha256_sum = hashlib.sha256(open(archive_file,"rb").read()).hexdigest() utils.make_file("%s.json"%(archive_file), "{", '"sha256":"%s",'%(sha256_sum), @@ -404,9 +418,10 @@ class script(script_common): '%s'%(credentialhelperscript)) # Create a release, if one is not present - list_of_releases=subprocess.check_output(['gh', - 'release', - 'list']) + if pythonversion=="2": + list_of_releases=subprocess.check_output(['gh', 'release', 'list']) + else: + list_of_releases=subprocess.check_output(['gh', 'release', 'list'], encoding='UTF-8' ) if github_release_name not in list_of_releases: utils.check_call('gh', diff --git a/ci_boost_status.py b/ci_boost_status.py index 56d44be..f79486b 100644 --- a/ci_boost_status.py +++ b/ci_boost_status.py @@ -11,6 +11,14 @@ import os.path from ci_boost_common import main, utils, script_common +# Check python version +if sys.version_info[0] == 2 : + pythonversion="2" + pythonbinary="python2" +else: + pythonversion="3" + pythonbinary="python3" + class script(script_common): def __init__(self, ci_klass, **kargs): @@ -45,7 +53,7 @@ class script(script_common): utils.check_call('chmod','+x','MakeBoostDistro.py') os.chdir(self.root_dir) - utils.check_call('python',os.path.join(self.build_dir,'MakeBoostDistro.py'), + utils.check_call(pythonbinary,os.path.join(self.build_dir,'MakeBoostDistro.py'), self.root_dir, 'release') self.root_dir = os.path.join( self.root_dir, 'release' ) diff --git a/git-mirror-and-export.py b/git-mirror-and-export.py index 96c4904..71f911c 100755 --- a/git-mirror-and-export.py +++ b/git-mirror-and-export.py @@ -1,6 +1,8 @@ #!/usr/bin/python # +from __future__ import print_function + import requests import json @@ -11,12 +13,25 @@ import errno import subprocess import re import shutil -from urlparse import urlparse, urljoin + +# For urllib +from future.standard_library import install_aliases +install_aliases() +from urllib.parse import urlparse, urljoin + +# For iteritems +from future.utils import iteritems + +# Check python version +if sys.version_info[0] == 2 : + pythonversion="2" +else: + pythonversion="3" # Update or create the github mirror. def mirror_boostorg(root_dir): mirror_dir = os.path.join(root_dir, 'mirror') - print "Updating mirror at %s" % mirror_dir + print("Updating mirror at %s" % mirror_dir) url = 'https://api.github.com/orgs/boostorg/repos' while (url) : @@ -25,7 +40,7 @@ def mirror_boostorg(root_dir): raise Exception("Error getting: " + url) for repo in json.loads(r.text or r.content): - print "Downloading " + repo['name'] + print("Downloading " + repo['name']) url = repo['clone_url'] # Not using os.path.join because url path is absolute. path = mirror_dir + urlparse(url).path @@ -51,8 +66,8 @@ def mirror_export(root_dir, dst_dir, branch = 'master', eol = 'lf'): [ module_settings[x]['path'] for x in module_settings ]) # Export child submodules - for name, module in module_settings.iteritems(): - print "Exporting submodule " + name + for name, module in iteritems(module_settings): + print("Exporting submodule " + name) if module['path'] not in hashes: raise Exception('No hash for module ' + name) export_single_repo( @@ -74,6 +89,8 @@ def get_submodule_settings(dst_dir): for line in subprocess.Popen( [ 'git', 'config', '-f', dst_dir + "/.gitmodules", "-l" ], stdout=subprocess.PIPE).stdout: + if pythonversion=="3": + line=line.decode(encoding="utf-8") result = re.match('submodule\.([^.]*)\.([^.]*)=(.*)', line) if result.group(1) not in module_settings: module_settings[result.group(1)] = { 'name': result.group(1) } @@ -86,6 +103,8 @@ def get_submodule_hashes(boost_module_dir, branch, paths): for line in subprocess.Popen( [ 'git', '--git-dir=' + boost_module_dir, 'ls-tree', branch ] + paths, stdout=subprocess.PIPE).stdout: + if pythonversion=="3": + line=line.decode(encoding="utf-8") result = re.match('160000 commit ([0-9a-zA-Z]+)\t(.*)', line) hashes[result.group(2)] = result.group(1) return hashes @@ -106,7 +125,7 @@ def mkdir_p(path): def export_boost(root_dir, branch, eol): dir = os.path.join(root_dir, branch + '-' + eol) - print "Exporting to %s" % dir + print("Exporting to %s" % dir) if os.path.isdir(dir): shutil.rmtree(dir) mirror_export(root_dir, dir, branch, eol) @@ -116,14 +135,14 @@ if len(sys.argv) > 1: else: root=os.path.dirname(sys.argv[0]) -print "Update mirror" -print +print("Update mirror") +print() mirror_boostorg(root) -print "Export master-crlf" -print +print("Export master-crlf") +print() export_boost(root, 'master', 'crlf') -print "Export master-lf" -print +print("Export master-lf") +print() export_boost(root, 'master', 'lf') diff --git a/inspect_libraries.py b/inspect_libraries.py index 04ce97d..49bc6a3 100755 --- a/inspect_libraries.py +++ b/inspect_libraries.py @@ -4,6 +4,8 @@ # Distributed under the Boost Software License, Version 1.0. # See http://www.boost.org/LICENSE_1_0.txt +from __future__ import print_function + import os import os.path import subprocess @@ -13,8 +15,8 @@ import argparse def cmnd(x): # execute a command rc = subprocess.call(x) if rc != 0: - print 'aborting command:' - print x + print('aborting command:') + print(x) exit(rc) argpsr = argparse.ArgumentParser( @@ -29,37 +31,37 @@ args = argpsr.parse_args() filepath = 'temp/' + args.filename if not os.path.exists('.svn'): - print 'Error: must be run in the boost root directory' + print('Error: must be run in the boost root directory') exit(1) -print 'Boost root directory detected' +print('Boost root directory detected') if not os.path.exists('temp'): - print 'Create temp directory...' + print('Create temp directory...') os.mkdir('temp') open('temp/boost-no-inspect', 'w').close() # cleanup clears locks or other residual problems (learned this the hard way!) -print 'Clean working copy ...' +print('Clean working copy ...') cmnd(['svn', 'cleanup']) -print 'Update working copy...' +print('Update working copy...') cmnd(['svn', 'up', '--non-interactive', '--trust-server-cert']) -print 'Build inspect program...' +print('Build inspect program...') cmnd(['bjam', 'tools/inspect/build']) -print 'Running inspect from ' + os.getcwd() +print('Running inspect from ' + os.getcwd()) subprocess.call('dist/bin/inspect', stdout=open(filepath, 'w')) -print 'FTP: Sign on...' +print('FTP: Sign on...') ftp = FTP(args.host, args.user, args.password) -print 'FTP: List current contents...' +print('FTP: List current contents...') ftp.retrlines('LIST') -print 'FTP: Upload web page via ftp...' +print('FTP: Upload web page via ftp...') ftp.storlines('STOR ' + args.filename, open(filepath)) -print 'FTP: List updated contents...' +print('FTP: List updated contents...') ftp.retrlines('LIST') ftp.quit() -print "Inspect and upload complete" +print("Inspect and upload complete") diff --git a/mergeTree.py b/mergeTree.py index 07286ae..bbf5a57 100755 --- a/mergeTree.py +++ b/mergeTree.py @@ -1,61 +1,67 @@ #!/usr/bin/python -import os, sys +from __future__ import print_function + +import os +import sys import shutil + def mergetree(src, dst, symlinks=False, ignore=None): - from shutil import copy2, copystat, Error - import os + from shutil import copy2, copystat, Error + import os - names = os.listdir(src) - if ignore is not None: - ignored_names = ignore(src, names) - else: - ignored_names = set() + names = os.listdir(src) + if ignore is not None: + ignored_names = ignore(src, names) + else: + ignored_names = set() - try: - os.makedirs(dst) - except OSError, exc: - # XXX - this is pretty ugly - if "file already exists" in exc[1]: # Windows - pass - elif "File exists" in exc[1]: # Linux - pass - else: - raise + try: + os.makedirs(dst) + except OSError as exc: + # XXX - this is pretty ugly + strexc=str(exc) + if "file already exists" in strexc: # Windows + pass + elif "File exists" in strexc: # Linux + pass + else: + raise + + errors = [] + for name in names: + if name in ignored_names: + continue + srcname = os.path.join(src, name) + dstname = os.path.join(dst, name) + try: + if symlinks and os.path.islink(srcname): + linkto = os.readlink(srcname) + os.symlink(linkto, dstname) + elif os.path.isdir(srcname): + mergetree(srcname, dstname, symlinks, ignore) + else: + copy2(srcname, dstname) + # XXX What about devices, sockets etc.? + except (IOError, os.error) as why: + errors.append((srcname, dstname, str(why))) + # catch the Error from the recursive mergetree so that we can + # continue with other files + except Error as err: + errors.extend(err.args[0]) + try: + copystat(src, dst) + except WindowsError: + # can't copy file access times on Windows + pass + except OSError as why: + errors.extend((src, dst, str(why))) + if errors: + raise Error(errors) - errors = [] - for name in names: - if name in ignored_names: - continue - srcname = os.path.join(src, name) - dstname = os.path.join(dst, name) - try: - if symlinks and os.path.islink(srcname): - linkto = os.readlink(srcname) - os.symlink(linkto, dstname) - elif os.path.isdir(srcname): - mergetree(srcname, dstname, symlinks, ignore) - else: - copy2(srcname, dstname) - # XXX What about devices, sockets etc.? - except (IOError, os.error), why: - errors.append((srcname, dstname, str(why))) - # catch the Error from the recursive mergetree so that we can - # continue with other files - except Error, err: - errors.extend(err.args[0]) - try: - copystat(src, dst) - except WindowsError: - # can't copy file access times on Windows - pass - except OSError, why: - errors.extend((src, dst, str(why))) - if errors: - raise Error, errors if len(sys.argv) == 3: - mergetree ( sys.argv[1], sys.argv[2] ) + mergetree(sys.argv[1], sys.argv[2]) else: - print "Usage %s " % sys.argv[0] + print("Usage %s " % sys.argv[0]) diff --git a/snapshot.py b/snapshot.py index 6388b92..4d587a6 100755 --- a/snapshot.py +++ b/snapshot.py @@ -38,14 +38,14 @@ def mergetree(src, dst, symlinks=False, ignore=None): try: os.makedirs(dst) - except OSError, exc: - # XXX - this is pretty ugly - if "file already exists" in exc[1]: # Windows - pass - elif "File exists" in exc[1]: # Linux - pass - else: - raise + except OSError as exc: + strexc=str(exc) + if "file already exists" in strexc: # Windows + pass + elif "File exists" in strexc: # Linux + pass + else: + raise errors = [] for name in names: @@ -62,21 +62,21 @@ def mergetree(src, dst, symlinks=False, ignore=None): else: copy2(srcname, dstname) # XXX What about devices, sockets etc.? - except (IOError, os.error), why: + except (IOError, os.error) as why: errors.append((srcname, dstname, str(why))) # catch the Error from the recursive mergetree so that we can # continue with other files - except Error, err: + except Error as err: errors.extend(err.args[0]) try: copystat(src, dst) except WindowsError: # can't copy file access times on Windows pass - except OSError, why: + except OSError as why: errors.extend((src, dst, str(why))) if errors: - raise Error, errors + raise Error(errors) def svnExport(url, eol, revisionStr, dest): command_arr = [ "svn", "export", "--non-interactive", "--native-eol", eol, "-r", revisionStr, url, dest ] @@ -102,7 +102,7 @@ def hash_file(filename): return hasher def print_hash(filename): - print "%s *%s" % (hash_file(filename).hexdigest(), filename) + print("%s *%s" % (hash_file(filename).hexdigest(), filename)) def compress_7z (dest, source): command_arr = [ k7zName, "a", "-r", dest + ".7z", source ]