python3 (#19)

MakeBoostDistro.py

@@ -10,9 +10,12 @@
 #
 #	Usage: %0 source dest
 
+from __future__ import print_function
+
 import os, sys
 import shutil
 import stat
+import six
 
 IgnoreFiles = shutil.ignore_patterns(
 	'[.]*',
@@ -51,7 +54,7 @@ def MergeTree(src, dst, symlinks = False):
 			MergeTree(s, d, symlinks)
 		else:
 			if os.path.exists(d):
-				print "## Overwriting file %s with %s" % (d, s)
+				print("## Overwriting file %s with %s" % (d, s))
 			shutil.copy2(s, d)
 
 
@@ -83,7 +86,7 @@ def CopyInclude(src, dst):
 			MergeTree(s, d, symlinks=False)
 		else:
 			if os.path.exists(d):
-				print "## Overwriting file %s with %s" % (d, s)
+				print("## Overwriting file %s with %s" % (d, s))
 			CopyFile(src, dst, item)
 	
 
@@ -138,15 +141,15 @@ BoostSpecialFolders = [ "doc", "more", "status", "tools" ]
 SourceRoot = sys.argv[1]
 DestRoot   = sys.argv[2]
 
-print "Source = %s" % SourceRoot
+print("Source = %s" % SourceRoot)
-print "Dest   = %s" % DestRoot
+print("Dest   = %s" % DestRoot)
 
 if not os.path.exists(SourceRoot):
-	print "## Error: %s does not exist" % SourceRoot
+	print("## Error: %s does not exist" % SourceRoot)
 	exit(1)
 
 if not os.path.exists(DestRoot):
-	print "Creating destination directory %s" % DestRoot
+	print("Creating destination directory %s" % DestRoot)
 	os.makedirs(DestRoot)
 
 DestHeaders = os.path.join(DestRoot, BoostHeaders)
@@ -187,7 +190,7 @@ for f in os.listdir(SourceLibs):
 						BoostSubProjects.add((f,s))
 
 for p in BoostSubProjects:
-	if isinstance(p, basestring):
+	if isinstance(p, six.string_types):
 		CopySubProject(SourceLibs, DestLibs, DestHeaders, p)
 	else:
 		NestedSource  = os.path.join(SourceRoot,"libs",p[0])

ci_boost_build.py

@@ -12,6 +12,14 @@ import os.path
 
 from ci_boost_common import main, utils, script_common
 
+# Check python version
+if sys.version_info[0] == 2 :
+    pythonversion="2"
+    pythonbinary="python2"
+else:
+    pythonversion="3"
+    pythonbinary="python3"
+
 class script(script_common):
 
     def __init__(self, ci_klass, **kargs):
@@ -43,7 +51,7 @@ class script(script_common):
             utils.check_call('chmod','+x','MakeBoostDistro.py')
 
             os.chdir(self.root_dir)
-            utils.check_call('python',os.path.join(self.build_dir,'MakeBoostDistro.py'),
+            utils.check_call(pythonbinary,os.path.join(self.build_dir,'MakeBoostDistro.py'),
                 self.root_dir, 'release')
 
             self.root_dir = os.path.join( self.root_dir, 'release' )

ci_boost_common.py

@@ -6,6 +6,8 @@
 # (See accompanying file LICENSE_1_0.txt or copy at
 # http://www.boost.org/LICENSE_1_0.txt)
 
+from __future__ import print_function
+
 import sys
 import inspect
 import optparse
@@ -18,6 +20,12 @@ import shutil
 import threading
 import distutils.dir_util
 
+# For urllib
+from future.standard_library import install_aliases
+install_aliases()
+
+from builtins import str
+
 class SystemCallError(Exception):
     def __init__(self, command, result):
         self.command = command
@@ -36,7 +44,7 @@ class utils:
         result = subprocess.call(command, **kargs)
         t = time.time()-t
         if result != 0:
-            print "Failed: '%s' ERROR = %s"%("' '".join(command), result)
+            print("Failed: '%s' ERROR = %s"%("' '".join(command), result))
         utils.call_stats.append((t,os.getcwd(),command,result))
         utils.log( "%s> '%s' execution time %s seconds"%(os.getcwd(), "' '".join(command), t) )
         return result
@@ -53,7 +61,7 @@ class utils:
         cwd = os.getcwd()
         result = utils.call(*command, **kargs)
         if result != 0:
-            raise(SystemCallError([cwd].extend(command), result))
+            raise SystemCallError([cwd].extend(command), result)
     
     @staticmethod
     def makedirs( path ):
@@ -65,7 +73,7 @@ class utils:
         frames = inspect.stack()
         level = 0
         for i in frames[ 3: ]:
-            if i[0].f_locals.has_key( '__log__' ):
+            if '__log__' in i[0].f_locals:
                 level = level + i[0].f_locals[ '__log__' ]
         return level
     
@@ -82,6 +90,11 @@ class utils:
             #~ shutil.rmtree( unicode( path ) )
             if sys.platform == 'win32':
                 os.system( 'del /f /s /q "%s" >nul 2>&1' % path )
+                # Python 3 compatibility hack
+                try:
+                    unicode('')
+                except NameError:
+                    unicode = str
                 shutil.rmtree( unicode( path ) )
             else:
                 os.system( 'rm -f -r "%s"' % path )
@@ -91,7 +104,7 @@ class utils:
         for attempts in range( max_attempts, -1, -1 ):
             try:
                 return f()
-            except Exception, msg:
+            except Exception as msg:
                 utils.log( '%s failed with message "%s"' % ( f.__name__, msg ) )
                 if attempts == 0:
                     utils.log( 'Giving up.' )
@@ -102,7 +115,7 @@ class utils:
 
     @staticmethod
     def web_get( source_url, destination_file, proxy = None ):
-        import urllib
+        import urllib.request, urllib.parse, urllib.error
 
         proxies = None
         if proxy is not None:
@@ -111,7 +124,7 @@ class utils:
                 'http' : proxy
                 }
 
-        src = urllib.urlopen( source_url, proxies = proxies )
+        src = urllib.request.urlopen( source_url, proxies = proxies )
 
         f = open( destination_file, 'wb' )
         while True:
@@ -164,7 +177,7 @@ class utils:
     @staticmethod
     def make_file(filename, *text):
         f = codecs.open( filename, 'w', 'utf-8' )
-        f.write( string.join( text, '\n' ) )
+        f.write( '\n'.join(text) )
         f.close()
     
     @staticmethod
@@ -188,7 +201,7 @@ class parallel_call(threading.Thread):
     def join(self):
         super(parallel_call,self).join()
         if self.result != 0:
-            raise(SystemCallError(self.command, self.result))
+            raise SystemCallError(self.command, self.result)
 
 class script_common(object):
     '''

ci_boost_release.py

@@ -6,7 +6,10 @@
 # (See accompanying file LICENSE_1_0.txt or copy at
 # http://www.boost.org/LICENSE_1_0.txt)
 
+from __future__ import print_function
+
 import os.path
+import sys
 import time
 import shutil
 import site
@@ -15,6 +18,14 @@ import subprocess
 
 from ci_boost_common import main, utils, script_common, parallel_call
 
+# Check python version
+if sys.version_info[0] == 2 :
+    pythonversion="2"
+    pythonbinary="python2"
+else:
+    pythonversion="3"
+    pythonbinary="python3"
+
 class script(script_common):
     '''
     Main script to build/test Boost C++ Libraries continuous releases. This base
@@ -98,7 +109,10 @@ class script(script_common):
         utils.check_call("gem","install","asciidoctor", "--version", "1.5.8")
         utils.check_call("asciidoctor","--version")
         utils.check_call("gem","install","rouge")
-        utils.check_call("gem","install","pygments.rb", "--version", "1.2.1")
+        if pythonversion=="2":
+            utils.check_call("gem","install","pygments.rb", "--version", "1.2.1")
+        else: 
+            utils.check_call("gem","install","pygments.rb", "--version", "2.1.0")
         utils.check_call("pip","install","--user","Pygments==2.1")
         utils.check_call("pip","install","--user","https://github.com/bfgroup/jam_pygments/archive/master.zip")
         os.chdir(self.root_dir)
@@ -220,10 +234,10 @@ class script(script_common):
 
         # Make the real distribution tree from the base tree.
         os.chdir(os.path.join(self.build_dir))
-        utils.check_call('wget','https://raw.githubusercontent.com/boostorg/release-tools/develop/MakeBoostDistro.py')
+        utils.check_call('wget','https://raw.githubusercontent.com/sdarwin/release-tools/python3/MakeBoostDistro.py','-O','MakeBoostDistro.py')
         utils.check_call('chmod','+x','MakeBoostDistro.py')
         os.chdir(os.path.dirname(self.root_dir))
-        utils.check_call('python',os.path.join(self.build_dir,'MakeBoostDistro.py'),
+        utils.check_call(pythonbinary,os.path.join(self.build_dir,'MakeBoostDistro.py'),
             self.root_dir,self.boost_release_name)
         
         packages = []
@@ -269,7 +283,7 @@ class script(script_common):
         
         # Create archive info data files.
         for archive_file in archive_files:
-            sha256_sum = hashlib.sha256(open(archive_file).read()).hexdigest()
+            sha256_sum = hashlib.sha256(open(archive_file,"rb").read()).hexdigest()
             utils.make_file("%s.json"%(archive_file),
                 "{",
                 '"sha256":"%s",'%(sha256_sum),
@@ -404,9 +418,10 @@ class script(script_common):
                     '%s'%(credentialhelperscript))
 
             # Create a release, if one is not present
-            list_of_releases=subprocess.check_output(['gh',
+            if pythonversion=="2":
-                'release',
+                list_of_releases=subprocess.check_output(['gh', 'release', 'list'])
-                'list'])
+            else:
+                list_of_releases=subprocess.check_output(['gh', 'release', 'list'], encoding='UTF-8' )
 
             if github_release_name not in list_of_releases:
                 utils.check_call('gh',

ci_boost_status.py

@@ -11,6 +11,14 @@ import os.path
 
 from ci_boost_common import main, utils, script_common
 
+# Check python version
+if sys.version_info[0] == 2 :
+    pythonversion="2"
+    pythonbinary="python2"
+else:
+    pythonversion="3"
+    pythonbinary="python3"
+
 class script(script_common):
 
     def __init__(self, ci_klass, **kargs):
@@ -45,7 +53,7 @@ class script(script_common):
             utils.check_call('chmod','+x','MakeBoostDistro.py')
 
             os.chdir(self.root_dir)
-            utils.check_call('python',os.path.join(self.build_dir,'MakeBoostDistro.py'),
+            utils.check_call(pythonbinary,os.path.join(self.build_dir,'MakeBoostDistro.py'),
                 self.root_dir, 'release')
 
             self.root_dir = os.path.join( self.root_dir, 'release' )

git-mirror-and-export.py

@@ -1,6 +1,8 @@
 #!/usr/bin/python
 #
 
+from __future__ import print_function
+
 import requests
 
 import json
@@ -11,12 +13,25 @@ import errno
 import subprocess
 import re
 import shutil
-from urlparse import urlparse, urljoin
+
+# For urllib
+from future.standard_library import install_aliases
+install_aliases()
+from urllib.parse import urlparse, urljoin
+
+# For iteritems
+from future.utils import iteritems
+
+# Check python version
+if sys.version_info[0] == 2 :
+    pythonversion="2"
+else:
+    pythonversion="3"
 
 # Update or create the github mirror.
 def mirror_boostorg(root_dir):
     mirror_dir = os.path.join(root_dir, 'mirror')
-    print "Updating mirror at %s" % mirror_dir
+    print("Updating mirror at %s" % mirror_dir)
     url = 'https://api.github.com/orgs/boostorg/repos'
 
     while (url) :
@@ -25,7 +40,7 @@ def mirror_boostorg(root_dir):
             raise Exception("Error getting: " + url)
 
         for repo in json.loads(r.text or r.content):
-            print "Downloading " + repo['name']
+            print("Downloading " + repo['name'])
             url = repo['clone_url']
             # Not using os.path.join because url path is absolute.
             path = mirror_dir + urlparse(url).path
@@ -51,8 +66,8 @@ def mirror_export(root_dir, dst_dir, branch = 'master', eol = 'lf'):
             [ module_settings[x]['path'] for x in module_settings ])
 
     # Export child submodules
-    for name, module in module_settings.iteritems():
+    for name, module in iteritems(module_settings):
-        print "Exporting submodule " + name
+        print("Exporting submodule " + name)
         if module['path'] not in hashes:
             raise Exception('No hash for module ' + name)
         export_single_repo(
@@ -74,6 +89,8 @@ def get_submodule_settings(dst_dir):
     for line in subprocess.Popen(
             [ 'git', 'config', '-f', dst_dir + "/.gitmodules", "-l" ],
             stdout=subprocess.PIPE).stdout:
+        if pythonversion=="3":
+            line=line.decode(encoding="utf-8")
         result = re.match('submodule\.([^.]*)\.([^.]*)=(.*)', line)
         if result.group(1) not in module_settings:
             module_settings[result.group(1)] = { 'name': result.group(1) }
@@ -86,6 +103,8 @@ def get_submodule_hashes(boost_module_dir, branch, paths):
     for line in subprocess.Popen(
             [ 'git', '--git-dir=' + boost_module_dir, 'ls-tree', branch ] + paths,
             stdout=subprocess.PIPE).stdout:
+        if pythonversion=="3":
+            line=line.decode(encoding="utf-8")
         result = re.match('160000 commit ([0-9a-zA-Z]+)\t(.*)', line)
         hashes[result.group(2)] = result.group(1)
     return hashes
@@ -106,7 +125,7 @@ def mkdir_p(path):
 
 def export_boost(root_dir, branch, eol):
     dir = os.path.join(root_dir, branch + '-' + eol)
-    print "Exporting to %s" % dir
+    print("Exporting to %s" % dir)
     if os.path.isdir(dir):
         shutil.rmtree(dir)
     mirror_export(root_dir, dir, branch, eol)
@@ -116,14 +135,14 @@ if len(sys.argv) > 1:
 else:
 	root=os.path.dirname(sys.argv[0])
 
-print "Update mirror"
+print("Update mirror")
-print
+print()
 mirror_boostorg(root)
 
-print "Export master-crlf"
+print("Export master-crlf")
-print
+print()
 export_boost(root, 'master', 'crlf')
 
-print "Export master-lf"
+print("Export master-lf")
-print
+print()
 export_boost(root, 'master', 'lf')

inspect_libraries.py

@@ -4,6 +4,8 @@
 # Distributed under the Boost Software License, Version 1.0.
 # See http://www.boost.org/LICENSE_1_0.txt
 
+from __future__ import print_function
+
 import os
 import os.path
 import subprocess
@@ -13,8 +15,8 @@ import argparse
 def cmnd(x):  # execute a command
   rc = subprocess.call(x)
   if rc != 0:
-    print 'aborting command:'
+    print('aborting command:')
-    print x
+    print(x)
     exit(rc)
     
 argpsr = argparse.ArgumentParser(
@@ -29,37 +31,37 @@ args = argpsr.parse_args()
 filepath = 'temp/' + args.filename  
 
 if not os.path.exists('.svn'):
-  print 'Error: must be run in the boost root directory'
+  print('Error: must be run in the boost root directory')
   exit(1)
-print 'Boost root directory detected'
+print('Boost root directory detected')
 
 if not os.path.exists('temp'):
-  print 'Create temp directory...'
+  print('Create temp directory...')
   os.mkdir('temp')
   open('temp/boost-no-inspect', 'w').close()
 
 # cleanup clears locks or other residual problems (learned this the hard way!)
-print 'Clean working copy ...'
+print('Clean working copy ...')
 cmnd(['svn', 'cleanup'])
 
-print 'Update working copy...'
+print('Update working copy...')
 cmnd(['svn', 'up', '--non-interactive', '--trust-server-cert'])
 
-print 'Build inspect program...'
+print('Build inspect program...')
 cmnd(['bjam', 'tools/inspect/build'])
 
-print 'Running inspect from ' + os.getcwd()
+print('Running inspect from ' + os.getcwd())
 subprocess.call('dist/bin/inspect', stdout=open(filepath, 'w'))
 
-print 'FTP: Sign on...'
+print('FTP: Sign on...')
 ftp = FTP(args.host, args.user, args.password)
-print 'FTP: List current contents...'
+print('FTP: List current contents...')
 ftp.retrlines('LIST')
-print 'FTP: Upload web page via ftp...'
+print('FTP: Upload web page via ftp...')
 ftp.storlines('STOR ' + args.filename, open(filepath)) 
-print 'FTP: List updated contents...'
+print('FTP: List updated contents...')
 ftp.retrlines('LIST')
 ftp.quit()
 
-print "Inspect and upload complete"
+print("Inspect and upload complete")
 

mergeTree.py

@@ -1,61 +1,67 @@
 #!/usr/bin/python
 
-import os, sys
+from __future__ import print_function
+
+import os
+import sys
 import shutil
 
+
 def mergetree(src, dst, symlinks=False, ignore=None):
-	from shutil import copy2, copystat, Error
+    from shutil import copy2, copystat, Error
-	import os
+    import os
 
-	names = os.listdir(src)
+    names = os.listdir(src)
-	if ignore is not None:
+    if ignore is not None:
-		ignored_names = ignore(src, names)
+        ignored_names = ignore(src, names)
-	else:
+    else:
-		ignored_names = set()
+        ignored_names = set()
 
-	try:
+    try:
-		os.makedirs(dst)
+        os.makedirs(dst)
-	except OSError, exc:
+    except OSError as exc:
-		# XXX - this is pretty ugly
+        # XXX - this is pretty ugly
-		if "file already exists" in exc[1]:	 # Windows
+        strexc=str(exc)
-			pass
+        if "file already exists" in strexc:	 # Windows
-		elif "File exists" in exc[1]:		 # Linux
+            pass
-			pass
+        elif "File exists" in strexc:		 # Linux
-		else:
+            pass
-			raise
+        else:
+            raise
+
+    errors = []
+    for name in names:
+        if name in ignored_names:
+            continue
+        srcname = os.path.join(src, name)
+        dstname = os.path.join(dst, name)
+        try:
+            if symlinks and os.path.islink(srcname):
+                linkto = os.readlink(srcname)
+                os.symlink(linkto, dstname)
+            elif os.path.isdir(srcname):
+                mergetree(srcname, dstname, symlinks, ignore)
+            else:
+                copy2(srcname, dstname)
+            # XXX What about devices, sockets etc.?
+        except (IOError, os.error) as why:
+            errors.append((srcname, dstname, str(why)))
+        # catch the Error from the recursive mergetree so that we can
+        # continue with other files
+        except Error as err:
+            errors.extend(err.args[0])
+    try:
+        copystat(src, dst)
+    except WindowsError:
+        # can't copy file access times on Windows
+        pass
+    except OSError as why:
+        errors.extend((src, dst, str(why)))
+    if errors:
+        raise Error(errors)
 
-	errors = []
-	for name in names:
-		if name in ignored_names:
-			continue
-		srcname = os.path.join(src, name)
-		dstname = os.path.join(dst, name)
-		try:
-			if symlinks and os.path.islink(srcname):
-				linkto = os.readlink(srcname)
-				os.symlink(linkto, dstname)
-			elif os.path.isdir(srcname):
-				mergetree(srcname, dstname, symlinks, ignore)
-			else:
-				copy2(srcname, dstname)
-			# XXX What about devices, sockets etc.?
-		except (IOError, os.error), why:
-			errors.append((srcname, dstname, str(why)))
-		# catch the Error from the recursive mergetree so that we can
-		# continue with other files
-		except Error, err:
-			errors.extend(err.args[0])
-	try:
-		copystat(src, dst)
-	except WindowsError:
-		# can't copy file access times on Windows
-		pass
-	except OSError, why:
-		errors.extend((src, dst, str(why)))
-	if errors:
-		raise Error, errors
 
 if len(sys.argv) == 3:
-	mergetree ( sys.argv[1], sys.argv[2] )
+    mergetree(sys.argv[1], sys.argv[2])
 else:
-	print "Usage %s <source> <dest>" % sys.argv[0]
+    print("Usage %s <source> <dest>" % sys.argv[0])

snapshot.py

@@ -38,14 +38,14 @@ def mergetree(src, dst, symlinks=False, ignore=None):
 
 	try:
 		os.makedirs(dst)
-	except OSError, exc:
+	except OSError as exc:
-		# XXX - this is pretty ugly
+            strexc=str(exc)
-		if "file already exists" in exc[1]:	 # Windows
+            if "file already exists" in strexc:      # Windows
-			pass
+                pass
-		elif "File exists" in exc[1]:		 # Linux
+            elif "File exists" in strexc:            # Linux
-			pass
+                pass
-		else:
+            else:
-			raise
+                raise
 
 	errors = []
 	for name in names:
@@ -62,21 +62,21 @@ def mergetree(src, dst, symlinks=False, ignore=None):
 			else:
 				copy2(srcname, dstname)
 			# XXX What about devices, sockets etc.?
-		except (IOError, os.error), why:
+		except (IOError, os.error) as why:
 			errors.append((srcname, dstname, str(why)))
 		# catch the Error from the recursive mergetree so that we can
 		# continue with other files
-		except Error, err:
+		except Error as err:
 			errors.extend(err.args[0])
 	try:
 		copystat(src, dst)
 	except WindowsError:
 		# can't copy file access times on Windows
 		pass
-	except OSError, why:
+	except OSError as why:
 		errors.extend((src, dst, str(why)))
 	if errors:
-		raise Error, errors
+		raise Error(errors)
 
 def svnExport(url, eol, revisionStr, dest):
 	command_arr = [ "svn", "export", "--non-interactive", "--native-eol", eol, "-r", revisionStr, url, dest ]
@@ -102,7 +102,7 @@ def hash_file(filename):
 	return hasher
 
 def print_hash(filename):
-	print "%s *%s" % (hash_file(filename).hexdigest(), filename)
+	print("%s *%s" % (hash_file(filename).hexdigest(), filename))
 
 def compress_7z (dest, source):
 	command_arr = [ k7zName, "a", "-r", dest + ".7z",  source ]