boost/release-tools
2
0
Fork 0
mirror of https://github.com/boostorg/release-tools.git synced 2026-01-19 04:42:10 +00:00
Code Issues Projects Releases Wiki Activity

python3 (#19)

Browse Source
This commit is contained in:
Sam Darwin
2021-10-25 13:22:46 -05:00
committed by GitHub
parent 68ca68eedd
commit 40e13028ef
9 changed files with 187 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
MakeBoostDistro.py
View File

@@ -10,9 +10,12 @@
#
# Usage: %0 source dest
from __future__ import print_function
import os, sys
import shutil
import stat
import six
IgnoreFiles = shutil.ignore_patterns(
'[.]*',
@@ -51,7 +54,7 @@ def MergeTree(src, dst, symlinks = False):
MergeTree(s, d, symlinks)
else:
if os.path.exists(d):
print "## Overwriting file %s with %s" % (d, s)
print("## Overwriting file %s with %s" % (d, s))
shutil.copy2(s, d)
@@ -83,7 +86,7 @@ def CopyInclude(src, dst):
MergeTree(s, d, symlinks=False)
else:
if os.path.exists(d):
print "## Overwriting file %s with %s" % (d, s)
print("## Overwriting file %s with %s" % (d, s))
CopyFile(src, dst, item)
@@ -138,15 +141,15 @@ BoostSpecialFolders = [ "doc", "more", "status", "tools" ]
SourceRoot = sys.argv[1]
DestRoot = sys.argv[2]
print "Source = %s" % SourceRoot
print "Dest = %s" % DestRoot
print("Source = %s" % SourceRoot)
print("Dest = %s" % DestRoot)
if not os.path.exists(SourceRoot):
print "## Error: %s does not exist" % SourceRoot
print("## Error: %s does not exist" % SourceRoot)
exit(1)
if not os.path.exists(DestRoot):
print "Creating destination directory %s" % DestRoot
print("Creating destination directory %s" % DestRoot)
os.makedirs(DestRoot)
DestHeaders = os.path.join(DestRoot, BoostHeaders)
@@ -187,7 +190,7 @@ for f in os.listdir(SourceLibs):
BoostSubProjects.add((f,s))
for p in BoostSubProjects:
if isinstance(p, basestring):
if isinstance(p, six.string_types):
CopySubProject(SourceLibs, DestLibs, DestHeaders, p)
else:
NestedSource = os.path.join(SourceRoot,"libs",p[0])

10
ci_boost_build.py
View File

@@ -12,6 +12,14 @@ import os.path
from ci_boost_common import main, utils, script_common
# Check python version
if sys.version_info[0] == 2 :
pythonversion="2"
pythonbinary="python2"
else:
pythonversion="3"
pythonbinary="python3"
class script(script_common):
def __init__(self, ci_klass, **kargs):
@@ -43,7 +51,7 @@ class script(script_common):
utils.check_call('chmod','+x','MakeBoostDistro.py')
os.chdir(self.root_dir)
utils.check_call('python',os.path.join(self.build_dir,'MakeBoostDistro.py'),
utils.check_call(pythonbinary,os.path.join(self.build_dir,'MakeBoostDistro.py'),
self.root_dir, 'release')
self.root_dir = os.path.join( self.root_dir, 'release' )

29
ci_boost_common.py
View File

@@ -6,6 +6,8 @@
# (See accompanying file LICENSE_1_0.txt or copy at
# http://www.boost.org/LICENSE_1_0.txt)
from __future__ import print_function
import sys
import inspect
import optparse
@@ -18,6 +20,12 @@ import shutil
import threading
import distutils.dir_util
# For urllib
from future.standard_library import install_aliases
install_aliases()
from builtins import str
class SystemCallError(Exception):
def __init__(self, command, result):
self.command = command
@@ -36,7 +44,7 @@ class utils:
result = subprocess.call(command, **kargs)
t = time.time()-t
if result != 0:
print "Failed: '%s' ERROR = %s"%("' '".join(command), result)
print("Failed: '%s' ERROR = %s"%("' '".join(command), result))
utils.call_stats.append((t,os.getcwd(),command,result))
utils.log( "%s> '%s' execution time %s seconds"%(os.getcwd(), "' '".join(command), t) )
return result
@@ -53,7 +61,7 @@ class utils:
cwd = os.getcwd()
result = utils.call(*command, **kargs)
if result != 0:
raise(SystemCallError([cwd].extend(command), result))
raise SystemCallError([cwd].extend(command), result)
@staticmethod
def makedirs( path ):
@@ -65,7 +73,7 @@ class utils:
frames = inspect.stack()
level = 0
for i in frames[ 3: ]:
if i[0].f_locals.has_key( '__log__' ):
if '__log__' in i[0].f_locals:
level = level + i[0].f_locals[ '__log__' ]
return level
@@ -82,6 +90,11 @@ class utils:
#~ shutil.rmtree( unicode( path ) )
if sys.platform == 'win32':
os.system( 'del /f /s /q "%s" >nul 2>&1' % path )
# Python 3 compatibility hack
try:
unicode('')
except NameError:
unicode = str
shutil.rmtree( unicode( path ) )
else:
os.system( 'rm -f -r "%s"' % path )
@@ -91,7 +104,7 @@ class utils:
for attempts in range( max_attempts, -1, -1 ):
try:
return f()
except Exception, msg:
except Exception as msg:
utils.log( '%s failed with message "%s"' % ( f.__name__, msg ) )
if attempts == 0:
utils.log( 'Giving up.' )
@@ -102,7 +115,7 @@ class utils:
@staticmethod
def web_get( source_url, destination_file, proxy = None ):
import urllib
import urllib.request, urllib.parse, urllib.error
proxies = None
if proxy is not None:
@@ -111,7 +124,7 @@ class utils:
'http' : proxy
}
src = urllib.urlopen( source_url, proxies = proxies )
src = urllib.request.urlopen( source_url, proxies = proxies )
f = open( destination_file, 'wb' )
while True:
@@ -164,7 +177,7 @@ class utils:
@staticmethod
def make_file(filename, *text):
f = codecs.open( filename, 'w', 'utf-8' )
f.write( string.join( text, '\n' ) )
f.write( '\n'.join(text) )
f.close()
@staticmethod
@@ -188,7 +201,7 @@ class parallel_call(threading.Thread):
def join(self):
super(parallel_call,self).join()
if self.result != 0:
raise(SystemCallError(self.command, self.result))
raise SystemCallError(self.command, self.result)
class script_common(object):
'''

29
ci_boost_release.py
View File

@@ -6,7 +6,10 @@
# (See accompanying file LICENSE_1_0.txt or copy at
# http://www.boost.org/LICENSE_1_0.txt)
from __future__ import print_function
import os.path
import sys
import time
import shutil
import site
@@ -15,6 +18,14 @@ import subprocess
from ci_boost_common import main, utils, script_common, parallel_call
# Check python version
if sys.version_info[0] == 2 :
pythonversion="2"
pythonbinary="python2"
else:
pythonversion="3"
pythonbinary="python3"
class script(script_common):
'''
Main script to build/test Boost C++ Libraries continuous releases. This base
@@ -98,7 +109,10 @@ class script(script_common):
utils.check_call("gem","install","asciidoctor", "--version", "1.5.8")
utils.check_call("asciidoctor","--version")
utils.check_call("gem","install","rouge")
utils.check_call("gem","install","pygments.rb", "--version", "1.2.1")
if pythonversion=="2":
utils.check_call("gem","install","pygments.rb", "--version", "1.2.1")
else:
utils.check_call("gem","install","pygments.rb", "--version", "2.1.0")
utils.check_call("pip","install","--user","Pygments==2.1")
utils.check_call("pip","install","--user","https://github.com/bfgroup/jam_pygments/archive/master.zip")
os.chdir(self.root_dir)
@@ -220,10 +234,10 @@ class script(script_common):
# Make the real distribution tree from the base tree.
os.chdir(os.path.join(self.build_dir))
utils.check_call('wget','https://raw.githubusercontent.com/boostorg/release-tools/develop/MakeBoostDistro.py')
utils.check_call('wget','https://raw.githubusercontent.com/sdarwin/release-tools/python3/MakeBoostDistro.py','-O','MakeBoostDistro.py')
utils.check_call('chmod','+x','MakeBoostDistro.py')
os.chdir(os.path.dirname(self.root_dir))
utils.check_call('python',os.path.join(self.build_dir,'MakeBoostDistro.py'),
utils.check_call(pythonbinary,os.path.join(self.build_dir,'MakeBoostDistro.py'),
self.root_dir,self.boost_release_name)
packages = []
@@ -269,7 +283,7 @@ class script(script_common):
# Create archive info data files.
for archive_file in archive_files:
sha256_sum = hashlib.sha256(open(archive_file).read()).hexdigest()
sha256_sum = hashlib.sha256(open(archive_file,"rb").read()).hexdigest()
utils.make_file("%s.json"%(archive_file),
"{",
'"sha256":"%s",'%(sha256_sum),
@@ -404,9 +418,10 @@ class script(script_common):
'%s'%(credentialhelperscript))
# Create a release, if one is not present
list_of_releases=subprocess.check_output(['gh',
'release',
'list'])
if pythonversion=="2":
list_of_releases=subprocess.check_output(['gh', 'release', 'list'])
else:
list_of_releases=subprocess.check_output(['gh', 'release', 'list'], encoding='UTF-8' )
if github_release_name not in list_of_releases:
utils.check_call('gh',

10
ci_boost_status.py
View File

@@ -11,6 +11,14 @@ import os.path
from ci_boost_common import main, utils, script_common
# Check python version
if sys.version_info[0] == 2 :
pythonversion="2"
pythonbinary="python2"
else:
pythonversion="3"
pythonbinary="python3"
class script(script_common):
def __init__(self, ci_klass, **kargs):
@@ -45,7 +53,7 @@ class script(script_common):
utils.check_call('chmod','+x','MakeBoostDistro.py')
os.chdir(self.root_dir)
utils.check_call('python',os.path.join(self.build_dir,'MakeBoostDistro.py'),
utils.check_call(pythonbinary,os.path.join(self.build_dir,'MakeBoostDistro.py'),
self.root_dir, 'release')
self.root_dir = os.path.join( self.root_dir, 'release' )

43
git-mirror-and-export.py
View File

@@ -1,6 +1,8 @@
#!/usr/bin/python
#
from __future__ import print_function
import requests
import json
@@ -11,12 +13,25 @@ import errno
import subprocess
import re
import shutil
from urlparse import urlparse, urljoin
# For urllib
from future.standard_library import install_aliases
install_aliases()
from urllib.parse import urlparse, urljoin
# For iteritems
from future.utils import iteritems
# Check python version
if sys.version_info[0] == 2 :
pythonversion="2"
else:
pythonversion="3"
# Update or create the github mirror.
def mirror_boostorg(root_dir):
mirror_dir = os.path.join(root_dir, 'mirror')
print "Updating mirror at %s" % mirror_dir
print("Updating mirror at %s" % mirror_dir)
url = 'https://api.github.com/orgs/boostorg/repos'
while (url) :
@@ -25,7 +40,7 @@ def mirror_boostorg(root_dir):
raise Exception("Error getting: " + url)
for repo in json.loads(r.text or r.content):
print "Downloading " + repo['name']
print("Downloading " + repo['name'])
url = repo['clone_url']
# Not using os.path.join because url path is absolute.
path = mirror_dir + urlparse(url).path
@@ -51,8 +66,8 @@ def mirror_export(root_dir, dst_dir, branch = 'master', eol = 'lf'):
[ module_settings[x]['path'] for x in module_settings ])
# Export child submodules
for name, module in module_settings.iteritems():
print "Exporting submodule " + name
for name, module in iteritems(module_settings):
print("Exporting submodule " + name)
if module['path'] not in hashes:
raise Exception('No hash for module ' + name)
export_single_repo(
@@ -74,6 +89,8 @@ def get_submodule_settings(dst_dir):
for line in subprocess.Popen(
[ 'git', 'config', '-f', dst_dir + "/.gitmodules", "-l" ],
stdout=subprocess.PIPE).stdout:
if pythonversion=="3":
line=line.decode(encoding="utf-8")
result = re.match('submodule\.([^.]*)\.([^.]*)=(.*)', line)
if result.group(1) not in module_settings:
module_settings[result.group(1)] = { 'name': result.group(1) }
@@ -86,6 +103,8 @@ def get_submodule_hashes(boost_module_dir, branch, paths):
for line in subprocess.Popen(
[ 'git', '--git-dir=' + boost_module_dir, 'ls-tree', branch ] + paths,
stdout=subprocess.PIPE).stdout:
if pythonversion=="3":
line=line.decode(encoding="utf-8")
result = re.match('160000 commit ([0-9a-zA-Z]+)\t(.*)', line)
hashes[result.group(2)] = result.group(1)
return hashes
@@ -106,7 +125,7 @@ def mkdir_p(path):
def export_boost(root_dir, branch, eol):
dir = os.path.join(root_dir, branch + '-' + eol)
print "Exporting to %s" % dir
print("Exporting to %s" % dir)
if os.path.isdir(dir):
shutil.rmtree(dir)
mirror_export(root_dir, dir, branch, eol)
@@ -116,14 +135,14 @@ if len(sys.argv) > 1:
else:
root=os.path.dirname(sys.argv[0])
print "Update mirror"
print
print("Update mirror")
print()
mirror_boostorg(root)
print "Export master-crlf"
print
print("Export master-crlf")
print()
export_boost(root, 'master', 'crlf')
print "Export master-lf"
print
print("Export master-lf")
print()
export_boost(root, 'master', 'lf')

30
inspect_libraries.py
View File

@@ -4,6 +4,8 @@
# Distributed under the Boost Software License, Version 1.0.
# See http://www.boost.org/LICENSE_1_0.txt
from __future__ import print_function
import os
import os.path
import subprocess
@@ -13,8 +15,8 @@ import argparse
def cmnd(x): # execute a command
rc = subprocess.call(x)
if rc != 0:
print 'aborting command:'
print x
print('aborting command:')
print(x)
exit(rc)
argpsr = argparse.ArgumentParser(
@@ -29,37 +31,37 @@ args = argpsr.parse_args()
filepath = 'temp/' + args.filename
if not os.path.exists('.svn'):
print 'Error: must be run in the boost root directory'
print('Error: must be run in the boost root directory')
exit(1)
print 'Boost root directory detected'
print('Boost root directory detected')
if not os.path.exists('temp'):
print 'Create temp directory...'
print('Create temp directory...')
os.mkdir('temp')
open('temp/boost-no-inspect', 'w').close()
# cleanup clears locks or other residual problems (learned this the hard way!)
print 'Clean working copy ...'
print('Clean working copy ...')
cmnd(['svn', 'cleanup'])
print 'Update working copy...'
print('Update working copy...')
cmnd(['svn', 'up', '--non-interactive', '--trust-server-cert'])
print 'Build inspect program...'
print('Build inspect program...')
cmnd(['bjam', 'tools/inspect/build'])
print 'Running inspect from ' + os.getcwd()
print('Running inspect from ' + os.getcwd())
subprocess.call('dist/bin/inspect', stdout=open(filepath, 'w'))
print 'FTP: Sign on...'
print('FTP: Sign on...')
ftp = FTP(args.host, args.user, args.password)
print 'FTP: List current contents...'
print('FTP: List current contents...')
ftp.retrlines('LIST')
print 'FTP: Upload web page via ftp...'
print('FTP: Upload web page via ftp...')
ftp.storlines('STOR ' + args.filename, open(filepath))
print 'FTP: List updated contents...'
print('FTP: List updated contents...')
ftp.retrlines('LIST')
ftp.quit()
print "Inspect and upload complete"
print("Inspect and upload complete")

106
mergeTree.py
View File

@@ -1,61 +1,67 @@
#!/usr/bin/python
import os, sys
from __future__ import print_function
import os
import sys
import shutil
def mergetree(src, dst, symlinks=False, ignore=None):
from shutil import copy2, copystat, Error
import os
from shutil import copy2, copystat, Error
import os
names = os.listdir(src)
if ignore is not None:
ignored_names = ignore(src, names)
else:
ignored_names = set()
names = os.listdir(src)
if ignore is not None:
ignored_names = ignore(src, names)
else:
ignored_names = set()
try:
os.makedirs(dst)
except OSError, exc:
# XXX - this is pretty ugly
if "file already exists" in exc[1]: # Windows
pass
elif "File exists" in exc[1]: # Linux
pass
else:
raise
try:
os.makedirs(dst)
except OSError as exc:
# XXX - this is pretty ugly
strexc=str(exc)
if "file already exists" in strexc: # Windows
pass
elif "File exists" in strexc: # Linux
pass
else:
raise
errors = []
for name in names:
if name in ignored_names:
continue
srcname = os.path.join(src, name)
dstname = os.path.join(dst, name)
try:
if symlinks and os.path.islink(srcname):
linkto = os.readlink(srcname)
os.symlink(linkto, dstname)
elif os.path.isdir(srcname):
mergetree(srcname, dstname, symlinks, ignore)
else:
copy2(srcname, dstname)
# XXX What about devices, sockets etc.?
except (IOError, os.error) as why:
errors.append((srcname, dstname, str(why)))
# catch the Error from the recursive mergetree so that we can
# continue with other files
except Error as err:
errors.extend(err.args[0])
try:
copystat(src, dst)
except WindowsError:
# can't copy file access times on Windows
pass
except OSError as why:
errors.extend((src, dst, str(why)))
if errors:
raise Error(errors)
errors = []
for name in names:
if name in ignored_names:
continue
srcname = os.path.join(src, name)
dstname = os.path.join(dst, name)
try:
if symlinks and os.path.islink(srcname):
linkto = os.readlink(srcname)
os.symlink(linkto, dstname)
elif os.path.isdir(srcname):
mergetree(srcname, dstname, symlinks, ignore)
else:
copy2(srcname, dstname)
# XXX What about devices, sockets etc.?
except (IOError, os.error), why:
errors.append((srcname, dstname, str(why)))
# catch the Error from the recursive mergetree so that we can
# continue with other files
except Error, err:
errors.extend(err.args[0])
try:
copystat(src, dst)
except WindowsError:
# can't copy file access times on Windows
pass
except OSError, why:
errors.extend((src, dst, str(why)))
if errors:
raise Error, errors
if len(sys.argv) == 3:
mergetree ( sys.argv[1], sys.argv[2] )
mergetree(sys.argv[1], sys.argv[2])
else:
print "Usage %s <source> <dest>" % sys.argv[0]
print("Usage %s <source> <dest>" % sys.argv[0])

26
snapshot.py
View File

@@ -38,14 +38,14 @@ def mergetree(src, dst, symlinks=False, ignore=None):
try:
os.makedirs(dst)
except OSError, exc:
# XXX - this is pretty ugly
if "file already exists" in exc[1]: # Windows
pass
elif "File exists" in exc[1]: # Linux
pass
else:
raise
except OSError as exc:
strexc=str(exc)
if "file already exists" in strexc: # Windows
pass
elif "File exists" in strexc: # Linux
pass
else:
raise
errors = []
for name in names:
@@ -62,21 +62,21 @@ def mergetree(src, dst, symlinks=False, ignore=None):
else:
copy2(srcname, dstname)
# XXX What about devices, sockets etc.?
except (IOError, os.error), why:
except (IOError, os.error) as why:
errors.append((srcname, dstname, str(why)))
# catch the Error from the recursive mergetree so that we can
# continue with other files
except Error, err:
except Error as err:
errors.extend(err.args[0])
try:
copystat(src, dst)
except WindowsError:
# can't copy file access times on Windows
pass
except OSError, why:
except OSError as why:
errors.extend((src, dst, str(why)))
if errors:
raise Error, errors
raise Error(errors)
def svnExport(url, eol, revisionStr, dest):
command_arr = [ "svn", "export", "--non-interactive", "--native-eol", eol, "-r", revisionStr, url, dest ]
@@ -102,7 +102,7 @@ def hash_file(filename):
return hasher
def print_hash(filename):
print "%s *%s" % (hash_file(filename).hexdigest(), filename)
print("%s *%s" % (hash_file(filename).hexdigest(), filename))
def compress_7z (dest, source):
command_arr = [ k7zName, "a", "-r", dest + ".7z", source ]