mirror of
https://github.com/boostorg/release-tools.git
synced 2026-01-19 04:42:10 +00:00
Add my Python files for creating Boost releases
This commit is contained in:
Marshall Clow
199
MakeBoostDistro.py
Executable file
199
MakeBoostDistro.py
Executable file
@@ -0,0 +1,199 @@
|
||||
#!/usr/bin/python
|
||||
#
|
||||
|
||||
# Prepare a boost checkout for release
|
||||
# 1) Copy all the files at the root level to the dest folder ($DEST)
|
||||
# 2) Copy all the "special" folders to the dest folder ($DEST)
|
||||
# 3) copy all the files from $SOURCE/libs to $DEST/libs
|
||||
# 4a) For each subproject, copy everything except "include" into $DEST/libs
|
||||
# 4b) For each subproject, copy the contents of the "includes" folder into $DEST/boost
|
||||
#
|
||||
# Usage: %0 source dest
|
||||
|
||||
import os, sys
|
||||
import shutil
|
||||
|
||||
## from <http://stackoverflow.com/questions/1868714/how-do-i-copy-an-entire-directory-of-files-into-an-existing-directory-using-pyth>
|
||||
def mergetree(src, dst, symlinks = False, ignore = None):
|
||||
if not os.path.exists(dst):
|
||||
os.makedirs(dst)
|
||||
shutil.copystat(src, dst)
|
||||
lst = os.listdir(src)
|
||||
if ignore:
|
||||
excl = ignore(src, lst)
|
||||
lst = [x for x in lst if x not in excl]
|
||||
for item in lst:
|
||||
s = os.path.join(src, item)
|
||||
d = os.path.join(dst, item)
|
||||
if symlinks and os.path.islink(s):
|
||||
if os.path.lexists(d):
|
||||
os.remove(d)
|
||||
os.symlink(os.readlink(s), d)
|
||||
try:
|
||||
st = os.lstat(s)
|
||||
mode = stat.S_IMODE(st.st_mode)
|
||||
os.lchmod(d, mode)
|
||||
except:
|
||||
pass # lchmod not available
|
||||
elif os.path.isdir(s):
|
||||
mergetree(s, d, symlinks, ignore)
|
||||
else:
|
||||
if os.path.exists(d):
|
||||
print "## Overwriting file %s with %s" % (d, s)
|
||||
shutil.copy2(s, d)
|
||||
|
||||
|
||||
def CopyFile (s, d, file):
|
||||
shutil.copy2(os.path.join(s,file), os.path.join(d,file))
|
||||
|
||||
def CopyDir (s, d, dir):
|
||||
shutil.copytree(os.path.join(s,dir), os.path.join(d,dir), symlinks=False, ignore=shutil.ignore_patterns('\.*'))
|
||||
|
||||
def MergeIf(s, d, dir):
|
||||
# if dir == 'detail':
|
||||
# print "MergeIf %s -> %s" % (os.path.join(s, dir), os.path.join(d, dir))
|
||||
if os.path.exists(os.path.join(s, dir)):
|
||||
mergetree(os.path.join(s, dir), os.path.join(d, dir), symlinks=False, ignore=shutil.ignore_patterns('\.*'))
|
||||
|
||||
def CopyInclude(src, dst):
|
||||
for item in os.listdir(src):
|
||||
if item[0] == '.':
|
||||
continue
|
||||
if item == 'pending':
|
||||
continue
|
||||
if item == 'detail':
|
||||
continue
|
||||
s = os.path.join(src, item)
|
||||
d = os.path.join(dst, item)
|
||||
if os.path.isdir(s):
|
||||
mergetree(s, d, symlinks=False, ignore=shutil.ignore_patterns('\.*'))
|
||||
else:
|
||||
if os.path.exists(d):
|
||||
print "## Overwriting file %s with %s" % (d, s)
|
||||
shutil.copy2(s, d)
|
||||
|
||||
|
||||
def CopySubProject(src, dst, headers, p):
|
||||
# First, everything except the "include" directory
|
||||
Source = os.path.join(src,p)
|
||||
Dest = os.path.join(dst,p)
|
||||
# print "CopySubProject %p" % p
|
||||
os.makedirs(Dest)
|
||||
items = [ f for f in os.listdir(Source) if f[0] != '.' ]
|
||||
for item in items:
|
||||
if os.path.isfile(os.path.join(Source, item)):
|
||||
CopyFile(Source, Dest, item)
|
||||
elif item != "include":
|
||||
CopyDir(Source, Dest, item)
|
||||
|
||||
# shutil.copytree(Source, Dest, symlinks=False, ignore=shutil.ignore_patterns('\.*', "include"))
|
||||
|
||||
# Now the includes
|
||||
Source = os.path.join(src, "%s/include/boost" % p)
|
||||
CopyInclude(Source, headers)
|
||||
# mergetree(Source, Dest, symlinks=False, ignore=shutil.ignore_patterns('\.*', 'detail', 'pending'))
|
||||
MergeIf(Source, headers, 'detail')
|
||||
MergeIf(Source, headers, 'pending')
|
||||
|
||||
|
||||
def CopyNestedProject(src, dst, headers, p):
|
||||
# First, everything except the "include" directory
|
||||
Source = os.path.join(src,p)
|
||||
Dest = os.path.join(dst,p)
|
||||
os.makedirs(Dest)
|
||||
items = [ f for f in os.listdir(Source) if f[0] != '.' ]
|
||||
for item in items:
|
||||
if os.path.isfile(os.path.join(Source, item)):
|
||||
CopyFile(Source, Dest, item)
|
||||
elif item != "include":
|
||||
CopyDir(Source, Dest, item)
|
||||
# shutil.copytree(Source, Dest, symlinks=False, ignore=shutil.ignore_patterns('\.*', "include"))
|
||||
|
||||
Source = os.path.join(src, "%s/include/boost/numeric" % p)
|
||||
# Dest = os.path.join(headers, p)
|
||||
# print "Installing headers from %s to %s" % (Source, headers)
|
||||
CopyInclude(Source, headers)
|
||||
# # mergetree(Source, Dest, symlinks=False, ignore=shutil.ignore_patterns('\.*', 'detail', 'pending'))
|
||||
# MergeIf(Source, headers, 'detail')
|
||||
# MergeIf(Source, headers, 'pending')
|
||||
return
|
||||
|
||||
BoostHeaders = "boost"
|
||||
BoostLibs = "libs"
|
||||
|
||||
BoostSpecialFolders = [ "doc", "more", "status", "tools" ]
|
||||
BoostSubProjects = [
|
||||
"accumulators", "align", "algorithm", "any", "array", "asio", "assert", "assign", "atomic",
|
||||
"bimap", "bind",
|
||||
"chrono", "circular_buffer", "compatibility", "concept_check", "config", "container", "context", "conversion", "core", "coroutine", "crc",
|
||||
"date_time", "detail", "disjoint_sets", "dynamic_bitset",
|
||||
"endian", "exception",
|
||||
"filesystem", "flyweight", "foreach", "format", "function", "function_types", "functional", "fusion",
|
||||
"geometry", "gil", "graph", "graph_parallel",
|
||||
"heap",
|
||||
"icl", "integer", "interprocess", "intrusive", "io", "iostreams", "iterator",
|
||||
"lambda", "lexical_cast", "local_function", "locale", "lockfree", "log", "logic",
|
||||
"math", "move", "mpi", "mpl", "msm", "multi_array", "multi_index", "multiprecision",
|
||||
# "numeric",
|
||||
"optional",
|
||||
"parameter", "phoenix", "polygon", "pool", "predef", "preprocessor", "program_options", "property_map", "property_tree", "proto", "ptr_container", "python",
|
||||
"random", "range", "ratio", "rational", "regex",
|
||||
"scope_exit", "serialization", "signals", "signals2", "smart_ptr", "sort", "spirit", "statechart", "static_assert", "system",
|
||||
"test", "thread", "throw_exception", "timer", "tokenizer", "tr1", "tti", "tuple", "type_index", "type_erasure", "type_traits", "typeof",
|
||||
"units", "unordered", "utility", "uuid",
|
||||
"variant",
|
||||
"wave", "winapi",
|
||||
"xpressive"
|
||||
]
|
||||
|
||||
NumericLibs = [ "conversion", "interval", "odeint", "ublas" ]
|
||||
|
||||
SourceRoot = sys.argv[1]
|
||||
DestRoot = sys.argv[2]
|
||||
|
||||
print "Source = %s" % SourceRoot
|
||||
print "Dest = %s" % DestRoot
|
||||
|
||||
if not os.path.exists(SourceRoot):
|
||||
print "## Error: %s does not exist" % SourceRoot
|
||||
exit(1)
|
||||
|
||||
if not os.path.exists(DestRoot):
|
||||
print "Creating destination directory %s" % DestRoot
|
||||
os.makedirs(DestRoot)
|
||||
|
||||
DestHeaders = os.path.join(DestRoot, BoostHeaders)
|
||||
DestLibs = os.path.join(DestRoot, BoostLibs)
|
||||
os.makedirs(DestHeaders)
|
||||
os.makedirs(DestLibs)
|
||||
|
||||
## Step 1
|
||||
files = [ f for f in os.listdir(SourceRoot) if os.path.isfile(os.path.join(SourceRoot,f)) and f[0] != '.' ]
|
||||
for f in files:
|
||||
CopyFile(SourceRoot, DestRoot, f)
|
||||
|
||||
## Step 2
|
||||
for d in BoostSpecialFolders:
|
||||
CopyDir(SourceRoot, DestRoot, d)
|
||||
|
||||
## Step 3
|
||||
SourceLibs = os.path.join(SourceRoot, BoostLibs)
|
||||
files = [ f for f in os.listdir(SourceLibs) if os.path.isfile(os.path.join(SourceLibs,f)) and f[0] != '.' ]
|
||||
for f in files:
|
||||
CopyFile(SourceLibs, DestLibs, f)
|
||||
|
||||
for p in BoostSubProjects:
|
||||
CopySubProject(SourceLibs, DestLibs, DestHeaders, p)
|
||||
|
||||
# ## Step 4
|
||||
NumericSource = os.path.join(SourceRoot, "libs/numeric")
|
||||
NumericDest = os.path.join(DestRoot, "libs/numeric")
|
||||
NumericHeaders = os.path.join(DestRoot, "boost/numeric")
|
||||
os.makedirs(NumericDest)
|
||||
os.makedirs(NumericHeaders)
|
||||
files = [ f for f in os.listdir(NumericSource) if os.path.isfile(os.path.join(NumericSource,f)) and f[0] != '.' ]
|
||||
for f in files:
|
||||
CopyFile(NumericSource, NumericDest, f)
|
||||
for p in NumericLibs:
|
||||
CopyNestedProject (NumericSource, NumericDest, NumericHeaders, p)
|
||||
|
||||
129
git-mirror-and-export.py
Executable file
129
git-mirror-and-export.py
Executable file
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/python
|
||||
#
|
||||
|
||||
import requests
|
||||
|
||||
import json
|
||||
import sys
|
||||
import os
|
||||
import os.path
|
||||
import errno
|
||||
import subprocess
|
||||
import re
|
||||
import shutil
|
||||
from urlparse import urlparse, urljoin
|
||||
|
||||
# Update or create the github mirror.
|
||||
def mirror_boostorg(root_dir):
|
||||
mirror_dir = os.path.join(root_dir, 'mirror')
|
||||
print "Updating mirror at %s" % mirror_dir
|
||||
url = 'https://api.github.com/orgs/boostorg/repos'
|
||||
|
||||
while (url) :
|
||||
r = requests.get(url)
|
||||
if (not r.ok):
|
||||
raise Exception("Error getting: " + url)
|
||||
|
||||
for repo in json.loads(r.text or r.content):
|
||||
print "Downloading " + repo['name']
|
||||
url = repo['clone_url']
|
||||
# Not using os.path.join because url path is absolute.
|
||||
path = mirror_dir + urlparse(url).path
|
||||
mkdir_p(os.path.join(path, os.pardir))
|
||||
|
||||
# TODO: Check that path is actually a git repo?
|
||||
if os.path.isdir(path):
|
||||
subprocess.check_call(["git", "--git-dir=" + path, "fetch"])
|
||||
else:
|
||||
subprocess.check_call(["git", "clone", "--mirror", url, path])
|
||||
|
||||
url = r.links['next']['url'] if 'next' in r.links else False
|
||||
|
||||
# Export the full tree from the mirror
|
||||
def mirror_export(root_dir, dst_dir, branch = 'master', eol = 'lf'):
|
||||
git_flags = "-c core.autocrlf=false -c core.eol="+ eol
|
||||
boost_module_dir = os.path.join(root_dir, 'mirror/boostorg/boost.git')
|
||||
|
||||
os.mkdir(dst_dir)
|
||||
export_single_repo(boost_module_dir, dst_dir, branch, eol)
|
||||
module_settings = get_submodule_settings(dst_dir)
|
||||
hashes = get_submodule_hashes(boost_module_dir, branch,
|
||||
[ module_settings[x]['path'] for x in module_settings ])
|
||||
|
||||
# Export child submodules
|
||||
for name, module in module_settings.iteritems():
|
||||
print "Exporting submodule " + name
|
||||
if module['path'] not in hashes:
|
||||
raise Exception('No hash for module ' + name)
|
||||
export_single_repo(
|
||||
urljoin(boost_module_dir + '/', module['url']),
|
||||
os.path.join(dst_dir, module['path']),
|
||||
hashes[module['path']], eol)
|
||||
|
||||
# Export from a single git repo
|
||||
def export_single_repo(git_dir, dst_dir, ref, eol):
|
||||
ps = subprocess.Popen(
|
||||
["git", "-c", "core.autocrlf=false", "-c", "core.eol=" + eol,
|
||||
"--git-dir=" + git_dir, "archive", ref],
|
||||
stdout=subprocess.PIPE)
|
||||
subprocess.check_call(['tar', '-x', '-C', dst_dir], stdin=ps.stdout)
|
||||
|
||||
# Load the submodule settings from an exported repo.
|
||||
def get_submodule_settings(dst_dir):
|
||||
module_settings = {}
|
||||
for line in subprocess.Popen(
|
||||
[ 'git', 'config', '-f', dst_dir + "/.gitmodules", "-l" ],
|
||||
stdout=subprocess.PIPE).stdout:
|
||||
result = re.match('submodule\.([^.]*)\.([^.]*)=(.*)', line)
|
||||
if result.group(1) not in module_settings:
|
||||
module_settings[result.group(1)] = { 'name': result.group(1) }
|
||||
module_settings[result.group(1)][result.group(2)] = result.group(3)
|
||||
return module_settings
|
||||
|
||||
# Load the submodule hashes from the given paths
|
||||
def get_submodule_hashes(boost_module_dir, branch, paths):
|
||||
hashes = {}
|
||||
for line in subprocess.Popen(
|
||||
[ 'git', '--git-dir=' + boost_module_dir, 'ls-tree', branch ] + paths,
|
||||
stdout=subprocess.PIPE).stdout:
|
||||
result = re.match('160000 commit ([0-9a-zA-Z]+)\t(.*)', line)
|
||||
hashes[result.group(2)] = result.group(1)
|
||||
return hashes
|
||||
|
||||
# Equivalent to mkdir -p
|
||||
# From http://stackoverflow.com/a/600612/2434
|
||||
def mkdir_p(path):
|
||||
path = os.path.realpath(path)
|
||||
try:
|
||||
os.makedirs(path)
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.EEXIST and os.path.isdir(path):
|
||||
pass
|
||||
else:
|
||||
raise
|
||||
|
||||
###############################################################################
|
||||
|
||||
def export_boost(root_dir, branch, eol):
|
||||
dir = os.path.join(root_dir, branch + '-' + eol)
|
||||
print "Exporting to %s" % dir
|
||||
if os.path.isdir(dir):
|
||||
shutil.rmtree(dir)
|
||||
mirror_export(root_dir, dir, branch, eol)
|
||||
|
||||
if len(sys.argv) > 1:
|
||||
root=sys.argv[1]
|
||||
else:
|
||||
root=os.path.dirname(sys.argv[0])
|
||||
|
||||
print "Update mirror"
|
||||
print
|
||||
mirror_boostorg(root)
|
||||
|
||||
print "Export master-crlf"
|
||||
print
|
||||
export_boost(root, 'master', 'crlf')
|
||||
|
||||
print "Export master-lf"
|
||||
print
|
||||
export_boost(root, 'master', 'lf')
|
||||
61
mergeTree.py
Executable file
61
mergeTree.py
Executable file
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/python
|
||||
|
||||
import os, sys
|
||||
import shutil
|
||||
|
||||
def mergetree(src, dst, symlinks=False, ignore=None):
|
||||
from shutil import copy2, copystat, Error
|
||||
import os
|
||||
|
||||
names = os.listdir(src)
|
||||
if ignore is not None:
|
||||
ignored_names = ignore(src, names)
|
||||
else:
|
||||
ignored_names = set()
|
||||
|
||||
try:
|
||||
os.makedirs(dst)
|
||||
except OSError, exc:
|
||||
# XXX - this is pretty ugly
|
||||
if "file already exists" in exc[1]: # Windows
|
||||
pass
|
||||
elif "File exists" in exc[1]: # Linux
|
||||
pass
|
||||
else:
|
||||
raise
|
||||
|
||||
errors = []
|
||||
for name in names:
|
||||
if name in ignored_names:
|
||||
continue
|
||||
srcname = os.path.join(src, name)
|
||||
dstname = os.path.join(dst, name)
|
||||
try:
|
||||
if symlinks and os.path.islink(srcname):
|
||||
linkto = os.readlink(srcname)
|
||||
os.symlink(linkto, dstname)
|
||||
elif os.path.isdir(srcname):
|
||||
mergetree(srcname, dstname, symlinks, ignore)
|
||||
else:
|
||||
copy2(srcname, dstname)
|
||||
# XXX What about devices, sockets etc.?
|
||||
except (IOError, os.error), why:
|
||||
errors.append((srcname, dstname, str(why)))
|
||||
# catch the Error from the recursive mergetree so that we can
|
||||
# continue with other files
|
||||
except Error, err:
|
||||
errors.extend(err.args[0])
|
||||
try:
|
||||
copystat(src, dst)
|
||||
except WindowsError:
|
||||
# can't copy file access times on Windows
|
||||
pass
|
||||
except OSError, why:
|
||||
errors.extend((src, dst, str(why)))
|
||||
if errors:
|
||||
raise Error, errors
|
||||
|
||||
if len(sys.argv) == 3:
|
||||
mergetree ( sys.argv[1], sys.argv[2] )
|
||||
else:
|
||||
print "Usage %s <source> <dest>" % sys.argv[0]
|
||||
Reference in New Issue
Block a user