mirror of
https://github.com/boostorg/release-tools.git
synced 2026-01-19 04:42:10 +00:00
149 lines
4.7 KiB
Python
Executable File
149 lines
4.7 KiB
Python
Executable File
#!/usr/bin/python
|
|
#
|
|
|
|
from __future__ import print_function
|
|
|
|
import requests
|
|
|
|
import json
|
|
import sys
|
|
import os
|
|
import os.path
|
|
import errno
|
|
import subprocess
|
|
import re
|
|
import shutil
|
|
|
|
# For urllib
|
|
from future.standard_library import install_aliases
|
|
install_aliases()
|
|
from urllib.parse import urlparse, urljoin
|
|
|
|
# For iteritems
|
|
from future.utils import iteritems
|
|
|
|
# Check python version
|
|
if sys.version_info[0] == 2 :
|
|
pythonversion="2"
|
|
else:
|
|
pythonversion="3"
|
|
|
|
# Update or create the github mirror.
|
|
def mirror_boostorg(root_dir):
|
|
mirror_dir = os.path.join(root_dir, 'mirror')
|
|
print("Updating mirror at %s" % mirror_dir)
|
|
url = 'https://api.github.com/orgs/boostorg/repos'
|
|
|
|
while (url) :
|
|
r = requests.get(url)
|
|
if (not r.ok):
|
|
raise Exception("Error getting: " + url)
|
|
|
|
for repo in json.loads(r.text or r.content):
|
|
print("Downloading " + repo['name'])
|
|
url = repo['clone_url']
|
|
# Not using os.path.join because url path is absolute.
|
|
path = mirror_dir + urlparse(url).path
|
|
mkdir_p(os.path.join(path, os.pardir))
|
|
|
|
# TODO: Check that path is actually a git repo?
|
|
if os.path.isdir(path):
|
|
subprocess.check_call(["git", "--git-dir=" + path, "fetch"])
|
|
else:
|
|
subprocess.check_call(["git", "clone", "--mirror", url, path])
|
|
|
|
url = r.links['next']['url'] if 'next' in r.links else False
|
|
|
|
# Export the full tree from the mirror
|
|
def mirror_export(root_dir, dst_dir, branch = 'master', eol = 'lf'):
|
|
git_flags = "-c core.autocrlf=false -c core.eol="+ eol
|
|
boost_module_dir = os.path.join(root_dir, 'mirror/boostorg/boost.git')
|
|
|
|
os.mkdir(dst_dir)
|
|
export_single_repo(boost_module_dir, dst_dir, branch, eol)
|
|
module_settings = get_submodule_settings(dst_dir)
|
|
hashes = get_submodule_hashes(boost_module_dir, branch,
|
|
[ module_settings[x]['path'] for x in module_settings ])
|
|
|
|
# Export child submodules
|
|
for name, module in iteritems(module_settings):
|
|
print("Exporting submodule " + name)
|
|
if module['path'] not in hashes:
|
|
raise Exception('No hash for module ' + name)
|
|
export_single_repo(
|
|
urljoin(boost_module_dir + '/', module['url']),
|
|
os.path.join(dst_dir, module['path']),
|
|
hashes[module['path']], eol)
|
|
|
|
# Export from a single git repo
|
|
def export_single_repo(git_dir, dst_dir, ref, eol):
|
|
ps = subprocess.Popen(
|
|
["git", "-c", "core.autocrlf=false", "-c", "core.eol=" + eol,
|
|
"--git-dir=" + git_dir, "archive", ref],
|
|
stdout=subprocess.PIPE)
|
|
subprocess.check_call(['tar', '-x', '-C', dst_dir], stdin=ps.stdout)
|
|
|
|
# Load the submodule settings from an exported repo.
|
|
def get_submodule_settings(dst_dir):
|
|
module_settings = {}
|
|
for line in subprocess.Popen(
|
|
[ 'git', 'config', '-f', dst_dir + "/.gitmodules", "-l" ],
|
|
stdout=subprocess.PIPE).stdout:
|
|
if pythonversion=="3":
|
|
line=line.decode(encoding="utf-8")
|
|
result = re.match('submodule\.([^.]*)\.([^.]*)=(.*)', line)
|
|
if result.group(1) not in module_settings:
|
|
module_settings[result.group(1)] = { 'name': result.group(1) }
|
|
module_settings[result.group(1)][result.group(2)] = result.group(3)
|
|
return module_settings
|
|
|
|
# Load the submodule hashes from the given paths
|
|
def get_submodule_hashes(boost_module_dir, branch, paths):
|
|
hashes = {}
|
|
for line in subprocess.Popen(
|
|
[ 'git', '--git-dir=' + boost_module_dir, 'ls-tree', branch ] + paths,
|
|
stdout=subprocess.PIPE).stdout:
|
|
if pythonversion=="3":
|
|
line=line.decode(encoding="utf-8")
|
|
result = re.match('160000 commit ([0-9a-zA-Z]+)\t(.*)', line)
|
|
hashes[result.group(2)] = result.group(1)
|
|
return hashes
|
|
|
|
# Equivalent to mkdir -p
|
|
# From http://stackoverflow.com/a/600612/2434
|
|
def mkdir_p(path):
|
|
path = os.path.realpath(path)
|
|
try:
|
|
os.makedirs(path)
|
|
except OSError as exc:
|
|
if exc.errno == errno.EEXIST and os.path.isdir(path):
|
|
pass
|
|
else:
|
|
raise
|
|
|
|
###############################################################################
|
|
|
|
def export_boost(root_dir, branch, eol):
|
|
dir = os.path.join(root_dir, branch + '-' + eol)
|
|
print("Exporting to %s" % dir)
|
|
if os.path.isdir(dir):
|
|
shutil.rmtree(dir)
|
|
mirror_export(root_dir, dir, branch, eol)
|
|
|
|
if len(sys.argv) > 1:
|
|
root=sys.argv[1]
|
|
else:
|
|
root=os.path.dirname(sys.argv[0])
|
|
|
|
print("Update mirror")
|
|
print()
|
|
mirror_boostorg(root)
|
|
|
|
print("Export master-crlf")
|
|
print()
|
|
export_boost(root, 'master', 'crlf')
|
|
|
|
print("Export master-lf")
|
|
print()
|
|
export_boost(root, 'master', 'lf')
|