mirror of
https://github.com/boostorg/release-tools.git
synced 2026-01-22 17:52:10 +00:00
94 lines
3.0 KiB
Python
94 lines
3.0 KiB
Python
#!/usr/bin/env -S python3 -u
|
|
|
|
# Download files from s3 to the local machine on a frequent cron schedule.
|
|
#
|
|
# Instructions
|
|
#
|
|
# This scripts reads a file /tmp/boostarchivesinfo/filelist.txt
|
|
#
|
|
# Each line of filelist.txt should contain a path to an artifactory file in this format:
|
|
# main/release/1.84.0/source/boost_1_84_0.tar.bz2
|
|
# main/release/1.84.0/source/boost_1_84_0.tar.bz2.json
|
|
#
|
|
# install dotenv:
|
|
# pip3 install python-dotenv
|
|
#
|
|
# install rclone:
|
|
# wget https://downloads.rclone.org/v1.66.0/rclone-v1.66.0-linux-amd64.deb; dpkg -i rclone-v1.66.0-linux-amd64.deb
|
|
#
|
|
# create ${HOME}/.config/rclone/rclone.conf
|
|
# [remote1]
|
|
# type = s3
|
|
# provider = AWS
|
|
# env_auth = true
|
|
# region = us-east-2
|
|
#
|
|
# create ${HOME}/.aws/credentials and config, with production profile
|
|
#
|
|
# Add a per-minute cron task:
|
|
#
|
|
# * * * * * ${HOME}/scripts/s3-file-sync.py > /tmp/s3-file-sync-output.txt 2>&1
|
|
#
|
|
# Run either s3-file-sync.py or jfrog-file-sync.py, but not both
|
|
#
|
|
|
|
import os
|
|
import subprocess
|
|
import pathlib
|
|
import re
|
|
from dotenv import load_dotenv
|
|
|
|
load_dotenv()
|
|
|
|
upload_to_s3 = True
|
|
s3_archives_bucket = "boost-archives"
|
|
debug = 1
|
|
source_file_lists = ["/tmp/boostarchivesinfo/filelist.txt", "/tmp/boostarchivesinfo/vsbinaries_filelist.txt"]
|
|
local_copy_of_archives = "/drive2/boostorg"
|
|
|
|
os.chdir(local_copy_of_archives)
|
|
|
|
for source_file_list in source_file_lists:
|
|
if not os.path.isdir(os.path.dirname(source_file_list)):
|
|
if debug > 0:
|
|
print(
|
|
"The directory of source_file_list is missing. Perhaps there are no files to sync right now. Exiting."
|
|
)
|
|
continue
|
|
|
|
if not os.path.isfile(source_file_list):
|
|
if debug > 0:
|
|
print(
|
|
"The source_file_list is missing. Perhaps there are no files to sync right now. Exiting."
|
|
)
|
|
continue
|
|
|
|
# Download files
|
|
with open(source_file_list, "r") as f:
|
|
# data = f.readlines()
|
|
data = f.read().splitlines()
|
|
# Remove/clean-up source_file_list so it won't be processed next time
|
|
pathlib.Path(source_file_list).unlink()
|
|
for file in data:
|
|
file = file.strip()
|
|
# Sanitize file. It exists, matches ordinary characters, doesn't contain two dots "..", doesn't start with absolute path "/"
|
|
if (
|
|
file
|
|
and re.match("^[a-zA-Z0-9_/.-]+$", file)
|
|
and not re.match(r"\.\.", file)
|
|
and not re.match("^/", file)
|
|
):
|
|
|
|
archivePathLocal = re.sub("^main/", "", file)
|
|
archivePathRemote = "remote1:" + s3_archives_bucket + "/" + archivePathLocal
|
|
result = subprocess.run(
|
|
"export AWS_PROFILE=%s;rclone -v --s3-no-check-bucket copyto --checksum %s %s"
|
|
% ("production", archivePathRemote, archivePathLocal),
|
|
check=True,
|
|
shell=True,
|
|
text=True,
|
|
)
|
|
|
|
if debug > 0:
|
|
print(result)
|