Python script to download the latest snapshot for CI testing

This commit is contained in:
Daniel James
2018-02-02 23:37:36 +00:00
parent cfd4bacc2c
commit 6491f6c10a
3 changed files with 70 additions and 19 deletions

View File

@ -16,10 +16,9 @@ environment:
TOOLSET: msvc-14.1
install:
- cd c:\projects
- curl -sSL -o boost.7z https://dl.bintray.com/boostorg/master/boost_1_67_0-snapshot.7z
- 7z x boost.7z
- set BOOST_ROOT=c:\projects\boost_1_67_0
- set BOOST_ROOT=c:\projects\boost
- cd c:\projects\
- python %APPVEYOR_BUILD_FOLDER%\build\download-boost-snapshot.py master
- rd /s /q %BOOST_ROOT%\boost\unordered
- cd %BOOST_ROOT%\tools\build
- cmd /c bootstrap

View File

@ -66,25 +66,11 @@ before_script:
snapshot=master
fi
# Set download_url for snapshot
if [ $snapshot == "stable" ]
then
download_url=https://sourceforge.net/projects/boost/files/boost/${BOOST_VERSION}/${BOOST_FILENAME}.tar.bz2/download
else
download_url=$(curl https://api.bintray.com/packages/boostorg/$snapshot/snapshot/files |
python -c "import os.path, sys, json; x = json.load(sys.stdin); print '\n'.join(a['path'] for a in x if os.path.splitext(a['path'])[1] == '.bz2')" |
head -n 1 |
sed "s/^/http:\/\/dl.bintray.com\/boostorg\/$snapshot\//")
fi
# Download and extract snapshot
echo "Downloading ${download_url}"
mkdir $HOME/download
mkdir $HOME/extract
cd $HOME/download
wget -O boost.tar.bz2 $download_url
cd $HOME/extract
tar -xjf $HOME/download/boost.tar.bz2
python ${TRAVIS_BUILD_DIR}/build/download-boost-snapshot.py $snapshot
mv * ${BOOST_ROOT}
- rm -r ${BOOST_ROOT}/boost/unordered
- cd ${BOOST_ROOT}/tools/build

View File

@ -0,0 +1,66 @@
#!/usr/bin/env python
import urllib, os, os.path, sys, json, tarfile, zipfile, tempfile
def download(snapshot):
if snapshot == 'stable':
# TODO: Default version/filename if not available?
downloads = [
"https://sourceforge.net/projects/boost/files/boost/%s/%s.tar.bz2/download" %
(os.environ['BOOST_VERSION'], os.environ['BOOST_FILENAME'])]
else:
json_response = urllib.urlopen('https://api.bintray.com/packages/boostorg/%s/snapshot/files' % (snapshot))
x = json.load(json_response)
extension_priorities = { '.bz2': 2, '.gz': 1, '.zip': 0 }
file_list = []
version_dates = {}
for file in x:
file_extension = os.path.splitext(file['path'])[1]
if (file_extension in extension_priorities):
file['priority'] = extension_priorities[file_extension]
file_list.append(file)
if not file['version'] in version_dates or file['created'] < version_dates[file['version']]:
version_dates[file['version']] = file['created']
file_list.sort(key=lambda x: (version_dates[x['version']], x['priority']), reverse=True)
downloads = ['http://dl.bintray.com/boostorg/%s/%s' % (snapshot, file['path']) for file in file_list]
filename = ''
for download_url in downloads:
try:
print "Downloading: " + download_url
(filename, headers) = urllib.urlretrieve(download_url)
print "Extracting: " + filename
dir = tempfile.mkdtemp()
extract(filename, dir)
os.remove(filename)
files = os.listdir(dir)
assert(len(files) == 1)
os.rename(os.path.join(dir, files[0]), 'boost')
return
except IOError:
print "Error opening URL: " + download_url
def extract(filename, path = '.'):
if (filename.endswith(".gz")):
tar = tarfile.open(filename, "r:gz")
tar.extractall(path)
tar.close
elif (filename.endswith(".bz2")):
tar = tarfile.open(filename, "r:bz2")
tar.extractall(path)
tar.close
elif (filename.endswith(".zip")):
zip = zipfile.ZipFile(filename, "r")
zip.extractall(path)
zip.close
else:
assert False
if len(sys.argv) == 1:
download('stable')
elif len(sys.argv) == 2:
download(sys.argv[1])
else:
print "Usage: %s [stable|branch-name]" % (sys.argv[0])