Python script to download the latest snapshot for CI testing

.appveyor.yml

@@ -16,10 +16,9 @@ environment:
       TOOLSET: msvc-14.1
 
 install:
-  - cd c:\projects
+  - set BOOST_ROOT=c:\projects\boost
-  - curl -sSL -o boost.7z https://dl.bintray.com/boostorg/master/boost_1_67_0-snapshot.7z
+  - cd c:\projects\
-  - 7z x boost.7z
+  - python %APPVEYOR_BUILD_FOLDER%\build\download-boost-snapshot.py master
-  - set BOOST_ROOT=c:\projects\boost_1_67_0
   - rd /s /q %BOOST_ROOT%\boost\unordered
   - cd %BOOST_ROOT%\tools\build
   - cmd /c bootstrap

.travis.yml

@@ -66,25 +66,11 @@ before_script:
             snapshot=master
         fi
 
-        # Set download_url for snapshot
-        if [ $snapshot == "stable" ]
-        then
-            download_url=https://sourceforge.net/projects/boost/files/boost/${BOOST_VERSION}/${BOOST_FILENAME}.tar.bz2/download
-        else
-            download_url=$(curl https://api.bintray.com/packages/boostorg/$snapshot/snapshot/files |
-                    python -c "import os.path, sys, json; x = json.load(sys.stdin); print '\n'.join(a['path'] for a in x if os.path.splitext(a['path'])[1] == '.bz2')" |
-                    head -n 1 |
-                    sed "s/^/http:\/\/dl.bintray.com\/boostorg\/$snapshot\//")
-        fi
-
         # Download and extract snapshot
         echo "Downloading ${download_url}"
         mkdir $HOME/download
-        mkdir $HOME/extract
         cd $HOME/download
-        wget -O boost.tar.bz2 $download_url
+        python ${TRAVIS_BUILD_DIR}/build/download-boost-snapshot.py $snapshot
-        cd $HOME/extract
-        tar -xjf $HOME/download/boost.tar.bz2
         mv * ${BOOST_ROOT}
     - rm -r ${BOOST_ROOT}/boost/unordered
     - cd ${BOOST_ROOT}/tools/build

build/download-boost-snapshot.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+
+import urllib, os, os.path, sys, json, tarfile, zipfile, tempfile
+
+def download(snapshot):
+    if snapshot == 'stable':
+        # TODO: Default version/filename if not available?
+        downloads = [
+            "https://sourceforge.net/projects/boost/files/boost/%s/%s.tar.bz2/download" %
+                (os.environ['BOOST_VERSION'], os.environ['BOOST_FILENAME'])]
+    else:
+        json_response = urllib.urlopen('https://api.bintray.com/packages/boostorg/%s/snapshot/files' % (snapshot))
+        x = json.load(json_response)
+
+        extension_priorities = { '.bz2': 2, '.gz': 1, '.zip': 0 }
+        file_list = []
+        version_dates = {}
+        for file in x:
+            file_extension = os.path.splitext(file['path'])[1]
+            if (file_extension in extension_priorities):
+                file['priority'] = extension_priorities[file_extension]
+                file_list.append(file)
+                if not file['version'] in version_dates or file['created'] < version_dates[file['version']]:
+                    version_dates[file['version']] = file['created']
+        file_list.sort(key=lambda x: (version_dates[x['version']], x['priority']), reverse=True)
+        downloads = ['http://dl.bintray.com/boostorg/%s/%s' % (snapshot, file['path']) for file in file_list]
+
+    filename = ''
+    for download_url in downloads:
+        try:
+            print "Downloading: " + download_url
+            (filename, headers) = urllib.urlretrieve(download_url)
+
+            print "Extracting: " + filename
+            dir = tempfile.mkdtemp()
+            extract(filename, dir)
+            os.remove(filename)
+            files = os.listdir(dir)
+            assert(len(files) == 1)
+            os.rename(os.path.join(dir, files[0]), 'boost')
+            return
+        except IOError:
+            print "Error opening URL: " + download_url
+
+def extract(filename, path = '.'):
+    if (filename.endswith(".gz")):
+        tar = tarfile.open(filename, "r:gz")
+        tar.extractall(path)
+        tar.close
+    elif (filename.endswith(".bz2")):
+        tar = tarfile.open(filename, "r:bz2")
+        tar.extractall(path)
+        tar.close
+    elif (filename.endswith(".zip")):
+        zip = zipfile.ZipFile(filename, "r")
+        zip.extractall(path)
+        zip.close
+    else:
+        assert False
+
+if len(sys.argv) == 1:
+    download('stable')
+elif len(sys.argv) == 2:
+    download(sys.argv[1])
+else:
+    print "Usage: %s [stable|branch-name]" % (sys.argv[0])