diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fda52b3..3fed165 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -2,7 +2,7 @@ # subject to the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) # -# Copyright Rene Rivera 2019. +# Copyright Rene Rivera 2019-2020. trigger: branches: @@ -19,185 +19,315 @@ variables: AZP: 1 AZP_REPO_DIR: $(Build.Repository.LocalPath) AZP_BRANCH: $(Build.SourceBranch) + AZP_BRANCH_NAME: $(Build.SourceBranchName) AZP_COMMIT: $(Build.SourceVersion) AZP_REPO: $(Build.Repository.Name) AZP_PULL_REQUEST: $(System.PullRequest.PullRequestNumber) -jobs: +stages: -- job: 'Linux' - pool: - vmImage: 'ubuntu-16.04' - strategy: - matrix: - GCC 8 (GNU): - TOOLSET: gcc-8 - CXXSTD: 03,11,14,17,2a - CXXDIALECT: gnu - GCC 8 (ISO): - TOOLSET: gcc-8 - CXXSTD: 03,11,14,17,2a - GCC 7: - TOOLSET: gcc-7 - GCC 6: - TOOLSET: gcc-6 - GCC 5: - TOOLSET: gcc-5 - GCC 4.9: - TOOLSET: gcc-4.9 - GCC 4.8: - TOOLSET: gcc-4.8 - GCC 4.7: - TOOLSET: gcc-4.7 - Clang 8: - TOOLSET: clang-8 - Clang 7: - TOOLSET: clang-7 - Clang 6: - TOOLSET: clang-6.0 - Clang 5: - TOOLSET: clang-5.0 - Clang 4: - TOOLSET: clang-4.0 - Clang 3.9: - TOOLSET: clang-3.9 - Clang 3.8: - TOOLSET: clang-3.8 - Clang 3.7: - TOOLSET: clang-3.7 - Clang 3.6: - TOOLSET: clang-3.6 - Clang 3.5: - TOOLSET: clang-3.5 - steps: - - task: UsePythonVersion@0 - - script: python tools/ci/library_test.py install - failOnStderr: false - displayName: Install - - script: python tools/ci/library_test.py script - failOnStderr: false - displayName: Test +- stage: Test + jobs: -- job: 'macOS' - strategy: - matrix: - Xcode 10.1: - TOOLSET: xcode-10.1 - XCODE_APP: /Applications/Xcode_10.1.app - VM_IMAGE: 'macOS-10.14' - Xcode 10.0: - TOOLSET: xcode-10.0 - XCODE_APP: /Applications/Xcode_10.app - VM_IMAGE: 'macOS-10.14' - Xcode 9.4.1: - TOOLSET: xcode-9.4.1 - XCODE_APP: /Applications/Xcode_9.4.1.app - VM_IMAGE: 'macOS-10.13' - Xcode 9.4: - TOOLSET: xcode-9.4 - XCODE_APP: /Applications/Xcode_9.4.app - VM_IMAGE: 'macOS-10.13' - Xcode 9.3.1: - TOOLSET: xcode-9.3.1 - XCODE_APP: /Applications/Xcode_9.3.1.app - VM_IMAGE: 'macOS-10.13' - Xcode 9.3: - TOOLSET: xcode-9.3 - XCODE_APP: /Applications/Xcode_9.3.app - VM_IMAGE: 'macOS-10.13' - Xcode 9.2: - TOOLSET: xcode-9.2 - XCODE_APP: /Applications/Xcode_9.2.app - VM_IMAGE: 'macOS-10.13' - Xcode 9.1: - TOOLSET: xcode-9.1 - XCODE_APP: /Applications/Xcode_9.1.app - VM_IMAGE: 'macOS-10.13' - Xcode 9.0.1: - TOOLSET: xcode-9.0.1 - XCODE_APP: /Applications/Xcode_9.0.1.app - VM_IMAGE: 'macOS-10.13' - Xcode 9.0: - TOOLSET: xcode-9.0 - XCODE_APP: /Applications/Xcode_9.app - VM_IMAGE: 'macOS-10.13' - Xcode 8.3.3: - TOOLSET: xcode-8.3 - XCODE_APP: /Applications/Xcode_8.3.3.app - VM_IMAGE: 'macOS-10.13' - pool: - vmImage: $(VM_IMAGE) - steps: - - task: UsePythonVersion@0 - - script: sudo xcode-select -switch ${XCODE_APP} - failOnStderr: false - displayName: Xcode Select - - script: python tools/ci/library_test.py install - failOnStderr: false - displayName: Install - - script: python tools/ci/library_test.py script - failOnStderr: false - displayName: Test + - job: 'Linux' + strategy: + matrix: + GCC 9 (GNU): + TOOLSET: gcc + CXX: g++-9 + PACKAGES: g++-9 + B2_ARGS: 'cxxstd=03,11,14,17,2a cxxstd-dialect=gnu' + VM_IMAGE: 'ubuntu-18.04' + GCC 9 (ISO): + TOOLSET: gcc + CXX: g++-9 + PACKAGES: g++-9 + B2_ARGS: 'cxxstd=03,11,14,17,2a cxxstd-dialect=iso' + VM_IMAGE: 'ubuntu-18.04' + GCC 9: + TOOLSET: gcc + CXX: g++-9 + PACKAGES: g++-9 + VM_IMAGE: 'ubuntu-18.04' + GCC 8: + TOOLSET: gcc + CXX: g++-8 + PACKAGES: g++-8 + VM_IMAGE: 'ubuntu-18.04' + GCC 7: + TOOLSET: gcc + CXX: g++-7 + PACKAGES: g++-7 + VM_IMAGE: 'ubuntu-18.04' + GCC 6: + TOOLSET: gcc + CXX: g++-6 + PACKAGES: g++-6 + VM_IMAGE: 'ubuntu-18.04' + GCC 5: + TOOLSET: gcc + CXX: g++-5 + PACKAGES: g++-5 + VM_IMAGE: 'ubuntu-18.04' + GCC 4.9: + TOOLSET: gcc + CXX: g++-4.9 + PACKAGES: g++-4.9 + VM_IMAGE: 'ubuntu-16.04' + GCC 4.8: + TOOLSET: gcc + CXX: g++-4.8 + PACKAGES: g++-4.8 + VM_IMAGE: 'ubuntu-16.04' + GCC 4.7: + TOOLSET: gcc + CXX: g++-4.7 + PACKAGES: g++-4.7 + VM_IMAGE: 'ubuntu-16.04' + Clang 9: + TOOLSET: clang + CXX: clang++-9 + PACKAGES: clang-9 + LLVM_REPO: llvm-toolchain-xenial-9 + VM_IMAGE: 'ubuntu-18.04' + Clang 8: + TOOLSET: clang + CXX: clang++-8 + PACKAGES: clang-8 + LLVM_REPO: llvm-toolchain-xenial-8 + VM_IMAGE: 'ubuntu-18.04' + Clang 7: + TOOLSET: clang + CXX: clang++-7 + PACKAGES: clang-7 + LLVM_REPO: llvm-toolchain-xenial-7 + VM_IMAGE: 'ubuntu-18.04' + Clang 6: + TOOLSET: clang + CXX: clang++-6.0 + PACKAGES: clang-6.0 + LLVM_REPO: llvm-toolchain-xenial-6.0 + VM_IMAGE: 'ubuntu-18.04' + Clang 5: + TOOLSET: clang + CXX: clang++-5.0 + PACKAGES: clang-5.0 + LLVM_REPO: llvm-toolchain-xenial-5.0 + VM_IMAGE: 'ubuntu-18.04' + Clang 4: + TOOLSET: clang + CXX: clang++-4.0 + PACKAGES: clang-4.0 + LLVM_REPO: llvm-toolchain-xenial-4.0 + VM_IMAGE: 'ubuntu-18.04' + Clang 3.9: + TOOLSET: clang + CXX: clang++-3.9 + PACKAGES: clang-3.9 + VM_IMAGE: 'ubuntu-18.04' + Clang 3.8: + TOOLSET: clang + CXX: clang++-3.8 + PACKAGES: clang-3.8 + VM_IMAGE: 'ubuntu-16.04' + Clang 3.7: + TOOLSET: clang + CXX: clang++-3.7 + PACKAGES: clang-3.7 + VM_IMAGE: 'ubuntu-16.04' + Clang 3.6: + TOOLSET: clang + CXX: clang++-3.6 + PACKAGES: clang-3.6 + VM_IMAGE: 'ubuntu-16.04' + Clang 3.5: + TOOLSET: clang + CXX: clang++-3.5 + PACKAGES: clang-3.5 + VM_IMAGE: 'ubuntu-16.04' + pool: + vmImage: $(VM_IMAGE) + steps: + - bash: | + set -e + uname -a + sudo -E apt-add-repository -y "ppa:ubuntu-toolchain-r/test" + if test -n "${LLVM_REPO}" ; then + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - + sudo -E apt-add-repository "deb http://apt.llvm.org/xenial/ ${LLVM_REPO} main" + fi + sudo -E apt-get update + sudo -E apt-get -yq --no-install-suggests --no-install-recommends install ${PACKAGES} + displayName: 'Install CXX' + - bash: | + set -e + pushd ${HOME} + wget -nv https://github.com/boostorg/build/archive/master.tar.gz + tar -zxf master.tar.gz + cd build-master + CXX= ./bootstrap.sh + sudo ./b2 install + popd + displayName: 'Install B2' + - bash: | + set -e + CXX_PATH=`which ${CXX}` + echo "using ${TOOLSET} : : ${CXX_PATH} ;" > ${HOME}/user-config.jam + pushd test + b2 --debug-configuration -a toolset=${TOOLSET} ${B2_ARGS} + popd + displayName: Test -- job: 'Windows' - strategy: - matrix: - VS 2019: - TOOLSET: vs-2019 - ADDRESS_MODEL: 32,64 - CXXSTD: 14,latest - VM_IMAGE: 'windows-2019' - VS 2019 (UWP DESKTOP): - TOOLSET: vs-2019 - ADDRESS_MODEL: 64 - CXXDEFS: _WIN32_WINNT=0x0A00,WINAPI_FAMILY=WINAPI_FAMILY_DESKTOP_APP - VM_IMAGE: 'windows-2019' - VS 2019 (UWP PHONE): - TOOLSET: vs-2019 - ADDRESS_MODEL: 64 - CXXDEFS: _WIN32_WINNT=0x0A00,WINAPI_FAMILY=WINAPI_FAMILY_PHONE_APP - VM_IMAGE: 'windows-2019' - VS 2019 (UWP STORE): - TOOLSET: vs-2019 - ADDRESS_MODEL: 64 - CXXDEFS: _WIN32_WINNT=0x0A00,WINAPI_FAMILY=WINAPI_FAMILY_PC_APP - VM_IMAGE: 'windows-2019' - VS 2019 (UWP SERVER): - TOOLSET: vs-2019 - ADDRESS_MODEL: 64 - CXXDEFS: _WIN32_WINNT=0x0A00,WINAPI_FAMILY=WINAPI_FAMILY_SERVER - VM_IMAGE: 'windows-2019' - VS 2019 (UWP SYSTEM): - TOOLSET: vs-2019 - ADDRESS_MODEL: 64 - CXXDEFS: _WIN32_WINNT=0x0A00,WINAPI_FAMILY=WINAPI_FAMILY_SYSTEM - VM_IMAGE: 'windows-2019' - VS 2017: - TOOLSET: vs-2017 - VM_IMAGE: 'vs2017-win2016' - VS 2015: - TOOLSET: vs-2015 - VM_IMAGE: 'vs2015-win2012r2' - VS 2013: - TOOLSET: vs-2013 - VM_IMAGE: 'vs2015-win2012r2' - MinGW 8.1.0: - TOOLSET: mingw-8 - VM_IMAGE: 'vs2017-win2016' - MinGW 8.1.0 (UWP DESKTOP): - TOOLSET: mingw-8 - CXXDEFS: _WIN32_WINNT=0x0A00,WINAPI_FAMILY=WINAPI_FAMILY_DESKTOP_APP - VM_IMAGE: 'vs2017-win2016' - MinGW 8.1.0 (UWP STORE): - TOOLSET: mingw-8 - CXXDEFS: _WIN32_WINNT=0x0A00,WINAPI_FAMILY=WINAPI_FAMILY_PC_APP - VM_IMAGE: 'vs2017-win2016' - pool: - vmImage: $(VM_IMAGE) - steps: - - script: python tools/ci/library_test.py install - failOnStderr: false - displayName: Install - - script: python tools/ci/library_test.py script - failOnStderr: false - displayName: Test + - job: 'macOS' + strategy: + matrix: + Xcode 11.4: + TOOLSET: clang + CXX: clang++ + XCODE_APP: /Applications/Xcode_11.4_beta.app + VM_IMAGE: 'macOS-10.15' + Xcode 11.3.1: + TOOLSET: clang + CXX: clang++ + XCODE_APP: /Applications/Xcode_11.3.1.app + VM_IMAGE: 'macOS-10.15' + Xcode 11.3: + TOOLSET: clang + CXX: clang++ + XCODE_APP: /Applications/Xcode_11.3.app + VM_IMAGE: 'macOS-10.15' + Xcode 11.2: + TOOLSET: clang + CXX: clang++ + XCODE_APP: /Applications/Xcode_11.2.app + VM_IMAGE: 'macOS-10.15' + Xcode 11.1: + TOOLSET: clang + CXX: clang++ + XCODE_APP: /Applications/Xcode_11.1.app + VM_IMAGE: 'macOS-10.15' + Xcode 11.0: + TOOLSET: clang + CXX: clang++ + XCODE_APP: /Applications/Xcode_11.app + VM_IMAGE: 'macOS-10.15' + Xcode 10.2.1: + TOOLSET: clang + CXX: clang++ + XCODE_APP: /Applications/Xcode_10.2.1.app + VM_IMAGE: 'macOS-10.14' + Xcode 10.2: + TOOLSET: clang + CXX: clang++ + XCODE_APP: /Applications/Xcode_10.2.app + VM_IMAGE: 'macOS-10.14' + Xcode 10.1: + TOOLSET: clang + CXX: clang++ + XCODE_APP: /Applications/Xcode_10.1.app + VM_IMAGE: 'macOS-10.14' + Xcode 10.0: + TOOLSET: clang + CXX: clang++ + XCODE_APP: /Applications/Xcode_10.app + VM_IMAGE: 'macOS-10.14' + pool: + vmImage: $(VM_IMAGE) + steps: + - bash: | + set -e + uname -a + sudo xcode-select -switch ${XCODE_APP} + which clang++ + clang++ --version + displayName: 'Install CXX' + - bash: | + set -e + pushd ${HOME} + wget -nv https://github.com/boostorg/build/archive/master.tar.gz + tar -zxf master.tar.gz + cd build-master + CXX= ./bootstrap.sh + sudo ./b2 install + popd + displayName: 'Install B2' + - bash: | + set -e + CXX_PATH=`which ${CXX}` + echo "using ${TOOLSET} : : ${CXX_PATH} ;" > ${HOME}/user-config.jam + pushd test + b2 --debug-configuration -a toolset=${TOOLSET} ${B2_ARGS} + popd + displayName: Test + + - job: 'Windows' + strategy: + matrix: + VS 2019: + TOOLSET: msvc + TOOLSET_VERSION: 14.2 + B2_ARGS: address-model=32,64 cxxstd=14,latest + VM_IMAGE: 'windows-2019' + VS 2019 (UWP DESKTOP): + TOOLSET: msvc + TOOLSET_VERSION: 14.2 + B2_ARGS: address-model=64 define=_WIN32_WINNT=0x0A00 define=WINAPI_FAMILY=WINAPI_FAMILY_DESKTOP_APP + VM_IMAGE: 'windows-2019' + VS 2019 (UWP PHONE): + TOOLSET: msvc + TOOLSET_VERSION: 14.2 + B2_ARGS: address-model=64 define=_WIN32_WINNT=0x0A00 define=WINAPI_FAMILY=WINAPI_FAMILY_PHONE_APP + VM_IMAGE: 'windows-2019' + VS 2019 (UWP STORE): + TOOLSET: msvc + TOOLSET_VERSION: 14.2 + B2_ARGS: address-model=64 define=_WIN32_WINNT=0x0A00 define=WINAPI_FAMILY=WINAPI_FAMILY_PC_APP + VM_IMAGE: 'windows-2019' + VS 2019 (UWP SERVER): + TOOLSET: msvc + TOOLSET_VERSION: 14.2 + B2_ARGS: address-model=64 define=_WIN32_WINNT=0x0A00 define=WINAPI_FAMILY=WINAPI_FAMILY_SERVER + VM_IMAGE: 'windows-2019' + VS 2019 (UWP SYSTEM): + TOOLSET: msvc + TOOLSET_VERSION: 14.2 + B2_ARGS: address-model=64 define=_WIN32_WINNT=0x0A00 define=WINAPI_FAMILY=WINAPI_FAMILY_SYSTEM + VM_IMAGE: 'windows-2019' + VS 2017: + TOOLSET: msvc + TOOLSET_VERSION: 14.1 + VM_IMAGE: 'vs2017-win2016' + MinGW 8.1.0: + TOOLSET: gcc + VM_IMAGE: 'windows-2019' + MinGW 8.1.0 (UWP DESKTOP): + TOOLSET: gcc + B2_ARGS: define=_WIN32_WINNT=0x0A00 define=WINAPI_FAMILY=WINAPI_FAMILY_DESKTOP_APP + VM_IMAGE: 'windows-2019' + MinGW 8.1.0 (UWP STORE): + TOOLSET: gcc + B2_ARGS: define=_WIN32_WINNT=0x0A00 define=WINAPI_FAMILY=WINAPI_FAMILY_PC_APP + VM_IMAGE: 'windows-2019' + pool: + vmImage: $(VM_IMAGE) + steps: + - script: | + cd %BUILD_SOURCESDIRECTORY%/.. + curl "https://github.com/boostorg/build/archive/master.zip" -L -o b2.zip + displayName: Download B2 + - task: ExtractFiles@1 + inputs: + archiveFilePatterns: $(Build.SourcesDirectory)/../b2.zip + destinationFolder: $(Build.SourcesDirectory)/.. + cleanDestinationFolder: false + displayName: Extract B2 + - script: | + echo using %TOOLSET% ^: %TOOLSET_VERSION% ^; > %HOMEDRIVE%%HOMEPATH%/user-config.jam + cd %BUILD_SOURCESDIRECTORY%/../build-master + cmd.exe /c bootstrap.bat + displayName: Install B2 + - script: | + set BOOST_BUILD_PATH=%BUILD_SOURCESDIRECTORY%/../build-master + cd %BUILD_SOURCESDIRECTORY%/test + %BUILD_SOURCESDIRECTORY%/../build-master/b2.exe --debug-configuration -a toolset=%TOOLSET% %B2_ARGS% + displayName: Test diff --git a/doc/index.html b/doc/index.html index 6bfc9d6..53b45d9 100644 --- a/doc/index.html +++ b/doc/index.html @@ -6746,6 +6746,9 @@ expressions. It defaults to "c++", but can be any of: "c", "cpp",

  • Remove deprecated BOOST_OS_ANDROID.

    • +
  • +

    Fix compile for Wine. (Kevin Puetz)

    +
    • @@ -7043,7 +7046,7 @@ Joel Falcou for managing the review of this library.

    https://www.boost.org/LICENSE_1_0.txt)

    -

    Copyright 2005-2019 Rene Rivera; Copyright 2015 Charly Chevalier; Copyright 2015 Joel Falcou

    +

    Copyright 2005-2020 Rene Rivera; Copyright 2015 Charly Chevalier; Copyright 2015 Joel Falcou

    diff --git a/doc/predef.adoc b/doc/predef.adoc index 7249f5f..cac33c8 100644 --- a/doc/predef.adoc +++ b/doc/predef.adoc @@ -1011,4 +1011,4 @@ Distributed under the Boost Software License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) -Copyright 2005-2019 Rene Rivera; Copyright 2015 Charly Chevalier; Copyright 2015 Joel Falcou \ No newline at end of file +Copyright 2005-2020 Rene Rivera; Copyright 2015 Charly Chevalier; Copyright 2015 Joel Falcou \ No newline at end of file diff --git a/tools/ci/build_log.py b/tools/ci/build_log.py deleted file mode 100644 index f862cc5..0000000 --- a/tools/ci/build_log.py +++ /dev/null @@ -1,446 +0,0 @@ - -# Copyright 2008-2019 Rene Rivera -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -from __future__ import print_function - -import re -import optparse -import time -import xml.dom.minidom -import xml.dom.pulldom -from xml.sax.saxutils import unescape, escape -import os.path -from pprint import pprint -from sys import exit - - -class BuildOutputXMLParsing(object): - ''' - XML parsing utilities for dealing with the Boost Build output - XML format. - ''' - - def get_child_data(self, root, tag=None, id=None, name=None, strip=False, default=None): - return self.get_data(self.get_child(root, tag=tag, id=id, name=name), strip=strip, default=default) - - def get_data(self, node, strip=False, default=None): - data = None - if node: - data_node = None - if not data_node: - data_node = self.get_child(node, tag='#text') - if not data_node: - data_node = self.get_child(node, tag='#cdata-section') - data = "" - while data_node: - data += data_node.data - data_node = data_node.nextSibling - if data_node: - if data_node.nodeName != '#text' \ - and data_node.nodeName != '#cdata-section': - data_node = None - if not data: - data = default - else: - if strip: - data = data.strip() - return data - - def get_child(self, root, tag=None, id=None, name=None, type=None): - return self.get_sibling(root.firstChild, tag=tag, id=id, name=name, type=type) - - def get_sibling(self, sibling, tag=None, id=None, name=None, type=None): - n = sibling - while n: - found = True - if type and found: - found = found and type == n.nodeType - if tag and found: - found = found and tag == n.nodeName - if (id or name) and found: - found = found and n.nodeType == xml.dom.Node.ELEMENT_NODE - if id and found: - if n.hasAttribute('id'): - found = found and n.getAttribute('id') == id - else: - found = found and n.hasAttribute( - 'id') and n.getAttribute('id') == id - if name and found: - found = found and n.hasAttribute( - 'name') and n.getAttribute('name') == name - if found: - return n - n = n.nextSibling - return None - - -class BuildOutputProcessor(BuildOutputXMLParsing): - - def __init__(self, inputs): - self.test = {} - self.target_to_test = {} - self.target = {} - self.parent = {} - self.timestamps = [] - for input in inputs: - self.add_input(input) - - def add_input(self, input): - ''' - Add a single build XML output file to our data. - ''' - events = xml.dom.pulldom.parse(input) - context = [] - for (event, node) in events: - if event == xml.dom.pulldom.START_ELEMENT: - context.append(node) - if node.nodeType == xml.dom.Node.ELEMENT_NODE: - x_f = self.x_name_(*context) - if x_f: - events.expandNode(node) - # expanding eats the end element, hence walking us out one level - context.pop() - # call handler - (x_f[1])(node) - elif event == xml.dom.pulldom.END_ELEMENT: - context.pop() - - def x_name_(self, *context, **kwargs): - node = None - names = [] - for c in context: - if c: - if not isinstance(c, xml.dom.Node): - suffix = '_'+c.replace('-', '_').replace('#', '_') - else: - suffix = '_'+c.nodeName.replace('-', '_').replace('#', '_') - node = c - names.append('x') - names = [x+suffix for x in names] - if node: - for name in names: - if hasattr(self, name): - return (name, getattr(self, name)) - return None - - def x_build_test(self, node): - ''' - Records the initial test information that will eventually - get expanded as we process the rest of the results. - ''' - test_node = node - test_name = test_node.getAttribute('name') - test_target = self.get_child_data(test_node, tag='target', strip=True) - # print ">>> %s %s" %(test_name,test_target) - self.test[test_name] = { - 'library': "/".join(test_name.split('/')[0:-1]), - 'test-name': test_name.split('/')[-1], - 'test-type': test_node.getAttribute('type').lower(), - 'test-program': self.get_child_data(test_node, tag='source', strip=True), - 'target': test_target, - 'info': self.get_child_data(test_node, tag='info', strip=True), - 'dependencies': [], - 'actions': [], - } - # Add a lookup for the test given the test target. - self.target_to_test[self.test[test_name]['target']] = test_name - return None - - def x_build_targets_target(self, node): - ''' - Process the target dependency DAG into an ancestry tree so we can look up - which top-level library and test targets specific build actions correspond to. - ''' - target_node = node - name = self.get_child_data(target_node, tag='name', strip=True) - path = self.get_child_data(target_node, tag='path', strip=True) - jam_target = self.get_child_data( - target_node, tag='jam-target', strip=True) - # ~ Map for jam targets to virtual targets. - self.target[jam_target] = { - 'name': name, - 'path': path - } - # ~ Create the ancestry. - dep_node = self.get_child(self.get_child( - target_node, tag='dependencies'), tag='dependency') - while dep_node: - child = self.get_data(dep_node, strip=True) - child_jam_target = '%s' % (path, child.split('//', 1)[1]) - self.parent[child_jam_target] = jam_target - dep_node = self.get_sibling(dep_node.nextSibling, tag='dependency') - return None - - def x_build_action(self, node): - ''' - Given a build action log, process into the corresponding test log and - specific test log sub-part. - ''' - action_node = node - name = self.get_child(action_node, tag='name') - if name: - name = self.get_data(name) - # ~ Based on the action, we decide what sub-section the log - # ~ should go into. - action_type = None - if re.match('[^%]+%[^.]+[.](compile)', name): - action_type = 'compile' - elif re.match('[^%]+%[^.]+[.](link|archive)', name): - action_type = 'link' - elif re.match('[^%]+%testing[.](capture-output)', name): - action_type = 'run' - elif re.match('[^%]+%testing[.](expect-failure|expect-success)', name): - action_type = 'result' - else: - # TODO: Enable to see what other actions can be included in the test results. - # action_type = None - action_type = 'other' - # ~ print "+ [%s] %s %s :: %s" %(action_type,name,'','') - if action_type: - # ~ Get the corresponding test. - (target, test) = self.get_test(action_node, type=action_type) - # ~ Skip action that have no corresponding test as they are - # ~ regular build actions and don't need to show up in the - # ~ regression results. - if not test: - # print "??? [%s] %s %s :: %s" %(action_type,name,target,test) - return None - # print "+++ [%s] %s %s :: %s" %(action_type,name,target,test) - # ~ Collect some basic info about the action. - action = { - 'command': self.get_action_command(action_node, action_type), - 'output': self.get_action_output(action_node, action_type), - 'info': self.get_action_info(action_node, action_type) - } - # ~ For the test result status we find the appropriate node - # ~ based on the type of test. Then adjust the result status - # ~ accordingly. This makes the result status reflect the - # ~ expectation as the result pages post processing does not - # ~ account for this inversion. - action['type'] = action_type - if action_type == 'result': - if re.match(r'^compile', test['test-type']): - action['type'] = 'compile' - elif re.match(r'^link', test['test-type']): - action['type'] = 'link' - elif re.match(r'^run', test['test-type']): - action['type'] = 'run' - # ~ The result sub-part we will add this result to. - if action_node.getAttribute('status') == '0': - action['result'] = 'succeed' - else: - action['result'] = 'fail' - # Add the action to the test. - test['actions'].append(action) - # Set the test result if this is the result action for the test. - if action_type == 'result': - test['result'] = action['result'] - return None - - def x_build_timestamp(self, node): - ''' - The time-stamp goes to the corresponding attribute in the result. - ''' - self.timestamps.append(self.get_data(node).strip()) - return None - - def get_test(self, node, type=None): - ''' - Find the test corresponding to an action. For testing targets these - are the ones pre-declared in the --dump-test option. For libraries - we create a dummy test as needed. - ''' - jam_target = self.get_child_data(node, tag='jam-target') - base = self.target[jam_target]['name'] - target = jam_target - while target in self.parent: - target = self.parent[target] - # ~ print "--- TEST: %s ==> %s" %(jam_target,target) - # ~ main-target-type is a precise indicator of what the build target is - # ~ originally meant to be. - # main_type = self.get_child_data(self.get_child(node,tag='properties'), - # name='main-target-type',strip=True) - main_type = None - if main_type == 'LIB' and type: - lib = self.target[target]['name'] - if not lib in self.test: - self.test[lib] = { - 'library': re.search(r'libs/([^/]+)', lib).group(1), - 'test-name': os.path.basename(lib), - 'test-type': 'lib', - 'test-program': os.path.basename(lib), - 'target': lib - } - test = self.test[lib] - else: - target_name_ = self.target[target]['name'] - if target_name_ in self.target_to_test: - test = self.test[self.target_to_test[target_name_]] - else: - test = None - return (base, test) - - # ~ The command executed for the action. For run actions we omit the command - # ~ as it's just noise. - def get_action_command(self, action_node, action_type): - if action_type != 'run': - return self.get_child_data(action_node, tag='command') - else: - return '' - - # ~ The command output. - def get_action_output(self, action_node, action_type): - return self.get_child_data(action_node, tag='output', default='') - - # ~ Some basic info about the action. - def get_action_info(self, action_node, action_type): - info = {} - # ~ The jam action and target. - info['name'] = self.get_child_data(action_node, tag='name') - info['path'] = self.get_child_data(action_node, tag='path') - # ~ The timing of the action. - info['time-start'] = action_node.getAttribute('start') - info['time-end'] = action_node.getAttribute('end') - info['time-user'] = action_node.getAttribute('user') - info['time-system'] = action_node.getAttribute('system') - # ~ Testing properties. - test_info_prop = self.get_child_data(self.get_child( - action_node, tag='properties'), name='test-info') - info['always_show_run_output'] = test_info_prop == 'always_show_run_output' - # ~ And for compiles some context that may be hidden if using response files. - if action_type == 'compile': - info['define'] = [] - define = self.get_child(self.get_child( - action_node, tag='properties'), name='define') - while define: - info['define'].append(self.get_data(define, strip=True)) - define = self.get_sibling(define.nextSibling, name='define') - return info - - -class BuildConsoleSummaryReport(object): - - HEADER = '\033[35m\033[1m' - INFO = '\033[34m' - OK = '\033[32m' - WARNING = '\033[33m' - FAIL = '\033[31m' - ENDC = '\033[0m' - - def __init__(self, bop, opt): - self.bop = bop - - def generate(self): - self.summary_info = { - 'total': 0, - 'success': 0, - 'failed': [], - } - self.header_print( - "======================================================================") - self.print_test_log() - self.print_summary() - self.header_print( - "======================================================================") - - @property - def failed(self): - return len(self.summary_info['failed']) > 0 - - def print_test_log(self): - self.header_print("Tests run..") - self.header_print( - "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") - for k in sorted(self.bop.test.keys()): - test = self.bop.test[k] - if len(test['actions']) > 0: - self.summary_info['total'] += 1 - # print ">>>> {0}".format(test['test-name']) - if 'result' in test: - succeed = test['result'] == 'succeed' - else: - succeed = test['actions'][-1]['result'] == 'succeed' - if succeed: - self.summary_info['success'] += 1 - else: - self.summary_info['failed'].append(test) - if succeed: - self.ok_print("[PASS] {0}", k) - else: - self.fail_print("[FAIL] {0}", k) - for action in test['actions']: - self.print_action(succeed, action) - - def print_action(self, test_succeed, action): - ''' - Print the detailed info of failed or always print tests. - ''' - #self.info_print(">>> {0}",action.keys()) - if not test_succeed or action['info']['always_show_run_output']: - output = action['output'].strip() - if output != "": - p = self.fail_print if action['result'] == 'fail' else self.p_print - self.info_print("") - self.info_print( - "({0}) {1}", action['info']['name'], action['info']['path']) - p("") - p("{0}", action['command'].strip()) - p("") - for line in output.splitlines(): - p("{0}", line.encode('utf-8')) - - def print_summary(self): - self.header_print("") - self.header_print("Testing summary..") - self.header_print( - "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") - self.p_print("Total: {0}", self.summary_info['total']) - self.p_print("Success: {0}", self.summary_info['success']) - if self.failed: - self.fail_print("Failed: {0}", len(self.summary_info['failed'])) - for test in self.summary_info['failed']: - self.fail_print( - " {0}/{1}", test['library'], test['test-name']) - - def p_print(self, format, *args, **kargs): - print(format.format(*args, **kargs)) - - def info_print(self, format, *args, **kargs): - print(self.INFO+format.format(*args, **kargs)+self.ENDC) - - def header_print(self, format, *args, **kargs): - print(self.HEADER+format.format(*args, **kargs)+self.ENDC) - - def ok_print(self, format, *args, **kargs): - print(self.OK+format.format(*args, **kargs)+self.ENDC) - - def warn_print(self, format, *args, **kargs): - print(self.WARNING+format.format(*args, **kargs)+self.ENDC) - - def fail_print(self, format, *args, **kargs): - print(self.FAIL+format.format(*args, **kargs)+self.ENDC) - - -class Main(object): - - def __init__(self, args=None): - op = optparse.OptionParser( - usage="%prog [options] input+") - op.add_option('--output', - help="type of output to generate") - (opt, inputs) = op.parse_args(args) - bop = BuildOutputProcessor(inputs) - output = None - if opt.output == 'console': - output = BuildConsoleSummaryReport(bop, opt) - if output: - output.generate() - self.failed = output.failed - - -if __name__ == '__main__': - m = Main() - if m.failed: - exit(-1) diff --git a/tools/ci/common.py b/tools/ci/common.py deleted file mode 100644 index 81b4afb..0000000 --- a/tools/ci/common.py +++ /dev/null @@ -1,1001 +0,0 @@ -# Copyright Rene Rivera 2016-2019 -# -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or copy at -# http://www.boost.org/LICENSE_1_0.txt) - -import sys -import inspect -import optparse -import os.path -import string -import time -import subprocess -import codecs -import shutil -import threading - -toolset_info = { - 'clang-3.4': { - 'ppa': ["ppa:ubuntu-toolchain-r/test"], - 'package': 'clang-3.4', - 'command': 'clang++-3.4', - 'toolset': 'clang', - 'version': '' - }, - 'clang-3.5': { - 'ppa': ["ppa:ubuntu-toolchain-r/test"], - 'package': 'clang-3.5', - 'command': 'clang++-3.5', - 'toolset': 'clang', - 'version': '' - }, - 'clang-3.6': { - 'ppa': ["ppa:ubuntu-toolchain-r/test"], - 'package': 'clang-3.6', - 'command': 'clang++-3.6', - 'toolset': 'clang', - 'version': '' - }, - 'clang-3.7': { - 'ppa': ["ppa:ubuntu-toolchain-r/test"], - 'package': 'clang-3.7', - 'command': 'clang++-3.7', - 'toolset': 'clang', - 'version': '' - }, - 'clang-3.8': { - 'ppa': ["ppa:ubuntu-toolchain-r/test"], - 'package': 'clang-3.8', - 'command': 'clang++-3.8', - 'toolset': 'clang', - 'version': '' - }, - 'clang-3.9': { - 'ppa': ["ppa:ubuntu-toolchain-r/test"], - 'package': 'clang-3.9', - 'command': 'clang++-3.9', - 'toolset': 'clang', - 'version': '' - }, - 'clang-4.0': { - 'deb': ["http://apt.llvm.org/xenial/", "llvm-toolchain-xenial-4.0", "main"], - 'apt-key': ['http://apt.llvm.org/llvm-snapshot.gpg.key'], - 'package': 'clang-4.0', - 'command': 'clang++-4.0', - 'toolset': 'clang', - 'version': '' - }, - 'clang-5.0': { - 'deb': ["http://apt.llvm.org/xenial/", "llvm-toolchain-xenial-5.0", "main"], - 'apt-key': ['http://apt.llvm.org/llvm-snapshot.gpg.key'], - 'package': 'clang-5.0', - 'command': 'clang++-5.0', - 'toolset': 'clang', - 'version': '' - }, - 'clang-6.0': { - 'deb': ["http://apt.llvm.org/xenial/", "llvm-toolchain-xenial-6.0", "main"], - 'apt-key': ['http://apt.llvm.org/llvm-snapshot.gpg.key'], - 'package': 'clang-6.0', - 'command': 'clang++-6.0', - 'toolset': 'clang', - 'version': '' - }, - 'clang-7': { - 'deb': ["http://apt.llvm.org/xenial/", "llvm-toolchain-xenial-7", "main"], - 'apt-key': ['http://apt.llvm.org/llvm-snapshot.gpg.key'], - 'package': 'clang-6.0', - 'command': 'clang++-6.0', - 'toolset': 'clang', - 'version': '' - }, - 'clang-8': { - 'deb': ["http://apt.llvm.org/xenial/", "llvm-toolchain-xenial-8", "main"], - 'apt-key': ['http://apt.llvm.org/llvm-snapshot.gpg.key'], - 'package': 'clang-6.0', - 'command': 'clang++-6.0', - 'toolset': 'clang', - 'version': '' - }, - 'gcc-4.7': { - 'ppa': ["ppa:ubuntu-toolchain-r/test"], - 'package': 'g++-4.7', - 'command': 'g++-4.7', - 'toolset': 'gcc', - 'version': '' - }, - 'gcc-4.8': { - 'bin': 'gcc-4.8', - 'ppa': ["ppa:ubuntu-toolchain-r/test"], - 'package': 'g++-4.8', - 'command': 'g++-4.8', - 'toolset': 'gcc', - 'version': '' - }, - 'gcc-4.9': { - 'ppa': ["ppa:ubuntu-toolchain-r/test"], - 'package': 'g++-4.9', - 'command': 'g++-4.9', - 'toolset': 'gcc', - 'version': '' - }, - 'gcc-5.1': { - 'ppa': ["ppa:ubuntu-toolchain-r/test"], - 'package': 'g++-5', - 'command': 'g++-5', - 'toolset': 'gcc', - 'version': '' - }, - 'gcc-5': { - 'ppa': ["ppa:ubuntu-toolchain-r/test"], - 'package': 'g++-5', - 'command': 'g++-5', - 'toolset': 'gcc', - 'version': '' - }, - 'gcc-6': { - 'ppa': ["ppa:ubuntu-toolchain-r/test"], - 'package': 'g++-6', - 'command': 'g++-6', - 'toolset': 'gcc', - 'version': '' - }, - 'gcc-7': { - 'ppa': ["ppa:ubuntu-toolchain-r/test"], - 'package': 'g++-7', - 'command': 'g++-7', - 'toolset': 'gcc', - 'version': '' - }, - 'gcc-8': { - 'ppa': ["ppa:ubuntu-toolchain-r/test"], - 'package': 'g++-8', - 'command': 'g++-8', - 'toolset': 'gcc', - 'version': '' - }, - 'mingw-8': { - 'toolset': 'gcc', - 'command': 'g++.exe', - 'version': '' - }, - 'vs-2008': { - 'toolset': 'msvc', - 'command': '', - 'version': '9.0' - }, - 'vs-2010': { - 'toolset': 'msvc', - 'command': '', - 'version': '10.0' - }, - 'vs-2012': { - 'toolset': 'msvc', - 'command': '', - 'version': '11.0' - }, - 'vs-2013': { - 'toolset': 'msvc', - 'command': '', - 'version': '12.0' - }, - 'vs-2015': { - 'toolset': 'msvc', - 'command': '', - 'version': '14.0' - }, - 'vs-2017': { - 'toolset': 'msvc', - 'command': '', - 'version': '14.1' - }, - 'vs-2019': { - 'toolset': 'msvc', - 'command': '', - 'version': '14.2' - }, - 'xcode-6.1': { - 'command': 'clang++', - 'toolset': 'clang', - 'version': '' - }, - 'xcode-6.2': { - 'command': 'clang++', - 'toolset': 'clang', - 'version': '' - }, - 'xcode-6.3': { - 'command': 'clang++', - 'toolset': 'clang', - 'version': '' - }, - 'xcode-6.4': { - 'command': 'clang++', - 'toolset': 'clang', - 'version': '' - }, - 'xcode-7.0': { - 'command': 'clang++', - 'toolset': 'clang', - 'version': '' - }, - 'xcode-7.1': { - 'command': 'clang++', - 'toolset': 'clang', - 'version': '' - }, - 'xcode-7.2': { - 'command': 'clang++', - 'toolset': 'clang', - 'version': '' - }, - 'xcode-7.3': { - 'command': 'clang++', - 'toolset': 'clang', - 'version': '' - }, - 'xcode-8.0': { - 'command': 'clang++', - 'toolset': 'clang', - 'version': '' - }, - 'xcode-8.1': { - 'command': 'clang++', - 'toolset': 'clang', - 'version': '' - }, - 'xcode-8.2': { - 'command': 'clang++', - 'toolset': 'clang', - 'version': '' - }, - 'xcode-8.3': { - 'command': 'clang++', - 'toolset': 'clang', - 'version': '' - }, - 'xcode-9.0': { - 'command': 'clang++', - 'toolset': 'clang', - 'version': '' - }, - 'xcode-9.0.1': { - 'command': 'clang++', - 'toolset': 'clang', - 'version': '' - }, - 'xcode-9.1': { - 'command': 'clang++', - 'toolset': 'clang', - 'version': '' - }, - 'xcode-9.2': { - 'command': 'clang++', - 'toolset': 'clang', - 'version': '' - }, - 'xcode-9.3': { - 'command': 'clang++', - 'toolset': 'clang', - 'version': '' - }, - 'xcode-9.3.1': { - 'command': 'clang++', - 'toolset': 'clang', - 'version': '' - }, - 'xcode-9.4': { - 'command': 'clang++', - 'toolset': 'clang', - 'version': '' - }, - 'xcode-9.4.1': { - 'command': 'clang++', - 'toolset': 'clang', - 'version': '' - }, - 'xcode-10.0': { - 'command': 'clang++', - 'toolset': 'clang', - 'version': '' - }, - 'xcode-10.1': { - 'command': 'clang++', - 'toolset': 'clang', - 'version': '' - }, -} - - -class SystemCallError(Exception): - def __init__(self, command, result): - self.command = command - self.result = result - - def __str__(self, *args, **kwargs): - return "'%s' ==> %s" % ("' '".join(self.command), self.result) - - -class utils: - - call_stats = [] - - @staticmethod - def call(*command, **kargs): - utils.log("%s> '%s'" % (os.getcwd(), "' '".join(command))) - t = time.time() - result = subprocess.call(command, **kargs) - t = time.time()-t - if result != 0: - print("Failed: '%s' ERROR = %s" % ("' '".join(command), result)) - utils.call_stats.append((t, os.getcwd(), command, result)) - utils.log("%s> '%s' execution time %s seconds" % - (os.getcwd(), "' '".join(command), t)) - return result - - @staticmethod - def print_call_stats(): - utils.log( - "================================================================================") - for j in sorted(utils.call_stats, reverse=True): - utils.log("{:>12.4f}\t{}> {} ==> {}".format(*j)) - utils.log( - "================================================================================") - - @staticmethod - def check_call(*command, **kargs): - cwd = os.getcwd() - result = utils.call(*command, **kargs) - if result != 0: - raise(SystemCallError([cwd].extend(command), result)) - - @staticmethod - def makedirs(path): - if not os.path.exists(path): - os.makedirs(path) - - @staticmethod - def log_level(): - frames = inspect.stack() - level = 0 - for i in frames[3:]: - if '__log__' in i[0].f_locals: - level = level + i[0].f_locals['__log__'] - return level - - @staticmethod - def log(message): - sys.stdout.flush() - sys.stdout.write('# ' + ' ' * utils.log_level() + message + '\n') - sys.stdout.flush() - - @staticmethod - def rmtree(path): - if os.path.exists(path): - #~ shutil.rmtree( unicode( path ) ) - if sys.platform == 'win32': - os.system('del /f /s /q "%s" >nul 2>&1' % path) - shutil.rmtree(unicode(path)) - else: - os.system('rm -f -r "%s"' % path) - - @staticmethod - def retry(f, max_attempts=5, sleep_secs=10): - for attempts in range(max_attempts, -1, -1): - try: - return f() - except Exception as msg: - utils.log('%s failed with message "%s"' % (f.__name__, msg)) - if attempts == 0: - utils.log('Giving up.') - raise - - utils.log('Retrying (%d more attempts).' % attempts) - time.sleep(sleep_secs) - - @staticmethod - def web_get(source_url, destination_file, proxy=None): - import urllib - - proxies = None - if proxy is not None: - proxies = { - 'https': proxy, - 'http': proxy - } - - src = urllib.urlopen(source_url, proxies=proxies) - - f = open(destination_file, 'wb') - while True: - data = src.read(16*1024) - if len(data) == 0: - break - f.write(data) - - f.close() - src.close() - - @staticmethod - def unpack_archive(archive_path): - utils.log('Unpacking archive ("%s")...' % archive_path) - - archive_name = os.path.basename(archive_path) - extension = archive_name[archive_name.find('.'):] - - if extension in (".tar.gz", ".tar.bz2"): - import tarfile - import stat - - mode = os.path.splitext(extension)[1][1:] - tar = tarfile.open(archive_path, 'r:%s' % mode) - for tarinfo in tar: - tar.extract(tarinfo) - if sys.platform == 'win32' and not tarinfo.isdir(): - # workaround what appears to be a Win32-specific bug in 'tarfile' - # (modification times for extracted files are not set properly) - f = os.path.join(os.curdir, tarinfo.name) - os.chmod(f, stat.S_IWRITE) - os.utime(f, (tarinfo.mtime, tarinfo.mtime)) - tar.close() - elif extension in (".zip"): - import zipfile - - z = zipfile.ZipFile(archive_path, 'r', zipfile.ZIP_DEFLATED) - for f in z.infolist(): - destination_file_path = os.path.join(os.curdir, f.filename) - if destination_file_path[-1] == "/": # directory - if not os.path.exists(destination_file_path): - os.makedirs(destination_file_path) - else: # file - result = open(destination_file_path, 'wb') - result.write(z.read(f.filename)) - result.close() - z.close() - else: - raise 'Do not know how to unpack archives with extension \"%s\"' % extension - - @staticmethod - def make_file(filename, *text): - text = '\n'.join(text) - with codecs.open(filename, 'w', 'utf-8') as f: - f.write(text) - - @staticmethod - def append_file(filename, *text): - with codecs.open(filename, 'a', 'utf-8') as f: - f.write('\n'.join(text)) - - @staticmethod - def mem_info(): - if sys.platform == "darwin": - utils.call("top", "-l", "1", "-s", "0", "-n", "0") - elif sys.platform.startswith("linux"): - utils.call("free", "-m", "-l") - - @staticmethod - def query_boost_version(boost_root): - ''' - Read in the Boost version from a given boost_root. - ''' - boost_version = None - if os.path.exists(os.path.join(boost_root, 'Jamroot')): - with codecs.open(os.path.join(boost_root, 'Jamroot'), 'r', 'utf-8') as f: - for line in f.readlines(): - parts = line.split() - if len(parts) >= 5 and parts[1] == 'BOOST_VERSION': - boost_version = parts[3] - break - if not boost_version: - boost_version = 'default' - return boost_version - - @staticmethod - def git_clone(owner, repo, branch, commit=None, repo_dir=None, submodules=False, url_format="https://github.com/%(owner)s/%(repo)s.git"): - ''' - This clone mimicks the way Travis-CI clones a project's repo. So far - Travis-CI is the most limiting in the sense of only fetching partial - history of the repo. - ''' - if not repo_dir: - repo_dir = os.path.join(os.getcwd(), owner+','+repo) - utils.makedirs(os.path.dirname(repo_dir)) - if not os.path.exists(os.path.join(repo_dir, '.git')): - utils.check_call("git", "clone", - "--depth=1", - "--branch=%s" % (branch), - url_format % {'owner': owner, 'repo': repo}, - repo_dir) - os.chdir(repo_dir) - else: - os.chdir(repo_dir) - utils.check_call("git", "pull", - # "--depth=1", # Can't do depth as we get merge errors. - "--quiet", "--no-recurse-submodules") - if commit: - utils.check_call("git", "checkout", "-qf", commit) - if os.path.exists(os.path.join('.git', 'modules')): - if sys.platform == 'win32': - utils.check_call('dir', os.path.join('.git', 'modules')) - else: - utils.check_call('ls', '-la', os.path.join('.git', 'modules')) - if submodules: - utils.check_call("git", "submodule", "--quiet", "update", - "--quiet", "--init", "--recursive", - ) - utils.check_call("git", "submodule", "--quiet", - "foreach", "git", "fetch") - return repo_dir - - -class parallel_call(threading.Thread): - ''' - Runs a synchronous command in a thread waiting for it to complete. - ''' - - def __init__(self, *command, **kargs): - super(parallel_call, self).__init__() - self.command = command - self.command_kargs = kargs - self.start() - - def run(self): - self.result = utils.call(*self.command, **self.command_kargs) - - def join(self): - super(parallel_call, self).join() - if self.result != 0: - raise(SystemCallError(self.command, self.result)) - - -def set_arg(args, k, v=None): - if not args.get(k): - args[k] = v - return args[k] - - -class script_common(object): - ''' - Main script to run continuous integration. - ''' - - def __init__(self, ci_klass, **kargs): - self.ci = ci_klass(self) - - opt = optparse.OptionParser( - usage="%prog [options] [commands]") - - # ~ Debug Options: - opt.add_option('--debug-level', - help="debugging level; controls the amount of debugging output printed", - type='int') - opt.add_option('-j', - help="maximum number of parallel jobs to use for building with b2", - type='int', dest='jobs') - opt.add_option('--branch') - opt.add_option('--commit') - kargs = self.init(opt, kargs) - kargs = self.ci.init(opt, kargs) - set_arg(kargs, 'debug_level', 0) - set_arg(kargs, 'jobs', 2) - set_arg(kargs, 'branch', None) - set_arg(kargs, 'commit', None) - set_arg(kargs, 'repo', None) - set_arg(kargs, 'repo_dir', None) - set_arg(kargs, 'actions', None) - set_arg(kargs, 'pull_request', None) - - #~ Defaults - for (k, v) in kargs.items(): - setattr(self, k, v) - (_opt_, self.actions) = opt.parse_args(None, self) - if not self.actions or self.actions == []: - self.actions = kargs.get('actions', None) - if not self.actions or self.actions == []: - self.actions = ['info'] - if not self.repo_dir: - self.repo_dir = os.getcwd() - self.build_dir = os.path.join(os.path.dirname(self.repo_dir), "build") - - # API keys. - self.bintray_key = os.getenv('BINTRAY_KEY') - - try: - self.start() - self.command_info() - self.main() - utils.print_call_stats() - except Exception as e: - utils.print_call_stats() - # raise - utils.log(str(e)) - exit(1) - - def init(self, opt, kargs): - return kargs - - def start(self): - pass - - def main(self): - for action in self.actions: - action_m = "command_"+action.replace('-', '_') - ci_command = getattr(self.ci, action_m, None) - ci_script = getattr(self, action_m, None) - if ci_command or ci_script: - utils.log("### %s.." % (action)) - if os.path.exists(self.repo_dir): - os.chdir(self.repo_dir) - if ci_command: - ci_command() - elif ci_script: - ci_script() - - def b2(self, *args, **kargs): - cmd = ['b2', '--debug-configuration', '-j%s' % (self.jobs)] - cmd.extend(args) - - if 'toolset' in kargs: - cmd.append('toolset=' + kargs['toolset']) - - if 'parallel' in kargs: - return parallel_call(*cmd) - else: - return utils.check_call(*cmd) - - # Common test commands in the order they should be executed.. - - def command_info(self): - pass - - def command_install(self): - utils.makedirs(self.build_dir) - os.chdir(self.build_dir) - - def command_install_toolset(self, toolset): - if self.ci and hasattr(self.ci, 'install_toolset'): - self.ci.install_toolset(toolset) - - def command_before_build(self): - pass - - def command_build(self): - pass - - def command_before_cache(self): - pass - - def command_after_success(self): - pass - - -class ci_cli(object): - ''' - This version of the script provides a way to do manual building. It sets up - additional environment and adds fetching of the git repos that would - normally be done by the CI system. - - The common way to use this variant is to invoke something like: - - mkdir ci - cd ci - python path-to/library_test.py --branch=develop [--repo=mylib] ... - - Status: In working order. - ''' - - def __init__(self, script): - if sys.platform == 'darwin': - # Requirements for running on OSX: - # https://www.stack.nl/~dimitri/doxygen/download.html#srcbin - # https://tug.org/mactex/morepackages.html - doxygen_path = "/Applications/Doxygen.app/Contents/Resources" - if os.path.isdir(doxygen_path): - os.environ["PATH"] = doxygen_path+':'+os.environ['PATH'] - self.script = script - self.repo_dir = os.getcwd() - self.exit_result = 0 - - def init(self, opt, kargs): - kargs['actions'] = [ - # 'clone', - 'install', - 'before_build', - 'build', - 'before_cache', - 'finish' - ] - return kargs - - def finish(self, result): - self.exit_result = result - - def command_finish(self): - exit(self.exit_result) - - -class ci_travis(object): - ''' - This variant build releases in the context of the Travis-CI service. - ''' - - def __init__(self, script): - self.script = script - - def init(self, opt, kargs): - set_arg(kargs, 'repo_dir', os.getenv("TRAVIS_BUILD_DIR")) - set_arg(kargs, 'branch', os.getenv("TRAVIS_BRANCH")) - set_arg(kargs, 'commit', os.getenv("TRAVIS_COMMIT")) - set_arg(kargs, 'repo', os.getenv("TRAVIS_REPO_SLUG").split("/")[1]) - set_arg(kargs, 'pull_request', - os.getenv('TRAVIS_PULL_REQUEST') - if os.getenv('TRAVIS_PULL_REQUEST') != 'false' else None) - return kargs - - def finish(self, result): - exit(result) - - def install_toolset(self, toolset): - ''' - Installs specific toolset on CI system. - ''' - info = toolset_info[toolset] - if sys.platform.startswith('linux'): - os.chdir(self.script.build_dir) - if 'ppa' in info: - for ppa in info['ppa']: - utils.check_call( - 'sudo', 'add-apt-repository', '--yes', ppa) - if 'deb' in info: - utils.make_file('sources.list', - "deb %s" % (' '.join(info['deb'])), - "deb-src %s" % (' '.join(info['deb']))) - utils.check_call('sudo', 'bash', '-c', - 'cat sources.list >> /etc/apt/sources.list') - if 'apt-key' in info: - for key in info['apt-key']: - utils.check_call('wget', key, '-O', 'apt.key') - utils.check_call('sudo', 'apt-key', 'add', 'apt.key') - utils.check_call( - 'sudo', 'apt-get', 'update', '-qq') - utils.check_call( - 'sudo', 'apt-get', 'install', '-qq', info['package']) - if 'debugpackage' in info and info['debugpackage']: - utils.check_call( - 'sudo', 'apt-get', 'install', '-qq', info['debugpackage']) - - # Travis-CI commands in the order they are executed. We need - # these to forward to our common commands, if they are different. - - def command_before_install(self): - pass - - def command_install(self): - self.script.command_install() - - def command_before_script(self): - self.script.command_before_build() - - def command_script(self): - self.script.command_build() - - def command_before_cache(self): - self.script.command_before_cache() - - def command_after_success(self): - self.script.command_after_success() - - def command_after_failure(self): - pass - - def command_before_deploy(self): - pass - - def command_after_deploy(self): - pass - - def command_after_script(self): - pass - - -class ci_circleci(object): - ''' - This variant build releases in the context of the CircleCI service. - ''' - - def __init__(self, script): - self.script = script - - def init(self, opt, kargs): - set_arg(kargs, 'repo_dir', os.path.join( - os.getenv("HOME"), os.getenv("CIRCLE_PROJECT_REPONAME"))) - set_arg(kargs, 'branch', os.getenv("CIRCLE_BRANCH")) - set_arg(kargs, 'commit', os.getenv("CIRCLE_SHA1")) - set_arg(kargs, 'repo', os.getenv( - "CIRCLE_PROJECT_REPONAME").split("/")[1]) - set_arg(kargs, 'pull_request', os.getenv('CIRCLE_PR_NUMBER')) - return kargs - - def finish(self, result): - exit(result) - - def command_machine_post(self): - # Apt update for the pckages installs we'll do later. - utils.check_call('sudo', 'apt-get', '-qq', 'update') - # Need PyYAML to read Travis yaml in a later step. - utils.check_call("pip", "install", "--user", "PyYAML") - - def command_checkout_post(self): - os.chdir(self.script.repo_dir) - utils.check_call("git", "submodule", "update", - "--quiet", "--init", "--recursive") - - def command_dependencies_pre(self): - # Read in .travis.yml for list of packages to install - # as CircleCI doesn't have a convenient apt install method. - import yaml - utils.check_call('sudo', '-E', 'apt-get', '-yqq', 'update') - utils.check_call('sudo', 'apt-get', '-yqq', 'purge', 'texlive*') - with open(os.path.join(self.script.repo_dir, '.travis.yml')) as yml: - travis_yml = yaml.load(yml) - utils.check_call('sudo', 'apt-get', '-yqq', - '--no-install-suggests', '--no-install-recommends', '--force-yes', 'install', - *travis_yml['addons']['apt']['packages']) - - def command_dependencies_override(self): - self.script.command_install() - - def command_dependencies_post(self): - pass - - def command_database_pre(self): - pass - - def command_database_override(self): - pass - - def command_database_post(self): - pass - - def command_test_pre(self): - self.script.command_install() - self.script.command_before_build() - - def command_test_override(self): - # CircleCI runs all the test subsets. So in order to avoid - # running the after_success we do it here as the build step - # will halt accordingly. - self.script.command_build() - self.script.command_before_cache() - self.script.command_after_success() - - def command_test_post(self): - pass - - -class ci_appveyor(object): - - def __init__(self, script): - self.script = script - - def init(self, opt, kargs): - set_arg(kargs, 'repo_dir', os.getenv("APPVEYOR_BUILD_FOLDER")) - set_arg(kargs, 'branch', os.getenv("APPVEYOR_REPO_BRANCH")) - set_arg(kargs, 'commit', os.getenv("APPVEYOR_REPO_COMMIT")) - set_arg(kargs, 'repo', os.getenv("APPVEYOR_REPO_NAME").split("/")[1]) - set_arg(kargs, 'address_model', os.getenv("PLATFORM", None)) - set_arg(kargs, 'variant', os.getenv("CONFIGURATION", "debug")) - set_arg(kargs, 'pull_request', os.getenv( - 'APPVEYOR_PULL_REQUEST_NUMBER')) - return kargs - - def finish(self, result): - exit(result) - - # Appveyor commands in the order they are executed. We need - # these to forward to our common commands, if they are different. - - def command_install(self): - self.script.command_install() - - def command_before_build(self): - os.chdir(self.script.repo_dir) - utils.check_call("git", "submodule", "update", - "--quiet", "--init", "--recursive") - self.script.command_before_build() - - def command_build_script(self): - self.script.command_build() - - def command_after_build(self): - self.script.command_before_cache() - - def command_before_test(self): - pass - - def command_test_script(self): - pass - - def command_after_test(self): - pass - - def command_on_success(self): - self.script.command_after_success() - - def command_on_failure(self): - pass - - def command_on_finish(self): - pass - - -class ci_azp(object): - ''' - This variant build releases in the context of the Azure Pipelines service. - ''' - - def __init__(self, script): - self.script = script - - def init(self, opt, kargs): - set_arg(kargs, 'repo_dir', os.getenv("AZP_REPO_DIR")) - set_arg(kargs, 'branch', os.getenv("AZP_BRANCH")) - set_arg(kargs, 'commit', os.getenv("AZP_COMMIT")) - set_arg(kargs, 'repo', os.getenv("AZP_REPO")) - set_arg(kargs, 'pull_request', os.getenv('AZP_PULL_REQUEST')) - return kargs - - def finish(self, result): - exit(result) - - def install_toolset(self, toolset): - ''' - Installs specific toolset on CI system. - ''' - info = toolset_info[toolset] - if sys.platform.startswith('linux'): - os.chdir(self.script.build_dir) - if 'ppa' in info: - for ppa in info['ppa']: - utils.check_call( - 'sudo', 'add-apt-repository', '--yes', ppa) - if 'deb' in info: - utils.check_call( - 'sudo', '-E', 'apt-add-repository', - 'deb %s' % (' '.join(info['deb']))) - if 'apt-key' in info: - for key in info['apt-key']: - utils.check_call('wget', key, '-O', 'apt.key') - utils.check_call('sudo', '-E', 'apt-key', 'add', 'apt.key') - utils.check_call( - 'sudo', '-E', 'apt-get', 'update') - utils.check_call( - 'sudo', '-E', 'apt-get', '-yq', '--no-install-suggests', - '--no-install-recommends', 'install', info['package']) - if 'debugpackage' in info and info['debugpackage']: - utils.check_call( - 'sudo', 'apt-get', '-yq', '--no-install-suggests', - '--no-install-recommends', 'install', info['debugpackage']) - - def command_install(self): - self.script.command_install() - self.script.command_before_build() - - def command_script(self): - self.script.command_build() - self.script.command_before_cache() - self.script.command_after_success() - - -def main(script_klass): - if os.getenv('TRAVIS', False): - script_klass(ci_travis) - elif os.getenv('CIRCLECI', False): - script_klass(ci_circleci) - elif os.getenv('APPVEYOR', False): - script_klass(ci_appveyor) - elif os.getenv('AZP', False): - script_klass(ci_azp) - else: - script_klass(ci_cli) diff --git a/tools/ci/library_test.py b/tools/ci/library_test.py deleted file mode 100644 index 16bc734..0000000 --- a/tools/ci/library_test.py +++ /dev/null @@ -1,152 +0,0 @@ - -# Copyright Rene Rivera 2016-2019 -# -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or copy at -# http://www.boost.org/LICENSE_1_0.txt) - -from __future__ import print_function - -import os.path -import shutil -import sys -from common import toolset_info, main, utils, script_common, ci_cli, set_arg - -__dirname__ = os.path.dirname(os.path.realpath(__file__)) - - -class script(script_common): - ''' - Main script to test a Boost C++ Library. - ''' - - def __init__(self, ci_klass, **kargs): - script_common.__init__(self, ci_klass, **kargs) - - def init(self, opt, kargs): - opt.add_option( - '--toolset', - help="single toolset to test with") - opt.add_option( - '--target', - help="test target to build for testing, defaults to TARGET or 'minimal'") - opt.add_option( - '--address-model', - help="address model to test, ie 64 or 32") - opt.add_option( - '--variant', - help="variant to test, ie debug, release") - set_arg(kargs, 'toolset', os.getenv("TOOLSET")) - set_arg(kargs, 'target', os.getenv('TARGET', 'minimal')) - set_arg(kargs, 'address_model', os.getenv("ADDRESS_MODEL", None)) - set_arg(kargs, 'variant', os.getenv("VARIANT", "debug")) - set_arg(kargs, 'cxxstd', os.getenv("CXXSTD", None)) - set_arg(kargs, 'cxxdialect', os.getenv("CXXDIALECT", None)) - set_arg(kargs, 'cxxdefs', os.getenv("CXXDEFS", None)) - return kargs - - def start(self): - script_common.start(self) - # Some setup we need to redo for each invocation. - self.b2_dir = os.path.join(self.build_dir, 'b2') - - def command_install(self): - script_common.command_install(self) - # Fetch & install toolset.. - utils.log("Install toolset: %s" % (self.toolset)) - if self.toolset: - self.command_install_toolset(self.toolset) - - def command_before_build(self): - script_common.command_before_build(self) - - # Fetch dependencies. - utils.git_clone('boostorg', 'build', 'develop', repo_dir=self.b2_dir) - - # Create config file for b2 toolset. - if not isinstance(self.ci, ci_cli): - utils.make_file(os.path.join(self.repo_dir, 'project-config.jam'), - """ -using %(toolset)s : %(version)s : %(command)s ; -""" % { - 'toolset': toolset_info[self.toolset]['toolset'], - 'version': toolset_info[self.toolset]['version'], - 'command': toolset_info[self.toolset]['command'], - }) - - # # "Convert" boostorg-predef into standalone b2 project. - # if os.path.exists(os.path.join(self.repo_dir, 'build.jam')) and not os.path.exists(os.path.join(self.repo_dir, 'project-root.jam')): - # os.rename(os.path.join(self.repo_dir, 'build.jam'), - # os.path.join(self.repo_dir, 'project-root.jam')) - - def command_build(self): - script_common.command_build(self) - - # Set up tools. - if not isinstance(self.ci, ci_cli) and toolset_info[self.toolset]['command']: - os.environ['PATH'] = os.pathsep.join([ - os.path.dirname(toolset_info[self.toolset]['command']), - os.environ['PATH']]) - - # Bootstrap Boost Build engine. - os.chdir(self.b2_dir) - if sys.platform == 'win32': - utils.check_call(".\\bootstrap.bat") - else: - utils.check_call("./bootstrap.sh") - os.environ['PATH'] = os.pathsep.join([self.b2_dir, os.environ['PATH']]) - os.environ['BOOST_BUILD_PATH'] = self.b2_dir - - # Run the limited tests. - print("--- Testing %s ---" % (self.repo_dir)) - os.chdir(os.path.join(self.repo_dir, 'test')) - toolset_to_test = "" - if self.toolset: - if not isinstance(self.ci, ci_cli): - toolset_to_test = toolset_info[self.toolset]['toolset'] - else: - toolset_to_test = self.toolset - cxxdefs = [] - if self.cxxdefs: - cxxdefs = ['define=%s' % (d) for d in self.cxxdefs.split(',')] - self.b2( - '-d1', - '-p0', - 'preserve-test-targets=off', - '--dump-tests', - '--verbose-test', - '--build-dir=%s' % (self.build_dir), - '--out-xml=%s' % (os.path.join(self.build_dir, 'regression.xml')), - '' if not toolset_to_test else 'toolset=%s' % (toolset_to_test), - '' if not self.address_model else 'address-model=%s' % ( - self.address_model), - 'variant=%s' % (self.variant), - '' if not self.cxxstd else 'cxxstd=%s' % ( - self.cxxstd), - '' if not self.cxxdialect else 'cxxstd-dialect=%s' % ( - self.cxxdialect), - self.target, - *cxxdefs - ) - - # Generate a readable test report. - import build_log - log_main = build_log.Main([ - '--output=console', - os.path.join(self.build_dir, 'regression.xml')]) - # And exit with an error if the report contains failures. - # This lets the CI notice the error and report a failed build. - # And hence trigger the failure machinery, like sending emails. - if log_main.failed: - self.ci.finish(-1) - - def command_before_cache(self): - script_common.command_before_cache(self) - os.chdir(self.b2_dir) - utils.check_call("git", "clean", "-dfqx") - utils.check_call("git", "status", "-bs") - # utils.check_call("git","submodule","--quiet","foreach","git","clean","-dfqx") - # utils.check_call("git","submodule","foreach","git","status","-bs") - - -main(script)