From 9e73c5529892d1e7eeb221bea4f76b3d075c2511 Mon Sep 17 00:00:00 2001 From: Rene Rivera Date: Sun, 13 May 2018 22:59:56 -0500 Subject: [PATCH] Switch to local CI scripts to make progress on CI errors. --- .project | 6 + .pydevproject | 5 + .travis.yml | 22 +- appveyor.yml | 26 +- tools/ci/build_log.py | 428 ++++++++++++++++++++ tools/ci/common.py | 845 +++++++++++++++++++++++++++++++++++++++ tools/ci/library_test.py | 205 ++++++++++ 7 files changed, 1506 insertions(+), 31 deletions(-) create mode 100644 .pydevproject create mode 100644 tools/ci/build_log.py create mode 100644 tools/ci/common.py create mode 100644 tools/ci/library_test.py diff --git a/.project b/.project index 8fe9304..2f7b668 100644 --- a/.project +++ b/.project @@ -5,7 +5,13 @@ + + org.python.pydev.PyDevBuilder + + + + org.python.pydev.pythonNature diff --git a/.pydevproject b/.pydevproject new file mode 100644 index 0000000..d001f0a --- /dev/null +++ b/.pydevproject @@ -0,0 +1,5 @@ + + +Default +python interpreter + diff --git a/.travis.yml b/.travis.yml index 0ffe948..e4fd420 100644 --- a/.travis.yml +++ b/.travis.yml @@ -99,21 +99,13 @@ matrix: os: osx osx_image: xcode8.3 -# These are the standard steps to bootstrap the Boost CI scripts -# and to forward the actions to the scripts. -before_install: - # Fetch the scripts to do the actual building/testing. - - | - wget "https://raw.githubusercontent.com/boostorg/regression/develop/ci/src/ci_boost_common.py" -P .. - wget "https://raw.githubusercontent.com/boostorg/regression/develop/ci/src/ci_boost_library_test.py" -P .. - -install: python "${TRAVIS_BUILD_DIR}/../ci_boost_library_test.py" install -before_script: python "${TRAVIS_BUILD_DIR}/../ci_boost_library_test.py" before_script -script: python "${TRAVIS_BUILD_DIR}/../ci_boost_library_test.py" script -before_cache: python "${TRAVIS_BUILD_DIR}/../ci_boost_library_test.py" before_cache -after_success: python "${TRAVIS_BUILD_DIR}/../ci_boost_library_test.py" after_success -after_failure: python "${TRAVIS_BUILD_DIR}/../ci_boost_library_test.py" after_failure -after_script: python "${TRAVIS_BUILD_DIR}/../ci_boost_library_test.py" after_script +install: python "${TRAVIS_BUILD_DIR}/tools/ci/library_test.py" install +before_script: python "${TRAVIS_BUILD_DIR}/tools/ci/library_test.py" before_script +script: python "${TRAVIS_BUILD_DIR}/tools/ci/library_test.py" script +before_cache: python "${TRAVIS_BUILD_DIR}/tools/ci/library_test.py" before_cache +after_success: python "${TRAVIS_BUILD_DIR}/tools/ci/library_test.py" after_success +after_failure: python "${TRAVIS_BUILD_DIR}/tools/ci/library_test.py" after_failure +after_script: python "${TRAVIS_BUILD_DIR}/tools/ci/library_test.py" after_script cache: directories: - $HOME/boostorg/boost diff --git a/appveyor.yml b/appveyor.yml index 5f23d61..36386d9 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -132,21 +132,15 @@ configuration: - debug - release -# These are the standard steps to bootstrap the Boost CI scripts -# and to forward the actions to the scripts. init: - # Fetch the scripts to do the actual building/testing. - - cd %APPVEYOR_BUILD_FOLDER%/.. - - appveyor DownloadFile "https://raw.githubusercontent.com/boostorg/regression/develop/ci/src/ci_boost_common.py" - - appveyor DownloadFile "https://raw.githubusercontent.com/boostorg/regression/develop/ci/src/ci_boost_library_test.py" - cd %APPVEYOR_BUILD_FOLDER% -install: python ../ci_boost_library_test.py install -before_build: python ../ci_boost_library_test.py before_build -build_script: python ../ci_boost_library_test.py build_script -after_build: python ../ci_boost_library_test.py after_build -before_test: python ../ci_boost_library_test.py before_test -test_script: python ../ci_boost_library_test.py test_script -after_test: python ../ci_boost_library_test.py after_test -on_success: python ../ci_boost_library_test.py on_success -on_failure: python ../ci_boost_library_test.py on_failure -on_finish: python ../ci_boost_library_test.py on_finish +install: python tools/ci/clibrary_test.py install +before_build: python tools/ci/library_test.py before_build +build_script: python tools/ci/library_test.py build_script +after_build: python tools/ci/library_test.py after_build +before_test: python tools/ci/library_test.py before_test +test_script: python tools/ci/library_test.py test_script +after_test: python tools/ci/library_test.py after_test +on_success: python tools/ci/library_test.py on_success +on_failure: python tools/ci/library_test.py on_failure +on_finish: python tools/ci/library_test.py on_finish diff --git a/tools/ci/build_log.py b/tools/ci/build_log.py new file mode 100644 index 0000000..6ae156b --- /dev/null +++ b/tools/ci/build_log.py @@ -0,0 +1,428 @@ +#!/usr/bin/env python + +# Copyright 2008 Rene Rivera +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) + +import re +import optparse +import time +import xml.dom.minidom +import xml.dom.pulldom +from xml.sax.saxutils import unescape, escape +import os.path +from pprint import pprint +from __builtin__ import exit + +class BuildOutputXMLParsing(object): + ''' + XML parsing utilities for dealing with the Boost Build output + XML format. + ''' + + def get_child_data( self, root, tag = None, id = None, name = None, strip = False, default = None ): + return self.get_data(self.get_child(root,tag=tag,id=id,name=name),strip=strip,default=default) + + def get_data( self, node, strip = False, default = None ): + data = None + if node: + data_node = None + if not data_node: + data_node = self.get_child(node,tag='#text') + if not data_node: + data_node = self.get_child(node,tag='#cdata-section') + data = "" + while data_node: + data += data_node.data + data_node = data_node.nextSibling + if data_node: + if data_node.nodeName != '#text' \ + and data_node.nodeName != '#cdata-section': + data_node = None + if not data: + data = default + else: + if strip: + data = data.strip() + return data + + def get_child( self, root, tag = None, id = None, name = None, type = None ): + return self.get_sibling(root.firstChild,tag=tag,id=id,name=name,type=type) + + def get_sibling( self, sibling, tag = None, id = None, name = None, type = None ): + n = sibling + while n: + found = True + if type and found: + found = found and type == n.nodeType + if tag and found: + found = found and tag == n.nodeName + if (id or name) and found: + found = found and n.nodeType == xml.dom.Node.ELEMENT_NODE + if id and found: + if n.hasAttribute('id'): + found = found and n.getAttribute('id') == id + else: + found = found and n.hasAttribute('id') and n.getAttribute('id') == id + if name and found: + found = found and n.hasAttribute('name') and n.getAttribute('name') == name + if found: + return n + n = n.nextSibling + return None + +class BuildOutputProcessor(BuildOutputXMLParsing): + + def __init__(self, inputs): + self.test = {} + self.target_to_test = {} + self.target = {} + self.parent = {} + self.timestamps = [] + for input in inputs: + self.add_input(input) + + def add_input(self, input): + ''' + Add a single build XML output file to our data. + ''' + events = xml.dom.pulldom.parse(input) + context = [] + for (event,node) in events: + if event == xml.dom.pulldom.START_ELEMENT: + context.append(node) + if node.nodeType == xml.dom.Node.ELEMENT_NODE: + x_f = self.x_name_(*context) + if x_f: + events.expandNode(node) + # expanding eats the end element, hence walking us out one level + context.pop() + # call handler + (x_f[1])(node) + elif event == xml.dom.pulldom.END_ELEMENT: + context.pop() + + def x_name_(self, *context, **kwargs): + node = None + names = [ ] + for c in context: + if c: + if not isinstance(c,xml.dom.Node): + suffix = '_'+c.replace('-','_').replace('#','_') + else: + suffix = '_'+c.nodeName.replace('-','_').replace('#','_') + node = c + names.append('x') + names = map(lambda x: x+suffix,names) + if node: + for name in names: + if hasattr(self,name): + return (name,getattr(self,name)) + return None + + def x_build_test(self, node): + ''' + Records the initial test information that will eventually + get expanded as we process the rest of the results. + ''' + test_node = node + test_name = test_node.getAttribute('name') + test_target = self.get_child_data(test_node,tag='target',strip=True) + ## print ">>> %s %s" %(test_name,test_target) + self.test[test_name] = { + 'library' : "/".join(test_name.split('/')[0:-1]), + 'test-name' : test_name.split('/')[-1], + 'test-type' : test_node.getAttribute('type').lower(), + 'test-program' : self.get_child_data(test_node,tag='source',strip=True), + 'target' : test_target, + 'info' : self.get_child_data(test_node,tag='info',strip=True), + 'dependencies' : [], + 'actions' : [], + } + # Add a lookup for the test given the test target. + self.target_to_test[self.test[test_name]['target']] = test_name + return None + + def x_build_targets_target( self, node ): + ''' + Process the target dependency DAG into an ancestry tree so we can look up + which top-level library and test targets specific build actions correspond to. + ''' + target_node = node + name = self.get_child_data(target_node,tag='name',strip=True) + path = self.get_child_data(target_node,tag='path',strip=True) + jam_target = self.get_child_data(target_node,tag='jam-target',strip=True) + #~ Map for jam targets to virtual targets. + self.target[jam_target] = { + 'name' : name, + 'path' : path + } + #~ Create the ancestry. + dep_node = self.get_child(self.get_child(target_node,tag='dependencies'),tag='dependency') + while dep_node: + child = self.get_data(dep_node,strip=True) + child_jam_target = '%s' % (path,child.split('//',1)[1]) + self.parent[child_jam_target] = jam_target + dep_node = self.get_sibling(dep_node.nextSibling,tag='dependency') + return None + + def x_build_action( self, node ): + ''' + Given a build action log, process into the corresponding test log and + specific test log sub-part. + ''' + action_node = node + name = self.get_child(action_node,tag='name') + if name: + name = self.get_data(name) + #~ Based on the action, we decide what sub-section the log + #~ should go into. + action_type = None + if re.match('[^%]+%[^.]+[.](compile)',name): + action_type = 'compile' + elif re.match('[^%]+%[^.]+[.](link|archive)',name): + action_type = 'link' + elif re.match('[^%]+%testing[.](capture-output)',name): + action_type = 'run' + elif re.match('[^%]+%testing[.](expect-failure|expect-success)',name): + action_type = 'result' + else: + # TODO: Enable to see what other actions can be included in the test results. + # action_type = None + action_type = 'other' + #~ print "+ [%s] %s %s :: %s" %(action_type,name,'','') + if action_type: + #~ Get the corresponding test. + (target,test) = self.get_test(action_node,type=action_type) + #~ Skip action that have no corresponding test as they are + #~ regular build actions and don't need to show up in the + #~ regression results. + if not test: + ##print "??? [%s] %s %s :: %s" %(action_type,name,target,test) + return None + ##print "+++ [%s] %s %s :: %s" %(action_type,name,target,test) + #~ Collect some basic info about the action. + action = { + 'command' : self.get_action_command(action_node,action_type), + 'output' : self.get_action_output(action_node,action_type), + 'info' : self.get_action_info(action_node,action_type) + } + #~ For the test result status we find the appropriate node + #~ based on the type of test. Then adjust the result status + #~ accordingly. This makes the result status reflect the + #~ expectation as the result pages post processing does not + #~ account for this inversion. + action['type'] = action_type + if action_type == 'result': + if re.match(r'^compile',test['test-type']): + action['type'] = 'compile' + elif re.match(r'^link',test['test-type']): + action['type'] = 'link' + elif re.match(r'^run',test['test-type']): + action['type'] = 'run' + #~ The result sub-part we will add this result to. + if action_node.getAttribute('status') == '0': + action['result'] = 'succeed' + else: + action['result'] = 'fail' + # Add the action to the test. + test['actions'].append(action) + # Set the test result if this is the result action for the test. + if action_type == 'result': + test['result'] = action['result'] + return None + + def x_build_timestamp( self, node ): + ''' + The time-stamp goes to the corresponding attribute in the result. + ''' + self.timestamps.append(self.get_data(node).strip()) + return None + + def get_test( self, node, type = None ): + ''' + Find the test corresponding to an action. For testing targets these + are the ones pre-declared in the --dump-test option. For libraries + we create a dummy test as needed. + ''' + jam_target = self.get_child_data(node,tag='jam-target') + base = self.target[jam_target]['name'] + target = jam_target + while target in self.parent: + target = self.parent[target] + #~ print "--- TEST: %s ==> %s" %(jam_target,target) + #~ main-target-type is a precise indicator of what the build target is + #~ originally meant to be. + #main_type = self.get_child_data(self.get_child(node,tag='properties'), + # name='main-target-type',strip=True) + main_type = None + if main_type == 'LIB' and type: + lib = self.target[target]['name'] + if not lib in self.test: + self.test[lib] = { + 'library' : re.search(r'libs/([^/]+)',lib).group(1), + 'test-name' : os.path.basename(lib), + 'test-type' : 'lib', + 'test-program' : os.path.basename(lib), + 'target' : lib + } + test = self.test[lib] + else: + target_name_ = self.target[target]['name'] + if self.target_to_test.has_key(target_name_): + test = self.test[self.target_to_test[target_name_]] + else: + test = None + return (base,test) + + #~ The command executed for the action. For run actions we omit the command + #~ as it's just noise. + def get_action_command( self, action_node, action_type ): + if action_type != 'run': + return self.get_child_data(action_node,tag='command') + else: + return '' + + #~ The command output. + def get_action_output( self, action_node, action_type ): + return self.get_child_data(action_node,tag='output',default='') + + #~ Some basic info about the action. + def get_action_info( self, action_node, action_type ): + info = {} + #~ The jam action and target. + info['name'] = self.get_child_data(action_node,tag='name') + info['path'] = self.get_child_data(action_node,tag='path') + #~ The timing of the action. + info['time-start'] = action_node.getAttribute('start') + info['time-end'] = action_node.getAttribute('end') + info['time-user'] = action_node.getAttribute('user') + info['time-system'] = action_node.getAttribute('system') + #~ Testing properties. + test_info_prop = self.get_child_data(self.get_child(action_node,tag='properties'),name='test-info') + info['always_show_run_output'] = test_info_prop == 'always_show_run_output' + #~ And for compiles some context that may be hidden if using response files. + if action_type == 'compile': + info['define'] = [] + define = self.get_child(self.get_child(action_node,tag='properties'),name='define') + while define: + info['define'].append(self.get_data(define,strip=True)) + define = self.get_sibling(define.nextSibling,name='define') + return info + +class BuildConsoleSummaryReport(object): + + HEADER = '\033[35m\033[1m' + INFO = '\033[34m' + OK = '\033[32m' + WARNING = '\033[33m' + FAIL = '\033[31m' + ENDC = '\033[0m' + + def __init__(self, bop, opt): + self.bop = bop + + def generate(self): + self.summary_info = { + 'total' : 0, + 'success' : 0, + 'failed' : [], + } + self.header_print("======================================================================") + self.print_test_log() + self.print_summary() + self.header_print("======================================================================") + + @property + def failed(self): + return len(self.summary_info['failed']) > 0 + + def print_test_log(self): + self.header_print("Tests run..") + self.header_print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + for k in sorted(self.bop.test.keys()): + test = self.bop.test[k] + if len(test['actions']) > 0: + self.summary_info['total'] += 1 + ##print ">>>> {0}".format(test['test-name']) + if 'result' in test: + succeed = test['result'] == 'succeed' + else: + succeed = test['actions'][-1]['result'] == 'succeed' + if succeed: + self.summary_info['success'] += 1 + else: + self.summary_info['failed'].append(test) + if succeed: + self.ok_print("[PASS] {0}",k) + else: + self.fail_print("[FAIL] {0}",k) + for action in test['actions']: + self.print_action(succeed, action) + + def print_action(self, test_succeed, action): + ''' + Print the detailed info of failed or always print tests. + ''' + #self.info_print(">>> {0}",action.keys()) + if not test_succeed or action['info']['always_show_run_output']: + output = action['output'].strip() + if output != "": + p = self.fail_print if action['result'] == 'fail' else self.p_print + self.info_print("") + self.info_print("({0}) {1}",action['info']['name'],action['info']['path']) + p("") + p("{0}",action['command'].strip()) + p("") + for line in output.splitlines(): + p("{0}",line.encode('utf-8')) + + def print_summary(self): + self.header_print("") + self.header_print("Testing summary..") + self.header_print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + self.p_print("Total: {0}",self.summary_info['total']) + self.p_print("Success: {0}",self.summary_info['success']) + if self.failed: + self.fail_print("Failed: {0}",len(self.summary_info['failed'])) + for test in self.summary_info['failed']: + self.fail_print(" {0}/{1}",test['library'],test['test-name']) + + def p_print(self, format, *args, **kargs): + print format.format(*args,**kargs) + + def info_print(self, format, *args, **kargs): + print self.INFO+format.format(*args,**kargs)+self.ENDC + + def header_print(self, format, *args, **kargs): + print self.HEADER+format.format(*args,**kargs)+self.ENDC + + def ok_print(self, format, *args, **kargs): + print self.OK+format.format(*args,**kargs)+self.ENDC + + def warn_print(self, format, *args, **kargs): + print self.WARNING+format.format(*args,**kargs)+self.ENDC + + def fail_print(self, format, *args, **kargs): + print self.FAIL+format.format(*args,**kargs)+self.ENDC + +class Main(object): + + def __init__(self,args=None): + op = optparse.OptionParser( + usage="%prog [options] input+") + op.add_option( '--output', + help="type of output to generate" ) + ( opt, inputs ) = op.parse_args(args) + bop = BuildOutputProcessor(inputs) + output = None + if opt.output == 'console': + output = BuildConsoleSummaryReport(bop, opt) + if output: + output.generate() + self.failed = output.failed + +if __name__ == '__main__': + m = Main() + if m.failed: + exit(-1) diff --git a/tools/ci/common.py b/tools/ci/common.py new file mode 100644 index 0000000..bb7a189 --- /dev/null +++ b/tools/ci/common.py @@ -0,0 +1,845 @@ +#!/usr/bin/env python + +# Copyright Rene Rivera 2016 +# +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# http://www.boost.org/LICENSE_1_0.txt) + +import sys +import inspect +import optparse +import os.path +import string +import time +import subprocess +import codecs +import shutil +import threading + +toolset_info = { + 'clang-3.4' : { + 'ppa' : ["ppa:h-rayflood/llvm"], + 'package' : 'clang-3.4', + 'command' : 'clang++-3.4', + 'toolset' : 'clang', + 'version' : '' + }, + 'clang-3.5' : { + 'ppa' : ["ppa:h-rayflood/llvm"], + 'package' : 'clang-3.5', + 'command' : 'clang++-3.5', + 'toolset' : 'clang', + 'version' : '' + }, + 'clang-3.6' : { + 'ppa' : ["ppa:h-rayflood/llvm"], + 'package' : 'clang-3.6', + 'command' : 'clang++-3.6', + 'toolset' : 'clang', + 'version' : '' + }, + 'clang-3.7' : { + 'deb' : ["http://apt.llvm.org/trusty/","llvm-toolchain-trusty-3.7","main"], + 'apt-key' : ['http://apt.llvm.org/llvm-snapshot.gpg.key'], + 'package' : 'clang-3.7', + 'command' : 'clang++-3.7', + 'toolset' : 'clang', + 'version' : '' + }, + 'clang-3.8' : { + 'deb' : ["http://apt.llvm.org/trusty/","llvm-toolchain-trusty-3.8","main"], + 'apt-key' : ['http://apt.llvm.org/llvm-snapshot.gpg.key'], + 'package' : 'clang-3.8', + 'command' : 'clang++-3.8', + 'toolset' : 'clang', + 'version' : '' + }, + 'clang-3.9' : { + 'deb' : ["http://apt.llvm.org/trusty/","llvm-toolchain-trusty-3.9","main"], + 'apt-key' : ['http://apt.llvm.org/llvm-snapshot.gpg.key'], + 'package' : 'clang-3.9', + 'command' : 'clang++-3.9', + 'toolset' : 'clang', + 'version' : '' + }, + 'clang-4.0' : { + 'deb' : ["http://apt.llvm.org/trusty/","llvm-toolchain-trusty-4.0","main"], + 'apt-key' : ['http://apt.llvm.org/llvm-snapshot.gpg.key'], + 'package' : 'clang-4.0', + 'command' : 'clang++-4.0', + 'toolset' : 'clang', + 'version' : '' + }, + 'gcc-4.7' : { + 'ppa' : ["ppa:ubuntu-toolchain-r/test"], + 'package' : 'g++-4.7', + 'command' : 'g++-4.7', + 'toolset' : 'gcc', + 'version' : '' + }, + 'gcc-4.8' : { + 'bin' : 'gcc-4.8', + 'ppa' : ["ppa:ubuntu-toolchain-r/test"], + 'package' : 'g++-4.8', + 'command' : 'g++-4.8', + 'toolset' : 'gcc', + 'version' : '' + }, + 'gcc-4.9' : { + 'ppa' : ["ppa:ubuntu-toolchain-r/test"], + 'package' : 'g++-4.9', + 'command' : 'g++-4.9', + 'toolset' : 'gcc', + 'version' : '' + }, + 'gcc-5.1' : { + 'ppa' : ["ppa:ubuntu-toolchain-r/test"], + 'package' : 'g++-5', + 'command' : 'g++-5', + 'toolset' : 'gcc', + 'version' : '' + }, + 'gcc-5' : { + 'ppa' : ["ppa:ubuntu-toolchain-r/test"], + 'package' : 'g++-5', + 'command' : 'g++-5', + 'toolset' : 'gcc', + 'version' : '' + }, + 'gcc-6' : { + 'ppa' : ["ppa:ubuntu-toolchain-r/test"], + 'package' : 'g++-6', + 'command' : 'g++-6', + 'toolset' : 'gcc', + 'version' : '' + }, + 'gcc-7' : { + 'ppa' : ["ppa:ubuntu-toolchain-r/test"], + 'package' : 'g++-7', + 'command' : 'g++-7', + 'toolset' : 'gcc', + 'version' : '' + }, + 'mingw-5' : { + 'toolset' : 'gcc', + 'command' : 'C:\\\\MinGW\\\\bin\\\\g++.exe', + 'version' : '' + }, + 'mingw64-6' : { + 'toolset' : 'gcc', + 'command' : 'C:\\\\mingw-w64\\\\x86_64-6.3.0-posix-seh-rt_v5-rev1\\\\mingw64\\\\bin\\\\g++.exe', + 'version' : '' + }, + 'vs-2008' : { + 'toolset' : 'msvc', + 'command' : '', + 'version' : '9.0' + }, + 'vs-2010' : { + 'toolset' : 'msvc', + 'command' : '', + 'version' : '10.0' + }, + 'vs-2012' : { + 'toolset' : 'msvc', + 'command' : '', + 'version' : '11.0' + }, + 'vs-2013' : { + 'toolset' : 'msvc', + 'command' : '', + 'version' : '12.0' + }, + 'vs-2015' : { + 'toolset' : 'msvc', + 'command' : '', + 'version' : '14.0' + }, + 'vs-2017' : { + 'toolset' : 'msvc', + 'command' : '', + 'version' : '14.1' + }, + 'xcode-6.1' : { + 'command' : 'clang++', + 'toolset' : 'clang', + 'version' : '' + }, + 'xcode-6.2' : { + 'command' : 'clang++', + 'toolset' : 'clang', + 'version' : '' + }, + 'xcode-6.3' : { + 'command' : 'clang++', + 'toolset' : 'clang', + 'version' : '' + }, + 'xcode-6.4' : { + 'command' : 'clang++', + 'toolset' : 'clang', + 'version' : '' + }, + 'xcode-7.0' : { + 'command' : 'clang++', + 'toolset' : 'clang', + 'version' : '' + }, + 'xcode-7.1' : { + 'command' : 'clang++', + 'toolset' : 'clang', + 'version' : '' + }, + 'xcode-7.2' : { + 'command' : 'clang++', + 'toolset' : 'clang', + 'version' : '' + }, + 'xcode-7.3' : { + 'command' : 'clang++', + 'toolset' : 'clang', + 'version' : '' + }, + 'xcode-8.0' : { + 'command' : 'clang++', + 'toolset' : 'clang', + 'version' : '' + }, + 'xcode-8.1' : { + 'command' : 'clang++', + 'toolset' : 'clang', + 'version' : '' + }, + 'xcode-8.2' : { + 'command' : 'clang++', + 'toolset' : 'clang', + 'version' : '' + }, + 'xcode-8.3' : { + 'command' : 'clang++', + 'toolset' : 'clang', + 'version' : '' + }, + } + +class SystemCallError(Exception): + def __init__(self, command, result): + self.command = command + self.result = result + def __str__(self, *args, **kwargs): + return "'%s' ==> %s"%("' '".join(self.command), self.result) + +class utils: + + call_stats = [] + + @staticmethod + def call(*command, **kargs): + utils.log( "%s> '%s'"%(os.getcwd(), "' '".join(command)) ) + t = time.time() + result = subprocess.call(command, **kargs) + t = time.time()-t + if result != 0: + print "Failed: '%s' ERROR = %s"%("' '".join(command), result) + utils.call_stats.append((t,os.getcwd(),command,result)) + utils.log( "%s> '%s' execution time %s seconds"%(os.getcwd(), "' '".join(command), t) ) + return result + + @staticmethod + def print_call_stats(): + utils.log("================================================================================") + for j in sorted(utils.call_stats, reverse=True): + utils.log("{:>12.4f}\t{}> {} ==> {}".format(*j)) + utils.log("================================================================================") + + @staticmethod + def check_call(*command, **kargs): + cwd = os.getcwd() + result = utils.call(*command, **kargs) + if result != 0: + raise(SystemCallError([cwd].extend(command), result)) + + @staticmethod + def makedirs( path ): + if not os.path.exists( path ): + os.makedirs( path ) + + @staticmethod + def log_level(): + frames = inspect.stack() + level = 0 + for i in frames[ 3: ]: + if i[0].f_locals.has_key( '__log__' ): + level = level + i[0].f_locals[ '__log__' ] + return level + + @staticmethod + def log( message ): + sys.stdout.flush() + sys.stderr.flush() + sys.stderr.write( '# ' + ' ' * utils.log_level() + message + '\n' ) + sys.stderr.flush() + + @staticmethod + def rmtree(path): + if os.path.exists( path ): + #~ shutil.rmtree( unicode( path ) ) + if sys.platform == 'win32': + os.system( 'del /f /s /q "%s" >nul 2>&1' % path ) + shutil.rmtree( unicode( path ) ) + else: + os.system( 'rm -f -r "%s"' % path ) + + @staticmethod + def retry( f, max_attempts=5, sleep_secs=10 ): + for attempts in range( max_attempts, -1, -1 ): + try: + return f() + except Exception, msg: + utils.log( '%s failed with message "%s"' % ( f.__name__, msg ) ) + if attempts == 0: + utils.log( 'Giving up.' ) + raise + + utils.log( 'Retrying (%d more attempts).' % attempts ) + time.sleep( sleep_secs ) + + @staticmethod + def web_get( source_url, destination_file, proxy = None ): + import urllib + + proxies = None + if proxy is not None: + proxies = { + 'https' : proxy, + 'http' : proxy + } + + src = urllib.urlopen( source_url, proxies = proxies ) + + f = open( destination_file, 'wb' ) + while True: + data = src.read( 16*1024 ) + if len( data ) == 0: break + f.write( data ) + + f.close() + src.close() + + @staticmethod + def unpack_archive( archive_path ): + utils.log( 'Unpacking archive ("%s")...' % archive_path ) + + archive_name = os.path.basename( archive_path ) + extension = archive_name[ archive_name.find( '.' ) : ] + + if extension in ( ".tar.gz", ".tar.bz2" ): + import tarfile + import stat + + mode = os.path.splitext( extension )[1][1:] + tar = tarfile.open( archive_path, 'r:%s' % mode ) + for tarinfo in tar: + tar.extract( tarinfo ) + if sys.platform == 'win32' and not tarinfo.isdir(): + # workaround what appears to be a Win32-specific bug in 'tarfile' + # (modification times for extracted files are not set properly) + f = os.path.join( os.curdir, tarinfo.name ) + os.chmod( f, stat.S_IWRITE ) + os.utime( f, ( tarinfo.mtime, tarinfo.mtime ) ) + tar.close() + elif extension in ( ".zip" ): + import zipfile + + z = zipfile.ZipFile( archive_path, 'r', zipfile.ZIP_DEFLATED ) + for f in z.infolist(): + destination_file_path = os.path.join( os.curdir, f.filename ) + if destination_file_path[-1] == "/": # directory + if not os.path.exists( destination_file_path ): + os.makedirs( destination_file_path ) + else: # file + result = open( destination_file_path, 'wb' ) + result.write( z.read( f.filename ) ) + result.close() + z.close() + else: + raise 'Do not know how to unpack archives with extension \"%s\"' % extension + + @staticmethod + def make_file(filename, *text): + text = string.join( text, '\n' ) + with codecs.open( filename, 'w', 'utf-8' ) as f: + f.write( text ) + + @staticmethod + def append_file(filename, *text): + with codecs.open( filename, 'a', 'utf-8' ) as f: + f.write( string.join( text, '\n' ) ) + + @staticmethod + def mem_info(): + if sys.platform == "darwin": + utils.call("top","-l","1","-s","0","-n","0") + elif sys.platform.startswith("linux"): + utils.call("free","-m","-l") + + @staticmethod + def query_boost_version(boost_root): + ''' + Read in the Boost version from a given boost_root. + ''' + boost_version = None + if os.path.exists(os.path.join(boost_root,'Jamroot')): + with codecs.open(os.path.join(boost_root,'Jamroot'), 'r', 'utf-8') as f: + for line in f.readlines(): + parts = line.split() + if len(parts) >= 5 and parts[1] == 'BOOST_VERSION': + boost_version = parts[3] + break + if not boost_version: + boost_version = 'default' + return boost_version + + @staticmethod + def git_clone(sub_repo, branch, commit = None, cwd = None, no_submodules = False): + ''' + This clone mimicks the way Travis-CI clones a project's repo. So far + Travis-CI is the most limiting in the sense of only fetching partial + history of the repo. + ''' + if not cwd: + cwd = cwd = os.getcwd() + root_dir = os.path.join(cwd,'boostorg',sub_repo) + if not os.path.exists(os.path.join(root_dir,'.git')): + utils.check_call("git","clone", + "--depth=1", + "--branch=%s"%(branch), + "https://github.com/boostorg/%s.git"%(sub_repo), + root_dir) + os.chdir(root_dir) + else: + os.chdir(root_dir) + utils.check_call("git","pull", + # "--depth=1", # Can't do depth as we get merge errors. + "--quiet","--no-recurse-submodules") + if commit: + utils.check_call("git","checkout","-qf",commit) + if os.path.exists(os.path.join('.git','modules')): + if sys.platform == 'win32': + utils.check_call('dir',os.path.join('.git','modules')) + else: + utils.check_call('ls','-la',os.path.join('.git','modules')) + if not no_submodules: + utils.check_call("git","submodule","--quiet","update", + "--quiet","--init","--recursive", + ) + utils.check_call("git","submodule","--quiet","foreach","git","fetch") + return root_dir + +class parallel_call(threading.Thread): + ''' + Runs a synchronous command in a thread waiting for it to complete. + ''' + + def __init__(self, *command, **kargs): + super(parallel_call,self).__init__() + self.command = command + self.command_kargs = kargs + self.start() + + def run(self): + self.result = utils.call(*self.command, **self.command_kargs) + + def join(self): + super(parallel_call,self).join() + if self.result != 0: + raise(SystemCallError(self.command, self.result)) + +def set_arg(args, k, v = None): + if not args.get(k): + args[k] = v + return args[k] + +class script_common(object): + ''' + Main script to run Boost C++ Libraries continuous integration. + ''' + + def __init__(self, ci_klass, **kargs): + self.ci = ci_klass(self) + + opt = optparse.OptionParser( + usage="%prog [options] [commands]") + + #~ Debug Options: + opt.add_option( '--debug-level', + help="debugging level; controls the amount of debugging output printed", + type='int' ) + opt.add_option( '-j', + help="maximum number of parallel jobs to use for building with b2", + type='int', dest='jobs') + opt.add_option('--branch') + opt.add_option('--commit') + kargs = self.init(opt,kargs) + kargs = self.ci.init(opt, kargs) + set_arg(kargs,'debug_level',0) + set_arg(kargs,'jobs',2) + set_arg(kargs,'branch',None) + set_arg(kargs,'commit',None) + set_arg(kargs,'repo',None) + set_arg(kargs,'root_dir',None) + set_arg(kargs,'actions',None) + set_arg(kargs,'pull_request', None) + + #~ Defaults + for (k,v) in kargs.iteritems(): + setattr(self,k,v) + ( _opt_, self.actions ) = opt.parse_args(None,self) + if not self.actions or self.actions == []: + self.actions = kargs.get('actions',None) + if not self.actions or self.actions == []: + self.actions = [ 'info' ] + if not self.root_dir: + self.root_dir = os.getcwd() + self.build_dir = os.path.join(os.path.dirname(self.root_dir), "build") + + # API keys. + self.bintray_key = os.getenv('BINTRAY_KEY') + + try: + self.start() + self.command_info() + self.main() + utils.print_call_stats() + except: + utils.print_call_stats() + raise + + def init(self, opt, kargs): + return kargs + + def start(self): + pass + + def main(self): + for action in self.actions: + action_m = "command_"+action.replace('-','_') + ci_command = getattr(self.ci, action_m, None) + ci_script = getattr(self, action_m, None) + if ci_command or ci_script: + utils.log( "### %s.."%(action) ) + if os.path.exists(self.root_dir): + os.chdir(self.root_dir) + if ci_command: + ci_command() + elif ci_script: + ci_script() + + def b2( self, *args, **kargs ): + cmd = ['b2','--debug-configuration', '-j%s'%(self.jobs)] + cmd.extend(args) + + if 'toolset' in kargs: + cmd.append('toolset=' + kargs['toolset']) + + if 'parallel' in kargs: + return parallel_call(*cmd) + else: + return utils.check_call(*cmd) + + # Common test commands in the order they should be executed.. + + def command_info(self): + pass + + def command_install(self): + utils.makedirs(self.build_dir) + os.chdir(self.build_dir) + + def command_install_toolset(self, toolset): + if self.ci and hasattr(self.ci,'install_toolset'): + self.ci.install_toolset(toolset) + + def command_before_build(self): + pass + + def command_build(self): + pass + + def command_before_cache(self): + pass + + def command_after_success(self): + pass + +class ci_cli(object): + ''' + This version of the script provides a way to do manual building. It sets up + additional environment and adds fetching of the git repos that would + normally be done by the CI system. + + The common way to use this variant is to invoke something like: + + mkdir boost-ci + cd boost-ci + python path-to/ci_boost_