From 4833dc3958e1e60feb9da5be253b19627bc22086 Mon Sep 17 00:00:00 2001 From: Nicholas Dudfield Date: Sat, 28 May 2016 12:21:40 +0700 Subject: [PATCH] Travis CI updates: * Run autobahn/valgrind tests when target branch in {master, develop} * Add coveralls * Show full stacktrace for usan (RIPD-1150) * Manual launch of coverage (RIPD-1152) * Use lldb on Darwin (RIPD-1152) * Set defaults if not CI (RIPD-1152) * Add autobahn result parser (RIPD-1147) --- .gitignore | 4 + .travis.yml | 55 +++---------- README.md | 3 +- scripts/blacklist.supp | 39 ++++++++++ scripts/build-and-test.sh | 134 ++++++++++++++++++++------------ scripts/install-boost.sh | 4 +- scripts/install-dependencies.sh | 12 ++- scripts/install-valgrind.sh | 4 +- scripts/parseautobahn.py | 43 ++++++++++ scripts/run-with-debugger.sh | 22 ++++++ 10 files changed, 222 insertions(+), 98 deletions(-) create mode 100644 scripts/blacklist.supp mode change 100644 => 100755 scripts/install-valgrind.sh create mode 100644 scripts/parseautobahn.py create mode 100755 scripts/run-with-debugger.sh diff --git a/.gitignore b/.gitignore index cc8f6cb3..e51312d9 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,7 @@ contents.xcworkspacedata .svn profile bin/ +node_modules/ +cov-int/ +nohup.out +venv/ diff --git a/.travis.yml b/.travis.yml index 04f04d4d..380cb741 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,13 +14,6 @@ env: packages: &gcc5_pkgs - gcc-5 - g++-5 - # - gcc-5-multilib - # - g++-5-multilib - # - gcc-multilib - # - g++-multilib - # - libgd2-xpm - # - ia32-libs - # - ia32-libs-multiarch - python-software-properties - libssl-dev - libffi-dev @@ -52,20 +45,7 @@ packages: &clang38_pkgs matrix: include: - # GCC/Debug - # - compiler: gcc - # env: GCC_VER=5 VARIANT=debug ADDRESS_MODEL=64 - # addons: &ao_gcc5 - # apt: - # sources: ['ubuntu-toolchain-r-test'] - # packages: *gcc5_pkgs - - # # GCC/Release - # - compiler: gcc - # env: GCC_VER=5 VARIANT=release ADDRESS_MODEL=64 - # addons: *ao_gcc5 - - # Coverage + # GCC/Coverage - compiler: gcc env: GCC_VER=5 VARIANT=coverage ADDRESS_MODEL=64 addons: &ao_gcc5 @@ -73,36 +53,25 @@ matrix: sources: ['ubuntu-toolchain-r-test'] packages: *gcc5_pkgs - # # Clang/Debug - # - compiler: clang - # env: GCC_VER=5 VARIANT=debug CLANG_VER=3.8 ADDRESS_MODEL=64 - # addons: &ao_clang38 - # apt: - # sources: ['ubuntu-toolchain-r-test', 'llvm-toolchain-precise-3.8'] - # packages: *clang38_pkgs + # # GCC/Debug + # - compiler: gcc + # env: GCC_VER=5 VARIANT=debug ADDRESS_MODEL=64 + # addons: *ao_gcc5 + # branches: # NOTE: this does NOT work, though it SHOULD + # - master + # - develop - # # Clang/Release - # - compiler: clang - # env: GCC_VER=5 VARIANT=release CLANG_VER=3.8 ADDRESS_MODEL=64 - # addons: *ao_clang38 - - # Clang/AddressSanitizer + # Clang/UndefinedBehaviourSanitizer - compiler: clang - env: GCC_VER=5 VARIANT=asan CLANG_VER=3.8 ADDRESS_MODEL=64 + env: GCC_VER=5 VARIANT=usan CLANG_VER=3.8 ADDRESS_MODEL=64 UBSAN_OPTIONS='print_stacktrace=1' addons: &ao_clang38 apt: sources: ['ubuntu-toolchain-r-test', 'llvm-toolchain-precise-3.8'] packages: *clang38_pkgs - # Clang/MemorySanitizer - # VFALCO Generates false positives unless libc++ is compiled with msan turned on - #- compiler: clang - # env: GCC_VER=5 VARIANT=msan CLANG_VER=3.8 ADDRESS_MODEL=64 MSAN_OPTIONS=poison_in_dtor=1,sanitize-memory-track-origins=2 - # addons: *ao_clang38 - - # Clang/UndefinedBehaviourSanitizer + # Clang/AddressSanitizer - compiler: clang - env: GCC_VER=5 VARIANT=usan CLANG_VER=3.8 ADDRESS_MODEL=64 + env: GCC_VER=5 VARIANT=asan CLANG_VER=3.8 ADDRESS_MODEL=64 addons: *ao_clang38 cache: diff --git a/README.md b/README.md index 87612d41..a9c8a923 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # Beast [![Build Status](https://travis-ci.org/vinniefalco/Beast.svg?branch=master)](https://travis-ci.org/vinniefalco/Beast) [![codecov] -(https://codecov.io/gh/vinniefalco/Beast/branch/master/graph/badge.svg)](https://codecov.io/gh/vinniefalco/Beast) [![Documentation] +(https://codecov.io/gh/vinniefalco/Beast/branch/master/graph/badge.svg)](https://codecov.io/gh/vinniefalco/Beast) [![coveralls] +(https://coveralls.io/repos/github/vinniefalco/Beast/badge.svg?branch=master)](https://coveralls.io/github/vinniefalco/Beast?branch=master) [![Documentation] (https://img.shields.io/badge/documentation-master-brightgreen.svg)](http://vinniefalco.github.io/beast/) [![License] (https://img.shields.io/badge/license-boost-brightgreen.svg)](LICENSE_1_0.txt) diff --git a/scripts/blacklist.supp b/scripts/blacklist.supp new file mode 100644 index 00000000..08968f04 --- /dev/null +++ b/scripts/blacklist.supp @@ -0,0 +1,39 @@ +# Remember that this blacklist file is GLOBAL to all sanitizers +# Be therefore extremely careful when considering to add a sanitizer +# filter here instead of using a runtime suppression +# +# Remember also that filters here quite literally completely +# remove instrumentation altogether, so filtering here means +# that sanitizers such as tsan will false positive on problems +# introduced by code filtered here. +# +# The main use for this file is ubsan, as it's the only sanitizer +# without a runtime suppression facility. +# +# Be ESPECIALLY careful when filtering out entire source files! +# Try if at all possible to filter only functions using fun:regex +# Remember you must use mangled symbol names with fun:regex + + +#### Compile time filters for ubsan #### + +## The well known ubsan failure in libstdc++ extant for years :) +# Line 96:24: runtime error: load of value 4294967221, which is not a valid value for type 'std::_Ios_Fmtflags' +fun:*_Ios_Fmtflags* + +# boost/any.hpp:259:16: runtime error: downcast of address 0x000004392e70 which does not point to an object of type 'any::holder' +fun:*any_cast* + +# boost/lexical_cast.hpp:1625:43: runtime error: downcast of address 0x7fbb4fffbce8 which does not point to an object of type 'buffer_t' (aka 'parser_buf >, char>') +fun:*shl_input_streamable* + + + + +#### Compile time filters for asan #### + + +#### Compile time filters for msan #### + + +#### Compile time filters for tsan #### diff --git a/scripts/build-and-test.sh b/scripts/build-and-test.sh index d71b6e23..0aa41040 100755 --- a/scripts/build-and-test.sh +++ b/scripts/build-and-test.sh @@ -1,8 +1,11 @@ -#!/bin/bash -u -# We use set -e and bash with -u to bail on first non zero exit code of any -# processes launched or upon any unbound variable +#!/usr/bin/env bash +# We use set -e to bail on first non zero exit code of any processes launched +# and -x to exit upon any unbound variable. -x will output command lines used +# (with variable expansion) +set -eux + +# brew install bash (4) to get this working on OSX! shopt -s globstar -set -ex ################################## ENVIRONMENT ################################# @@ -17,36 +20,90 @@ else export PATH=$VALGRIND_ROOT/bin:$LCOV_ROOT/usr/bin:$PATH fi +MAIN_BRANCH="0" +# For builds not triggered by a pull request TRAVIS_BRANCH is the name of the +# branch currently being built; whereas for builds triggered by a pull request +# it is the name of the branch targeted by the pull request (in many cases this +# will be master). +if [[ $TRAVIS_BRANCH == "master" || $TRAVIS_BRANCH == "develop" ]]; then + MAIN_BRANCH="1" +fi + +num_jobs="1" +if [[ $(uname) == "Darwin" ]]; then + num_jobs=$(sysctl -n hw.physicalcpu) +elif [[ $(uname -s) == "Linux" ]]; then + # CircleCI returns 32 phys procs, but 2 virt proc + num_proc_units=$(nproc) + # Physical cores + num_jobs=$(lscpu -p | grep -v '^#' | sort -u -t, -k 2,4 | wc -l) + if (("$num_proc_units" < "$num_jobs")); then + num_jobs=$num_proc_units + fi +fi + echo "using toolset: $CC" echo "using variant: $VARIANT" echo "using address-model: $ADDRESS_MODEL" echo "using PATH: $PATH" +echo "using MAIN_BRANCH: $MAIN_BRANCH" +echo "using BOOST_ROOT: $BOOST_ROOT" #################################### HELPERS ################################### -function run_tests_with_gdb { - for x in bin/**/*-tests; do scripts/run-with-gdb.sh "$x"; done +function run_tests_with_debugger { + for x in bin/**/$VARIANT/**/*-tests; do + scripts/run-with-debugger.sh "$x" + done } function run_tests { - for x in bin/**/*-tests; do "$x"; done + for x in bin/**/$VARIANT/**/*-tests; do + $x + done } -num_procs=1 -if [[ $(uname) == "Darwin" ]]; then - num_procs=$(sysctl -n hw.ncpu) -elif [[ $(uname -s) == "Linux" ]]; then - num_procs=$(lscpu -p | grep -v '^#' | sort -u -t, -k 2,4 | wc -l) # physical cores - virt_num_procs=$(nproc) # CircleCI returns 32 phys procs, but 1 virt proc - if (("$virt_num_procs" < "$num_procs")); then - num_procs=$virt_num_procs +function run_tests_with_valgrind { + for x in bin/**/$VARIANT/**/*-tests; do + if [[ $(basename $x) == "bench-tests" ]]; then + $x + else + # TODO --max-stackframe=8388608 + # see: https://travis-ci.org/vinniefalco/Beast/jobs/132486245 + valgrind --error-exitcode=1 "$x" fi -fi + done +} function build_beast { $BOOST_ROOT/bjam toolset=$CC \ variant=$VARIANT \ - address-model=$ADDRESS_MODEL -j${num_procs} + address-model=$ADDRESS_MODEL \ + -j${num_jobs} +} + +function run_autobahn_test_suite { + # Run autobahn tests + wsecho=$(find bin -name "websocket-echo" | grep /$VARIANT/) + nohup $wsecho& + + # We need to wait a while so wstest can connect! + sleep 5 + cd scripts && wstest -m fuzzingclient + cd .. + # Show the output + cat nohup.out + rm nohup.out + # Show what jobs are running + jobs + # Wait a while for things to wind down before issuing a kill + sleep 5 + # Kill it gracefully + kill -INT %1 + # Wait for all the jobs to finish + wait + # Parse the test results, with python>=2.5<3 script + python scripts/parseautobahn.py scripts/autoresults/index.json } ##################################### BUILD #################################### @@ -62,24 +119,12 @@ if [[ $VARIANT == "coverage" ]]; then lcov --no-external -c -i -d . -o baseline.info > /dev/null # Perform test - run_tests - - # Run autobahn tests - export SERVER=$(find . -name "websocket-echo") - nohup $SERVER& - - # We need to wait a while so wstest can connect! - sleep 5 - cd scripts && wstest -m fuzzingclient - cd .. - # Show the output - cat nohup.out - rm nohup.out - jobs - sleep 5 - # Kill it gracefully - kill -INT %1 - wait + if [[ $MAIN_BRANCH == "1" ]]; then + run_tests_with_valgrind + run_autobahn_test_suite + else + run_tests + fi # Create test coverage data file lcov --no-external -c -d . -o testrun.info > /dev/null @@ -88,20 +133,13 @@ if [[ $VARIANT == "coverage" ]]; then lcov -a baseline.info -a testrun.info -o lcov-all.info > /dev/null # Extract only include/beast, and don\'t report on examples/test - lcov -e "lcov-all.info" "*/include/beast/*" -o lcov.info > /dev/null + lcov -e "lcov-all.info" "$PWD/include/beast/*" -o lcov.info > /dev/null ~/.local/bin/codecov -X gcov -else - # TODO: make a function - run_tests_with_gdb + cat lcov.info | node_modules/.bin/coveralls - if [[ $VARIANT == "debug" ]]; then - for x in bin/**/*-tests; do - # if [[ $x != "bench-tests" ]]; then - valgrind --error-exitcode=1 "$x" - ## declare -i RESULT=$RESULT + $? - # fi - done - echo - fi + # Clean up these stragglers so BOOST_ROOT cache is clean + find $BOOST_ROOT/bin.v2 -name "*.gcda" | xargs rm -f +else + run_tests_with_debugger fi diff --git a/scripts/install-boost.sh b/scripts/install-boost.sh index 9dd3a4f6..8365c2d5 100755 --- a/scripts/install-boost.sh +++ b/scripts/install-boost.sh @@ -1,4 +1,4 @@ -#!/bin/bash -u +#!/usr/bin/env bash # Assumptions: # 1) BOOST_ROOT and BOOST_URL are already defined, # and contain valid values. @@ -6,7 +6,7 @@ # folder name internal to boost's .tar.gz # When testing you can force a boost build by clearing travis caches: # https://travis-ci.org/ripple/rippled/caches -set -e +set -eu if [ ! -d "$BOOST_ROOT/lib" ] then wget $BOOST_URL -O /tmp/boost.tar.gz diff --git a/scripts/install-dependencies.sh b/scripts/install-dependencies.sh index 34eb8604..4130e2b4 100755 --- a/scripts/install-dependencies.sh +++ b/scripts/install-dependencies.sh @@ -1,6 +1,9 @@ -#!/bin/bash -u +#!/usr/bin/env bash # Exit if anything fails. -set -e +set -eux + +HERE=$PWD + # Override gcc version to $GCC_VER. # Put an appropriate symlink at the front of the path. mkdir -v $HOME/bin @@ -44,3 +47,8 @@ tar xfvz lcov-1.12.tar.gz -C $HOME # Set install path mkdir -p $LCOV_ROOT cd $HOME/lcov-1.12 && make install PREFIX=$LCOV_ROOT + +# Install coveralls reporter +cd $HERE +mkdir -p node_modules +npm install coveralls diff --git a/scripts/install-valgrind.sh b/scripts/install-valgrind.sh old mode 100644 new mode 100755 index f1779a2b..943eb867 --- a/scripts/install-valgrind.sh +++ b/scripts/install-valgrind.sh @@ -1,7 +1,7 @@ -#!/bin/bash -u +#!/usr/bin/env bash # Assumptions: # 1) VALGRIND_ROOT is already defined, and contains a valid values -set -e +set -eu if [ ! -d "$VALGRIND_ROOT/bin" ] then # These are specified in the addons/apt section of .travis.yml diff --git a/scripts/parseautobahn.py b/scripts/parseautobahn.py new file mode 100644 index 00000000..6b91e8c6 --- /dev/null +++ b/scripts/parseautobahn.py @@ -0,0 +1,43 @@ +import os +import json +import sys + +VARIANT = os.environ.get('VARIANT', 'release') +EXPECTED_BEHAVIOR = ('OK', 'UNIMPLEMENTED', 'INFORMATIONAL') +EXPECTED_BEHAVIOR_CLOSE = ('OK', 'INFORMATIONAL') +WARNINGS = ("peer did not respond (in time) in closing handshake", ) + +args = sys.argv[1:] +fn = os.path.abspath(args[0]) +indexPath = os.path.dirname(fn) +relativeToIndex = lambda f: os.path.join(indexPath, f) +print "index", fn + + +failures = [] +warnings = [] + +with open(fn, 'r') as fh: + index = json.load(fh) + for servername, serverResults in index.items(): + for test in serverResults: + result = serverResults[test] + if ((result['behavior'] not in EXPECTED_BEHAVIOR) or + result['behaviorClose'] not in EXPECTED_BEHAVIOR_CLOSE): + with open(relativeToIndex(result['reportfile'])) as rh: + report = json.load(rh) + if (report.get('wasNotCleanReason', '') in WARNINGS and + VARIANT != 'release'): + warnings.append(report) + else: + failures.append(report) + + +if warnings: + print >> sys.stderr, json.dumps(warnings, indent=2) + print >> sys.stderr, 'there was %s warnings' % len(warnings) + +if failures: + print >> sys.stderr, json.dumps(failures, indent=2) + print >> sys.stderr, 'there was %s failures' % len(failures) + sys.exit(1) diff --git a/scripts/run-with-debugger.sh b/scripts/run-with-debugger.sh new file mode 100755 index 00000000..6a8b55d8 --- /dev/null +++ b/scripts/run-with-debugger.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -eu + +if [[ $(uname) == "Darwin" ]]; then + # -o runs after loading the binary + # -k runs after any crash + # We use a ghetto appromixation of --return-child-result, exiting with + # 1 on a crash + lldb --batch \ + -o 'run' \ + -k 'thread backtrace all' \ + -k 'script import os; os._exit(1)' \ + $@ +else + gdb --silent \ + --batch \ + --return-child-result \ + -ex="set print thread-events off" \ + -ex=run \ + -ex="thread apply all bt full" \ + --args $@ +fi