Travis CI updates:

* Run autobahn/valgrind tests when target branch in {master, develop}
* Add coveralls
* Show full stacktrace for usan (RIPD-1150)
* Manual launch of coverage (RIPD-1152)
* Use lldb on Darwin (RIPD-1152)
* Set defaults if not CI (RIPD-1152)
* Add autobahn result parser (RIPD-1147)

.gitignore

@@ -24,3 +24,7 @@ contents.xcworkspacedata
 .svn
 profile
 bin/
+node_modules/
+cov-int/
+nohup.out
+venv/

.travis.yml

@@ -14,13 +14,6 @@ env:
 packages: &gcc5_pkgs
   - gcc-5
   - g++-5
-  # - gcc-5-multilib
-  # - g++-5-multilib
-  # - gcc-multilib
-  # - g++-multilib
-  # - libgd2-xpm
-  # - ia32-libs
-  # - ia32-libs-multiarch
   - python-software-properties
   - libssl-dev
   - libffi-dev
@@ -52,20 +45,7 @@ packages: &clang38_pkgs
 
 matrix:
   include:
-    # GCC/Debug
-    # - compiler: gcc
-    #   env: GCC_VER=5 VARIANT=debug ADDRESS_MODEL=64
-    #   addons: &ao_gcc5
-    #     apt:
-    #       sources: ['ubuntu-toolchain-r-test']
-    #       packages: *gcc5_pkgs
-
-    # # GCC/Release
-    # - compiler: gcc
-    #   env: GCC_VER=5 VARIANT=release ADDRESS_MODEL=64
-    #   addons: *ao_gcc5
-
-    # Coverage
+    # GCC/Coverage
     - compiler: gcc
       env: GCC_VER=5 VARIANT=coverage ADDRESS_MODEL=64
       addons: &ao_gcc5
@@ -73,36 +53,25 @@ matrix:
           sources: ['ubuntu-toolchain-r-test']
           packages: *gcc5_pkgs
 
-    # # Clang/Debug
-    # - compiler: clang
-    #   env: GCC_VER=5 VARIANT=debug CLANG_VER=3.8 ADDRESS_MODEL=64
-    #   addons: &ao_clang38
-    #     apt:
-    #       sources: ['ubuntu-toolchain-r-test', 'llvm-toolchain-precise-3.8']
-    #       packages: *clang38_pkgs
+    # # GCC/Debug
+    # - compiler: gcc
+    #   env: GCC_VER=5 VARIANT=debug ADDRESS_MODEL=64
+    #   addons: *ao_gcc5
+    #   branches: # NOTE: this does NOT work, though it SHOULD
+    #     - master
+    #     - develop
 
-    # # Clang/Release
-    # - compiler: clang
-    #   env: GCC_VER=5 VARIANT=release CLANG_VER=3.8 ADDRESS_MODEL=64
-    #   addons: *ao_clang38
-
-    # Clang/AddressSanitizer
+    # Clang/UndefinedBehaviourSanitizer
     - compiler: clang
-      env: GCC_VER=5 VARIANT=asan CLANG_VER=3.8 ADDRESS_MODEL=64
+      env: GCC_VER=5 VARIANT=usan CLANG_VER=3.8 ADDRESS_MODEL=64 UBSAN_OPTIONS='print_stacktrace=1'
       addons: &ao_clang38
         apt:
           sources: ['ubuntu-toolchain-r-test', 'llvm-toolchain-precise-3.8']
           packages: *clang38_pkgs
 
-    # Clang/MemorySanitizer
-    # VFALCO Generates false positives unless libc++ is compiled with msan turned on
-    #- compiler: clang
-    #  env: GCC_VER=5 VARIANT=msan CLANG_VER=3.8 ADDRESS_MODEL=64 MSAN_OPTIONS=poison_in_dtor=1,sanitize-memory-track-origins=2
-    #  addons: *ao_clang38
-
-    # Clang/UndefinedBehaviourSanitizer
+    # Clang/AddressSanitizer
     - compiler: clang
-      env: GCC_VER=5 VARIANT=usan CLANG_VER=3.8 ADDRESS_MODEL=64
+      env: GCC_VER=5 VARIANT=asan CLANG_VER=3.8 ADDRESS_MODEL=64
       addons: *ao_clang38
 
 cache:

README.md

@@ -1,7 +1,8 @@
 # Beast
 
 [![Build Status](https://travis-ci.org/vinniefalco/Beast.svg?branch=master)](https://travis-ci.org/vinniefalco/Beast) [![codecov]
-(https://codecov.io/gh/vinniefalco/Beast/branch/master/graph/badge.svg)](https://codecov.io/gh/vinniefalco/Beast) [![Documentation]
+(https://codecov.io/gh/vinniefalco/Beast/branch/master/graph/badge.svg)](https://codecov.io/gh/vinniefalco/Beast) [![coveralls]
+(https://coveralls.io/repos/github/vinniefalco/Beast/badge.svg?branch=master)](https://coveralls.io/github/vinniefalco/Beast?branch=master) [![Documentation]
 (https://img.shields.io/badge/documentation-master-brightgreen.svg)](http://vinniefalco.github.io/beast/) [![License]
 (https://img.shields.io/badge/license-boost-brightgreen.svg)](LICENSE_1_0.txt)
 

scripts/blacklist.supp

@@ -0,0 +1,39 @@
+# Remember that this blacklist file is GLOBAL to all sanitizers
+# Be therefore extremely careful when considering to add a sanitizer
+# filter here instead of using a runtime suppression
+#
+# Remember also that filters here quite literally completely
+# remove instrumentation altogether, so filtering here means
+# that sanitizers such as tsan will false positive on problems
+# introduced by code filtered here.
+#
+# The main use for this file is ubsan, as it's the only sanitizer
+# without a runtime suppression facility.
+#
+# Be ESPECIALLY careful when filtering out entire source files!
+# Try if at all possible to filter only functions using fun:regex
+# Remember you must use mangled symbol names with fun:regex
+
+
+#### Compile time filters for ubsan ####
+
+## The well known ubsan failure in libstdc++ extant for years :)
+# Line 96:24: runtime error: load of value 4294967221, which is not a valid value for type 'std::_Ios_Fmtflags'
+fun:*_Ios_Fmtflags*
+
+# boost/any.hpp:259:16: runtime error: downcast of address 0x000004392e70 which does not point to an object of type 'any::holder<int>'
+fun:*any_cast*
+
+# boost/lexical_cast.hpp:1625:43: runtime error: downcast of address 0x7fbb4fffbce8 which does not point to an object of type 'buffer_t' (aka 'parser_buf<std::basic_streambuf<char, char_traits<char> >, char>')
+fun:*shl_input_streamable*
+
+
+
+
+#### Compile time filters for asan ####
+
+
+#### Compile time filters for msan ####
+
+
+#### Compile time filters for tsan ####

scripts/build-and-test.sh

@@ -1,8 +1,11 @@
-#!/bin/bash -u
-# We use set -e and bash with -u to bail on first non zero exit code of any
-# processes launched or upon any unbound variable
+#!/usr/bin/env bash
+# We use set -e to bail on first non zero exit code of any processes launched
+# and -x to exit upon any unbound variable. -x will output command lines used
+# (with variable expansion)
+set -eux
+
+# brew install bash (4) to get this working on OSX!
 shopt -s globstar
-set -ex
 
 ################################## ENVIRONMENT #################################
 
@@ -17,36 +20,90 @@ else
   export PATH=$VALGRIND_ROOT/bin:$LCOV_ROOT/usr/bin:$PATH
 fi
 
+MAIN_BRANCH="0"
+# For builds not triggered by a pull request TRAVIS_BRANCH is the name of the
+# branch currently being built; whereas for builds triggered by a pull request
+# it is the name of the branch targeted by the pull request (in many cases this
+# will be master).
+if [[ $TRAVIS_BRANCH == "master" || $TRAVIS_BRANCH == "develop" ]]; then
+    MAIN_BRANCH="1"
+fi
+
+num_jobs="1"
+if [[ $(uname) == "Darwin" ]]; then
+  num_jobs=$(sysctl -n hw.physicalcpu)
+elif [[ $(uname -s) == "Linux" ]]; then
+  # CircleCI returns 32 phys procs, but 2 virt proc
+  num_proc_units=$(nproc)
+  # Physical cores
+  num_jobs=$(lscpu -p | grep -v '^#' | sort -u -t, -k 2,4 | wc -l)
+  if (("$num_proc_units" < "$num_jobs")); then
+    num_jobs=$num_proc_units
+  fi
+fi
+
 echo "using toolset: $CC"
 echo "using variant: $VARIANT"
 echo "using address-model: $ADDRESS_MODEL"
 echo "using PATH: $PATH"
+echo "using MAIN_BRANCH: $MAIN_BRANCH"
+echo "using BOOST_ROOT: $BOOST_ROOT"
 
 #################################### HELPERS ###################################
 
-function run_tests_with_gdb {
-  for x in bin/**/*-tests; do scripts/run-with-gdb.sh "$x"; done
+function run_tests_with_debugger {
+  for x in bin/**/$VARIANT/**/*-tests; do
+    scripts/run-with-debugger.sh "$x"
+  done
 }
 
 function run_tests {
-    for x in bin/**/*-tests; do "$x"; done
+  for x in bin/**/$VARIANT/**/*-tests; do
+    $x
+  done
 }
 
-num_procs=1
-if [[ $(uname) == "Darwin" ]]; then
-    num_procs=$(sysctl -n hw.ncpu)
-elif [[ $(uname -s) == "Linux" ]]; then
-    num_procs=$(lscpu -p | grep -v '^#' | sort -u -t, -k 2,4 | wc -l) # physical cores
-    virt_num_procs=$(nproc) # CircleCI returns 32 phys procs, but 1 virt proc
-    if (("$virt_num_procs" < "$num_procs")); then
-        num_procs=$virt_num_procs
+function run_tests_with_valgrind {
+  for x in bin/**/$VARIANT/**/*-tests; do
+    if [[ $(basename $x) == "bench-tests" ]]; then
+      $x
+    else
+      # TODO --max-stackframe=8388608
+      # see: https://travis-ci.org/vinniefalco/Beast/jobs/132486245
+      valgrind --error-exitcode=1 "$x"
     fi
-fi
+  done
+}
 
 function build_beast {
   $BOOST_ROOT/bjam toolset=$CC \
                variant=$VARIANT \
-               address-model=$ADDRESS_MODEL -j${num_procs}
+               address-model=$ADDRESS_MODEL \
+               -j${num_jobs}
+}
+
+function run_autobahn_test_suite {
+  # Run autobahn tests
+  wsecho=$(find bin -name "websocket-echo" | grep /$VARIANT/)
+  nohup $wsecho&
+
+  # We need to wait a while so wstest can connect!
+  sleep 5
+  cd scripts && wstest -m fuzzingclient
+  cd ..
+  # Show the output
+  cat nohup.out
+  rm nohup.out
+  # Show what jobs are running
+  jobs
+  # Wait a while for things to wind down before issuing a kill
+  sleep 5
+  # Kill it gracefully
+  kill -INT %1
+  # Wait for all the jobs to finish
+  wait
+  # Parse the test results, with python>=2.5<3 script
+  python scripts/parseautobahn.py scripts/autoresults/index.json
 }
 
 ##################################### BUILD ####################################
@@ -62,24 +119,12 @@ if [[ $VARIANT == "coverage" ]]; then
   lcov --no-external -c -i -d . -o baseline.info > /dev/null
 
   # Perform test
-  run_tests
-
-  # Run autobahn tests
-  export SERVER=$(find . -name "websocket-echo")
-  nohup $SERVER&
-
-  # We need to wait a while so wstest can connect!
-  sleep 5
-  cd scripts && wstest -m fuzzingclient
-  cd ..
-  # Show the output
-  cat nohup.out
-  rm nohup.out
-  jobs
-  sleep 5
-  # Kill it gracefully
-  kill -INT %1
-  wait
+  if [[ $MAIN_BRANCH == "1" ]]; then
+    run_tests_with_valgrind
+    run_autobahn_test_suite
+  else
+    run_tests
+  fi
 
   # Create test coverage data file
   lcov --no-external -c -d . -o testrun.info > /dev/null
@@ -88,20 +133,13 @@ if [[ $VARIANT == "coverage" ]]; then
   lcov -a baseline.info -a testrun.info -o lcov-all.info > /dev/null
 
   # Extract only include/beast, and don\'t report on examples/test
-  lcov -e "lcov-all.info" "*/include/beast/*" -o lcov.info > /dev/null
+  lcov -e "lcov-all.info" "$PWD/include/beast/*" -o lcov.info > /dev/null
 
   ~/.local/bin/codecov -X gcov
-else
-  # TODO: make a function
-  run_tests_with_gdb
+  cat lcov.info | node_modules/.bin/coveralls
 
-  if [[ $VARIANT == "debug" ]]; then
-    for x in bin/**/*-tests; do
-      # if [[ $x != "bench-tests" ]]; then
-      valgrind --error-exitcode=1 "$x"
-        ## declare -i RESULT=$RESULT + $?
-      # fi
-    done
-    echo
-  fi
+  # Clean up these stragglers so BOOST_ROOT cache is clean
+  find $BOOST_ROOT/bin.v2 -name "*.gcda" | xargs rm -f
+else
+  run_tests_with_debugger
 fi

scripts/install-boost.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -u
+#!/usr/bin/env bash
 # Assumptions:
 # 1) BOOST_ROOT and BOOST_URL are already defined,
 # and contain valid values.
@@ -6,7 +6,7 @@
 # folder name internal to boost's .tar.gz
 # When testing you can force a boost build by clearing travis caches:
 # https://travis-ci.org/ripple/rippled/caches
-set -e
+set -eu
 if [ ! -d "$BOOST_ROOT/lib" ]
 then
   wget $BOOST_URL -O /tmp/boost.tar.gz

scripts/install-dependencies.sh

@@ -1,6 +1,9 @@
-#!/bin/bash -u
+#!/usr/bin/env bash
 # Exit if anything fails.
-set -e
+set -eux
+
+HERE=$PWD
+
 # Override gcc version to $GCC_VER.
 # Put an appropriate symlink at the front of the path.
 mkdir -v $HOME/bin
@@ -44,3 +47,8 @@ tar xfvz lcov-1.12.tar.gz -C $HOME
 # Set install path
 mkdir -p $LCOV_ROOT
 cd $HOME/lcov-1.12 && make install PREFIX=$LCOV_ROOT
+
+# Install coveralls reporter
+cd $HERE
+mkdir -p node_modules
+npm install coveralls

scripts/install-valgrind.sh

@@ -1,7 +1,7 @@
-#!/bin/bash -u
+#!/usr/bin/env bash
 # Assumptions:
 # 1) VALGRIND_ROOT is already defined, and contains a valid values
-set -e
+set -eu
 if [ ! -d "$VALGRIND_ROOT/bin" ]
 then
   # These are specified in the addons/apt section of .travis.yml

scripts/parseautobahn.py

@@ -0,0 +1,43 @@
+import os
+import json
+import sys
+
+VARIANT = os.environ.get('VARIANT', 'release')
+EXPECTED_BEHAVIOR = ('OK', 'UNIMPLEMENTED', 'INFORMATIONAL')
+EXPECTED_BEHAVIOR_CLOSE = ('OK', 'INFORMATIONAL')
+WARNINGS = ("peer did not respond (in time) in closing handshake", )
+
+args = sys.argv[1:]
+fn = os.path.abspath(args[0])
+indexPath = os.path.dirname(fn)
+relativeToIndex = lambda f: os.path.join(indexPath, f)
+print "index", fn
+
+
+failures = []
+warnings = []
+
+with open(fn, 'r') as fh:
+    index = json.load(fh)
+    for servername, serverResults in index.items():
+        for test in serverResults:
+            result = serverResults[test]
+            if ((result['behavior'] not in EXPECTED_BEHAVIOR) or
+                 result['behaviorClose'] not in EXPECTED_BEHAVIOR_CLOSE):
+                with open(relativeToIndex(result['reportfile'])) as rh:
+                    report = json.load(rh)
+                    if (report.get('wasNotCleanReason', '') in WARNINGS and
+                        VARIANT != 'release'):
+                        warnings.append(report)
+                    else:
+                        failures.append(report)
+
+
+if warnings:
+    print >> sys.stderr, json.dumps(warnings, indent=2)
+    print >> sys.stderr, 'there was %s warnings' % len(warnings)
+
+if failures:
+    print >> sys.stderr, json.dumps(failures, indent=2)
+    print >> sys.stderr, 'there was %s failures' % len(failures)
+    sys.exit(1)

scripts/run-with-debugger.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -eu
+
+if [[ $(uname) == "Darwin" ]]; then
+  # -o runs after loading the binary
+  # -k runs after any crash
+  # We use a ghetto appromixation of --return-child-result, exiting with
+  # 1 on a crash
+  lldb --batch \
+       -o 'run' \
+       -k 'thread backtrace all' \
+       -k 'script import os; os._exit(1)' \
+       $@
+else
+  gdb --silent \
+      --batch \
+      --return-child-result \
+      -ex="set print thread-events off" \
+      -ex=run \
+      -ex="thread apply all bt full" \
+      --args $@
+fi