travis: Reorganize tests a bit to move non-integration tests out

We were running the integration tests and the install tests with every
test-group, which doesn't increase test coverage at all. So now we bundle them
with the source check which are short and remove them from the overloaded
configurations.

.travis.yml

@@ -8,33 +8,33 @@ before_install:
   sudo apt-get install -y libsqlite3-dev cppcheck valgrind gcc-4.8 gettext libc6-dev-armhf-cross libc6-dev-arm64-cross gcc-aarch64-linux-gnu gcc-arm-linux-gnueabihf qemu-user-static binfmt-support libsecp256k1-dev
 
 env:
-  - ARCH=64 SOURCE_CHECK_ONLY=true COPTFLAGS="-O3"
-  - ARCH=32 SOURCE_CHECK_ONLY=true DEVELOPER=0
-  - ARCH=64 SOURCE_CHECK_ONLY=true DEVELOPER=0 COMPILER=gcc-4.8
-  - ARCH=64 SOURCE_CHECK_ONLY=true DEVELOPER=1 EXPERIMENTAL_FEATURES=1
-#  - VALGRIND=0 ARCH=32 DEVELOPER=1 COMPILER=gcc TEST_GROUP=1 TEST_GROUP_COUNT=2 SOURCE_CHECK_ONLY=false
-#  - VALGRIND=0 ARCH=32 DEVELOPER=1 COMPILER=gcc TEST_GROUP=2 TEST_GROUP_COUNT=2 SOURCE_CHECK_ONLY=false
-  - VALGRIND=0 ARCH=64 DEVELOPER=1 COMPILER=gcc SOURCE_CHECK_ONLY=false
-  - VALGRIND=0 ARCH=64 DEVELOPER=0 COMPILER=gcc COMPAT=0 TEST_GROUP=1 TEST_GROUP_COUNT=2 SOURCE_CHECK_ONLY=false
-  - VALGRIND=0 ARCH=64 DEVELOPER=0 COMPILER=gcc COMPAT=0 TEST_GROUP=2 TEST_GROUP_COUNT=2 SOURCE_CHECK_ONLY=false
-#  - VALGRIND=1 ARCH=64 DEVELOPER=0 COMPILER=gcc TEST_GROUP=1 TEST_GROUP_COUNT=3 SOURCE_CHECK_ONLY=false
-#  - VALGRIND=1 ARCH=64 DEVELOPER=0 COMPILER=gcc TEST_GROUP=2 TEST_GROUP_COUNT=3 SOURCE_CHECK_ONLY=false
-#  - VALGRIND=1 ARCH=64 DEVELOPER=0 COMPILER=gcc TEST_GROUP=3 TEST_GROUP_COUNT=3 SOURCE_CHECK_ONLY=false
-  - VALGRIND=0 ARCH=64 DEVELOPER=1 COMPILER=clang SOURCE_CHECK_ONLY=false
-  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=1 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
-  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=2 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
-  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=3 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
-  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=4 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
-  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=5 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
-  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=6 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
-  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=7 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
-  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=8 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
-  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=9 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
-  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=10 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
-  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=11 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
-  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=12 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
-  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=13 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
-  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=14 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
+  # These run superficial checks that don't take too long on their own
+  - ARCH=64 TEST_CMD="make check-source check-units installcheck" COPTFLAGS="-O3"
+  - ARCH=32 TEST_CMD="make check-source check-units installcheck" DEVELOPER=0
+  - ARCH=64 TEST_CMD="make check-source check-units installcheck" DEVELOPER=0 COMPILER=gcc-4.8
+  - ARCH=64 TEST_CMD="make check-source check-units installcheck" DEVELOPER=1 EXPERIMENTAL_FEATURES=1
+
+  # All of the following will just run `make pytest`
+  - VALGRIND=0 ARCH=64 DEVELOPER=1 COMPILER=clang
+  - VALGRIND=0 ARCH=64 DEVELOPER=1 COMPILER=gcc
+  - VALGRIND=0 ARCH=64 DEVELOPER=0 COMPILER=gcc COMPAT=0 TEST_GROUP=1 TEST_GROUP_COUNT=2
+  - VALGRIND=0 ARCH=64 DEVELOPER=0 COMPILER=gcc COMPAT=0 TEST_GROUP=2 TEST_GROUP_COUNT=2
+
+  ## All of these are pretty slow since they run under valgrind
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=1 TEST_GROUP_COUNT=14
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=2 TEST_GROUP_COUNT=14
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=3 TEST_GROUP_COUNT=14
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=4 TEST_GROUP_COUNT=14
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=5 TEST_GROUP_COUNT=14
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=6 TEST_GROUP_COUNT=14
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=7 TEST_GROUP_COUNT=14
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=8 TEST_GROUP_COUNT=14
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=9 TEST_GROUP_COUNT=14
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=10 TEST_GROUP_COUNT=14
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=11 TEST_GROUP_COUNT=14
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=12 TEST_GROUP_COUNT=14
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=13 TEST_GROUP_COUNT=14
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=14 TEST_GROUP_COUNT=14
   - VALGRIND=0 ARCH=arm32v7 DEVELOPER=1 TARGET_HOST=arm-linux-gnueabihf
   - VALGRIND=0 ARCH=arm64v8 DEVELOPER=1 TARGET_HOST=aarch64-linux-gnu
 cache:

.travis/build.sh

@@ -11,11 +11,12 @@ export COMPAT=${COMPAT:-1}
 export PATH=$CWD/dependencies/bin:"$HOME"/.local/bin:"$PATH"
 export PYTEST_PAR=2
 export PYTEST_SENTRY_ALWAYS_REPORT=1
-
+export BOLTDIR=lightning-rfc
 # Allow up to 4 concurrent tests when not under valgrind, which might run out of memory.
 if [ "$VALGRIND" = 0 ]; then
     PYTEST_PAR=4
 fi
+export TEST_CMD=${TEST_CMD:-"make -j $PYTEST_PAR pytest"}
 
 mkdir -p dependencies/bin || true
 
@@ -55,6 +56,8 @@ cat > pytest.ini << EOF
 addopts=-p no:logging --color=no --reruns=5
 EOF
 
+git clone https://github.com/lightningnetwork/lightning-rfc.git
+
 if [ "$TARGET_HOST" == "arm-linux-gnueabihf" ] || [ "$TARGET_HOST" == "aarch64-linux-gnu" ]
 then
     export QEMU_LD_PREFIX=/usr/"$TARGET_HOST"/
@@ -103,18 +106,12 @@ then
     #echo -en 'travis_fold:start:script.3\\r'
     #make -j$PYTEST_PAR check-units
     #echo -en 'travis_fold:end:script.3\\r'
-elif [ "$SOURCE_CHECK_ONLY" == "false" ]; then
+else
     echo -en 'travis_fold:start:script.2\\r'
-    make -j3 > /dev/null
+    make -j8 > /dev/null
     echo -en 'travis_fold:end:script.2\\r'
 
     echo -en 'travis_fold:start:script.3\\r'
-    make -j$PYTEST_PAR check
+    $TEST_CMD
     echo -en 'travis_fold:end:script.3\\r'
-else
-    git clone https://github.com/lightningnetwork/lightning-rfc.git
-    echo -en 'travis_fold:start:script.2\\r'
-    make -j3 > /dev/null
-    echo -en 'travis_fold:end:script.2\\r'
-    make check-source BOLTDIR=lightning-rfc
 fi