Tweak test ordering in all.sh

tests/scripts/all.sh

@@ -6,8 +6,8 @@
 # CMake configuration. After this script is run, the CMake cache is lost and
 # CMake is not initialised any more!
 #
-# Assumes gcc, clang (recent enough for using ASan) are available, as weel as
+# Assumes gcc and clang (recent enough for using ASan) are available,
-# cmake. Also assumes valgrind is available if --memcheck is used.
+# as well as cmake and valgrind.
 
 # Abort on errors (and uninitiliased variables)
 set -eu
@@ -54,54 +54,72 @@ msg()
     echo "******************************************************************"
 }
 
-# Step 1: various build types
+# The test ordering tries to optimize for the following criteria:
+# 1. Catch possible problems early, by running first test that run quickly
+#    and/or are more likely to fail than others.
+# 2. Minimize total running time, by avoiding useless rebuilds
+#
+# Indicative running times are given for reference.
 
-msg "Unix make, default compiler and flags"
+msg "build: cmake, gcc with lots of warnings" # ~ 1 min
-cleanup
-make
-
-msg "cmake, gcc with lots of warnings"
 cleanup
 CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Check .
 make
 
-msg "cmake, clang with lots of warnings"
+msg "test: main suites with valgrind" # ~ 2 min 10s
+make memcheck
+
+msg "build: with ASan" # ~ 1 min
+cleanup
+cmake -D CMAKE_BUILD_TYPE:String=ASan .
+make
+
+msg "test: ssl-opt.sh (ASan build)" # ~ 1 min 10s
+cd tests
+./ssl-opt.sh
+cd ..
+
+msg "test: main suites and selftest (ASan build)" # ~ 10s + 30s
+make test
+programs/test/selftest
+
+msg "test: ref-configs (ASan build)" # ~ 4 min 45 s
+tests/scripts/test-ref-configs.pl
+
+# Most issues are likely to be caught at this point
+
+msg "build: with ASan (rebuild after ref-configs)" # ~ 1 min
+make
+
+msg "test: compat.sh (ASan build)" # ~ 7 min 30s
+cd tests
+./compat.sh
+cd ..
+
+msg "build: cmake, clang with lots of warnings" # ~ 40s
 cleanup
 CC=clang cmake -D CMAKE_BUILD_TYPE:String=Check .
 make
 
-# Step 2: Full tests, with ASan
+msg "build: Unix make, -O2" # ~ 30s
-
-msg "ASan build and full tests"
 cleanup
-cmake -D CMAKE_BUILD_TYPE:String=ASan .
 make
-make test
-programs/test/selftest
-cd tests
-./compat.sh
-./ssl-opt.sh
-cd ..
-tests/scripts/test-ref-configs.pl
 
-# Step 3: using valgrind's memcheck
+# Optional parts that take a long time to run
-
-msg "Release build, test suites with valgrind's memcheck"
-cleanup
-# optimized build to compensate a bit for valgrind slowdown
-cmake -D CMAKE_BUILD_TYPE:String=Release .
-make
-make memcheck
 
 if [ "$MEMORY" -gt 0 ]; then
+    msg "test: ssl-opt --memcheck (-02 build)" # ~ 8 min
     cd tests
     ./ssl-opt.sh --memcheck
-    [ "$MEMORY" -gt 1 ] && ./compat.sh --memcheck
     cd ..
-    # no test-ref-configs: doesn't have a memcheck option (yet?)
-fi
 
-# Done
+    if [ "$MEMORY" -gt 1 ]; then
+        msg "test: compat --memcheck (-02 build)" # ~ 42 min
+        cd tests
+        ./compat.sh --memcheck
+        cd ..
+    fi
+fi
 
 echo "Done."
 cleanup