Merge branch 'mbedtls-2.1' into mbedtls-2.1-restricted

* mbedtls-2.1:
  all.sh: add some documentation
  all.sh: new option --no-armcc
  all.sh: --keep-going mode
  all.sh: cleaned up usage output
  all.sh: indent

tests/scripts/all.sh

@@ -4,20 +4,77 @@
 #
 # This file is part of mbed TLS (https://tls.mbed.org)
 #
-# Copyright (c) 2014-2016, ARM Limited, All Rights Reserved
+# Copyright (c) 2014-2017, ARM Limited, All Rights Reserved
-#
+
+
+
+################################################################
+#### Documentation
+################################################################
+
 # Purpose
+# -------
 #
 # To run all tests possible or available on the platform.
 #
+# Notes for users
+# ---------------
+#
 # Warning: the test is destructive. It includes various build modes and
 # configurations, and can and will arbitrarily change the current CMake
-# configuration. After this script has been run, the CMake cache will be lost
+# configuration. The following files must be committed into git:
-# and CMake will no longer be initialised.
+#    * include/mbedtls/config.h
+#    * Makefile, library/Makefile, programs/Makefile, tests/Makefile
+# After running this script, the CMake cache will be lost and CMake
+# will no longer be initialised.
 #
-# The script assumes the presence of gcc and clang (recent enough for using
+# The script assumes the presence of a number of tools:
-# ASan with gcc and MemSan with clang, or valgrind) are available, as well as
+#   * Basic Unix tools (Windows users note: a Unix-style find must be before
-# cmake and a "good" find.
+#     the Windows find in the PATH)
+#   * Perl
+#   * GNU Make
+#   * CMake
+#   * GCC and Clang (recent enough for using ASan with gcc and MemSan with clang, or valgrind)
+#   * arm-gcc and mingw-gcc
+#   * ArmCC 5 and ArmCC 6, unless invoked with --no-armcc
+#   * OpenSSL and GnuTLS command line tools, recent enough for the
+#     interoperability tests. If they don't support SSLv3 then a legacy
+#     version of these tools must be present as well (search for LEGACY
+#     below).
+# See the invocation of check_tools below for details.
+#
+# This script must be invoked from the toplevel directory of a git
+# working copy of Mbed TLS.
+#
+# Note that the output is not saved. You may want to run
+#   script -c tests/scripts/all.sh
+# or
+#   tests/scripts/all.sh >all.log 2>&1
+#
+# Notes for maintainers
+# ---------------------
+#
+# The tests are roughly in order from fastest to slowest. This doesn't
+# have to be exact, but in general you should add slower tests towards
+# the end and fast checks near the beginning.
+#
+# Sanity checks have the following form:
+#   1. msg "short description of what is about to be done"
+#   2. run sanity check (failure stops the script)
+#
+# Build or build-and-test steps have the following form:
+#   1. msg "short description of what is about to be done"
+#   2. cleanup
+#   3. preparation (config.pl, cmake, ...) (failure stops the script)
+#   4. make
+#   5. Run tests if relevant. All tests must be prefixed with
+#      if_build_successful for the sake of --keep-going.
+
+
+
+################################################################
+#### Initialization and command line parsing
+################################################################
 
 # Abort on errors (and uninitialised variables)
 set -eu
@@ -32,7 +89,9 @@ CONFIG_BAK="$CONFIG_H.bak"
 
 MEMORY=0
 FORCE=0
+KEEP_GOING=0
 RELEASE=0
+RUN_ARMCC=1
 
 # Default commands, can be overriden by the environment
 : ${OPENSSL:="openssl"}
@@ -45,25 +104,34 @@ RELEASE=0
 
 usage()
 {
-    printf "Usage: $0\n"
+    cat <<EOF
-    printf "  -h|--help\t\tPrint this help.\n"
+Usage: $0 [OPTION]...
-    printf "  -m|--memory\t\tAdditional optional memory tests.\n"
+  -h|--help             Print this help.
-    printf "  -f|--force\t\tForce the tests to overwrite any modified files.\n"
+
-    printf "  -s|--seed\t\tInteger seed value to use for this test run.\n"
+General options:
-    printf "  -r|--release-test\t\tRun this script in release mode. This fixes the seed value to 1.\n"
+  -f|--force            Force the tests to overwrite any modified files.
-    printf "     --out-of-source-dir=<path>\t\tDirectory used for CMake out-of-source build tests."
+  -k|--keep-going       Run all tests and report errors at the end.
-    printf "     --openssl=<OpenSSL_path>\t\tPath to OpenSSL executable to use for most tests.\n"
+  -m|--memory           Additional optional memory tests.
-    printf "     --openssl-legacy=<OpenSSL_path>\t\tPath to OpenSSL executable to use for legacy tests e.g. SSLv3.\n"
+     --armcc            Run ARM Compiler builds (on by default).
-    printf "     --gnutls-cli=<GnuTLS_cli_path>\t\tPath to GnuTLS client executable to use for most tests.\n"
+     --no-armcc         Skip ARM Compiler builds.
-    printf "     --gnutls-serv=<GnuTLS_serv_path>\t\tPath to GnuTLS server executable to use for most tests.\n"
+     --out-of-source-dir=<path>  Directory used for CMake out-of-source build tests.
-    printf "     --gnutls-legacy-cli=<GnuTLS_cli_path>\t\tPath to GnuTLS client executable to use for legacy tests.\n"
+  -r|--release-test     Run this script in release mode. This fixes the seed value to 1.
-    printf "     --gnutls-legacy-serv=<GnuTLS_serv_path>\t\tPath to GnuTLS server executable to use for legacy tests.\n"
+  -s|--seed             Integer seed value to use for this test run.
+
+Tool path options:
+     --gnutls-cli=<GnuTLS_cli_path>             GnuTLS client executable to use for most tests.
+     --gnutls-serv=<GnuTLS_serv_path>           GnuTLS server executable to use for most tests.
+     --gnutls-legacy-cli=<GnuTLS_cli_path>      GnuTLS client executable to use for legacy tests.
+     --gnutls-legacy-serv=<GnuTLS_serv_path>    GnuTLS server executable to use for legacy tests.
+     --openssl=<OpenSSL_path>                   OpenSSL executable to use for most tests.
+     --openssl-legacy=<OpenSSL_path>            OpenSSL executable to use for legacy tests e.g. SSLv3.
+EOF
 }
 
 # remove built files as well as the cmake cache/config
 cleanup()
 {
-    make clean
+    command make clean
 
     find . -name yotta -prune -o -iname '*cmake*' -not -name CMakeLists.txt -exec rm -rf {} \+
     rm -f include/Makefile include/mbedtls/Makefile programs/*/Makefile
@@ -75,7 +143,21 @@ cleanup()
     fi
 }
 
-trap cleanup INT TERM HUP
+# Executed on exit. May be redefined depending on command line options.
+final_report () {
+    :
+}
+
+fatal_signal () {
+    cleanup
+    final_report $1
+    trap - $1
+    kill -$1 $$
+}
+
+trap 'fatal_signal HUP' HUP
+trap 'fatal_signal INT' INT
+trap 'fatal_signal TERM' TERM
 
 msg()
 {
@@ -84,6 +166,7 @@ msg()
     echo "* $1 "
     printf "* "; date
     echo "******************************************************************"
+    current_section=$1
 }
 
 err_msg()
@@ -103,39 +186,16 @@ check_tools()
 
 while [ $# -gt 0 ]; do
     case "$1" in
-        --memory|-m*)
+        --armcc)
-            MEMORY=${1#-m}
+            RUN_ARMCC=1
             ;;
         --force|-f)
             FORCE=1
             ;;
-        --seed|-s)
-            shift
-            SEED="$1"
-            ;;
-        --release-test|-r)
-            RELEASE=1
-            ;;
-        --out-of-source-dir)
-            shift
-            OUT_OF_SOURCE_DIR="$1"
-            ;;
-        --openssl)
-            shift
-            OPENSSL="$1"
-            ;;
-        --openssl-legacy)
-            shift
-            OPENSSL_LEGACY="$1"
-            ;;
         --gnutls-cli)
             shift
             GNUTLS_CLI="$1"
             ;;
-        --gnutls-serv)
-            shift
-            GNUTLS_SERV="$1"
-            ;;
         --gnutls-legacy-cli)
             shift
             GNUTLS_LEGACY_CLI="$1"
@@ -144,9 +204,46 @@ while [ $# -gt 0 ]; do
             shift
             GNUTLS_LEGACY_SERV="$1"
             ;;
-        --help|-h|*)
+        --gnutls-serv)
+            shift
+            GNUTLS_SERV="$1"
+            ;;
+        --help|-h)
             usage
-            exit 1
+            exit
+            ;;
+        --keep-going|-k)
+            KEEP_GOING=1
+            ;;
+        --memory|-m)
+            MEMORY=1
+            ;;
+        --no-armcc)
+            RUN_ARMCC=0
+            ;;
+        --openssl)
+            shift
+            OPENSSL="$1"
+            ;;
+        --openssl-legacy)
+            shift
+            OPENSSL_LEGACY="$1"
+            ;;
+        --out-of-source-dir)
+            shift
+            OUT_OF_SOURCE_DIR="$1"
+            ;;
+        --release-test|-r)
+            RELEASE=1
+            ;;
+        --seed|-s)
+            shift
+            SEED="$1"
+            ;;
+        *)
+            echo >&2 "Unknown option: $1"
+            echo >&2 "Run $0 --help for usage."
+            exit 120
             ;;
     esac
     shift
@@ -173,7 +270,6 @@ else
     fi
 
     if ! git diff-files --quiet include/mbedtls/config.h; then
-        echo $?
         err_msg "Warning - the configuration file 'include/mbedtls/config.h' has been edited. "
         echo "You can either delete or preserve your work, or force the test by rerunning the"
         echo "script as: $0 --force"
@@ -181,6 +277,71 @@ else
     fi
 fi
 
+build_status=0
+if [ $KEEP_GOING -eq 1 ]; then
+    failure_summary=
+    failure_count=0
+    start_red=
+    end_color=
+    if [ -t 1 ]; then
+        case "$TERM" in
+            *color*|cygwin|linux|rxvt*|screen|[Eex]term*)
+                start_red=$(printf '\033[31m')
+                end_color=$(printf '\033[0m')
+                ;;
+        esac
+    fi
+    record_status () {
+        if "$@"; then
+            last_status=0
+        else
+            last_status=$?
+            text="$current_section: $* -> $last_status"
+            failure_summary="$failure_summary
+$text"
+            failure_count=$((failure_count + 1))
+            echo "${start_red}^^^^$text^^^^${end_color}"
+        fi
+    }
+    make () {
+        case "$*" in
+            *test|*check)
+                if [ $build_status -eq 0 ]; then
+                    record_status command make "$@"
+                else
+                    echo "(skipped because the build failed)"
+                fi
+                ;;
+            *)
+                record_status command make "$@"
+                build_status=$last_status
+                ;;
+        esac
+    }
+    final_report () {
+        if [ $failure_count -gt 0 ]; then
+            echo
+            echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+            echo "${start_red}FAILED: $failure_count${end_color}$failure_summary"
+            echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+        elif [ -z "${1-}" ]; then
+            echo "SUCCESS :)"
+        fi
+        if [ -n "${1-}" ]; then
+            echo "Killed by SIG$1."
+        fi
+    }
+else
+    record_status () {
+        "$@"
+    }
+fi
+if_build_succeeded () {
+    if [ $build_status -eq 0 ]; then
+        record_status "$@"
+    fi
+}
+
 if [ $RELEASE -eq 1 ]; then
     # Fix the seed value to 1 to ensure that the tests are deterministic.
     SEED=1
@@ -209,7 +370,16 @@ export GNUTLS_SERV="$GNUTLS_SERV"
 # Make sure the tools we need are available.
 check_tools "$OPENSSL" "$OPENSSL_LEGACY" "$GNUTLS_CLI" "$GNUTLS_SERV" \
             "$GNUTLS_LEGACY_CLI" "$GNUTLS_LEGACY_SERV" "doxygen" "dot" \
-    "arm-none-eabi-gcc" "armcc"
+            "arm-none-eabi-gcc"
+if [ $RUN_ARMCC -ne 0 ]; then
+    check_tools "armcc"
+fi
+
+
+
+################################################################
+#### Basic checks
+################################################################
 
 #
 # Test Suites to be executed
@@ -235,10 +405,16 @@ msg "test/build: declared and exported names" # < 3s
 cleanup
 tests/scripts/check-names.sh
 
+
+
+################################################################
+#### Build and test many configurations and targets
+################################################################
+
 # Yotta not supported in 2.1 branch
 #msg "build: create and build yotta module" # ~ 30s
 #cleanup
-#tests/scripts/yotta-build.sh
+#record_status tests/scripts/yotta-build.sh
 
 msg "build: cmake, gcc, ASan" # ~ 1 min 50s
 cleanup
@@ -250,16 +426,16 @@ make test
 programs/test/selftest
 
 msg "test: ssl-opt.sh (ASan build)" # ~ 1 min
-tests/ssl-opt.sh
+if_build_succeeded tests/ssl-opt.sh
 
 msg "test/build: ref-configs (ASan build)" # ~ 6 min 20s
-tests/scripts/test-ref-configs.pl
+if_build_succeeded tests/scripts/test-ref-configs.pl
 
 msg "build: with ASan (rebuild after ref-configs)" # ~ 1 min
 make
 
 msg "test: compat.sh (ASan build)" # ~ 6 min
-tests/compat.sh
+if_build_succeeded tests/compat.sh
 
 msg "build: default config except MFL extension (ASan build)" # ~ 30s
 cleanup
@@ -283,11 +459,11 @@ make test
 programs/test/selftest
 
 msg "build: SSLv3 - compat.sh (ASan build)" # ~ 6 min
-tests/compat.sh -m 'tls1 tls1_1 tls1_2 dtls1 dtls1_2'
+if_build_succeeded tests/compat.sh -m 'tls1 tls1_1 tls1_2 dtls1 dtls1_2'
-OPENSSL_CMD="$OPENSSL_LEGACY" tests/compat.sh -m 'ssl3'
+if_build_succeeded env OPENSSL_CMD="$OPENSSL_LEGACY" tests/compat.sh -m 'ssl3'
 
 msg "build: SSLv3 - ssl-opt.sh (ASan build)" # ~ 6 min
-tests/ssl-opt.sh
+if_build_succeeded tests/ssl-opt.sh
 
 msg "build: Default + !MBEDTLS_SSL_RENEGOTIATION (ASan build)" # ~ 6 min
 cleanup
@@ -300,7 +476,7 @@ msg "test: !MBEDTLS_SSL_RENEGOTIATION - main suites (inc. selftests) (ASan build
 make test
 
 msg "test: !MBEDTLS_SSL_RENEGOTIATION - ssl-opt.sh (ASan build)" # ~ 6 min
-tests/ssl-opt.sh
+if_build_succeeded tests/ssl-opt.sh
 
 msg "build: cmake, full config, clang" # ~ 50s
 cleanup
@@ -314,24 +490,24 @@ msg "test: main suites (full config)" # ~ 5s
 make test
 
 msg "test: ssl-opt.sh default (full config)" # ~ 1s
-tests/ssl-opt.sh -f Default
+if_build_succeeded tests/ssl-opt.sh -f Default
 
 msg "test: compat.sh RC4, DES & NULL (full config)" # ~ 2 min
-OPENSSL_CMD="$OPENSSL_LEGACY" GNUTLS_CLI="$GNUTLS_LEGACY_CLI" GNUTLS_SERV="$GNUTLS_LEGACY_SERV" tests/compat.sh -e '3DES\|DES-CBC3' -f 'NULL\|DES\|RC4\|ARCFOUR'
+if_build_succeeded env OPENSSL_CMD="$OPENSSL_LEGACY" GNUTLS_CLI="$GNUTLS_LEGACY_CLI" GNUTLS_SERV="$GNUTLS_LEGACY_SERV" tests/compat.sh -e '3DES\|DES-CBC3' -f 'NULL\|DES\|RC4\|ARCFOUR'
 
 msg "test/build: curves.pl (gcc)" # ~ 4 min
 cleanup
 cmake -D CMAKE_BUILD_TYPE:String=Debug .
-tests/scripts/curves.pl
+if_build_succeeded tests/scripts/curves.pl
 
 msg "test/build: key-exchanges (gcc)" # ~ 1 min
 cleanup
 cmake -D CMAKE_BUILD_TYPE:String=Check .
-tests/scripts/key-exchanges.pl
+if_build_succeeded tests/scripts/key-exchanges.pl
 
 msg "build: Unix make, -Os (gcc)" # ~ 30s
 cleanup
-CC=gcc CFLAGS='-Werror -Os' make
+make CC=gcc CFLAGS='-Werror -Os'
 
 # this is meant to cath missing #define mbedtls_printf etc
 # disable fsio to catch some more missing #include <stdio.h>
@@ -347,7 +523,7 @@ scripts/config.pl unset MBEDTLS_PLATFORM_SNPRINTF_ALT
 scripts/config.pl unset MBEDTLS_PLATFORM_EXIT_ALT
 scripts/config.pl unset MBEDTLS_MEMORY_BUFFER_ALLOC_C
 scripts/config.pl unset MBEDTLS_FS_IO
-CC=gcc CFLAGS='-Werror -O0' make
+make CC=gcc CFLAGS='-Werror -O0'
 
 # catch compile bugs in _uninit functions
 msg "build: full config with NO_STD_FUNCTION, make, gcc" # ~ 30s
@@ -355,21 +531,21 @@ cleanup
 cp "$CONFIG_H" "$CONFIG_BAK"
 scripts/config.pl full
 scripts/config.pl set MBEDTLS_PLATFORM_NO_STD_FUNCTIONS
-CC=gcc CFLAGS='-Werror -O0' make
+make CC=gcc CFLAGS='-Werror -O0'
 
 msg "build: full config except ssl_srv.c, make, gcc" # ~ 30s
 cleanup
 cp "$CONFIG_H" "$CONFIG_BAK"
 scripts/config.pl full
 scripts/config.pl unset MBEDTLS_SSL_SRV_C
-CC=gcc CFLAGS='-Werror -O0' make
+make CC=gcc CFLAGS='-Werror -O0'
 
 msg "build: full config except ssl_cli.c, make, gcc" # ~ 30s
 cleanup
 cp "$CONFIG_H" "$CONFIG_BAK"
 scripts/config.pl full
 scripts/config.pl unset MBEDTLS_SSL_CLI_C
-CC=gcc CFLAGS='-Werror -O0' make
+make CC=gcc CFLAGS='-Werror -O0'
 
 msg "build: full config except net.c, make, gcc -std=c99 -pedantic" # ~ 30s
 cleanup
@@ -377,7 +553,7 @@ cp "$CONFIG_H" "$CONFIG_BAK"
 scripts/config.pl full
 scripts/config.pl unset MBEDTLS_NET_C # getaddrinfo() undeclared, etc.
 scripts/config.pl set MBEDTLS_NO_PLATFORM_ENTROPY # uses syscall() on GNU/Linux
-CC=gcc CFLAGS='-Werror -O0 -std=c99 -pedantic' make lib
+make CC=gcc CFLAGS='-Werror -O0 -std=c99 -pedantic' lib
 
 if uname -a | grep -F Linux >/dev/null; then
     msg "build/test: make shared" # ~ 40s
@@ -388,7 +564,7 @@ fi
 if uname -a | grep -F x86_64 >/dev/null; then
     msg "build: i386, make, gcc" # ~ 30s
     cleanup
-CC=gcc CFLAGS='-Werror -m32' make
+    make CC=gcc CFLAGS='-Werror -m32'
 fi # x86_64
 
 msg "build: arm-none-eabi-gcc, make" # ~ 10s
@@ -405,8 +581,9 @@ scripts/config.pl unset MBEDTLS_THREADING_PTHREAD
 scripts/config.pl unset MBEDTLS_THREADING_C
 scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE # execinfo.h
 scripts/config.pl unset MBEDTLS_MEMORY_BUFFER_ALLOC_C # calls exit
-CC=arm-none-eabi-gcc AR=arm-none-eabi-ar LD=arm-none-eabi-ld CFLAGS=-Werror make lib
+make CC=arm-none-eabi-gcc AR=arm-none-eabi-ar LD=arm-none-eabi-ld CFLAGS=-Werror lib
 
+if [ $RUN_ARMCC -ne 0 ]; then
     msg "build: armcc, make"
     cleanup
     cp "$CONFIG_H" "$CONFIG_BAK"
@@ -424,24 +601,25 @@ scripts/config.pl unset MBEDTLS_THREADING_PTHREAD
     scripts/config.pl unset MBEDTLS_THREADING_C
     scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE # execinfo.h
     scripts/config.pl unset MBEDTLS_MEMORY_BUFFER_ALLOC_C # calls exit
-CC=armcc AR=armar WARNING_CFLAGS= make lib
+    make CC=armcc AR=armar WARNING_CFLAGS= lib
+fi
 
 msg "build: allow SHA1 in certificates by default"
 cleanup
 cp "$CONFIG_H" "$CONFIG_BAK"
 scripts/config.pl set MBEDTLS_TLS_DEFAULT_ALLOW_SHA1_IN_CERTIFICATES
-CFLAGS='-Werror -Wall -Wextra' make
+make CFLAGS='-Werror -Wall -Wextra'
 msg "test: allow SHA1 in certificates by default"
 make test
-tests/ssl-opt.sh -f SHA-1
+if_build_succeeded tests/ssl-opt.sh -f SHA-1
 
 if which i686-w64-mingw32-gcc >/dev/null; then
     msg "build: cross-mingw64, make" # ~ 30s
     cleanup
-CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS=-Werror WINDOWS_BUILD=1 make
+    make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS=-Werror WINDOWS_BUILD=1
-WINDOWS_BUILD=1 make clean
+    make WINDOWS_BUILD=1 clean
-CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS=-Werror WINDOWS_BUILD=1 SHARED=1 make
+    make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS=-Werror WINDOWS_BUILD=1 SHARED=1
-WINDOWS_BUILD=1 make clean
+    make WINDOWS_BUILD=1 clean
 fi
 
 # MemSan currently only available on Linux 64 bits
@@ -458,13 +636,13 @@ msg "test: main suites (MSan)" # ~ 10s
     make test
 
     msg "test: ssl-opt.sh (MSan)" # ~ 1 min
-tests/ssl-opt.sh
+    if_build_succeeded tests/ssl-opt.sh
 
     # Optional part(s)
 
     if [ "$MEMORY" -gt 0 ]; then
         msg "test: compat.sh (MSan)" # ~ 6 min 20s
-    tests/compat.sh
+        if_build_succeeded tests/compat.sh
     fi
 
 else # no MemSan
@@ -483,12 +661,12 @@ make memcheck
 
     if [ "$MEMORY" -gt 0 ]; then
         msg "test: ssl-opt.sh --memcheck (Release)"
-    tests/ssl-opt.sh --memcheck
+        if_build_succeeded tests/ssl-opt.sh --memcheck
     fi
 
     if [ "$MEMORY" -gt 1 ]; then
         msg "test: compat.sh --memcheck (Release)"
-    tests/compat.sh --memcheck
+        if_build_succeeded tests/compat.sh --memcheck
     fi
 
 fi # MemSan
@@ -506,5 +684,13 @@ make test
 cd "$MBEDTLS_ROOT_DIR"
 rm -rf "$OUT_OF_SOURCE_DIR"
 
+
+
+################################################################
+#### Termination
+################################################################
+
 msg "Done, cleaning up"
 cleanup
+
+final_report