From 57db6ff938d043e5ed6bc3cd0ab87f80659c3470 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 8 Jan 2019 22:04:31 +0100 Subject: [PATCH 01/22] Move the code into functions. No behavior change. Move almost all the code of this script into functions. There is no intended behavior change. The goal of this commit is to make subsequent improvements easier to follow. A very large number of lines have been reintended. To see what's going on, ignore whitespace differences (e.g. diff -w). I followed the following rules: * Minimize the amount of code that gets moved. * Don't change anything to what gets executed or displayed. * Almost all the code must end up in a function. This commit is in preparation for breaking up the sequence of tests into individual components that can run independently. --- tests/scripts/all.sh | 1258 ++++++++++++++++++++++-------------------- 1 file changed, 649 insertions(+), 609 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 391de195b..d76e6f805 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -80,38 +80,42 @@ # Abort on errors (and uninitialised variables) set -eu -if [ "$( uname )" != "Linux" ]; then - echo "This script only works in Linux" >&2 - exit 1 -elif [ -d library -a -d include -a -d tests ]; then :; else - echo "Must be run from mbed TLS root" >&2 - exit 1 -fi +pre_check_environment () { + if [ "$( uname )" != "Linux" ]; then + echo "This script only works in Linux" >&2 + exit 1 + elif [ -d library -a -d include -a -d tests ]; then :; else + echo "Must be run from mbed TLS root" >&2 + exit 1 + fi +} -CONFIG_H='include/mbedtls/config.h' -CONFIG_BAK="$CONFIG_H.bak" +pre_initialize_variables () { + CONFIG_H='include/mbedtls/config.h' + CONFIG_BAK="$CONFIG_H.bak" -MEMORY=0 -FORCE=0 -KEEP_GOING=0 -RUN_ARMCC=1 -YOTTA=1 + MEMORY=0 + FORCE=0 + KEEP_GOING=0 + RUN_ARMCC=1 + YOTTA=1 -# Default commands, can be overriden by the environment -: ${OPENSSL:="openssl"} -: ${OPENSSL_LEGACY:="$OPENSSL"} -: ${GNUTLS_CLI:="gnutls-cli"} -: ${GNUTLS_SERV:="gnutls-serv"} -: ${GNUTLS_LEGACY_CLI:="$GNUTLS_CLI"} -: ${GNUTLS_LEGACY_SERV:="$GNUTLS_SERV"} -: ${OUT_OF_SOURCE_DIR:=./mbedtls_out_of_source_build} -: ${ARMC5_BIN_DIR:=/usr/bin} -: ${ARMC6_BIN_DIR:=/usr/bin} + # Default commands, can be overriden by the environment + : ${OPENSSL:="openssl"} + : ${OPENSSL_LEGACY:="$OPENSSL"} + : ${GNUTLS_CLI:="gnutls-cli"} + : ${GNUTLS_SERV:="gnutls-serv"} + : ${GNUTLS_LEGACY_CLI:="$GNUTLS_CLI"} + : ${GNUTLS_LEGACY_SERV:="$GNUTLS_SERV"} + : ${OUT_OF_SOURCE_DIR:=./mbedtls_out_of_source_build} + : ${ARMC5_BIN_DIR:=/usr/bin} + : ${ARMC6_BIN_DIR:=/usr/bin} -# if MAKEFLAGS is not set add the -j option to speed up invocations of make -if [ -n "${MAKEFLAGS+set}" ]; then - export MAKEFLAGS="-j" -fi + # if MAKEFLAGS is not set add the -j option to speed up invocations of make + if [ -n "${MAKEFLAGS+set}" ]; then + export MAKEFLAGS="-j" + fi +} usage() { @@ -198,17 +202,15 @@ msg() current_section=$1 } -if [ $RUN_ARMCC -ne 0 ]; then - armc6_build_test() - { - FLAGS="$1" +armc6_build_test() +{ + FLAGS="$1" - msg "build: ARM Compiler 6 ($FLAGS), make" - ARM_TOOL_VARIANT="ult" CC="$ARMC6_CC" AR="$ARMC6_AR" CFLAGS="$FLAGS" \ - WARNING_CFLAGS='-xc -std=c99' make lib - make clean - } -fi + msg "build: ARM Compiler 6 ($FLAGS), make" + ARM_TOOL_VARIANT="ult" CC="$ARMC6_CC" AR="$ARMC6_AR" CFLAGS="$FLAGS" \ + WARNING_CFLAGS='-xc -std=c99' make lib + make clean +} err_msg() { @@ -225,72 +227,75 @@ check_tools() done } -while [ $# -gt 0 ]; do - case "$1" in - --armcc) RUN_ARMCC=1;; - --armc5-bin-dir) shift; ARMC5_BIN_DIR="$1";; - --armc6-bin-dir) shift; ARMC6_BIN_DIR="$1";; - --force|-f) FORCE=1;; - --gnutls-cli) shift; GNUTLS_CLI="$1";; - --gnutls-legacy-cli) shift; GNUTLS_LEGACY_CLI="$1";; - --gnutls-legacy-serv) shift; GNUTLS_LEGACY_SERV="$1";; - --gnutls-serv) shift; GNUTLS_SERV="$1";; - --help|-h) usage; exit;; - --keep-going|-k) KEEP_GOING=1;; - --memory|-m) MEMORY=1;; - --no-armcc) RUN_ARMCC=0;; - --no-force) FORCE=0;; - --no-keep-going) KEEP_GOING=0;; - --no-memory) MEMORY=0;; - --no-yotta) YOTTA=0;; - --openssl) shift; OPENSSL="$1";; - --openssl-legacy) shift; OPENSSL_LEGACY="$1";; - --out-of-source-dir) shift; OUT_OF_SOURCE_DIR="$1";; - --random-seed) unset SEED;; - --release-test|-r) SEED=1;; - --seed|-s) shift; SEED="$1";; - --yotta) YOTTA=1;; - *) - echo >&2 "Unknown option: $1" - echo >&2 "Run $0 --help for usage." - exit 120 - ;; - esac - shift -done +pre_parse_command_line () { + while [ $# -gt 0 ]; do + case "$1" in + --armcc) RUN_ARMCC=1;; + --armc5-bin-dir) shift; ARMC5_BIN_DIR="$1";; + --armc6-bin-dir) shift; ARMC6_BIN_DIR="$1";; + --force|-f) FORCE=1;; + --gnutls-cli) shift; GNUTLS_CLI="$1";; + --gnutls-legacy-cli) shift; GNUTLS_LEGACY_CLI="$1";; + --gnutls-legacy-serv) shift; GNUTLS_LEGACY_SERV="$1";; + --gnutls-serv) shift; GNUTLS_SERV="$1";; + --help|-h) usage; exit;; + --keep-going|-k) KEEP_GOING=1;; + --memory|-m) MEMORY=1;; + --no-armcc) RUN_ARMCC=0;; + --no-force) FORCE=0;; + --no-keep-going) KEEP_GOING=0;; + --no-memory) MEMORY=0;; + --no-yotta) YOTTA=0;; + --openssl) shift; OPENSSL="$1";; + --openssl-legacy) shift; OPENSSL_LEGACY="$1";; + --out-of-source-dir) shift; OUT_OF_SOURCE_DIR="$1";; + --random-seed) unset SEED;; + --release-test|-r) SEED=1;; + --seed|-s) shift; SEED="$1";; + --yotta) YOTTA=1;; + *) + echo >&2 "Unknown option: $1" + echo >&2 "Run $0 --help for usage." + exit 120 + ;; + esac + shift + done +} -if [ $FORCE -eq 1 ]; then - if [ $YOTTA -eq 1 ]; then - rm -rf yotta/module "$OUT_OF_SOURCE_DIR" +pre_check_git () { + if [ $FORCE -eq 1 ]; then + if [ $YOTTA -eq 1 ]; then + rm -rf yotta/module "$OUT_OF_SOURCE_DIR" + fi + git checkout-index -f -q $CONFIG_H + cleanup + else + + if [ $YOTTA -ne 0 ] && [ -d yotta/module ]; then + err_msg "Warning - there is an existing yotta module in the directory 'yotta/module'" + echo "You can either delete your work and retry, or force the test to overwrite the" + echo "test by rerunning the script as: $0 --force" + exit 1 + fi + + if [ -d "$OUT_OF_SOURCE_DIR" ]; then + echo "Warning - there is an existing directory at '$OUT_OF_SOURCE_DIR'" >&2 + echo "You can either delete this directory manually, or force the test by rerunning" + echo "the script as: $0 --force --out-of-source-dir $OUT_OF_SOURCE_DIR" + exit 1 + fi + + if ! git diff-files --quiet include/mbedtls/config.h; then + err_msg "Warning - the configuration file 'include/mbedtls/config.h' has been edited. " + echo "You can either delete or preserve your work, or force the test by rerunning the" + echo "script as: $0 --force" + exit 1 + fi fi - git checkout-index -f -q $CONFIG_H - cleanup -else +} - if [ $YOTTA -ne 0 ] && [ -d yotta/module ]; then - err_msg "Warning - there is an existing yotta module in the directory 'yotta/module'" - echo "You can either delete your work and retry, or force the test to overwrite the" - echo "test by rerunning the script as: $0 --force" - exit 1 - fi - - if [ -d "$OUT_OF_SOURCE_DIR" ]; then - echo "Warning - there is an existing directory at '$OUT_OF_SOURCE_DIR'" >&2 - echo "You can either delete this directory manually, or force the test by rerunning" - echo "the script as: $0 --force --out-of-source-dir $OUT_OF_SOURCE_DIR" - exit 1 - fi - - if ! git diff-files --quiet include/mbedtls/config.h; then - err_msg "Warning - the configuration file 'include/mbedtls/config.h' has been edited. " - echo "You can either delete or preserve your work, or force the test by rerunning the" - echo "script as: $0 --force" - exit 1 - fi -fi - -build_status=0 -if [ $KEEP_GOING -eq 1 ]; then +pre_setup_keep_going () { failure_summary= failure_count=0 start_red= @@ -344,53 +349,60 @@ $text" echo "Killed by SIG$1." fi } -else - record_status () { - "$@" - } -fi +} + if_build_succeeded () { if [ $build_status -eq 0 ]; then record_status "$@" fi } -msg "info: $0 configuration" -echo "MEMORY: $MEMORY" -echo "FORCE: $FORCE" -echo "SEED: ${SEED-"UNSET"}" -echo "OPENSSL: $OPENSSL" -echo "OPENSSL_LEGACY: $OPENSSL_LEGACY" -echo "GNUTLS_CLI: $GNUTLS_CLI" -echo "GNUTLS_SERV: $GNUTLS_SERV" -echo "GNUTLS_LEGACY_CLI: $GNUTLS_LEGACY_CLI" -echo "GNUTLS_LEGACY_SERV: $GNUTLS_LEGACY_SERV" -echo "ARMC5_BIN_DIR: $ARMC5_BIN_DIR" -echo "ARMC6_BIN_DIR: $ARMC6_BIN_DIR" - -ARMC5_CC="$ARMC5_BIN_DIR/armcc" -ARMC5_AR="$ARMC5_BIN_DIR/armar" -ARMC6_CC="$ARMC6_BIN_DIR/armclang" -ARMC6_AR="$ARMC6_BIN_DIR/armar" - -# To avoid setting OpenSSL and GnuTLS for each call to compat.sh and ssl-opt.sh -# we just export the variables they require -export OPENSSL_CMD="$OPENSSL" -export GNUTLS_CLI="$GNUTLS_CLI" -export GNUTLS_SERV="$GNUTLS_SERV" - -# Avoid passing --seed flag in every call to ssl-opt.sh -if [ -n "${SEED-}" ]; then - export SEED -fi +pre_print_configuration () { + msg "info: $0 configuration" + echo "MEMORY: $MEMORY" + echo "FORCE: $FORCE" + echo "SEED: ${SEED-"UNSET"}" + echo "OPENSSL: $OPENSSL" + echo "OPENSSL_LEGACY: $OPENSSL_LEGACY" + echo "GNUTLS_CLI: $GNUTLS_CLI" + echo "GNUTLS_SERV: $GNUTLS_SERV" + echo "GNUTLS_LEGACY_CLI: $GNUTLS_LEGACY_CLI" + echo "GNUTLS_LEGACY_SERV: $GNUTLS_LEGACY_SERV" + echo "ARMC5_BIN_DIR: $ARMC5_BIN_DIR" + echo "ARMC6_BIN_DIR: $ARMC6_BIN_DIR" +} # Make sure the tools we need are available. -check_tools "$OPENSSL" "$OPENSSL_LEGACY" "$GNUTLS_CLI" "$GNUTLS_SERV" \ - "$GNUTLS_LEGACY_CLI" "$GNUTLS_LEGACY_SERV" "doxygen" "dot" \ - "arm-none-eabi-gcc" "i686-w64-mingw32-gcc" -if [ $RUN_ARMCC -ne 0 ]; then - check_tools "$ARMC5_CC" "$ARMC5_AR" "$ARMC6_CC" "$ARMC6_AR" -fi +pre_check_tools () { + ARMC5_CC="$ARMC5_BIN_DIR/armcc" + ARMC5_AR="$ARMC5_BIN_DIR/armar" + ARMC6_CC="$ARMC6_BIN_DIR/armclang" + ARMC6_AR="$ARMC6_BIN_DIR/armar" + + # To avoid setting OpenSSL and GnuTLS for each call to compat.sh and ssl-opt.sh + # we just export the variables they require + export OPENSSL_CMD="$OPENSSL" + export GNUTLS_CLI="$GNUTLS_CLI" + export GNUTLS_SERV="$GNUTLS_SERV" + + # Avoid passing --seed flag in every call to ssl-opt.sh + if [ -n "${SEED-}" ]; then + export SEED + fi + + check_tools "$OPENSSL" "$OPENSSL_LEGACY" "$GNUTLS_CLI" "$GNUTLS_SERV" \ + "$GNUTLS_LEGACY_CLI" "$GNUTLS_LEGACY_SERV" "doxygen" "dot" \ + "arm-none-eabi-gcc" "i686-w64-mingw32-gcc" + if [ $RUN_ARMCC -ne 0 ]; then + check_tools "$ARMC5_CC" "$ARMC5_AR" "$ARMC6_CC" "$ARMC6_AR" + fi + + msg "info: output_env.sh" + OPENSSL="$OPENSSL" OPENSSL_LEGACY="$OPENSSL_LEGACY" GNUTLS_CLI="$GNUTLS_CLI" \ + GNUTLS_SERV="$GNUTLS_SERV" GNUTLS_LEGACY_CLI="$GNUTLS_LEGACY_CLI" \ + GNUTLS_LEGACY_SERV="$GNUTLS_LEGACY_SERV" ARMC5_CC="$ARMC5_CC" \ + ARMC6_CC="$ARMC6_CC" RUN_ARMCC="$RUN_ARMCC" scripts/output_env.sh +} @@ -409,502 +421,499 @@ fi # # Indicative running times are given for reference. -msg "info: output_env.sh" -OPENSSL="$OPENSSL" OPENSSL_LEGACY="$OPENSSL_LEGACY" GNUTLS_CLI="$GNUTLS_CLI" \ - GNUTLS_SERV="$GNUTLS_SERV" GNUTLS_LEGACY_CLI="$GNUTLS_LEGACY_CLI" \ - GNUTLS_LEGACY_SERV="$GNUTLS_LEGACY_SERV" ARMC5_CC="$ARMC5_CC" \ - ARMC6_CC="$ARMC6_CC" RUN_ARMCC="$RUN_ARMCC" scripts/output_env.sh +run_all_the_tests () { -msg "test: recursion.pl" # < 1s -record_status tests/scripts/recursion.pl library/*.c + msg "test: recursion.pl" # < 1s + record_status tests/scripts/recursion.pl library/*.c -msg "test: freshness of generated source files" # < 1s -record_status tests/scripts/check-generated-files.sh + msg "test: freshness of generated source files" # < 1s + record_status tests/scripts/check-generated-files.sh -msg "test: doxygen markup outside doxygen blocks" # < 1s -record_status tests/scripts/check-doxy-blocks.pl + msg "test: doxygen markup outside doxygen blocks" # < 1s + record_status tests/scripts/check-doxy-blocks.pl -msg "test: check-files.py" # < 1s -cleanup -record_status tests/scripts/check-files.py - -msg "test/build: declared and exported names" # < 3s -cleanup -record_status tests/scripts/check-names.sh - -msg "test: doxygen warnings" # ~ 3s -cleanup -record_status tests/scripts/doxygen.sh - - - -################################################################ -#### Build and test many configurations and targets -################################################################ - -if [ $RUN_ARMCC -ne 0 ] && [ $YOTTA -ne 0 ]; then - # Note - use of yotta is deprecated, and yotta also requires armcc to be on the - # path, and uses whatever version of armcc it finds there. - msg "build: create and build yotta module" # ~ 30s + msg "test: check-files.py" # < 1s cleanup - record_status tests/scripts/yotta-build.sh -fi + record_status tests/scripts/check-files.py -msg "build: cmake, gcc, ASan" # ~ 1 min 50s -cleanup -CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . -make - -msg "test: main suites (inc. selftests) (ASan build)" # ~ 50s -make test - -msg "test: ssl-opt.sh (ASan build)" # ~ 1 min -if_build_succeeded tests/ssl-opt.sh - -msg "test/build: ref-configs (ASan build)" # ~ 6 min 20s -record_status tests/scripts/test-ref-configs.pl - -msg "build: with ASan (rebuild after ref-configs)" # ~ 1 min -make - -msg "test: compat.sh (ASan build)" # ~ 6 min -if_build_succeeded tests/compat.sh - -msg "build: Default + SSLv3 (ASan build)" # ~ 6 min -cleanup -cp "$CONFIG_H" "$CONFIG_BAK" -scripts/config.pl set MBEDTLS_SSL_PROTO_SSL3 -CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . -make - -msg "test: SSLv3 - main suites (inc. selftests) (ASan build)" # ~ 50s -make test - -msg "build: SSLv3 - compat.sh (ASan build)" # ~ 6 min -if_build_succeeded tests/compat.sh -m 'tls1 tls1_1 tls1_2 dtls1 dtls1_2' -if_build_succeeded env OPENSSL_CMD="$OPENSSL_LEGACY" tests/compat.sh -m 'ssl3' - -msg "build: SSLv3 - ssl-opt.sh (ASan build)" # ~ 6 min -if_build_succeeded tests/ssl-opt.sh - -msg "build: Default + !MBEDTLS_SSL_RENEGOTIATION (ASan build)" # ~ 6 min -cleanup -cp "$CONFIG_H" "$CONFIG_BAK" -scripts/config.pl unset MBEDTLS_SSL_RENEGOTIATION -CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . -make - -msg "test: !MBEDTLS_SSL_RENEGOTIATION - main suites (inc. selftests) (ASan build)" # ~ 50s -make test - -msg "test: !MBEDTLS_SSL_RENEGOTIATION - ssl-opt.sh (ASan build)" # ~ 6 min -if_build_succeeded tests/ssl-opt.sh - -msg "build: Default + RSA_NO_CRT (ASan build)" # ~ 6 min -cleanup -cp "$CONFIG_H" "$CONFIG_BAK" -scripts/config.pl set MBEDTLS_RSA_NO_CRT -CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . -make - -msg "test: RSA_NO_CRT - main suites (inc. selftests) (ASan build)" # ~ 50s -make test - -msg "test: RSA_NO_CRT - RSA-related part of ssl-opt.sh (ASan build)" # ~ 5s -if_build_succeeded tests/ssl-opt.sh -f RSA - -msg "test: RSA_NO_CRT - RSA-related part of compat.sh (ASan build)" # ~ 3 min -if_build_succeeded tests/compat.sh -t RSA - -msg "build: cmake, full config, clang" # ~ 50s -cleanup -cp "$CONFIG_H" "$CONFIG_BAK" -scripts/config.pl full -scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE # too slow for tests -CC=clang cmake -D CMAKE_BUILD_TYPE:String=Check -D ENABLE_TESTING=On . -make - -msg "test: main suites (full config)" # ~ 5s -make test - -msg "test: ssl-opt.sh default (full config)" # ~ 1s -if_build_succeeded tests/ssl-opt.sh -f Default - -msg "test: compat.sh RC4, DES & NULL (full config)" # ~ 2 min -if_build_succeeded env OPENSSL_CMD="$OPENSSL_LEGACY" GNUTLS_CLI="$GNUTLS_LEGACY_CLI" GNUTLS_SERV="$GNUTLS_LEGACY_SERV" tests/compat.sh -e '3DES\|DES-CBC3' -f 'NULL\|DES\|RC4\|ARCFOUR' - -msg "build: make, full config + DEPRECATED_WARNING, gcc -O" # ~ 30s -cleanup -cp "$CONFIG_H" "$CONFIG_BAK" -scripts/config.pl full -scripts/config.pl set MBEDTLS_DEPRECATED_WARNING -# Build with -O -Wextra to catch a maximum of issues. -make CC=gcc CFLAGS='-O -Werror -Wall -Wextra' lib programs -make CC=gcc CFLAGS='-O -Werror -Wall -Wextra -Wno-unused-function' tests - -msg "build: make, full config + DEPRECATED_REMOVED, clang -O" # ~ 30s -# No cleanup, just tweak the configuration and rebuild -make clean -scripts/config.pl unset MBEDTLS_DEPRECATED_WARNING -scripts/config.pl set MBEDTLS_DEPRECATED_REMOVED -# Build with -O -Wextra to catch a maximum of issues. -make CC=clang CFLAGS='-O -Werror -Wall -Wextra' lib programs -make CC=clang CFLAGS='-O -Werror -Wall -Wextra -Wno-unused-function' tests - -msg "test/build: curves.pl (gcc)" # ~ 4 min -cleanup -record_status tests/scripts/curves.pl - -msg "test/build: depends-hashes.pl (gcc)" # ~ 2 min -cleanup -record_status tests/scripts/depends-hashes.pl - -msg "test/build: depends-pkalgs.pl (gcc)" # ~ 2 min -cleanup -record_status tests/scripts/depends-pkalgs.pl - -msg "test/build: key-exchanges (gcc)" # ~ 1 min -cleanup -record_status tests/scripts/key-exchanges.pl - -msg "build: Unix make, -Os (gcc)" # ~ 30s -cleanup -make CC=gcc CFLAGS='-Werror -Wall -Wextra -Os' - -# Full configuration build, without platform support, file IO and net sockets. -# This should catch missing mbedtls_printf definitions, and by disabling file -# IO, it should catch missing '#include ' -msg "build: full config except platform/fsio/net, make, gcc, C99" # ~ 30s -cleanup -cp "$CONFIG_H" "$CONFIG_BAK" -scripts/config.pl full -scripts/config.pl unset MBEDTLS_PLATFORM_C -scripts/config.pl unset MBEDTLS_NET_C -scripts/config.pl unset MBEDTLS_PLATFORM_MEMORY -scripts/config.pl unset MBEDTLS_PLATFORM_PRINTF_ALT -scripts/config.pl unset MBEDTLS_PLATFORM_FPRINTF_ALT -scripts/config.pl unset MBEDTLS_PLATFORM_SNPRINTF_ALT -scripts/config.pl unset MBEDTLS_PLATFORM_TIME_ALT -scripts/config.pl unset MBEDTLS_PLATFORM_EXIT_ALT -scripts/config.pl unset MBEDTLS_ENTROPY_NV_SEED -scripts/config.pl unset MBEDTLS_MEMORY_BUFFER_ALLOC_C -scripts/config.pl unset MBEDTLS_FS_IO -# Note, _DEFAULT_SOURCE needs to be defined for platforms using glibc version >2.19, -# to re-enable platform integration features otherwise disabled in C99 builds -make CC=gcc CFLAGS='-Werror -Wall -Wextra -std=c99 -pedantic -O0 -D_DEFAULT_SOURCE' lib programs -make CC=gcc CFLAGS='-Werror -Wall -Wextra -O0' test - -# catch compile bugs in _uninit functions -msg "build: full config with NO_STD_FUNCTION, make, gcc" # ~ 30s -cleanup -cp "$CONFIG_H" "$CONFIG_BAK" -scripts/config.pl full -scripts/config.pl set MBEDTLS_PLATFORM_NO_STD_FUNCTIONS -scripts/config.pl unset MBEDTLS_ENTROPY_NV_SEED -make CC=gcc CFLAGS='-Werror -Wall -Wextra -O0' - -msg "build: full config except ssl_srv.c, make, gcc" # ~ 30s -cleanup -cp "$CONFIG_H" "$CONFIG_BAK" -scripts/config.pl full -scripts/config.pl unset MBEDTLS_SSL_SRV_C -make CC=gcc CFLAGS='-Werror -Wall -Wextra -O0' - -msg "build: full config except ssl_cli.c, make, gcc" # ~ 30s -cleanup -cp "$CONFIG_H" "$CONFIG_BAK" -scripts/config.pl full -scripts/config.pl unset MBEDTLS_SSL_CLI_C -make CC=gcc CFLAGS='-Werror -Wall -Wextra -O0' - -# Note, C99 compliance can also be tested with the sockets support disabled, -# as that requires a POSIX platform (which isn't the same as C99). -msg "build: full config except net_sockets.c, make, gcc -std=c99 -pedantic" # ~ 30s -cleanup -cp "$CONFIG_H" "$CONFIG_BAK" -scripts/config.pl full -scripts/config.pl unset MBEDTLS_NET_C # getaddrinfo() undeclared, etc. -scripts/config.pl set MBEDTLS_NO_PLATFORM_ENTROPY # uses syscall() on GNU/Linux -make CC=gcc CFLAGS='-Werror -Wall -Wextra -O0 -std=c99 -pedantic' lib - -msg "build: default config except MFL extension (ASan build)" # ~ 30s -cleanup -cp "$CONFIG_H" "$CONFIG_BAK" -scripts/config.pl unset MBEDTLS_SSL_MAX_FRAGMENT_LENGTH -CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . -make - -msg "test: ssl-opt.sh, MFL-related tests" -if_build_succeeded tests/ssl-opt.sh -f "Max fragment length" - -msg "build: default config with MBEDTLS_TEST_NULL_ENTROPY (ASan build)" -cleanup -cp "$CONFIG_H" "$CONFIG_BAK" -scripts/config.pl set MBEDTLS_TEST_NULL_ENTROPY -scripts/config.pl set MBEDTLS_NO_DEFAULT_ENTROPY_SOURCES -scripts/config.pl set MBEDTLS_ENTROPY_C -scripts/config.pl unset MBEDTLS_ENTROPY_NV_SEED -scripts/config.pl unset MBEDTLS_ENTROPY_HARDWARE_ALT -scripts/config.pl unset MBEDTLS_HAVEGE_C -CC=gcc cmake -D UNSAFE_BUILD=ON -D CMAKE_C_FLAGS:String="-fsanitize=address -fno-common -O3" . -make - -msg "test: MBEDTLS_TEST_NULL_ENTROPY - main suites (inc. selftests) (ASan build)" -make test - -msg "build: MBEDTLS_PLATFORM_{CALLOC/FREE}_MACRO enabled (ASan build)" -cleanup -cp "$CONFIG_H" "$CONFIG_BAK" -scripts/config.pl set MBEDTLS_PLATFORM_MEMORY -scripts/config.pl set MBEDTLS_PLATFORM_CALLOC_MACRO calloc -scripts/config.pl set MBEDTLS_PLATFORM_FREE_MACRO free -CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . -make - -msg "test: MBEDTLS_PLATFORM_{CALLOC/FREE}_MACRO enabled (ASan build)" -make test - -if uname -a | grep -F Linux >/dev/null; then - msg "build/test: make shared" # ~ 40s + msg "test/build: declared and exported names" # < 3s cleanup - make SHARED=1 all check -fi + record_status tests/scripts/check-names.sh -if uname -a | grep -F x86_64 >/dev/null; then - # Build once with -O0, to compile out the i386 specific inline assembly - msg "build: i386, make, gcc -O0 (ASan build)" # ~ 30s + msg "test: doxygen warnings" # ~ 3s cleanup - cp "$CONFIG_H" "$CONFIG_BAK" - scripts/config.pl full - make CC=gcc CFLAGS='-O0 -Werror -Wall -Wextra -m32 -fsanitize=address' + record_status tests/scripts/doxygen.sh - msg "test: i386, make, gcc -O0 (ASan build)" - make test - # Build again with -O1, to compile in the i386 specific inline assembly - msg "build: i386, make, gcc -O1 (ASan build)" # ~ 30s + + ################################################################ + #### Build and test many configurations and targets + ################################################################ + + if [ $RUN_ARMCC -ne 0 ] && [ $YOTTA -ne 0 ]; then + # Note - use of yotta is deprecated, and yotta also requires armcc to be on the + # path, and uses whatever version of armcc it finds there. + msg "build: create and build yotta module" # ~ 30s + cleanup + record_status tests/scripts/yotta-build.sh + fi + + msg "build: cmake, gcc, ASan" # ~ 1 min 50s cleanup - cp "$CONFIG_H" "$CONFIG_BAK" - scripts/config.pl full - make CC=gcc CFLAGS='-O1 -Werror -Wall -Wextra -m32 -fsanitize=address' - - msg "test: i386, make, gcc -O1 (ASan build)" - make test - - msg "build: 64-bit ILP32, make, gcc" # ~ 30s - cleanup - cp "$CONFIG_H" "$CONFIG_BAK" - scripts/config.pl full - make CC=gcc CFLAGS='-Werror -Wall -Wextra -mx32' - - msg "test: 64-bit ILP32, make, gcc" - make test -fi # x86_64 - -msg "build: gcc, force 32-bit bignum limbs" -cleanup -cp "$CONFIG_H" "$CONFIG_BAK" -scripts/config.pl unset MBEDTLS_HAVE_ASM -scripts/config.pl unset MBEDTLS_AESNI_C -scripts/config.pl unset MBEDTLS_PADLOCK_C -make CC=gcc CFLAGS='-Werror -Wall -Wextra -DMBEDTLS_HAVE_INT32' - -msg "test: gcc, force 32-bit bignum limbs" -make test - -msg "build: gcc, force 64-bit bignum limbs" -cleanup -cp "$CONFIG_H" "$CONFIG_BAK" -scripts/config.pl unset MBEDTLS_HAVE_ASM -scripts/config.pl unset MBEDTLS_AESNI_C -scripts/config.pl unset MBEDTLS_PADLOCK_C -make CC=gcc CFLAGS='-Werror -Wall -Wextra -DMBEDTLS_HAVE_INT64' - -msg "test: gcc, force 64-bit bignum limbs" -make test - -msg "build: arm-none-eabi-gcc, make" # ~ 10s -cleanup -cp "$CONFIG_H" "$CONFIG_BAK" -scripts/config.pl full -scripts/config.pl unset MBEDTLS_NET_C -scripts/config.pl unset MBEDTLS_TIMING_C -scripts/config.pl unset MBEDTLS_FS_IO -scripts/config.pl unset MBEDTLS_ENTROPY_NV_SEED -scripts/config.pl set MBEDTLS_NO_PLATFORM_ENTROPY -# following things are not in the default config -scripts/config.pl unset MBEDTLS_HAVEGE_C # depends on timing.c -scripts/config.pl unset MBEDTLS_THREADING_PTHREAD -scripts/config.pl unset MBEDTLS_THREADING_C -scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE # execinfo.h -scripts/config.pl unset MBEDTLS_MEMORY_BUFFER_ALLOC_C # calls exit -make CC=arm-none-eabi-gcc AR=arm-none-eabi-ar LD=arm-none-eabi-ld CFLAGS='-Werror -Wall -Wextra' lib - -msg "build: arm-none-eabi-gcc -DMBEDTLS_NO_UDBL_DIVISION, make" # ~ 10s -cleanup -cp "$CONFIG_H" "$CONFIG_BAK" -scripts/config.pl full -scripts/config.pl unset MBEDTLS_NET_C -scripts/config.pl unset MBEDTLS_TIMING_C -scripts/config.pl unset MBEDTLS_FS_IO -scripts/config.pl unset MBEDTLS_ENTROPY_NV_SEED -scripts/config.pl set MBEDTLS_NO_PLATFORM_ENTROPY -# following things are not in the default config -scripts/config.pl unset MBEDTLS_HAVEGE_C # depends on timing.c -scripts/config.pl unset MBEDTLS_THREADING_PTHREAD -scripts/config.pl unset MBEDTLS_THREADING_C -scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE # execinfo.h -scripts/config.pl unset MBEDTLS_MEMORY_BUFFER_ALLOC_C # calls exit -scripts/config.pl set MBEDTLS_NO_UDBL_DIVISION -make CC=arm-none-eabi-gcc AR=arm-none-eabi-ar LD=arm-none-eabi-ld CFLAGS='-Werror -Wall -Wextra' lib -echo "Checking that software 64-bit division is not required" -! grep __aeabi_uldiv library/*.o - -msg "build: ARM Compiler 5, make" -cleanup -cp "$CONFIG_H" "$CONFIG_BAK" -scripts/config.pl full -scripts/config.pl unset MBEDTLS_NET_C -scripts/config.pl unset MBEDTLS_TIMING_C -scripts/config.pl unset MBEDTLS_FS_IO -scripts/config.pl unset MBEDTLS_ENTROPY_NV_SEED -scripts/config.pl unset MBEDTLS_HAVE_TIME -scripts/config.pl unset MBEDTLS_HAVE_TIME_DATE -scripts/config.pl set MBEDTLS_NO_PLATFORM_ENTROPY -# following things are not in the default config -scripts/config.pl unset MBEDTLS_DEPRECATED_WARNING -scripts/config.pl unset MBEDTLS_HAVEGE_C # depends on timing.c -scripts/config.pl unset MBEDTLS_THREADING_PTHREAD -scripts/config.pl unset MBEDTLS_THREADING_C -scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE # execinfo.h -scripts/config.pl unset MBEDTLS_MEMORY_BUFFER_ALLOC_C # calls exit -scripts/config.pl unset MBEDTLS_PLATFORM_TIME_ALT # depends on MBEDTLS_HAVE_TIME - -if [ $RUN_ARMCC -ne 0 ]; then - make CC="$ARMC5_CC" AR="$ARMC5_AR" WARNING_CFLAGS='--strict --c99' lib - make clean - - # ARM Compiler 6 - Target ARMv7-A - armc6_build_test "--target=arm-arm-none-eabi -march=armv7-a" - - # ARM Compiler 6 - Target ARMv7-M - armc6_build_test "--target=arm-arm-none-eabi -march=armv7-m" - - # ARM Compiler 6 - Target ARMv8-A - AArch32 - armc6_build_test "--target=arm-arm-none-eabi -march=armv8.2-a" - - # ARM Compiler 6 - Target ARMv8-M - armc6_build_test "--target=arm-arm-none-eabi -march=armv8-m.main" - - # ARM Compiler 6 - Target ARMv8-A - AArch64 - armc6_build_test "--target=aarch64-arm-none-eabi -march=armv8.2-a" -fi - -msg "build: allow SHA1 in certificates by default" -cleanup -cp "$CONFIG_H" "$CONFIG_BAK" -scripts/config.pl set MBEDTLS_TLS_DEFAULT_ALLOW_SHA1_IN_CERTIFICATES -make CFLAGS='-Werror -Wall -Wextra' -msg "test: allow SHA1 in certificates by default" -make test -if_build_succeeded tests/ssl-opt.sh -f SHA-1 - -msg "build: Default + MBEDTLS_RSA_NO_CRT (ASan build)" # ~ 6 min -cleanup -cp "$CONFIG_H" "$CONFIG_BAK" -scripts/config.pl set MBEDTLS_RSA_NO_CRT -CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . -make - -msg "test: MBEDTLS_RSA_NO_CRT - main suites (inc. selftests) (ASan build)" -make test - -msg "build: Windows cross build - mingw64, make (Link Library)" # ~ 30s -cleanup -make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 lib programs - -# note Make tests only builds the tests, but doesn't run them -make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror' WINDOWS_BUILD=1 tests -make WINDOWS_BUILD=1 clean - -msg "build: Windows cross build - mingw64, make (DLL)" # ~ 30s -make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 SHARED=1 lib programs -make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 SHARED=1 tests -make WINDOWS_BUILD=1 clean - -# MemSan currently only available on Linux 64 bits -if uname -a | grep 'Linux.*x86_64' >/dev/null; then - - msg "build: MSan (clang)" # ~ 1 min 20s - cleanup - cp "$CONFIG_H" "$CONFIG_BAK" - scripts/config.pl unset MBEDTLS_AESNI_C # memsan doesn't grok asm - CC=clang cmake -D CMAKE_BUILD_TYPE:String=MemSan . + CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . make - msg "test: main suites (MSan)" # ~ 10s + msg "test: main suites (inc. selftests) (ASan build)" # ~ 50s make test - msg "test: ssl-opt.sh (MSan)" # ~ 1 min + msg "test: ssl-opt.sh (ASan build)" # ~ 1 min if_build_succeeded tests/ssl-opt.sh - # Optional part(s) + msg "test/build: ref-configs (ASan build)" # ~ 6 min 20s + record_status tests/scripts/test-ref-configs.pl - if [ "$MEMORY" -gt 0 ]; then - msg "test: compat.sh (MSan)" # ~ 6 min 20s - if_build_succeeded tests/compat.sh - fi - -else # no MemSan - - msg "build: Release (clang)" - cleanup - CC=clang cmake -D CMAKE_BUILD_TYPE:String=Release . + msg "build: with ASan (rebuild after ref-configs)" # ~ 1 min make - msg "test: main suites valgrind (Release)" - make memcheck + msg "test: compat.sh (ASan build)" # ~ 6 min + if_build_succeeded tests/compat.sh - # Optional part(s) - # Currently broken, programs don't seem to receive signals - # under valgrind on OS X + msg "build: Default + SSLv3 (ASan build)" # ~ 6 min + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl set MBEDTLS_SSL_PROTO_SSL3 + CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . + make - if [ "$MEMORY" -gt 0 ]; then - msg "test: ssl-opt.sh --memcheck (Release)" - if_build_succeeded tests/ssl-opt.sh --memcheck + msg "test: SSLv3 - main suites (inc. selftests) (ASan build)" # ~ 50s + make test + + msg "build: SSLv3 - compat.sh (ASan build)" # ~ 6 min + if_build_succeeded tests/compat.sh -m 'tls1 tls1_1 tls1_2 dtls1 dtls1_2' + if_build_succeeded env OPENSSL_CMD="$OPENSSL_LEGACY" tests/compat.sh -m 'ssl3' + + msg "build: SSLv3 - ssl-opt.sh (ASan build)" # ~ 6 min + if_build_succeeded tests/ssl-opt.sh + + msg "build: Default + !MBEDTLS_SSL_RENEGOTIATION (ASan build)" # ~ 6 min + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl unset MBEDTLS_SSL_RENEGOTIATION + CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . + make + + msg "test: !MBEDTLS_SSL_RENEGOTIATION - main suites (inc. selftests) (ASan build)" # ~ 50s + make test + + msg "test: !MBEDTLS_SSL_RENEGOTIATION - ssl-opt.sh (ASan build)" # ~ 6 min + if_build_succeeded tests/ssl-opt.sh + + msg "build: Default + RSA_NO_CRT (ASan build)" # ~ 6 min + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl set MBEDTLS_RSA_NO_CRT + CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . + make + + msg "test: RSA_NO_CRT - main suites (inc. selftests) (ASan build)" # ~ 50s + make test + + msg "test: RSA_NO_CRT - RSA-related part of ssl-opt.sh (ASan build)" # ~ 5s + if_build_succeeded tests/ssl-opt.sh -f RSA + + msg "test: RSA_NO_CRT - RSA-related part of compat.sh (ASan build)" # ~ 3 min + if_build_succeeded tests/compat.sh -t RSA + + msg "build: cmake, full config, clang" # ~ 50s + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl full + scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE # too slow for tests + CC=clang cmake -D CMAKE_BUILD_TYPE:String=Check -D ENABLE_TESTING=On . + make + + msg "test: main suites (full config)" # ~ 5s + make test + + msg "test: ssl-opt.sh default (full config)" # ~ 1s + if_build_succeeded tests/ssl-opt.sh -f Default + + msg "test: compat.sh RC4, DES & NULL (full config)" # ~ 2 min + if_build_succeeded env OPENSSL_CMD="$OPENSSL_LEGACY" GNUTLS_CLI="$GNUTLS_LEGACY_CLI" GNUTLS_SERV="$GNUTLS_LEGACY_SERV" tests/compat.sh -e '3DES\|DES-CBC3' -f 'NULL\|DES\|RC4\|ARCFOUR' + + msg "build: make, full config + DEPRECATED_WARNING, gcc -O" # ~ 30s + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl full + scripts/config.pl set MBEDTLS_DEPRECATED_WARNING + # Build with -O -Wextra to catch a maximum of issues. + make CC=gcc CFLAGS='-O -Werror -Wall -Wextra' lib programs + make CC=gcc CFLAGS='-O -Werror -Wall -Wextra -Wno-unused-function' tests + + msg "build: make, full config + DEPRECATED_REMOVED, clang -O" # ~ 30s + # No cleanup, just tweak the configuration and rebuild + make clean + scripts/config.pl unset MBEDTLS_DEPRECATED_WARNING + scripts/config.pl set MBEDTLS_DEPRECATED_REMOVED + # Build with -O -Wextra to catch a maximum of issues. + make CC=clang CFLAGS='-O -Werror -Wall -Wextra' lib programs + make CC=clang CFLAGS='-O -Werror -Wall -Wextra -Wno-unused-function' tests + + msg "test/build: curves.pl (gcc)" # ~ 4 min + cleanup + record_status tests/scripts/curves.pl + + msg "test/build: depends-hashes.pl (gcc)" # ~ 2 min + cleanup + record_status tests/scripts/depends-hashes.pl + + msg "test/build: depends-pkalgs.pl (gcc)" # ~ 2 min + cleanup + record_status tests/scripts/depends-pkalgs.pl + + msg "test/build: key-exchanges (gcc)" # ~ 1 min + cleanup + record_status tests/scripts/key-exchanges.pl + + msg "build: Unix make, -Os (gcc)" # ~ 30s + cleanup + make CC=gcc CFLAGS='-Werror -Wall -Wextra -Os' + + # Full configuration build, without platform support, file IO and net sockets. + # This should catch missing mbedtls_printf definitions, and by disabling file + # IO, it should catch missing '#include ' + msg "build: full config except platform/fsio/net, make, gcc, C99" # ~ 30s + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl full + scripts/config.pl unset MBEDTLS_PLATFORM_C + scripts/config.pl unset MBEDTLS_NET_C + scripts/config.pl unset MBEDTLS_PLATFORM_MEMORY + scripts/config.pl unset MBEDTLS_PLATFORM_PRINTF_ALT + scripts/config.pl unset MBEDTLS_PLATFORM_FPRINTF_ALT + scripts/config.pl unset MBEDTLS_PLATFORM_SNPRINTF_ALT + scripts/config.pl unset MBEDTLS_PLATFORM_TIME_ALT + scripts/config.pl unset MBEDTLS_PLATFORM_EXIT_ALT + scripts/config.pl unset MBEDTLS_ENTROPY_NV_SEED + scripts/config.pl unset MBEDTLS_MEMORY_BUFFER_ALLOC_C + scripts/config.pl unset MBEDTLS_FS_IO + # Note, _DEFAULT_SOURCE needs to be defined for platforms using glibc version >2.19, + # to re-enable platform integration features otherwise disabled in C99 builds + make CC=gcc CFLAGS='-Werror -Wall -Wextra -std=c99 -pedantic -O0 -D_DEFAULT_SOURCE' lib programs + make CC=gcc CFLAGS='-Werror -Wall -Wextra -O0' test + + # catch compile bugs in _uninit functions + msg "build: full config with NO_STD_FUNCTION, make, gcc" # ~ 30s + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl full + scripts/config.pl set MBEDTLS_PLATFORM_NO_STD_FUNCTIONS + scripts/config.pl unset MBEDTLS_ENTROPY_NV_SEED + make CC=gcc CFLAGS='-Werror -Wall -Wextra -O0' + + msg "build: full config except ssl_srv.c, make, gcc" # ~ 30s + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl full + scripts/config.pl unset MBEDTLS_SSL_SRV_C + make CC=gcc CFLAGS='-Werror -Wall -Wextra -O0' + + msg "build: full config except ssl_cli.c, make, gcc" # ~ 30s + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl full + scripts/config.pl unset MBEDTLS_SSL_CLI_C + make CC=gcc CFLAGS='-Werror -Wall -Wextra -O0' + + # Note, C99 compliance can also be tested with the sockets support disabled, + # as that requires a POSIX platform (which isn't the same as C99). + msg "build: full config except net_sockets.c, make, gcc -std=c99 -pedantic" # ~ 30s + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl full + scripts/config.pl unset MBEDTLS_NET_C # getaddrinfo() undeclared, etc. + scripts/config.pl set MBEDTLS_NO_PLATFORM_ENTROPY # uses syscall() on GNU/Linux + make CC=gcc CFLAGS='-Werror -Wall -Wextra -O0 -std=c99 -pedantic' lib + + msg "build: default config except MFL extension (ASan build)" # ~ 30s + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl unset MBEDTLS_SSL_MAX_FRAGMENT_LENGTH + CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . + make + + msg "test: ssl-opt.sh, MFL-related tests" + if_build_succeeded tests/ssl-opt.sh -f "Max fragment length" + + msg "build: default config with MBEDTLS_TEST_NULL_ENTROPY (ASan build)" + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl set MBEDTLS_TEST_NULL_ENTROPY + scripts/config.pl set MBEDTLS_NO_DEFAULT_ENTROPY_SOURCES + scripts/config.pl set MBEDTLS_ENTROPY_C + scripts/config.pl unset MBEDTLS_ENTROPY_NV_SEED + scripts/config.pl unset MBEDTLS_ENTROPY_HARDWARE_ALT + scripts/config.pl unset MBEDTLS_HAVEGE_C + CC=gcc cmake -D UNSAFE_BUILD=ON -D CMAKE_C_FLAGS:String="-fsanitize=address -fno-common -O3" . + make + + msg "test: MBEDTLS_TEST_NULL_ENTROPY - main suites (inc. selftests) (ASan build)" + make test + + msg "build: MBEDTLS_PLATFORM_{CALLOC/FREE}_MACRO enabled (ASan build)" + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl set MBEDTLS_PLATFORM_MEMORY + scripts/config.pl set MBEDTLS_PLATFORM_CALLOC_MACRO calloc + scripts/config.pl set MBEDTLS_PLATFORM_FREE_MACRO free + CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . + make + + msg "test: MBEDTLS_PLATFORM_{CALLOC/FREE}_MACRO enabled (ASan build)" + make test + + if uname -a | grep -F Linux >/dev/null; then + msg "build/test: make shared" # ~ 40s + cleanup + make SHARED=1 all check fi - if [ "$MEMORY" -gt 1 ]; then - msg "test: compat.sh --memcheck (Release)" - if_build_succeeded tests/compat.sh --memcheck + if uname -a | grep -F x86_64 >/dev/null; then + # Build once with -O0, to compile out the i386 specific inline assembly + msg "build: i386, make, gcc -O0 (ASan build)" # ~ 30s + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl full + make CC=gcc CFLAGS='-O0 -Werror -Wall -Wextra -m32 -fsanitize=address' + + msg "test: i386, make, gcc -O0 (ASan build)" + make test + + # Build again with -O1, to compile in the i386 specific inline assembly + msg "build: i386, make, gcc -O1 (ASan build)" # ~ 30s + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl full + make CC=gcc CFLAGS='-O1 -Werror -Wall -Wextra -m32 -fsanitize=address' + + msg "test: i386, make, gcc -O1 (ASan build)" + make test + + msg "build: 64-bit ILP32, make, gcc" # ~ 30s + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl full + make CC=gcc CFLAGS='-Werror -Wall -Wextra -mx32' + + msg "test: 64-bit ILP32, make, gcc" + make test + fi # x86_64 + + msg "build: gcc, force 32-bit bignum limbs" + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl unset MBEDTLS_HAVE_ASM + scripts/config.pl unset MBEDTLS_AESNI_C + scripts/config.pl unset MBEDTLS_PADLOCK_C + make CC=gcc CFLAGS='-Werror -Wall -Wextra -DMBEDTLS_HAVE_INT32' + + msg "test: gcc, force 32-bit bignum limbs" + make test + + msg "build: gcc, force 64-bit bignum limbs" + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl unset MBEDTLS_HAVE_ASM + scripts/config.pl unset MBEDTLS_AESNI_C + scripts/config.pl unset MBEDTLS_PADLOCK_C + make CC=gcc CFLAGS='-Werror -Wall -Wextra -DMBEDTLS_HAVE_INT64' + + msg "test: gcc, force 64-bit bignum limbs" + make test + + msg "build: arm-none-eabi-gcc, make" # ~ 10s + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl full + scripts/config.pl unset MBEDTLS_NET_C + scripts/config.pl unset MBEDTLS_TIMING_C + scripts/config.pl unset MBEDTLS_FS_IO + scripts/config.pl unset MBEDTLS_ENTROPY_NV_SEED + scripts/config.pl set MBEDTLS_NO_PLATFORM_ENTROPY + # following things are not in the default config + scripts/config.pl unset MBEDTLS_HAVEGE_C # depends on timing.c + scripts/config.pl unset MBEDTLS_THREADING_PTHREAD + scripts/config.pl unset MBEDTLS_THREADING_C + scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE # execinfo.h + scripts/config.pl unset MBEDTLS_MEMORY_BUFFER_ALLOC_C # calls exit + make CC=arm-none-eabi-gcc AR=arm-none-eabi-ar LD=arm-none-eabi-ld CFLAGS='-Werror -Wall -Wextra' lib + + msg "build: arm-none-eabi-gcc -DMBEDTLS_NO_UDBL_DIVISION, make" # ~ 10s + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl full + scripts/config.pl unset MBEDTLS_NET_C + scripts/config.pl unset MBEDTLS_TIMING_C + scripts/config.pl unset MBEDTLS_FS_IO + scripts/config.pl unset MBEDTLS_ENTROPY_NV_SEED + scripts/config.pl set MBEDTLS_NO_PLATFORM_ENTROPY + # following things are not in the default config + scripts/config.pl unset MBEDTLS_HAVEGE_C # depends on timing.c + scripts/config.pl unset MBEDTLS_THREADING_PTHREAD + scripts/config.pl unset MBEDTLS_THREADING_C + scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE # execinfo.h + scripts/config.pl unset MBEDTLS_MEMORY_BUFFER_ALLOC_C # calls exit + scripts/config.pl set MBEDTLS_NO_UDBL_DIVISION + make CC=arm-none-eabi-gcc AR=arm-none-eabi-ar LD=arm-none-eabi-ld CFLAGS='-Werror -Wall -Wextra' lib + echo "Checking that software 64-bit division is not required" + ! grep __aeabi_uldiv library/*.o + + msg "build: ARM Compiler 5, make" + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl full + scripts/config.pl unset MBEDTLS_NET_C + scripts/config.pl unset MBEDTLS_TIMING_C + scripts/config.pl unset MBEDTLS_FS_IO + scripts/config.pl unset MBEDTLS_ENTROPY_NV_SEED + scripts/config.pl unset MBEDTLS_HAVE_TIME + scripts/config.pl unset MBEDTLS_HAVE_TIME_DATE + scripts/config.pl set MBEDTLS_NO_PLATFORM_ENTROPY + # following things are not in the default config + scripts/config.pl unset MBEDTLS_DEPRECATED_WARNING + scripts/config.pl unset MBEDTLS_HAVEGE_C # depends on timing.c + scripts/config.pl unset MBEDTLS_THREADING_PTHREAD + scripts/config.pl unset MBEDTLS_THREADING_C + scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE # execinfo.h + scripts/config.pl unset MBEDTLS_MEMORY_BUFFER_ALLOC_C # calls exit + scripts/config.pl unset MBEDTLS_PLATFORM_TIME_ALT # depends on MBEDTLS_HAVE_TIME + + if [ $RUN_ARMCC -ne 0 ]; then + make CC="$ARMC5_CC" AR="$ARMC5_AR" WARNING_CFLAGS='--strict --c99' lib + make clean + + # ARM Compiler 6 - Target ARMv7-A + armc6_build_test "--target=arm-arm-none-eabi -march=armv7-a" + + # ARM Compiler 6 - Target ARMv7-M + armc6_build_test "--target=arm-arm-none-eabi -march=armv7-m" + + # ARM Compiler 6 - Target ARMv8-A - AArch32 + armc6_build_test "--target=arm-arm-none-eabi -march=armv8.2-a" + + # ARM Compiler 6 - Target ARMv8-M + armc6_build_test "--target=arm-arm-none-eabi -march=armv8-m.main" + + # ARM Compiler 6 - Target ARMv8-A - AArch64 + armc6_build_test "--target=aarch64-arm-none-eabi -march=armv8.2-a" fi -fi # MemSan + msg "build: allow SHA1 in certificates by default" + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl set MBEDTLS_TLS_DEFAULT_ALLOW_SHA1_IN_CERTIFICATES + make CFLAGS='-Werror -Wall -Wextra' + msg "test: allow SHA1 in certificates by default" + make test + if_build_succeeded tests/ssl-opt.sh -f SHA-1 -msg "build: cmake 'out-of-source' build" -cleanup -MBEDTLS_ROOT_DIR="$PWD" -mkdir "$OUT_OF_SOURCE_DIR" -cd "$OUT_OF_SOURCE_DIR" -cmake "$MBEDTLS_ROOT_DIR" -make + msg "build: Default + MBEDTLS_RSA_NO_CRT (ASan build)" # ~ 6 min + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl set MBEDTLS_RSA_NO_CRT + CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . + make -msg "test: cmake 'out-of-source' build" -make test -# Test an SSL option that requires an auxiliary script in test/scripts/. -# Also ensure that there are no error messages such as -# "No such file or directory", which would indicate that some required -# file is missing (ssl-opt.sh tolerates the absence of some files so -# may exit with status 0 but emit errors). -if_build_succeeded ./tests/ssl-opt.sh -f 'Fallback SCSV: beginning of list' 2>ssl-opt.err -if [ -s ssl-opt.err ]; then - cat ssl-opt.err >&2 - record_status [ ! -s ssl-opt.err ] - rm ssl-opt.err -fi -cd "$MBEDTLS_ROOT_DIR" -rm -rf "$OUT_OF_SOURCE_DIR" -unset MBEDTLS_ROOT_DIR + msg "test: MBEDTLS_RSA_NO_CRT - main suites (inc. selftests) (ASan build)" + make test + + msg "build: Windows cross build - mingw64, make (Link Library)" # ~ 30s + cleanup + make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 lib programs + + # note Make tests only builds the tests, but doesn't run them + make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror' WINDOWS_BUILD=1 tests + make WINDOWS_BUILD=1 clean + + msg "build: Windows cross build - mingw64, make (DLL)" # ~ 30s + make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 SHARED=1 lib programs + make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 SHARED=1 tests + make WINDOWS_BUILD=1 clean + + # MemSan currently only available on Linux 64 bits + if uname -a | grep 'Linux.*x86_64' >/dev/null; then + + msg "build: MSan (clang)" # ~ 1 min 20s + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl unset MBEDTLS_AESNI_C # memsan doesn't grok asm + CC=clang cmake -D CMAKE_BUILD_TYPE:String=MemSan . + make + + msg "test: main suites (MSan)" # ~ 10s + make test + + msg "test: ssl-opt.sh (MSan)" # ~ 1 min + if_build_succeeded tests/ssl-opt.sh + + # Optional part(s) + + if [ "$MEMORY" -gt 0 ]; then + msg "test: compat.sh (MSan)" # ~ 6 min 20s + if_build_succeeded tests/compat.sh + fi + + else # no MemSan + + msg "build: Release (clang)" + cleanup + CC=clang cmake -D CMAKE_BUILD_TYPE:String=Release . + make + + msg "test: main suites valgrind (Release)" + make memcheck + + # Optional part(s) + # Currently broken, programs don't seem to receive signals + # under valgrind on OS X + + if [ "$MEMORY" -gt 0 ]; then + msg "test: ssl-opt.sh --memcheck (Release)" + if_build_succeeded tests/ssl-opt.sh --memcheck + fi + + if [ "$MEMORY" -gt 1 ]; then + msg "test: compat.sh --memcheck (Release)" + if_build_succeeded tests/compat.sh --memcheck + fi + + fi # MemSan + + msg "build: cmake 'out-of-source' build" + cleanup + MBEDTLS_ROOT_DIR="$PWD" + mkdir "$OUT_OF_SOURCE_DIR" + cd "$OUT_OF_SOURCE_DIR" + cmake "$MBEDTLS_ROOT_DIR" + make + + msg "test: cmake 'out-of-source' build" + make test + # Test an SSL option that requires an auxiliary script in test/scripts/. + # Also ensure that there are no error messages such as + # "No such file or directory", which would indicate that some required + # file is missing (ssl-opt.sh tolerates the absence of some files so + # may exit with status 0 but emit errors). + if_build_succeeded ./tests/ssl-opt.sh -f 'Fallback SCSV: beginning of list' 2>ssl-opt.err + if [ -s ssl-opt.err ]; then + cat ssl-opt.err >&2 + record_status [ ! -s ssl-opt.err ] + rm ssl-opt.err + fi + cd "$MBEDTLS_ROOT_DIR" + rm -rf "$OUT_OF_SOURCE_DIR" + unset MBEDTLS_ROOT_DIR +} @@ -912,7 +921,38 @@ unset MBEDTLS_ROOT_DIR #### Termination ################################################################ -msg "Done, cleaning up" +post_report () { + msg "Done, cleaning up" + cleanup + + final_report +} + + + +################################################################ +#### Run all the things +################################################################ + +# Preliminary setup +pre_check_environment +pre_initialize_variables +pre_parse_command_line "$@" + +pre_check_git +build_status=0 +if [ $KEEP_GOING -eq 1 ]; then + pre_setup_keep_going +else + record_status () { + "$@" + } +fi +pre_print_configuration +pre_check_tools cleanup -final_report +run_all_the_tests + +# We're done. +post_report From 1a2ca72ddcbf871d0ae26c287e8c3b12fa8587a3 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 8 Jan 2019 22:35:16 +0100 Subject: [PATCH 02/22] Break up the tests into components Split the long list of tests into individual functions. Each time the test code called the `cleanup` function, I start a new function called `component_xxx`. Run all the components by enumerating the `component_xxx` functions. After running each component, call `cleanup`. A few sanity checks didn't have calls to `cleanup` because they didn't need them. I put them into separate components anyway, so there are now a few extra harmless calls to `cleanup`. --- tests/scripts/all.sh | 156 ++++++++++++++++++++++++++++++------------- 1 file changed, 108 insertions(+), 48 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index d76e6f805..64ae3c8b9 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -115,6 +115,12 @@ pre_initialize_variables () { if [ -n "${MAKEFLAGS+set}" ]; then export MAKEFLAGS="-j" fi + + # Gather the list of available components. These are the functions + # defined in this script whose name starts with "component_". + # Parse the script with sed, because in sh there is no way to list + # defined functions. + ALL_COMPONENTS=$(sed -n 's/^ *component_\([0-9A-Z_a-z]*\) *().*/\1/p' <"$0") } usage() @@ -421,45 +427,52 @@ pre_check_tools () { # # Indicative running times are given for reference. -run_all_the_tests () { - +component_check_recursion () { msg "test: recursion.pl" # < 1s record_status tests/scripts/recursion.pl library/*.c +} +component_check_generated_files () { msg "test: freshness of generated source files" # < 1s record_status tests/scripts/check-generated-files.sh +} +component_check_doxy_blocks () { msg "test: doxygen markup outside doxygen blocks" # < 1s record_status tests/scripts/check-doxy-blocks.pl +} +component_check_files () { msg "test: check-files.py" # < 1s - cleanup record_status tests/scripts/check-files.py +} +component_check_names () { msg "test/build: declared and exported names" # < 3s - cleanup record_status tests/scripts/check-names.sh +} +component_check_doxygen_warnings () { msg "test: doxygen warnings" # ~ 3s - cleanup record_status tests/scripts/doxygen.sh +} +################################################################ +#### Build and test many configurations and targets +################################################################ - ################################################################ - #### Build and test many configurations and targets - ################################################################ - +component_build_yotta () { if [ $RUN_ARMCC -ne 0 ] && [ $YOTTA -ne 0 ]; then # Note - use of yotta is deprecated, and yotta also requires armcc to be on the # path, and uses whatever version of armcc it finds there. msg "build: create and build yotta module" # ~ 30s - cleanup record_status tests/scripts/yotta-build.sh fi +} +component_test_default_cmake_gcc_asan () { msg "build: cmake, gcc, ASan" # ~ 1 min 50s - cleanup CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . make @@ -477,9 +490,10 @@ run_all_the_tests () { msg "test: compat.sh (ASan build)" # ~ 6 min if_build_succeeded tests/compat.sh +} +component_test_sslv3 () { msg "build: Default + SSLv3 (ASan build)" # ~ 6 min - cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl set MBEDTLS_SSL_PROTO_SSL3 CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . @@ -494,9 +508,10 @@ run_all_the_tests () { msg "build: SSLv3 - ssl-opt.sh (ASan build)" # ~ 6 min if_build_succeeded tests/ssl-opt.sh +} +component_test_no_renegotiation () { msg "build: Default + !MBEDTLS_SSL_RENEGOTIATION (ASan build)" # ~ 6 min - cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl unset MBEDTLS_SSL_RENEGOTIATION CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . @@ -507,9 +522,10 @@ run_all_the_tests () { msg "test: !MBEDTLS_SSL_RENEGOTIATION - ssl-opt.sh (ASan build)" # ~ 6 min if_build_succeeded tests/ssl-opt.sh +} +component_test_rsa_no_crt () { msg "build: Default + RSA_NO_CRT (ASan build)" # ~ 6 min - cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl set MBEDTLS_RSA_NO_CRT CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . @@ -523,9 +539,10 @@ run_all_the_tests () { msg "test: RSA_NO_CRT - RSA-related part of compat.sh (ASan build)" # ~ 3 min if_build_succeeded tests/compat.sh -t RSA +} +component_test_full_cmake_clang () { msg "build: cmake, full config, clang" # ~ 50s - cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE # too slow for tests @@ -540,9 +557,10 @@ run_all_the_tests () { msg "test: compat.sh RC4, DES & NULL (full config)" # ~ 2 min if_build_succeeded env OPENSSL_CMD="$OPENSSL_LEGACY" GNUTLS_CLI="$GNUTLS_LEGACY_CLI" GNUTLS_SERV="$GNUTLS_LEGACY_SERV" tests/compat.sh -e '3DES\|DES-CBC3' -f 'NULL\|DES\|RC4\|ARCFOUR' +} +component_build_deprecated () { msg "build: make, full config + DEPRECATED_WARNING, gcc -O" # ~ 30s - cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full scripts/config.pl set MBEDTLS_DEPRECATED_WARNING @@ -558,32 +576,38 @@ run_all_the_tests () { # Build with -O -Wextra to catch a maximum of issues. make CC=clang CFLAGS='-O -Werror -Wall -Wextra' lib programs make CC=clang CFLAGS='-O -Werror -Wall -Wextra -Wno-unused-function' tests +} +component_test_depends_curves () { msg "test/build: curves.pl (gcc)" # ~ 4 min - cleanup record_status tests/scripts/curves.pl +} +component_test_depends_hashes () { msg "test/build: depends-hashes.pl (gcc)" # ~ 2 min - cleanup record_status tests/scripts/depends-hashes.pl +} +component_test_depends_pkalgs () { msg "test/build: depends-pkalgs.pl (gcc)" # ~ 2 min - cleanup record_status tests/scripts/depends-pkalgs.pl +} +component_build_key_exchanges () { msg "test/build: key-exchanges (gcc)" # ~ 1 min - cleanup record_status tests/scripts/key-exchanges.pl +} +component_build_default_make_gcc () { msg "build: Unix make, -Os (gcc)" # ~ 30s - cleanup make CC=gcc CFLAGS='-Werror -Wall -Wextra -Os' +} +component_test_no_platform () { # Full configuration build, without platform support, file IO and net sockets. # This should catch missing mbedtls_printf definitions, and by disabling file # IO, it should catch missing '#include ' msg "build: full config except platform/fsio/net, make, gcc, C99" # ~ 30s - cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full scripts/config.pl unset MBEDTLS_PLATFORM_C @@ -601,42 +625,47 @@ run_all_the_tests () { # to re-enable platform integration features otherwise disabled in C99 builds make CC=gcc CFLAGS='-Werror -Wall -Wextra -std=c99 -pedantic -O0 -D_DEFAULT_SOURCE' lib programs make CC=gcc CFLAGS='-Werror -Wall -Wextra -O0' test +} +component_build_no_std_function () { # catch compile bugs in _uninit functions msg "build: full config with NO_STD_FUNCTION, make, gcc" # ~ 30s - cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full scripts/config.pl set MBEDTLS_PLATFORM_NO_STD_FUNCTIONS scripts/config.pl unset MBEDTLS_ENTROPY_NV_SEED make CC=gcc CFLAGS='-Werror -Wall -Wextra -O0' +} +component_build_no_ssl_srv () { msg "build: full config except ssl_srv.c, make, gcc" # ~ 30s - cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full scripts/config.pl unset MBEDTLS_SSL_SRV_C make CC=gcc CFLAGS='-Werror -Wall -Wextra -O0' +} +component_build_no_ssl_cli () { msg "build: full config except ssl_cli.c, make, gcc" # ~ 30s - cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full scripts/config.pl unset MBEDTLS_SSL_CLI_C make CC=gcc CFLAGS='-Werror -Wall -Wextra -O0' +} +component_build_no_sockets () { # Note, C99 compliance can also be tested with the sockets support disabled, # as that requires a POSIX platform (which isn't the same as C99). msg "build: full config except net_sockets.c, make, gcc -std=c99 -pedantic" # ~ 30s - cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full scripts/config.pl unset MBEDTLS_NET_C # getaddrinfo() undeclared, etc. scripts/config.pl set MBEDTLS_NO_PLATFORM_ENTROPY # uses syscall() on GNU/Linux make CC=gcc CFLAGS='-Werror -Wall -Wextra -O0 -std=c99 -pedantic' lib +} +component_test_no_max_fragment_length () { msg "build: default config except MFL extension (ASan build)" # ~ 30s - cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl unset MBEDTLS_SSL_MAX_FRAGMENT_LENGTH CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . @@ -644,9 +673,10 @@ run_all_the_tests () { msg "test: ssl-opt.sh, MFL-related tests" if_build_succeeded tests/ssl-opt.sh -f "Max fragment length" +} +component_test_null_entropy () { msg "build: default config with MBEDTLS_TEST_NULL_ENTROPY (ASan build)" - cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl set MBEDTLS_TEST_NULL_ENTROPY scripts/config.pl set MBEDTLS_NO_DEFAULT_ENTROPY_SOURCES @@ -659,9 +689,10 @@ run_all_the_tests () { msg "test: MBEDTLS_TEST_NULL_ENTROPY - main suites (inc. selftests) (ASan build)" make test +} +component_test_platform_calloc_macro () { msg "build: MBEDTLS_PLATFORM_{CALLOC/FREE}_MACRO enabled (ASan build)" - cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl set MBEDTLS_PLATFORM_MEMORY scripts/config.pl set MBEDTLS_PLATFORM_CALLOC_MACRO calloc @@ -671,36 +702,45 @@ run_all_the_tests () { msg "test: MBEDTLS_PLATFORM_{CALLOC/FREE}_MACRO enabled (ASan build)" make test +} +component_test_make_shared () { if uname -a | grep -F Linux >/dev/null; then msg "build/test: make shared" # ~ 40s - cleanup make SHARED=1 all check fi +} + +component_test_m32_o0 () { if uname -a | grep -F x86_64 >/dev/null; then # Build once with -O0, to compile out the i386 specific inline assembly msg "build: i386, make, gcc -O0 (ASan build)" # ~ 30s - cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full make CC=gcc CFLAGS='-O0 -Werror -Wall -Wextra -m32 -fsanitize=address' msg "test: i386, make, gcc -O0 (ASan build)" make test + fi # x86_64 +} +component_test_m32_o1 () { + if uname -a | grep -F x86_64 >/dev/null; then # Build again with -O1, to compile in the i386 specific inline assembly msg "build: i386, make, gcc -O1 (ASan build)" # ~ 30s - cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full make CC=gcc CFLAGS='-O1 -Werror -Wall -Wextra -m32 -fsanitize=address' msg "test: i386, make, gcc -O1 (ASan build)" make test + fi # x86_64 +} +component_test_mx32 () { + if uname -a | grep -F x86_64 >/dev/null; then msg "build: 64-bit ILP32, make, gcc" # ~ 30s - cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full make CC=gcc CFLAGS='-Werror -Wall -Wextra -mx32' @@ -708,9 +748,10 @@ run_all_the_tests () { msg "test: 64-bit ILP32, make, gcc" make test fi # x86_64 +} +component_test_have_int32 () { msg "build: gcc, force 32-bit bignum limbs" - cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl unset MBEDTLS_HAVE_ASM scripts/config.pl unset MBEDTLS_AESNI_C @@ -719,9 +760,10 @@ run_all_the_tests () { msg "test: gcc, force 32-bit bignum limbs" make test +} +component_test_have_int64 () { msg "build: gcc, force 64-bit bignum limbs" - cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl unset MBEDTLS_HAVE_ASM scripts/config.pl unset MBEDTLS_AESNI_C @@ -730,9 +772,10 @@ run_all_the_tests () { msg "test: gcc, force 64-bit bignum limbs" make test +} +component_build_arm_none_eabi_gcc () { msg "build: arm-none-eabi-gcc, make" # ~ 10s - cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full scripts/config.pl unset MBEDTLS_NET_C @@ -747,9 +790,10 @@ run_all_the_tests () { scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE # execinfo.h scripts/config.pl unset MBEDTLS_MEMORY_BUFFER_ALLOC_C # calls exit make CC=arm-none-eabi-gcc AR=arm-none-eabi-ar LD=arm-none-eabi-ld CFLAGS='-Werror -Wall -Wextra' lib +} +component_build_arm_none_eabi_gcc_no_udbl_division () { msg "build: arm-none-eabi-gcc -DMBEDTLS_NO_UDBL_DIVISION, make" # ~ 10s - cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full scripts/config.pl unset MBEDTLS_NET_C @@ -767,9 +811,10 @@ run_all_the_tests () { make CC=arm-none-eabi-gcc AR=arm-none-eabi-ar LD=arm-none-eabi-ld CFLAGS='-Werror -Wall -Wextra' lib echo "Checking that software 64-bit division is not required" ! grep __aeabi_uldiv library/*.o +} +component_build_armcc () { msg "build: ARM Compiler 5, make" - cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full scripts/config.pl unset MBEDTLS_NET_C @@ -807,18 +852,20 @@ run_all_the_tests () { # ARM Compiler 6 - Target ARMv8-A - AArch64 armc6_build_test "--target=aarch64-arm-none-eabi -march=armv8.2-a" fi +} +component_test_allow_sha1 () { msg "build: allow SHA1 in certificates by default" - cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl set MBEDTLS_TLS_DEFAULT_ALLOW_SHA1_IN_CERTIFICATES make CFLAGS='-Werror -Wall -Wextra' msg "test: allow SHA1 in certificates by default" make test if_build_succeeded tests/ssl-opt.sh -f SHA-1 +} +component_test_rsa_no_crt_again () { msg "build: Default + MBEDTLS_RSA_NO_CRT (ASan build)" # ~ 6 min - cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl set MBEDTLS_RSA_NO_CRT CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . @@ -826,9 +873,10 @@ run_all_the_tests () { msg "test: MBEDTLS_RSA_NO_CRT - main suites (inc. selftests) (ASan build)" make test +} +component_build_mingw () { msg "build: Windows cross build - mingw64, make (Link Library)" # ~ 30s - cleanup make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 lib programs # note Make tests only builds the tests, but doesn't run them @@ -839,12 +887,12 @@ run_all_the_tests () { make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 SHARED=1 lib programs make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 SHARED=1 tests make WINDOWS_BUILD=1 clean +} +component_test_memsan () { # MemSan currently only available on Linux 64 bits if uname -a | grep 'Linux.*x86_64' >/dev/null; then - msg "build: MSan (clang)" # ~ 1 min 20s - cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl unset MBEDTLS_AESNI_C # memsan doesn't grok asm CC=clang cmake -D CMAKE_BUILD_TYPE:String=MemSan . @@ -862,11 +910,13 @@ run_all_the_tests () { msg "test: compat.sh (MSan)" # ~ 6 min 20s if_build_succeeded tests/compat.sh fi + fi +} - else # no MemSan - +component_test_memcheck () { + # Only run if MemSan is not available + if ! uname -a | grep 'Linux.*x86_64' >/dev/null; then msg "build: Release (clang)" - cleanup CC=clang cmake -D CMAKE_BUILD_TYPE:String=Release . make @@ -888,9 +938,10 @@ run_all_the_tests () { fi fi # MemSan +} +component_test_cmake_out_of_source () { msg "build: cmake 'out-of-source' build" - cleanup MBEDTLS_ROOT_DIR="$PWD" mkdir "$OUT_OF_SOURCE_DIR" cd "$OUT_OF_SOURCE_DIR" @@ -934,6 +985,12 @@ post_report () { #### Run all the things ################################################################ +# Run one component and clean up afterwards. +run_component () { + "$@" + cleanup +} + # Preliminary setup pre_check_environment pre_initialize_variables @@ -952,7 +1009,10 @@ pre_print_configuration pre_check_tools cleanup -run_all_the_tests +# Run all the test components. +for component in $ALL_COMPONENTS; do + run_component "component_$component" +done # We're done. post_report From 2f300dbb2f635e2195bfb4492391cceeaec14a37 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 8 Jan 2019 22:37:40 +0100 Subject: [PATCH 03/22] Remove duplicate component for RSA_NO_CRT --- tests/scripts/all.sh | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 64ae3c8b9..b1a76acb6 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -864,17 +864,6 @@ component_test_allow_sha1 () { if_build_succeeded tests/ssl-opt.sh -f SHA-1 } -component_test_rsa_no_crt_again () { - msg "build: Default + MBEDTLS_RSA_NO_CRT (ASan build)" # ~ 6 min - cp "$CONFIG_H" "$CONFIG_BAK" - scripts/config.pl set MBEDTLS_RSA_NO_CRT - CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . - make - - msg "test: MBEDTLS_RSA_NO_CRT - main suites (inc. selftests) (ASan build)" - make test -} - component_build_mingw () { msg "build: Windows cross build - mingw64, make (Link Library)" # ~ 30s make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 lib programs From 3484ed87974e4b187c44c5192dd6fe88bd9f0753 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 8 Jan 2019 22:51:19 +0100 Subject: [PATCH 04/22] Move test-ref-configs into its own component --- tests/scripts/all.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index b1a76acb6..2f891f779 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -482,16 +482,16 @@ component_test_default_cmake_gcc_asan () { msg "test: ssl-opt.sh (ASan build)" # ~ 1 min if_build_succeeded tests/ssl-opt.sh - msg "test/build: ref-configs (ASan build)" # ~ 6 min 20s - record_status tests/scripts/test-ref-configs.pl - - msg "build: with ASan (rebuild after ref-configs)" # ~ 1 min - make - msg "test: compat.sh (ASan build)" # ~ 6 min if_build_succeeded tests/compat.sh } +component_test_ref_configs () { + msg "test/build: ref-configs (ASan build)" # ~ 6 min 20s + CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . + record_status tests/scripts/test-ref-configs.pl +} + component_test_sslv3 () { msg "build: Default + SSLv3 (ASan build)" # ~ 6 min cp "$CONFIG_H" "$CONFIG_BAK" From 91bd8b78ed1f55b437896bce9e90f0849cf34876 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 8 Jan 2019 23:01:34 +0100 Subject: [PATCH 05/22] all.sh: with non-option arguments, run only these components --- tests/scripts/all.sh | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 2f891f779..d5c6ddc2a 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -94,6 +94,7 @@ pre_initialize_variables () { CONFIG_H='include/mbedtls/config.h' CONFIG_BAK="$CONFIG_H.bak" + COMPONENTS= MEMORY=0 FORCE=0 KEEP_GOING=0 @@ -126,8 +127,12 @@ pre_initialize_variables () { usage() { cat <&2 "Unknown option: $1" echo >&2 "Run $0 --help for usage." exit 120 ;; + *) + COMPONENTS="$COMPONENTS $1";; esac shift done + + if [ -z "$COMPONENTS" ]; then + COMPONENTS="$ALL_COMPONENTS" + fi } pre_check_git () { @@ -999,7 +1010,7 @@ pre_check_tools cleanup # Run all the test components. -for component in $ALL_COMPONENTS; do +for component in $COMPONENTS; do run_component "component_$component" done From 6e9842315aacaf485bea3ff9ef7d721b57f64344 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 27 Nov 2018 21:37:53 +0100 Subject: [PATCH 06/22] Add --except mode: run all components except a list Allow the list to use wildcards, e.g. you can run the sanity checks with all.sh --except "test_*" "build_*" --- tests/scripts/all.sh | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index d5c6ddc2a..d2eb47deb 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -94,7 +94,7 @@ pre_initialize_variables () { CONFIG_H='include/mbedtls/config.h' CONFIG_BAK="$CONFIG_H.bak" - COMPONENTS= + ALL_EXCEPT=0 MEMORY=0 FORCE=0 KEEP_GOING=0 @@ -124,6 +124,19 @@ pre_initialize_variables () { ALL_COMPONENTS=$(sed -n 's/^ *component_\([0-9A-Z_a-z]*\) *().*/\1/p' <"$0") } +# Test whether $1 is excluded via $COMPONENTS (a space-separated list of +# wildcard patterns). +is_component_excluded() +{ + set -f + for pattern in $COMPONENTS; do + set +f + case ${1#component_} in $pattern) return 0;; esac + done + set +f + return 1 +} + usage() { cat < Date: Tue, 4 Dec 2018 12:49:28 +0100 Subject: [PATCH 07/22] Add the current component name to msg output and the final report --- tests/scripts/all.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index d2eb47deb..f3a1da1d1 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -223,12 +223,16 @@ trap 'fatal_signal TERM' TERM msg() { + if [ -n "${current_component:-}" ]; then + current_section="${current_component#component_}: $1" + else + current_section="$1" + fi echo "" echo "******************************************************************" - echo "* $1 " + echo "* $current_section " printf "* "; date echo "******************************************************************" - current_section=$1 } armc6_build_test() @@ -1017,6 +1021,7 @@ post_report () { # Run one component and clean up afterwards. run_component () { + current_component="$1" "$@" cleanup } From 72adb432bce30f01915c742d99350114f5426026 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 2 Jan 2019 18:57:02 +0100 Subject: [PATCH 08/22] Back up and restore config.h systematically In all.sh, always save config.h before running a component, instead of doing it manually in each component that requires it (except when we forget, which has happened). This would break a script that requires config.h.bak not to exist, but we don't have any of those. --- tests/scripts/all.sh | 26 +++----------------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index f3a1da1d1..215a93d0f 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -539,7 +539,6 @@ component_test_ref_configs () { component_test_sslv3 () { msg "build: Default + SSLv3 (ASan build)" # ~ 6 min - cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl set MBEDTLS_SSL_PROTO_SSL3 CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . make @@ -557,7 +556,6 @@ component_test_sslv3 () { component_test_no_renegotiation () { msg "build: Default + !MBEDTLS_SSL_RENEGOTIATION (ASan build)" # ~ 6 min - cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl unset MBEDTLS_SSL_RENEGOTIATION CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . make @@ -571,7 +569,6 @@ component_test_no_renegotiation () { component_test_rsa_no_crt () { msg "build: Default + RSA_NO_CRT (ASan build)" # ~ 6 min - cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl set MBEDTLS_RSA_NO_CRT CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . make @@ -588,7 +585,6 @@ component_test_rsa_no_crt () { component_test_full_cmake_clang () { msg "build: cmake, full config, clang" # ~ 50s - cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE # too slow for tests CC=clang cmake -D CMAKE_BUILD_TYPE:String=Check -D ENABLE_TESTING=On . @@ -606,7 +602,6 @@ component_test_full_cmake_clang () { component_build_deprecated () { msg "build: make, full config + DEPRECATED_WARNING, gcc -O" # ~ 30s - cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full scripts/config.pl set MBEDTLS_DEPRECATED_WARNING # Build with -O -Wextra to catch a maximum of issues. @@ -653,7 +648,6 @@ component_test_no_platform () { # This should catch missing mbedtls_printf definitions, and by disabling file # IO, it should catch missing '#include ' msg "build: full config except platform/fsio/net, make, gcc, C99" # ~ 30s - cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full scripts/config.pl unset MBEDTLS_PLATFORM_C scripts/config.pl unset MBEDTLS_NET_C @@ -675,7 +669,6 @@ component_test_no_platform () { component_build_no_std_function () { # catch compile bugs in _uninit functions msg "build: full config with NO_STD_FUNCTION, make, gcc" # ~ 30s - cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full scripts/config.pl set MBEDTLS_PLATFORM_NO_STD_FUNCTIONS scripts/config.pl unset MBEDTLS_ENTROPY_NV_SEED @@ -684,7 +677,6 @@ component_build_no_std_function () { component_build_no_ssl_srv () { msg "build: full config except ssl_srv.c, make, gcc" # ~ 30s - cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full scripts/config.pl unset MBEDTLS_SSL_SRV_C make CC=gcc CFLAGS='-Werror -Wall -Wextra -O0' @@ -692,7 +684,6 @@ component_build_no_ssl_srv () { component_build_no_ssl_cli () { msg "build: full config except ssl_cli.c, make, gcc" # ~ 30s - cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full scripts/config.pl unset MBEDTLS_SSL_CLI_C make CC=gcc CFLAGS='-Werror -Wall -Wextra -O0' @@ -702,7 +693,6 @@ component_build_no_sockets () { # Note, C99 compliance can also be tested with the sockets support disabled, # as that requires a POSIX platform (which isn't the same as C99). msg "build: full config except net_sockets.c, make, gcc -std=c99 -pedantic" # ~ 30s - cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full scripts/config.pl unset MBEDTLS_NET_C # getaddrinfo() undeclared, etc. scripts/config.pl set MBEDTLS_NO_PLATFORM_ENTROPY # uses syscall() on GNU/Linux @@ -711,7 +701,6 @@ component_build_no_sockets () { component_test_no_max_fragment_length () { msg "build: default config except MFL extension (ASan build)" # ~ 30s - cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl unset MBEDTLS_SSL_MAX_FRAGMENT_LENGTH CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . make @@ -722,7 +711,6 @@ component_test_no_max_fragment_length () { component_test_null_entropy () { msg "build: default config with MBEDTLS_TEST_NULL_ENTROPY (ASan build)" - cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl set MBEDTLS_TEST_NULL_ENTROPY scripts/config.pl set MBEDTLS_NO_DEFAULT_ENTROPY_SOURCES scripts/config.pl set MBEDTLS_ENTROPY_C @@ -738,7 +726,6 @@ component_test_null_entropy () { component_test_platform_calloc_macro () { msg "build: MBEDTLS_PLATFORM_{CALLOC/FREE}_MACRO enabled (ASan build)" - cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl set MBEDTLS_PLATFORM_MEMORY scripts/config.pl set MBEDTLS_PLATFORM_CALLOC_MACRO calloc scripts/config.pl set MBEDTLS_PLATFORM_FREE_MACRO free @@ -761,7 +748,6 @@ component_test_m32_o0 () { if uname -a | grep -F x86_64 >/dev/null; then # Build once with -O0, to compile out the i386 specific inline assembly msg "build: i386, make, gcc -O0 (ASan build)" # ~ 30s - cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full make CC=gcc CFLAGS='-O0 -Werror -Wall -Wextra -m32 -fsanitize=address' @@ -774,7 +760,6 @@ component_test_m32_o1 () { if uname -a | grep -F x86_64 >/dev/null; then # Build again with -O1, to compile in the i386 specific inline assembly msg "build: i386, make, gcc -O1 (ASan build)" # ~ 30s - cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full make CC=gcc CFLAGS='-O1 -Werror -Wall -Wextra -m32 -fsanitize=address' @@ -786,7 +771,6 @@ component_test_m32_o1 () { component_test_mx32 () { if uname -a | grep -F x86_64 >/dev/null; then msg "build: 64-bit ILP32, make, gcc" # ~ 30s - cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full make CC=gcc CFLAGS='-Werror -Wall -Wextra -mx32' @@ -797,7 +781,6 @@ component_test_mx32 () { component_test_have_int32 () { msg "build: gcc, force 32-bit bignum limbs" - cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl unset MBEDTLS_HAVE_ASM scripts/config.pl unset MBEDTLS_AESNI_C scripts/config.pl unset MBEDTLS_PADLOCK_C @@ -809,7 +792,6 @@ component_test_have_int32 () { component_test_have_int64 () { msg "build: gcc, force 64-bit bignum limbs" - cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl unset MBEDTLS_HAVE_ASM scripts/config.pl unset MBEDTLS_AESNI_C scripts/config.pl unset MBEDTLS_PADLOCK_C @@ -821,7 +803,6 @@ component_test_have_int64 () { component_build_arm_none_eabi_gcc () { msg "build: arm-none-eabi-gcc, make" # ~ 10s - cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full scripts/config.pl unset MBEDTLS_NET_C scripts/config.pl unset MBEDTLS_TIMING_C @@ -839,7 +820,6 @@ component_build_arm_none_eabi_gcc () { component_build_arm_none_eabi_gcc_no_udbl_division () { msg "build: arm-none-eabi-gcc -DMBEDTLS_NO_UDBL_DIVISION, make" # ~ 10s - cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full scripts/config.pl unset MBEDTLS_NET_C scripts/config.pl unset MBEDTLS_TIMING_C @@ -860,7 +840,6 @@ component_build_arm_none_eabi_gcc_no_udbl_division () { component_build_armcc () { msg "build: ARM Compiler 5, make" - cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full scripts/config.pl unset MBEDTLS_NET_C scripts/config.pl unset MBEDTLS_TIMING_C @@ -901,7 +880,6 @@ component_build_armcc () { component_test_allow_sha1 () { msg "build: allow SHA1 in certificates by default" - cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl set MBEDTLS_TLS_DEFAULT_ALLOW_SHA1_IN_CERTIFICATES make CFLAGS='-Werror -Wall -Wextra' msg "test: allow SHA1 in certificates by default" @@ -927,7 +905,6 @@ component_test_memsan () { # MemSan currently only available on Linux 64 bits if uname -a | grep 'Linux.*x86_64' >/dev/null; then msg "build: MSan (clang)" # ~ 1 min 20s - cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl unset MBEDTLS_AESNI_C # memsan doesn't grok asm CC=clang cmake -D CMAKE_BUILD_TYPE:String=MemSan . make @@ -1021,6 +998,9 @@ post_report () { # Run one component and clean up afterwards. run_component () { + # Back up the configuration in case the component modifies it. + # The cleanup function will restore it. + cp -p "$CONFIG_H" "$CONFIG_BAK" current_component="$1" "$@" cleanup From 4e7b323fd816f34a418f6b738172fd9e262b81cc Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 6 Jan 2019 19:48:30 +0000 Subject: [PATCH 09/22] Use CMAKE_BUILD_TYPE to do Asan builds Use `cmake -D CMAKE_BUILD_TYPE=Asan` rather than manually setting `-fsanitize=address`. This lets cmake determine the necessary compiler and linker flags. With UNSAFE_BUILD on, force -Wno-error. This is necessary to build with MBEDTLS_TEST_NULL_ENTROPY. --- library/CMakeLists.txt | 6 ++++++ tests/scripts/all.sh | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/library/CMakeLists.txt b/library/CMakeLists.txt index 59df9589a..3afdcc53c 100644 --- a/library/CMakeLists.txt +++ b/library/CMakeLists.txt @@ -91,6 +91,12 @@ if(CMAKE_COMPILER_IS_CLANG) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wmissing-declarations -Wmissing-prototypes -Wdocumentation -Wno-documentation-deprecated-sync -Wunreachable-code") endif(CMAKE_COMPILER_IS_CLANG) +if(UNSAFE_BUILD) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-error") + set(CMAKE_C_FLAGS_ASAN "${CMAKE_C_FLAGS_ASAN} -Wno-error") + set(CMAKE_C_FLAGS_ASANDBG "${CMAKE_C_FLAGS_ASANDBG} -Wno-error") +endif(UNSAFE_BUILD) + if(WIN32) set(libs ${libs} ws2_32) endif(WIN32) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 215a93d0f..5e4421e4b 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -717,7 +717,7 @@ component_test_null_entropy () { scripts/config.pl unset MBEDTLS_ENTROPY_NV_SEED scripts/config.pl unset MBEDTLS_ENTROPY_HARDWARE_ALT scripts/config.pl unset MBEDTLS_HAVEGE_C - CC=gcc cmake -D UNSAFE_BUILD=ON -D CMAKE_C_FLAGS:String="-fsanitize=address -fno-common -O3" . + CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan -D UNSAFE_BUILD=ON . make msg "test: MBEDTLS_TEST_NULL_ENTROPY - main suites (inc. selftests) (ASan build)" From 770ad7e2c967f2531e73175f8a12cd69ae99ca61 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 8 Jan 2019 23:19:08 +0100 Subject: [PATCH 10/22] all.sh: don't insist on Linux; always run Valgrind Don't bail out of all.sh if the OS isn't Linux. We only expect everything to pass on a recent Linux x86_64, but it's useful to call all.sh to run some components on any platform. In all.sh, always run both MemorySanitizer and Valgrind. Valgrind is slower than ASan and MSan but finds some things that they don't. Run MSan unconditionally, not just on Linux/x86_64. MSan is supported on some other OSes and CPUs these days. Use `all.sh --except test_memsan` if you want to omit MSan because it isn't supported on your platform. Use `all.sh --except test_memcheck` if you want to omit Valgrind because it's too slow. Portability: ecognize amd64 (FreeBSD arch string) as well as x86_64 (Linux arch string) for `uname -m`. The `make` utility must still be GNU make. --- tests/scripts/all.sh | 131 +++++++++++++++++++------------------------ 1 file changed, 58 insertions(+), 73 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 5e4421e4b..cc5ab9569 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -81,10 +81,7 @@ set -eu pre_check_environment () { - if [ "$( uname )" != "Linux" ]; then - echo "This script only works in Linux" >&2 - exit 1 - elif [ -d library -a -d include -a -d tests ]; then :; else + if [ -d library -a -d include -a -d tests ]; then :; else echo "Must be run from mbed TLS root" >&2 exit 1 fi @@ -737,47 +734,42 @@ component_test_platform_calloc_macro () { } component_test_make_shared () { - if uname -a | grep -F Linux >/dev/null; then - msg "build/test: make shared" # ~ 40s - make SHARED=1 all check - fi - + msg "build/test: make shared" # ~ 40s + make SHARED=1 all check } -component_test_m32_o0 () { - if uname -a | grep -F x86_64 >/dev/null; then - # Build once with -O0, to compile out the i386 specific inline assembly - msg "build: i386, make, gcc -O0 (ASan build)" # ~ 30s - scripts/config.pl full - make CC=gcc CFLAGS='-O0 -Werror -Wall -Wextra -m32 -fsanitize=address' +case $(uname -m) in + amd64|x86_64) + component_test_m32_o0 () { + # Build once with -O0, to compile out the i386 specific inline assembly + msg "build: i386, make, gcc -O0 (ASan build)" # ~ 30s + scripts/config.pl full + make CC=gcc CFLAGS='-O0 -Werror -Wall -Wextra -m32 -fsanitize=address' - msg "test: i386, make, gcc -O0 (ASan build)" - make test - fi # x86_64 -} + msg "test: i386, make, gcc -O0 (ASan build)" + make test + } -component_test_m32_o1 () { - if uname -a | grep -F x86_64 >/dev/null; then - # Build again with -O1, to compile in the i386 specific inline assembly - msg "build: i386, make, gcc -O1 (ASan build)" # ~ 30s - scripts/config.pl full - make CC=gcc CFLAGS='-O1 -Werror -Wall -Wextra -m32 -fsanitize=address' + component_test_m32_o1 () { + # Build again with -O1, to compile in the i386 specific inline assembly + msg "build: i386, make, gcc -O1 (ASan build)" # ~ 30s + scripts/config.pl full + make CC=gcc CFLAGS='-O1 -Werror -Wall -Wextra -m32 -fsanitize=address' - msg "test: i386, make, gcc -O1 (ASan build)" - make test - fi # x86_64 -} + msg "test: i386, make, gcc -O1 (ASan build)" + make test + } -component_test_mx32 () { - if uname -a | grep -F x86_64 >/dev/null; then - msg "build: 64-bit ILP32, make, gcc" # ~ 30s - scripts/config.pl full - make CC=gcc CFLAGS='-Werror -Wall -Wextra -mx32' + component_test_mx32 () { + msg "build: 64-bit ILP32, make, gcc" # ~ 30s + scripts/config.pl full + make CC=gcc CFLAGS='-Werror -Wall -Wextra -mx32' - msg "test: 64-bit ILP32, make, gcc" - make test - fi # x86_64 -} + msg "test: 64-bit ILP32, make, gcc" + make test + } + ;; +esac component_test_have_int32 () { msg "build: gcc, force 32-bit bignum limbs" @@ -902,53 +894,46 @@ component_build_mingw () { } component_test_memsan () { - # MemSan currently only available on Linux 64 bits - if uname -a | grep 'Linux.*x86_64' >/dev/null; then - msg "build: MSan (clang)" # ~ 1 min 20s - scripts/config.pl unset MBEDTLS_AESNI_C # memsan doesn't grok asm - CC=clang cmake -D CMAKE_BUILD_TYPE:String=MemSan . - make + msg "build: MSan (clang)" # ~ 1 min 20s + scripts/config.pl unset MBEDTLS_AESNI_C # memsan doesn't grok asm + CC=clang cmake -D CMAKE_BUILD_TYPE:String=MemSan . + make - msg "test: main suites (MSan)" # ~ 10s - make test + msg "test: main suites (MSan)" # ~ 10s + make test - msg "test: ssl-opt.sh (MSan)" # ~ 1 min - if_build_succeeded tests/ssl-opt.sh + msg "test: ssl-opt.sh (MSan)" # ~ 1 min + if_build_succeeded tests/ssl-opt.sh - # Optional part(s) + # Optional part(s) - if [ "$MEMORY" -gt 0 ]; then - msg "test: compat.sh (MSan)" # ~ 6 min 20s - if_build_succeeded tests/compat.sh - fi + if [ "$MEMORY" -gt 0 ]; then + msg "test: compat.sh (MSan)" # ~ 6 min 20s + if_build_succeeded tests/compat.sh fi } component_test_memcheck () { - # Only run if MemSan is not available - if ! uname -a | grep 'Linux.*x86_64' >/dev/null; then - msg "build: Release (clang)" - CC=clang cmake -D CMAKE_BUILD_TYPE:String=Release . - make + msg "build: Release (clang)" + CC=clang cmake -D CMAKE_BUILD_TYPE:String=Release . + make - msg "test: main suites valgrind (Release)" - make memcheck + msg "test: main suites valgrind (Release)" + make memcheck - # Optional part(s) - # Currently broken, programs don't seem to receive signals - # under valgrind on OS X + # Optional part(s) + # Currently broken, programs don't seem to receive signals + # under valgrind on OS X - if [ "$MEMORY" -gt 0 ]; then - msg "test: ssl-opt.sh --memcheck (Release)" - if_build_succeeded tests/ssl-opt.sh --memcheck - fi + if [ "$MEMORY" -gt 0 ]; then + msg "test: ssl-opt.sh --memcheck (Release)" + if_build_succeeded tests/ssl-opt.sh --memcheck + fi - if [ "$MEMORY" -gt 1 ]; then - msg "test: compat.sh --memcheck (Release)" - if_build_succeeded tests/compat.sh --memcheck - fi - - fi # MemSan + if [ "$MEMORY" -gt 1 ]; then + msg "test: compat.sh --memcheck (Release)" + if_build_succeeded tests/compat.sh --memcheck + fi } component_test_cmake_out_of_source () { From 7120f7788961ad8031f7c6758e3f4c5b5a38cc63 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 6 Jan 2019 20:15:26 +0000 Subject: [PATCH 11/22] all.sh: fix MAKEFLAGS setting MAKEFLAGS was set to -j if it was already set, instead of being set if not previously set as intended. So now all.sh will do parallel builds if invoked without MAKEFLAGS in the environment. --- tests/scripts/all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index cc5ab9569..2b9825fe1 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -110,7 +110,7 @@ pre_initialize_variables () { : ${ARMC6_BIN_DIR:=/usr/bin} # if MAKEFLAGS is not set add the -j option to speed up invocations of make - if [ -n "${MAKEFLAGS+set}" ]; then + if [ -z "${MAKEFLAGS+set}" ]; then export MAKEFLAGS="-j" fi From 3fbdd21ca5fbbe5214860788a6829a2d88795c6e Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 8 Jan 2019 23:33:45 +0100 Subject: [PATCH 12/22] Add conditional component inclusion facility Add a conditional execution facility: if a function support_xxx exists and returns false then component_xxx is not executed (except when the command line lists an explicit set of components to execute). Use this facility for the 64-bit-specific or amd64-specific components. --- tests/scripts/all.sh | 82 ++++++++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 30 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 2b9825fe1..6d3dea533 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -119,12 +119,23 @@ pre_initialize_variables () { # Parse the script with sed, because in sh there is no way to list # defined functions. ALL_COMPONENTS=$(sed -n 's/^ *component_\([0-9A-Z_a-z]*\) *().*/\1/p' <"$0") + + # Exclude components that are not supported on this platform. + SUPPORTED_COMPONENTS= + for component in $ALL_COMPONENTS; do + case $(type "support_$component" 2>&1) in + *' function'*) + if ! support_$component; then continue; fi;; + esac + SUPPORTED_COMPONENTS="$SUPPORTED_COMPONENTS $component" + done } -# Test whether $1 is excluded via $COMPONENTS (a space-separated list of -# wildcard patterns). +# Test whether $1 is excluded via the command line. is_component_excluded() { + # Is $1 excluded via $COMPONENTS (a space-separated list of wildcard + # patterns)? set -f for pattern in $COMPONENTS; do set +f @@ -299,7 +310,7 @@ pre_parse_command_line () { if [ $ALL_EXCEPT -ne 0 ]; then RUN_COMPONENTS= - for component in $ALL_COMPONENTS; do + for component in $SUPPORTED_COMPONENTS; do if ! is_component_excluded "$component"; then RUN_COMPONENTS="$RUN_COMPONENTS $component" fi @@ -738,38 +749,49 @@ component_test_make_shared () { make SHARED=1 all check } -case $(uname -m) in - amd64|x86_64) - component_test_m32_o0 () { - # Build once with -O0, to compile out the i386 specific inline assembly - msg "build: i386, make, gcc -O0 (ASan build)" # ~ 30s - scripts/config.pl full - make CC=gcc CFLAGS='-O0 -Werror -Wall -Wextra -m32 -fsanitize=address' +component_test_m32_o0 () { + # Build once with -O0, to compile out the i386 specific inline assembly + msg "build: i386, make, gcc -O0 (ASan build)" # ~ 30s + scripts/config.pl full + make CC=gcc CFLAGS='-O0 -Werror -Wall -Wextra -m32 -fsanitize=address' - msg "test: i386, make, gcc -O0 (ASan build)" - make test - } + msg "test: i386, make, gcc -O0 (ASan build)" + make test +} +support_test_m32_o0 () { + case $(uname -m) in + *64*) true;; + *) false;; + esac +} - component_test_m32_o1 () { - # Build again with -O1, to compile in the i386 specific inline assembly - msg "build: i386, make, gcc -O1 (ASan build)" # ~ 30s - scripts/config.pl full - make CC=gcc CFLAGS='-O1 -Werror -Wall -Wextra -m32 -fsanitize=address' +component_test_m32_o1 () { + # Build again with -O1, to compile in the i386 specific inline assembly + msg "build: i386, make, gcc -O1 (ASan build)" # ~ 30s + scripts/config.pl full + make CC=gcc CFLAGS='-O1 -Werror -Wall -Wextra -m32 -fsanitize=address' - msg "test: i386, make, gcc -O1 (ASan build)" - make test - } + msg "test: i386, make, gcc -O1 (ASan build)" + make test +} +support_test_m32_o1 () { + support_test_m32_o0 "$@" +} - component_test_mx32 () { - msg "build: 64-bit ILP32, make, gcc" # ~ 30s - scripts/config.pl full - make CC=gcc CFLAGS='-Werror -Wall -Wextra -mx32' +component_test_mx32 () { + msg "build: 64-bit ILP32, make, gcc" # ~ 30s + scripts/config.pl full + make CC=gcc CFLAGS='-Werror -Wall -Wextra -mx32' - msg "test: 64-bit ILP32, make, gcc" - make test - } - ;; -esac + msg "test: 64-bit ILP32, make, gcc" + make test +} +support_test_mx32 () { + case $(uname -m) in + amd64|x86_64) true;; + *) false;; + esac +} component_test_have_int32 () { msg "build: gcc, force 32-bit bignum limbs" From eb39b9b729c37027c924373c7722b4153dc23270 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 8 Jan 2019 23:41:00 +0100 Subject: [PATCH 13/22] Minor cleanups to component name gathering Bring the code in line with the version in Mbed TLS 2.16+. --- tests/scripts/all.sh | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 6d3dea533..89ccfc866 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -91,7 +91,6 @@ pre_initialize_variables () { CONFIG_H='include/mbedtls/config.h' CONFIG_BAK="$CONFIG_H.bak" - ALL_EXCEPT=0 MEMORY=0 FORCE=0 KEEP_GOING=0 @@ -137,7 +136,7 @@ is_component_excluded() # Is $1 excluded via $COMPONENTS (a space-separated list of wildcard # patterns)? set -f - for pattern in $COMPONENTS; do + for pattern in $COMMAND_LINE_COMPONENTS; do set +f case ${1#component_} in $pattern) return 0;; esac done @@ -269,14 +268,15 @@ check_tools() } pre_parse_command_line () { - COMPONENTS= + COMMAND_LINE_COMPONENTS= + all_except= while [ $# -gt 0 ]; do case "$1" in --armcc) RUN_ARMCC=1;; --armc5-bin-dir) shift; ARMC5_BIN_DIR="$1";; --armc6-bin-dir) shift; ARMC6_BIN_DIR="$1";; - --except) ALL_EXCEPT=1;; + --except) all_except=1;; --force|-f) FORCE=1;; --gnutls-cli) shift; GNUTLS_CLI="$1";; --gnutls-legacy-cli) shift; GNUTLS_LEGACY_CLI="$1";; @@ -302,24 +302,28 @@ pre_parse_command_line () { echo >&2 "Run $0 --help for usage." exit 120 ;; - *) - COMPONENTS="$COMPONENTS $1";; + *) COMMAND_LINE_COMPONENTS="$COMMAND_LINE_COMPONENTS $1";; esac shift done - if [ $ALL_EXCEPT -ne 0 ]; then + if [ -z "$COMMAND_LINE_COMPONENTS" ]; then + all_except=1 + fi + + # Build the list of components to run. + if [ -n "$all_except" ]; then RUN_COMPONENTS= for component in $SUPPORTED_COMPONENTS; do if ! is_component_excluded "$component"; then RUN_COMPONENTS="$RUN_COMPONENTS $component" fi done - elif [ -z "$COMPONENTS" ]; then - RUN_COMPONENTS="$ALL_COMPONENTS" else - RUN_COMPONENTS="$COMPONENTS" + RUN_COMPONENTS="$COMMAND_LINE_COMPONENTS" fi + + unset all_except } pre_check_git () { @@ -1031,7 +1035,7 @@ pre_print_configuration pre_check_tools cleanup -# Run all the test components. +# Run the requested tests. for component in $RUN_COMPONENTS; do run_component "component_$component" done From b3241cbea74d381cb6e44a7f044b1f03433e6ff3 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 8 Jan 2019 23:44:07 +0100 Subject: [PATCH 14/22] Add command line options to list available components --- tests/scripts/all.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 89ccfc866..038657296 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -153,6 +153,8 @@ By default, run all tests. With one or more COMPONENT, run only those. Special options: -h|--help Print this help and exit. + --list-all-components List all available test components and exit. + --list-components List components supported on this platform and exit. General options: -f|--force Force the tests to overwrite any modified files. @@ -284,6 +286,8 @@ pre_parse_command_line () { --gnutls-serv) shift; GNUTLS_SERV="$1";; --help|-h) usage; exit;; --keep-going|-k) KEEP_GOING=1;; + --list-all-components) printf '%s\n' $ALL_COMPONENTS; exit;; + --list-components) printf '%s\n' $SUPPORTED_COMPONENTS; exit;; --memory|-m) MEMORY=1;; --no-armcc) RUN_ARMCC=0;; --no-force) FORCE=0;; From 53084872ab71f513e197b82516e9c8c7f39ad760 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 9 Jan 2019 23:13:54 +0100 Subject: [PATCH 15/22] all.sh: only look for armcc if it is used Only look for armcc if component_build_armcc or component_build_yotta is to be executed, instead of requiring the option --no-armcc. You can still pass --no-armcc, but it's no longer required when listing components to run. With no list of components or an exclude list on the command line, --no-armcc is equivalent to having build_armcc in the exclude list. Omit the yotta pre-checks if the build_yotta component is not going to be executed. This makes --no-yotta equivalent to specifying a list of components to run that doesn't include build_yotta. --- tests/scripts/all.sh | 75 ++++++++++++++++++++++++++------------------ 1 file changed, 45 insertions(+), 30 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 038657296..b723f02b4 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -94,7 +94,6 @@ pre_initialize_variables () { MEMORY=0 FORCE=0 KEEP_GOING=0 - RUN_ARMCC=1 YOTTA=1 # Default commands, can be overriden by the environment @@ -272,10 +271,11 @@ check_tools() pre_parse_command_line () { COMMAND_LINE_COMPONENTS= all_except= + no_armcc= while [ $# -gt 0 ]; do case "$1" in - --armcc) RUN_ARMCC=1;; + --armcc) no_armcc=;; --armc5-bin-dir) shift; ARMC5_BIN_DIR="$1";; --armc6-bin-dir) shift; ARMC6_BIN_DIR="$1";; --except) all_except=1;; @@ -289,7 +289,7 @@ pre_parse_command_line () { --list-all-components) printf '%s\n' $ALL_COMPONENTS; exit;; --list-components) printf '%s\n' $SUPPORTED_COMPONENTS; exit;; --memory|-m) MEMORY=1;; - --no-armcc) RUN_ARMCC=0;; + --no-armcc) no_armcc=1;; --no-force) FORCE=0;; --no-keep-going) KEEP_GOING=0;; --no-memory) MEMORY=0;; @@ -315,6 +315,14 @@ pre_parse_command_line () { all_except=1 fi + # --no-armcc is a legacy option. The modern way is --except '*_armcc*'. + # Ignore it if components are listed explicitly on the command line. + if [ -n "$no_armcc" ] && [ -n "$all_except" ]; then + COMMAND_LINE_COMPONENTS="$COMMAND_LINE_COMPONENTS *_armcc*" + # --no-armcc also disables yotta. + COMMAND_LINE_COMPONENTS="$COMMAND_LINE_COMPONENTS *_yotta*" + fi + # Build the list of components to run. if [ -n "$all_except" ]; then RUN_COMPONENTS= @@ -328,6 +336,7 @@ pre_parse_command_line () { fi unset all_except + unset no_armcc } pre_check_git () { @@ -460,15 +469,22 @@ pre_check_tools () { check_tools "$OPENSSL" "$OPENSSL_LEGACY" "$GNUTLS_CLI" "$GNUTLS_SERV" \ "$GNUTLS_LEGACY_CLI" "$GNUTLS_LEGACY_SERV" "doxygen" "dot" \ "arm-none-eabi-gcc" "i686-w64-mingw32-gcc" - if [ $RUN_ARMCC -ne 0 ]; then - check_tools "$ARMC5_CC" "$ARMC5_AR" "$ARMC6_CC" "$ARMC6_AR" - fi + case $RUN_COMPONENTS in + *_armcc*|*_yotta*) + check_tools "$ARMC5_CC" "$ARMC5_AR" "$ARMC6_CC" "$ARMC6_AR";; + esac msg "info: output_env.sh" - OPENSSL="$OPENSSL" OPENSSL_LEGACY="$OPENSSL_LEGACY" GNUTLS_CLI="$GNUTLS_CLI" \ - GNUTLS_SERV="$GNUTLS_SERV" GNUTLS_LEGACY_CLI="$GNUTLS_LEGACY_CLI" \ - GNUTLS_LEGACY_SERV="$GNUTLS_LEGACY_SERV" ARMC5_CC="$ARMC5_CC" \ - ARMC6_CC="$ARMC6_CC" RUN_ARMCC="$RUN_ARMCC" scripts/output_env.sh + set env + set "$@" OPENSSL="$OPENSSL" OPENSSL_LEGACY="$OPENSSL_LEGACY" + set "$@" GNUTLS_CLI="$GNUTLS_CLI" GNUTLS_SERV="$GNUTLS_SERV" + set "$@" GNUTLS_LEGACY_CLI="$GNUTLS_LEGACY_CLI" GNUTLS_LEGACY_SERV="$GNUTLS_LEGACY_SERV" + case $RUN_COMPONENTS in + *_armcc*|*_yotta*) + set "$@" ARMC5_CC="$ARMC5_CC" ARMC6_CC="$ARMC6_CC" RUN_ARMCC=1;; + *) set "$@" RUN_ARMCC=0;; + esac + "$@" scripts/output_env.sh } @@ -524,12 +540,13 @@ component_check_doxygen_warnings () { ################################################################ component_build_yotta () { - if [ $RUN_ARMCC -ne 0 ] && [ $YOTTA -ne 0 ]; then - # Note - use of yotta is deprecated, and yotta also requires armcc to be on the - # path, and uses whatever version of armcc it finds there. - msg "build: create and build yotta module" # ~ 30s - record_status tests/scripts/yotta-build.sh - fi + # Note - use of yotta is deprecated, and yotta also requires armcc to be on the + # path, and uses whatever version of armcc it finds there. + msg "build: create and build yotta module" # ~ 30s + record_status tests/scripts/yotta-build.sh +} +support_build_yotta () { + [ $YOTTA -ne 0 ] } component_test_default_cmake_gcc_asan () { @@ -879,25 +896,23 @@ component_build_armcc () { scripts/config.pl unset MBEDTLS_MEMORY_BUFFER_ALLOC_C # calls exit scripts/config.pl unset MBEDTLS_PLATFORM_TIME_ALT # depends on MBEDTLS_HAVE_TIME - if [ $RUN_ARMCC -ne 0 ]; then - make CC="$ARMC5_CC" AR="$ARMC5_AR" WARNING_CFLAGS='--strict --c99' lib - make clean + make CC="$ARMC5_CC" AR="$ARMC5_AR" WARNING_CFLAGS='--strict --c99' lib + make clean - # ARM Compiler 6 - Target ARMv7-A - armc6_build_test "--target=arm-arm-none-eabi -march=armv7-a" + # ARM Compiler 6 - Target ARMv7-A + armc6_build_test "--target=arm-arm-none-eabi -march=armv7-a" - # ARM Compiler 6 - Target ARMv7-M - armc6_build_test "--target=arm-arm-none-eabi -march=armv7-m" + # ARM Compiler 6 - Target ARMv7-M + armc6_build_test "--target=arm-arm-none-eabi -march=armv7-m" - # ARM Compiler 6 - Target ARMv8-A - AArch32 - armc6_build_test "--target=arm-arm-none-eabi -march=armv8.2-a" + # ARM Compiler 6 - Target ARMv8-A - AArch32 + armc6_build_test "--target=arm-arm-none-eabi -march=armv8.2-a" - # ARM Compiler 6 - Target ARMv8-M - armc6_build_test "--target=arm-arm-none-eabi -march=armv8-m.main" + # ARM Compiler 6 - Target ARMv8-M + armc6_build_test "--target=arm-arm-none-eabi -march=armv8-m.main" - # ARM Compiler 6 - Target ARMv8-A - AArch64 - armc6_build_test "--target=aarch64-arm-none-eabi -march=armv8.2-a" - fi + # ARM Compiler 6 - Target ARMv8-A - AArch64 + armc6_build_test "--target=aarch64-arm-none-eabi -march=armv8.2-a" } component_test_allow_sha1 () { From 541fb1e33be9c8a290eb5d4af7c203c9c458b259 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 6 Jan 2019 22:40:00 +0000 Subject: [PATCH 16/22] all.sh: only check tools that are going to be used Don't require openssl, mingw, etc. if we aren't going to run a component that uses them. --- tests/scripts/all.sh | 62 +++++++++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 21 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index b723f02b4..1d92c8042 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -450,35 +450,55 @@ pre_print_configuration () { # Make sure the tools we need are available. pre_check_tools () { - ARMC5_CC="$ARMC5_BIN_DIR/armcc" - ARMC5_AR="$ARMC5_BIN_DIR/armar" - ARMC6_CC="$ARMC6_BIN_DIR/armclang" - ARMC6_AR="$ARMC6_BIN_DIR/armar" + # Build the list of variables to pass to output_env.sh. + set env - # To avoid setting OpenSSL and GnuTLS for each call to compat.sh and ssl-opt.sh - # we just export the variables they require - export OPENSSL_CMD="$OPENSSL" - export GNUTLS_CLI="$GNUTLS_CLI" - export GNUTLS_SERV="$GNUTLS_SERV" + case " $RUN_COMPONENTS " in + # Require OpenSSL and GnuTLS if running any tests (as opposed to + # only doing builds). Not all tests run OpenSSL and GnuTLS, but this + # is a good enough approximation in practice. + *" test_"*) + # To avoid setting OpenSSL and GnuTLS for each call to compat.sh + # and ssl-opt.sh, we just export the variables they require. + export OPENSSL_CMD="$OPENSSL" + export GNUTLS_CLI="$GNUTLS_CLI" + export GNUTLS_SERV="$GNUTLS_SERV" + # Avoid passing --seed flag in every call to ssl-opt.sh + if [ -n "${SEED-}" ]; then + export SEED + fi + set "$@" OPENSSL="$OPENSSL" OPENSSL_LEGACY="$OPENSSL_LEGACY" + set "$@" GNUTLS_CLI="$GNUTLS_CLI" GNUTLS_SERV="$GNUTLS_SERV" + set "$@" GNUTLS_LEGACY_CLI="$GNUTLS_LEGACY_CLI" + set "$@" GNUTLS_LEGACY_SERV="$GNUTLS_LEGACY_SERV" + check_tools "$OPENSSL" "$OPENSSL_LEGACY" \ + "$GNUTLS_CLI" "$GNUTLS_SERV" \ + "$GNUTLS_LEGACY_CLI" "$GNUTLS_LEGACY_SERV" + ;; + esac - # Avoid passing --seed flag in every call to ssl-opt.sh - if [ -n "${SEED-}" ]; then - export SEED - fi + case " $RUN_COMPONENTS " in + *_doxygen[_\ ]*) check_tools "doxygen" "dot";; + esac - check_tools "$OPENSSL" "$OPENSSL_LEGACY" "$GNUTLS_CLI" "$GNUTLS_SERV" \ - "$GNUTLS_LEGACY_CLI" "$GNUTLS_LEGACY_SERV" "doxygen" "dot" \ - "arm-none-eabi-gcc" "i686-w64-mingw32-gcc" - case $RUN_COMPONENTS in + case " $RUN_COMPONENTS " in + *_arm_none_eabi_gcc[_\ ]*) check_tools "arm-none-eabi-gcc";; + esac + + case " $RUN_COMPONENTS " in + *_mingw[_\ ]*) check_tools "i686-w64-mingw32-gcc";; + esac + + case " $RUN_COMPONENTS " in *_armcc*|*_yotta*) + ARMC5_CC="$ARMC5_BIN_DIR/armcc" + ARMC5_AR="$ARMC5_BIN_DIR/armar" + ARMC6_CC="$ARMC6_BIN_DIR/armclang" + ARMC6_AR="$ARMC6_BIN_DIR/armar" check_tools "$ARMC5_CC" "$ARMC5_AR" "$ARMC6_CC" "$ARMC6_AR";; esac msg "info: output_env.sh" - set env - set "$@" OPENSSL="$OPENSSL" OPENSSL_LEGACY="$OPENSSL_LEGACY" - set "$@" GNUTLS_CLI="$GNUTLS_CLI" GNUTLS_SERV="$GNUTLS_SERV" - set "$@" GNUTLS_LEGACY_CLI="$GNUTLS_LEGACY_CLI" GNUTLS_LEGACY_SERV="$GNUTLS_LEGACY_SERV" case $RUN_COMPONENTS in *_armcc*|*_yotta*) set "$@" ARMC5_CC="$ARMC5_CC" ARMC6_CC="$ARMC6_CC" RUN_ARMCC=1;; From 2906a0ae8a112b3edd9d857e7ed8bec52fceb7f8 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 9 Jan 2019 22:29:17 +0100 Subject: [PATCH 17/22] all.sh: Update the maintainer documentation --- tests/scripts/all.sh | 49 +++++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 1d92c8042..e03efe8ef 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -55,21 +55,46 @@ # Notes for maintainers # --------------------- # +# The bulk of the code is organized into functions that follow one of the +# following naming conventions: +# * pre_XXX: things to do before running the tests, in order. +# * component_XXX: independent components. They can be run in any order. +# * component_check_XXX: quick tests that aren't worth parallelizing. +# * component_build_XXX: build things but don't run them. +# * component_test_XXX: build and test. +# * support_XXX: if support_XXX exists and returns false then +# component_XXX is not run by default. +# * post_XXX: things to do after running the tests. +# * other: miscellaneous support functions. +# +# Each component must start by invoking `msg` with a short informative message. +# +# The framework performs some cleanup tasks after each component. This +# means that components can assume that the working directory is in a +# cleaned-up state, and don't need to perform the cleanup themselves. +# * Run `make clean`. +# * Restore `include/mbedtks/config.h` from a backup made before running +# the component. +# * Check out `Makefile`, `library/Makefile`, `programs/Makefile` and +# `tests/Makefile` from git. This cleans up after an in-tree use of +# CMake. +# +# Any command that is expected to fail must be protected so that the +# script keeps running in --keep-going mode despite `set -e`. In keep-going +# mode, if a protected command fails, this is logged as a failure and the +# script will exit with a failure status once it has run all components. +# Commands can be protected in any of the following ways: +# * `make` is a function which runs the `make` command with protection. +# Note that you must write `make VAR=value`, not `VAR=value make`, +# because the `VAR=value make` syntax doesn't work with functions. +# * Put `report_status` before the command to protect it. +# * Put `if_build_successful` before a command. This protects it, and +# additionally skips it if a prior invocation of `make` in the same +# component failed. +# # The tests are roughly in order from fastest to slowest. This doesn't # have to be exact, but in general you should add slower tests towards # the end and fast checks near the beginning. -# -# Sanity checks have the following form: -# 1. msg "short description of what is about to be done" -# 2. run sanity check (failure stops the script) -# -# Build or build-and-test steps have the following form: -# 1. msg "short description of what is about to be done" -# 2. cleanup -# 3. preparation (config.pl, cmake, ...) (failure stops the script) -# 4. make -# 5. Run tests if relevant. All tests must be prefixed with -# if_build_successful for the sake of --keep-going. From c9663b16852927c2fa43cd2f747a024ede213898 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 9 Jan 2019 22:30:01 +0100 Subject: [PATCH 18/22] Fix sometimes-spurious warning about changed config.h After backing up and restoring config.h, `git diff-files` may report it as potentially-changed because it isn't sure whether the index is up to date. To avoid this, make sure that the git index is up-to-date. This fixes the warning about changed config.h that you might get when you run all.sh twice in succession, yet if you run `git status` or `git diff` everything seems up to date and you no longer get the warning because these git commands update the index. https://stackoverflow.com/questions/36367190/git-diff-files-output-changes-after-git-status --- tests/scripts/all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index e03efe8ef..b8a7da55d 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -387,7 +387,7 @@ pre_check_git () { exit 1 fi - if ! git diff-files --quiet include/mbedtls/config.h; then + if ! git diff --quiet include/mbedtls/config.h; then err_msg "Warning - the configuration file 'include/mbedtls/config.h' has been edited. " echo "You can either delete or preserve your work, or force the test by rerunning the" echo "script as: $0 --force" From c780095901ec2d96fae274e69afce7aab5ad5c4f Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 9 Jan 2019 23:14:09 +0100 Subject: [PATCH 19/22] Delete $OUT_OF_SOURCE_DIR under --force even without Yotta The deletion of "$OUT_OF_SOURCE_DIR" had mistakenly been lumped together with Yotta. --- tests/scripts/all.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index b8a7da55d..1ae6a3d8f 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -366,8 +366,9 @@ pre_parse_command_line () { pre_check_git () { if [ $FORCE -eq 1 ]; then + rm -rf "$OUT_OF_SOURCE_DIR" if [ $YOTTA -eq 1 ]; then - rm -rf yotta/module "$OUT_OF_SOURCE_DIR" + rm -rf yotta/module fi git checkout-index -f -q $CONFIG_H cleanup From 30bc3851240e4a711875dc2cd107345e271878e9 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 9 Jan 2019 23:25:25 +0100 Subject: [PATCH 20/22] Add missing protection on __aeabi_uldiv check under --keep-going Partial backport of 2adb375c50e2db5f44dd1ce8b7cb4b33b035563a "Add option to avoid 64-bit multiplication" --- tests/scripts/all.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 1ae6a3d8f..50b3750c5 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -459,6 +459,13 @@ if_build_succeeded () { fi } +# to be used instead of ! for commands run with +# record_status or if_build_succeeded +not() { + ! "$@" +} + + pre_print_configuration () { msg "info: $0 configuration" echo "MEMORY: $MEMORY" @@ -920,7 +927,7 @@ component_build_arm_none_eabi_gcc_no_udbl_division () { scripts/config.pl set MBEDTLS_NO_UDBL_DIVISION make CC=arm-none-eabi-gcc AR=arm-none-eabi-ar LD=arm-none-eabi-ld CFLAGS='-Werror -Wall -Wextra' lib echo "Checking that software 64-bit division is not required" - ! grep __aeabi_uldiv library/*.o + if_build_succeeded not grep __aeabi_uldiv library/*.o } component_build_armcc () { From ff7238f4ad7eb8c8114073c2505334b59b6736d4 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 10 Jan 2019 00:05:18 +0100 Subject: [PATCH 21/22] Support wildcard patterns with a positive list of components to run Wildcard patterns now work with command line COMPONENT arguments without --except as well as with. You can now run e.g. `all.sh "check_*` to run all the sanity checks. --- tests/scripts/all.sh | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 50b3750c5..83f6cc75e 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -154,11 +154,9 @@ pre_initialize_variables () { done } -# Test whether $1 is excluded via the command line. -is_component_excluded() +# Test whether the component $1 is included in the command line patterns. +is_component_included() { - # Is $1 excluded via $COMPONENTS (a space-separated list of wildcard - # patterns)? set -f for pattern in $COMMAND_LINE_COMPONENTS; do set +f @@ -174,6 +172,13 @@ usage() Usage: $0 [OPTION]... [COMPONENT]... Run mbedtls release validation tests. By default, run all tests. With one or more COMPONENT, run only those. +COMPONENT can be the name of a component or a shell wildcard pattern. + +Examples: + $0 "check_*" + Run all sanity checks. + $0 --no-armcc --except test_memsan + Run everything except builds that require armcc and MemSan. Special options: -h|--help Print this help and exit. @@ -185,11 +190,8 @@ General options: -k|--keep-going Run all tests and report errors at the end. -m|--memory Additional optional memory tests. --armcc Run ARM Compiler builds (on by default). - --except If some components are passed on the command line, - run all the tests except for these components. In - this mode, you can pass shell wildcard patterns as - component names, e.g. "$0 --except 'test_*'" to - exclude all components that run tests. + --except Exclude the COMPONENTs listed on the command line, + instead of running only those. --no-armcc Skip ARM Compiler builds. --no-force Refuse to overwrite modified files (default). --no-keep-going Stop at the first error (default). @@ -295,7 +297,7 @@ check_tools() pre_parse_command_line () { COMMAND_LINE_COMPONENTS= - all_except= + all_except=0 no_armcc= while [ $# -gt 0 ]; do @@ -336,29 +338,26 @@ pre_parse_command_line () { shift done + # With no list of components, run everything. if [ -z "$COMMAND_LINE_COMPONENTS" ]; then all_except=1 fi # --no-armcc is a legacy option. The modern way is --except '*_armcc*'. # Ignore it if components are listed explicitly on the command line. - if [ -n "$no_armcc" ] && [ -n "$all_except" ]; then + if [ -n "$no_armcc" ] && [ $all_except -eq 1 ]; then COMMAND_LINE_COMPONENTS="$COMMAND_LINE_COMPONENTS *_armcc*" # --no-armcc also disables yotta. COMMAND_LINE_COMPONENTS="$COMMAND_LINE_COMPONENTS *_yotta*" fi # Build the list of components to run. - if [ -n "$all_except" ]; then - RUN_COMPONENTS= - for component in $SUPPORTED_COMPONENTS; do - if ! is_component_excluded "$component"; then - RUN_COMPONENTS="$RUN_COMPONENTS $component" - fi - done - else - RUN_COMPONENTS="$COMMAND_LINE_COMPONENTS" - fi + RUN_COMPONENTS= + for component in $SUPPORTED_COMPONENTS; do + if is_component_included "$component"; [ $? -eq $all_except ]; then + RUN_COMPONENTS="$RUN_COMPONENTS $component" + fi + done unset all_except unset no_armcc From 9f55364ec744da38a3b6a0263a0ad0bef8ce1165 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 10 Jan 2019 00:11:42 +0100 Subject: [PATCH 22/22] Rename test_memcheck to test_valgrind Valgrind is what it does. `memcheck` is how it's implemented. --- tests/scripts/all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 83f6cc75e..070e623e6 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -1010,7 +1010,7 @@ component_test_memsan () { fi } -component_test_memcheck () { +component_test_valgrind () { msg "build: Release (clang)" CC=clang cmake -D CMAKE_BUILD_TYPE:String=Release . make