Merge remote-tracking branch 'myfork-public/development' into merge-crypto-development-20191115

First deal with deleted files.

* Files deleted by us: keep them deleted.
* Files deleted by them, whether modified by us or not: keep our version.

```
git rm $(git status -s | sed -n 's/^DU //p')
git reset -- $(git status -s | sed -n 's/^D  //p')
git checkout -- $(git status -s | sed -n 's/^ D //p')
git add -- $(git status -s | sed -n 's/^UD //p')
```

Individual files with conflicts:

* `3rdparty/everest/library/Hacl_Curve25519_joined.c`: spurious conflict because git mistakenly identified this file as a rename. Keep our version.
* `README.md`: conflict due to their change in a paragraph that doesn't exist in our version. Keep our version of this paragraph.
* `docs/architecture/Makefile`: near-identical additions. Adapt the definition of `all_markdown` and include the clean target.
* `doxygen/input/docs_mainpage.h`: conflict in the version number. Keep our version number.
* `include/mbedtls/config.h`: two delete/modify conflicts. Keep the removed chunks out.
* `library/CMakeLists.txt`: discard all their changes as they are not relevant.
* `library/Makefile`:
    * Discard the added chunk about the crypto submodule starting with `INCLUDING_FROM_MBEDTLS:=1`.
    * delete/modify: keep the removed chunk out.
    * library build: This is almost delete/modify. Their changes are mostly not applicable. Do keep the `libmbedcrypto.$(DLEXT): | libmbedcrypto.a` order dependency.
    * `.c.o`: `-o` was added on both sides but in a different place. Change to their place.
* `library/error.c`: to be regenerated.
* `library/version_features.c`: to be regenerated.
* `programs/Makefile`: Most of the changes are not relevant. The one relevant change is in the `clean` target for Windows; adapt it by removing `/S` from our version.
* `programs/test/query_config.c`: to be regenerated.
* `scripts/config.py`: added in parallel on both sides. Keep our version.
* `scripts/footprint.sh`: parallel changes. Keep our version.
* `scripts/generate_visualc_files.pl`: one delete/modify conflict. Keep the removed chunks out.
* `tests/Makefile`: discard all of their changes.
* `tests/scripts/all.sh`:
    * `pre_initialize_variables` add `append_outcome`: add it.
    * `pre_initialize_variables` add `ASAN_CFLAGS`: already there, keep our version.
    * `pre_parse_command_line` add `--no-append-outcome`: add it.
    * `pre_parse_command_line` add `--outcome-file`: add it.
    * `pre_print_configuration`: add `MBEDTLS_TEST_OUTCOME_FILE`.
    * Several changes in SSL-specific components: keep our version without them.
    * Several changes where `config.pl` was changed to `config.py` and there was an adjacent difference: keep our version.
    * Changes regarding the inclusion of `MBEDTLS_MEMORY_xxx`: ignore them here, they will be normalized in a subsequent commit.
    * `component_test_full_cmake_gcc_asan`: add it without the TLS tests.
    * `component_test_no_use_psa_crypto_full_cmake_asan`: keep the fixed `msg`, discard other changes.
    * `component_test_memory_buffer_allocator_backtrace`, `component_test_memory_buffer_allocator`: add them without the TLS tests.
    * `component_test_m32_everest`: added in parallel on both sides. Keep our version.
* `tests/scripts/check-names.sh`, `tests/scripts/list-enum-consts.pl`, `tests/scripts/list-identifiers.sh`, ``tests/scripts/list-macros.sh`: discard all of their changes.
* `tests/scripts/test-ref-configs.pl`: the change in the conflict is not relevant, so keep our version there.
* `visualc/VS2010/*.vcxproj`: to be regenerated.

Regenerate files:

```
scripts/generate_visualc_files.pl
git add visualc/VS2010/*.vcxproj
scripts/generate_errors.pl
git add library/error.c
scripts/generate_features.pl
git add library/version_features.c
scripts/generate_query_config.pl
git add programs/test/query_config.c
```

Rejected changes in non-conflicting files:

* `CMakeLists.txt`: discard their addition which has already been side-ported.
* `doxygen/mbedtls.doxyfile`: keep the version number change. Discard the changes related to `../crypto` paths.

Keep the following changes after examination:

* `.travis.yml`: all of their changes are relevant.
* `include/mbedtls/error.h`: do keep their changes. Even though Crypto doesn't use TLS errors, it must not encroach on TLS's allocated numbers.
* `tests/scripts/check-test-cases.py`: keep the code dealing with `ssl-opt.sh`. It works correctly when the file is not present.

.travis.yml

@@ -5,12 +5,6 @@ compiler:
 sudo: false
 cache: ccache
 
-# blocklist
-branches:
-  except:
-  - development-psa
-  - coverity_scan
-
 script:
 - tests/scripts/recursion.pl library/*.c
 - tests/scripts/check-generated-files.sh
@@ -27,7 +21,7 @@ script:
 env:
   global:
     - SEED=1
-    - secure: "barHldniAfXyoWOD/vcO+E6/Xm4fmcaUoC9BeKW+LwsHqlDMLvugaJnmLXkSpkbYhVL61Hzf3bo0KPJn88AFc5Rkf8oYHPjH4adMnVXkf3B9ghHCgznqHsAH3choo6tnPxaFgOwOYmLGb382nQxfE5lUdvnM/W/psQjWt66A1+k="
+    - secure: "FrI5d2s+ckckC17T66c8jm2jV6i2DkBPU5nyWzwbedjmEBeocREfQLd/x8yKpPzLDz7ghOvr+/GQvsPPn0dVkGlNzm3Q+hGHc/ujnASuUtGrcuMM+0ALnJ3k4rFr9xEvjJeWb4SmhJO5UCAZYvTItW4k7+bj9L+R6lt3TzQbXzg="
 
 addons:
   apt:
@@ -37,7 +31,7 @@ addons:
   coverity_scan:
     project:
       name: "ARMmbed/mbedtls"
-    notification_email: simon.butcher@arm.com
+    notification_email: support-mbedtls@arm.com
     build_command_prepend:
     build_command: make
     branch_pattern: coverity_scan

docs/architecture/Makefile

@@ -5,6 +5,7 @@ default: all
 all_markdown = \
 	       mbed-crypto-storage-specification.md \
 	       testing/driver-interface-test-strategy.md \
+	       testing/test-framework.md \
 	       # This line is intentionally left blank
 
 html: $(all_markdown:.md=.html)

docs/architecture/testing/test-framework.md

@@ -0,0 +1,58 @@
+# Mbed TLS test framework
+
+This document is an overview of the Mbed TLS test framework and test tools.
+
+This document is incomplete. You can help by expanding it.
+
+## Unit tests
+
+See <https://tls.mbed.org/kb/development/test_suites>
+
+### Unit test descriptions
+
+Each test case has a description which succinctly describes for a human audience what the test does. The first non-comment line of each paragraph in a `.data` file is the test description. The following rules and guidelines apply:
+
+* Test descriptions may not contain semicolons, line breaks and other control characters, or non-ASCII characters. <br>
+  Rationale: keep the tools that process test descriptions (`generate_test_code.py`, [outcome file](#outcome-file) tools) simple.
+* Test descriptions must be unique within a `.data` file. If you can't think of a better description, the convention is to append `#1`, `#2`, etc. <br>
+  Rationale: make it easy to relate a failure log to the test data. Avoid confusion between cases in the [outcome file](#outcome-file).
+* Test descriptions should be a maximum of **66 characters**. <br>
+  Rationale: 66 characters is what our various tools assume (leaving room for 14 more characters on an 80-column line). Longer descriptions may be truncated or may break a visual alignment. <br>
+  We have a lot of test cases with longer descriptions, but they should be avoided. At least please make sure that the first 66 characters describe the test uniquely.
+* Make the description descriptive. “foo: x=2, y=4” is more descriptive than “foo #2”. “foo: 0<x<y, both even” is even better if these inequalities and parities are why this particular test data was chosen.
+* Avoid changing the description of an existing test case without a good reason. This breaks the tracking of failures across CI runs, since this tracking is based on the descriptions.
+
+`tests/scripts/check-test-cases.py` enforces some rules and warns if some guidelines are violated.
+
+## TLS tests
+
+### SSL extension tests
+
+#### SSL test case descriptions
+
+Each test case in `ssl-opt.sh` has a description which succinctly describes for a human audience what the test does. The test description is the first parameter to `run_tests`.
+
+The same rules and guidelines apply as for [unit test descriptions](#unit-test-descriptions). In addition, the description must be written on the same line as `run_test`, in double quotes, for the sake of `check-test-cases.py`.
+
+## Running tests
+
+### Outcome file
+
+#### Generating an outcome file
+
+Unit tests and `ssl-opt.sh` record the outcome of each test case in a **test outcome file**. This feature is enabled if the environment variable `MBEDTLS_TEST_OUTCOME_FILE` is set. Set it to the path of the desired file.
+
+If you run `all.sh --outcome-file test-outcome.csv`, this collects the outcome of all the test cases in `test-outcome.csv`.
+
+#### Outcome file format
+
+The outcome file is in a CSV format using `;` (semicolon) as the delimiter and no quoting. This means that fields may not contain newlines or semicolons. There is no title line.
+
+The outcome file has 6 fields:
+
+* **Platform**: a description of the platform, e.g. `Linux-x86_64` or `Linux-x86_64-gcc7-msan`.
+* **Configuration**: a unique description of the configuration (`config.h`).
+* **Test suite**: `test_suite_xxx` or `ssl-opt`.
+* **Test case**: the description of the test case.
+* **Result**: one of `PASS`, `SKIP` or `FAIL`.
+* **Cause**: more information explaining the result.

doxygen/mbedtls.doxyfile

@@ -28,7 +28,7 @@ DOXYFILE_ENCODING      = UTF-8
 # identify the project. Note that if you do not use Doxywizard you need
 # to put quotes around the project name if it contains spaces.
 
-PROJECT_NAME           = "mbed TLS v2.17.0"
+PROJECT_NAME           = "mbed TLS v2.19.1"
 
 # The PROJECT_NUMBER tag can be used to enter a project or revision number.
 # This could be handy for archiving the generated documentation or

include/mbedtls/check_config.h

@@ -237,6 +237,14 @@
 #error "MBEDTLS_MEMORY_BUFFER_ALLOC_C defined, but not all prerequisites"
 #endif
 
+#if defined(MBEDTLS_MEMORY_BACKTRACE) && !defined(MBEDTLS_MEMORY_BUFFER_ALLOC_C)
+#error "MBEDTLS_MEMORY_BACKTRACE defined, but not all prerequesites"
+#endif
+
+#if defined(MBEDTLS_MEMORY_DEBUG) && !defined(MBEDTLS_MEMORY_BUFFER_ALLOC_C)
+#error "MBEDTLS_MEMORY_DEBUG defined, but not all prerequesites"
+#endif
+
 #if defined(MBEDTLS_PADLOCK_C) && !defined(MBEDTLS_HAVE_ASM)
 #error "MBEDTLS_PADLOCK_C defined, but not all prerequisites"
 #endif

include/mbedtls/error.h

@@ -100,6 +100,7 @@
  * ECP       4   10 (Started from top)
  * MD        5   5
  * HKDF      5   1 (Started from top)
+ * SSL       5   1 (Started from 0x5F00)
  * CIPHER    6   8 (Started from 0x6080)
  * SSL       6   24 (Started from top, plus 0x6000)
  * SSL       7   32

include/mbedtls/version.h

@@ -39,17 +39,17 @@
  * Major, Minor, Patchlevel
  */
 #define MBEDTLS_VERSION_MAJOR  2
-#define MBEDTLS_VERSION_MINOR  17
+#define MBEDTLS_VERSION_MINOR  19
-#define MBEDTLS_VERSION_PATCH  0
+#define MBEDTLS_VERSION_PATCH  1
 
 /**
  * The single version number has the following structure:
  *    MMNNPP00
  *    Major version | Minor version | Patch version
  */
-#define MBEDTLS_VERSION_NUMBER         0x02110000
+#define MBEDTLS_VERSION_NUMBER         0x02130100
-#define MBEDTLS_VERSION_STRING         "2.17.0"
+#define MBEDTLS_VERSION_STRING         "2.19.1"
-#define MBEDTLS_VERSION_STRING_FULL    "mbed TLS 2.17.0"
+#define MBEDTLS_VERSION_STRING_FULL    "mbed TLS 2.19.1"
 
 #if defined(MBEDTLS_VERSION_C)
 

library/Makefile

@@ -146,9 +146,11 @@ libmbedcrypto.dll: $(OBJS_CRYPTO)
 	echo "  LD    $@"
 	$(CC) -shared -Wl,-soname,$@ -Wl,--out-implib,$@.a -o $@ $(OBJS_CRYPTO) -lws2_32 -lwinmm -lgdi32 -static-libgcc $(LOCAL_LDFLAGS) $(LDFLAGS)
 
+libmbedcrypto.$(DLEXT): | libmbedcrypto.a
+
 .c.o:
 	echo "  CC    $<"
-	$(CC) $(LOCAL_CFLAGS) $(CFLAGS) -c $< -o $@
+	$(CC) $(LOCAL_CFLAGS) $(CFLAGS) -o $@ -c $<
 
 clean:
 ifndef WINDOWS

programs/Makefile

@@ -249,8 +249,8 @@ ifndef WINDOWS
 	rm -f $(APPS) $(EXTRA_GENERATED)
 	-rm -f test/cpp_dummy_build$(EXEXT)
 else
-	if exist *.o del /S /Q /F *.o
+	if exist *.o del /Q /F *.o
-	if exist *.exe del /S /Q /F *.exe
+	if exist *.exe del /Q /F *.exe
 	if exist $(EXTRA_GENERATED) del /S /Q /F $(EXTRA_GENERATED)
 endif
 

programs/pkey/pk_sign.c

@@ -60,7 +60,6 @@ int main( void )
 #include <stdio.h>
 #include <string.h>
 
-
 int main( int argc, char *argv[] )
 {
     FILE *f;

tests/scripts/all.sh

@@ -112,9 +112,15 @@ pre_initialize_variables () {
     CONFIG_H='include/mbedtls/config.h'
     CONFIG_BAK="$CONFIG_H.bak"
 
+    append_outcome=0
     FORCE=0
     KEEP_GOING=0
 
+    : ${MBEDTLS_TEST_OUTCOME_FILE=}
+    : ${MBEDTLS_TEST_PLATFORM="$(uname -s | tr -c \\n0-9A-Za-z _)-$(uname -m | tr -c \\n0-9A-Za-z _)"}
+    export MBEDTLS_TEST_OUTCOME_FILE
+    export MBEDTLS_TEST_PLATFORM
+
     # Default commands, can be overridden by the environment
     : ${OUT_OF_SOURCE_DIR:=./mbedtls_out_of_source_build}
     : ${ARMC5_BIN_DIR:=/usr/bin}
@@ -183,14 +189,18 @@ General options:
   -f|--force            Force the tests to overwrite any modified files.
   -k|--keep-going       Run all tests and report errors at the end.
   -m|--memory           Additional optional memory tests.
+     --append-outcome   Append to the outcome file (if used).
      --armcc            Run ARM Compiler builds (on by default).
      --except           Exclude the COMPONENTs listed on the command line,
                         instead of running only those.
+     --no-append-outcome    Write a new outcome file and analyze it (default).
      --no-armcc         Skip ARM Compiler builds.
      --no-force         Refuse to overwrite modified files (default).
      --no-keep-going    Stop at the first error (default).
      --no-memory        No additional memory tests (default).
      --out-of-source-dir=<path>  Directory used for CMake out-of-source build tests.
+     --outcome-file=<path>  File where test outcomes are written (not done if
+                            empty; default: \$MBEDTLS_TEST_OUTCOME_FILE).
      --random-seed      Use a random seed value for randomized tests (default).
   -r|--release-test     Run this script in release mode. This fixes the seed value to 1.
   -s|--seed             Integer seed value to use for this test run.
@@ -309,6 +319,7 @@ pre_parse_command_line () {
     # all.sh will still run and work properly.
     while [ $# -gt 0 ]; do
         case "$1" in
+            --append-outcome) append_outcome=1;;
             --armcc) no_armcc=;;
             --armc5-bin-dir) shift; ARMC5_BIN_DIR="$1";;
             --armc6-bin-dir) shift; ARMC6_BIN_DIR="$1";;
@@ -323,6 +334,7 @@ pre_parse_command_line () {
             --list-all-components) printf '%s\n' $ALL_COMPONENTS; exit;;
             --list-components) printf '%s\n' $SUPPORTED_COMPONENTS; exit;;
             --memory|-m) ;;
+            --no-append-outcome) append_outcome=0;;
             --no-armcc) no_armcc=1;;
             --no-force) FORCE=0;;
             --no-keep-going) KEEP_GOING=0;;
@@ -330,6 +342,7 @@ pre_parse_command_line () {
             --openssl) shift;;
             --openssl-legacy) shift;;
             --openssl-next) shift;;
+            --outcome-file) shift; MBEDTLS_TEST_OUTCOME_FILE="$1";;
             --out-of-source-dir) shift; OUT_OF_SOURCE_DIR="$1";;
             --random-seed) ;;
             --release-test|-r) ;;
@@ -464,9 +477,19 @@ not() {
     ! "$@"
 }
 
+pre_prepare_outcome_file () {
+    case "$MBEDTLS_TEST_OUTCOME_FILE" in
+      [!/]*) MBEDTLS_TEST_OUTCOME_FILE="$PWD/$MBEDTLS_TEST_OUTCOME_FILE";;
+    esac
+    if [ -n "$MBEDTLS_TEST_OUTCOME_FILE" ] && [ "$append_outcome" -eq 0 ]; then
+        rm -f "$MBEDTLS_TEST_OUTCOME_FILE"
+    fi
+}
+
 pre_print_configuration () {
     msg "info: $0 configuration"
     echo "FORCE: $FORCE"
+    echo "MBEDTLS_TEST_OUTCOME_FILE: ${MBEDTLS_TEST_OUTCOME_FILE:-(none)}"
     echo "ARMC5_BIN_DIR: $ARMC5_BIN_DIR"
     echo "ARMC6_BIN_DIR: $ARMC6_BIN_DIR"
 }
@@ -528,32 +551,37 @@ pre_check_tools () {
 # Indicative running times are given for reference.
 
 component_check_recursion () {
-    msg "test: recursion.pl" # < 1s
+    msg "Check: recursion.pl" # < 1s
     record_status tests/scripts/recursion.pl library/*.c
 }
 
 component_check_generated_files () {
-    msg "test: freshness of generated source files" # < 1s
+    msg "Check: freshness of generated source files" # < 1s
     record_status tests/scripts/check-generated-files.sh
 }
 
 component_check_doxy_blocks () {
-    msg "test: doxygen markup outside doxygen blocks" # < 1s
+    msg "Check: doxygen markup outside doxygen blocks" # < 1s
     record_status tests/scripts/check-doxy-blocks.pl
 }
 
 component_check_files () {
-    msg "test: check-files.py" # < 1s
+    msg "Check: file sanity checks (permissions, encodings)" # < 1s
     record_status tests/scripts/check-files.py
 }
 
 component_check_names () {
-    msg "test/build: declared and exported names" # < 3s
+    msg "Check: declared and exported names (builds the library)" # < 3s
     record_status tests/scripts/check-names.sh -v
 }
 
+component_check_test_cases () {
+    msg "Check: test case descriptions" # < 1s
+    record_status tests/scripts/check-test-cases.py
+}
+
 component_check_doxygen_warnings () {
-    msg "test: doxygen warnings" # ~ 3s
+    msg "Check: doxygen warnings (builds the documentation)" # ~ 3s
     record_status tests/scripts/doxygen.sh
 }
 
@@ -565,12 +593,18 @@ component_check_doxygen_warnings () {
 component_test_default_out_of_box () {
     msg "build: make, default config (out-of-box)" # ~1min
     make
+    # Disable fancy stuff
+    SAVE_MBEDTLS_TEST_OUTCOME_FILE="$MBEDTLS_TEST_OUTCOME_FILE"
+    unset MBEDTLS_TEST_OUTCOME_FILE
 
     msg "test: main suites make, default config (out-of-box)" # ~10s
     make test
 
     msg "selftest: make, default config (out-of-box)" # ~10s
     programs/test/selftest
+
+    export MBEDTLS_TEST_OUTCOME_FILE="$SAVE_MBEDTLS_TEST_OUTCOME_FILE"
+    unset SAVE_MBEDTLS_TEST_OUTCOME_FILE
 }
 
 component_test_default_cmake_gcc_asan () {
@@ -582,6 +616,16 @@ component_test_default_cmake_gcc_asan () {
     make test
 }
 
+component_test_full_cmake_gcc_asan () {
+    msg "build: full config, cmake, gcc, ASan"
+    scripts/config.py full
+    CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan .
+    make
+
+    msg "test: main suites (inc. selftests) (full config, ASan build)"
+    make test
+}
+
 component_test_ref_configs () {
     msg "test/build: ref-configs (ASan build)" # ~ 6 min 20s
     CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan .
@@ -712,7 +756,7 @@ component_build_default_make_gcc_and_cxx () {
 
 component_test_no_use_psa_crypto_full_cmake_asan() {
     # full minus MBEDTLS_USE_PSA_CRYPTO: run the same set of tests as basic-build-test.sh
-    msg "build: cmake, full config + MBEDTLS_USE_PSA_CRYPTO, ASan"
+    msg "build: cmake, full config minus MBEDTLS_USE_PSA_CRYPTO, ASan"
     scripts/config.py full
     scripts/config.py unset MBEDTLS_MEMORY_BUFFER_ALLOC_C
     scripts/config.py set MBEDTLS_ECP_RESTARTABLE  # not using PSA, so enable restartable ECC
@@ -840,6 +884,30 @@ component_test_malloc_0_null () {
     if_build_succeeded programs/test/selftest calloc
 }
 
+component_test_memory_buffer_allocator_backtrace () {
+    msg "build: default config with memory buffer allocator and backtrace enabled"
+    scripts/config.py set MBEDTLS_MEMORY_BUFFER_ALLOC_C
+    scripts/config.py set MBEDTLS_PLATFORM_MEMORY
+    scripts/config.py set MBEDTLS_MEMORY_BACKTRACE
+    scripts/config.py set MBEDTLS_MEMORY_DEBUG
+    CC=gcc cmake .
+    make
+
+    msg "test: MBEDTLS_MEMORY_BUFFER_ALLOC_C and MBEDTLS_MEMORY_BACKTRACE"
+    make test
+}
+
+component_test_memory_buffer_allocator () {
+    msg "build: default config with memory buffer allocator"
+    scripts/config.py set MBEDTLS_MEMORY_BUFFER_ALLOC_C
+    scripts/config.py set MBEDTLS_PLATFORM_MEMORY
+    CC=gcc cmake .
+    make
+
+    msg "test: MBEDTLS_MEMORY_BUFFER_ALLOC_C"
+    make test
+}
+
 component_test_aes_fewer_tables () {
     msg "build: default config with AES_FEWER_TABLES enabled"
     scripts/config.py set MBEDTLS_AES_FEWER_TABLES
@@ -890,7 +958,7 @@ component_test_se_full () {
 
 component_test_make_shared () {
     msg "build/test: make shared" # ~ 40s
-    make SHARED=1 all check -j1
+    make SHARED=1 all check
     ldd programs/util/strerror | grep libmbedcrypto
 }
 
@@ -1087,15 +1155,15 @@ component_build_armcc () {
 
 component_build_mingw () {
     msg "build: Windows cross build - mingw64, make (Link Library)" # ~ 30s
-    make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 lib programs -j1
+    make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 lib programs
 
     # note Make tests only builds the tests, but doesn't run them
-    make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror' WINDOWS_BUILD=1 tests -j1
+    make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror' WINDOWS_BUILD=1 tests
     make WINDOWS_BUILD=1 clean
 
     msg "build: Windows cross build - mingw64, make (DLL)" # ~ 30s
-    make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 SHARED=1 lib programs -j1
+    make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 SHARED=1 lib programs
-    make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 SHARED=1 tests -j1
+    make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 SHARED=1 tests
     make WINDOWS_BUILD=1 clean
 }
 support_build_mingw() {
@@ -1219,6 +1287,7 @@ run_component () {
     # The cleanup function will restore it.
     cp -p "$CONFIG_H" "$CONFIG_BAK"
     current_component="$1"
+    export MBEDTLS_TEST_CONFIGURATION="$current_component"
     "$@"
     cleanup
 }
@@ -1239,6 +1308,7 @@ else
         "$@"
     }
 fi
+pre_prepare_outcome_file
 pre_print_configuration
 pre_check_tools
 cleanup

tests/scripts/check-test-cases.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+
+"""Sanity checks for test data.
+"""
+
+# Copyright (C) 2019, Arm Limited, All Rights Reserved
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This file is part of Mbed TLS (https://tls.mbed.org)
+
+import glob
+import os
+import re
+import sys
+
+class Results:
+    def __init__(self):
+        self.errors = 0
+        self.warnings = 0
+
+    def error(self, file_name, line_number, fmt, *args):
+        sys.stderr.write(('{}:{}:ERROR:' + fmt + '\n').
+                         format(file_name, line_number, *args))
+        self.errors += 1
+
+    def warning(self, file_name, line_number, fmt, *args):
+        sys.stderr.write(('{}:{}:Warning:' + fmt + '\n')
+                         .format(file_name, line_number, *args))
+        self.warnings += 1
+
+def collect_test_directories():
+    if os.path.isdir('tests'):
+        tests_dir = 'tests'
+    elif os.path.isdir('suites'):
+        tests_dir = '.'
+    elif os.path.isdir('../suites'):
+        tests_dir = '..'
+    directories = [tests_dir]
+    crypto_tests_dir = os.path.normpath(os.path.join(tests_dir,
+                                                     '../crypto/tests'))
+    if os.path.isdir(crypto_tests_dir):
+        directories.append(crypto_tests_dir)
+    return directories
+
+def check_description(results, seen, file_name, line_number, description):
+    if description in seen:
+        results.error(file_name, line_number,
+                      'Duplicate description (also line {})',
+                      seen[description])
+        return
+    if re.search(br'[\t;]', description):
+        results.error(file_name, line_number,
+                      'Forbidden character \'{}\' in description',
+                      re.search(br'[\t;]', description).group(0).decode('ascii'))
+    if re.search(br'[^ -~]', description):
+        results.error(file_name, line_number,
+                      'Non-ASCII character in description')
+    if len(description) > 66:
+        results.warning(file_name, line_number,
+                        'Test description too long ({} > 66)',
+                        len(description))
+    seen[description] = line_number
+
+def check_test_suite(results, data_file_name):
+    in_paragraph = False
+    descriptions = {}
+    with open(data_file_name, 'rb') as data_file:
+        for line_number, line in enumerate(data_file, 1):
+            line = line.rstrip(b'\r\n')
+            if not line:
+                in_paragraph = False
+                continue
+            if line.startswith(b'#'):
+                continue
+            if not in_paragraph:
+                # This is a test case description line.
+                check_description(results, descriptions,
+                                  data_file_name, line_number, line)
+            in_paragraph = True
+
+def check_ssl_opt_sh(results, file_name):
+    descriptions = {}
+    with open(file_name, 'rb') as file_contents:
+        for line_number, line in enumerate(file_contents, 1):
+            # Assume that all run_test calls have the same simple form
+            # with the test description entirely on the same line as the
+            # function name.
+            m = re.match(br'\s*run_test\s+"((?:[^\\"]|\\.)*)"', line)
+            if not m:
+                continue
+            description = m.group(1)
+            check_description(results, descriptions,
+                              file_name, line_number, description)
+
+def main():
+    test_directories = collect_test_directories()
+    results = Results()
+    for directory in test_directories:
+        for data_file_name in glob.glob(os.path.join(directory, 'suites',
+                                                     '*.data')):
+            check_test_suite(results, data_file_name)
+        ssl_opt_sh = os.path.join(directory, 'ssl-opt.sh')
+        if os.path.exists(ssl_opt_sh):
+            check_ssl_opt_sh(results, ssl_opt_sh)
+    if results.warnings or results.errors:
+        sys.stderr.write('{}: {} errors, {} warnings\n'
+                         .format(sys.argv[0], results.errors, results.warnings))
+    sys.exit(1 if results.errors else 0)
+
+if __name__ == '__main__':
+    main()

tests/scripts/curves.pl

@@ -51,6 +51,7 @@ for my $curve (@curves) {
     print "\n******************************************\n";
     print "* Testing without curve: $curve\n";
     print "******************************************\n";
+    $ENV{MBEDTLS_TEST_CONFIGURATION} = "-$curve";
 
     system( "scripts/config.py unset $curve" )
         and abort "Failed to disable $curve\n";

tests/scripts/depends-hashes.pl

@@ -57,6 +57,7 @@ for my $hash (@hashes) {
     print "\n******************************************\n";
     print "* Testing without hash: $hash\n";
     print "******************************************\n";
+    $ENV{MBEDTLS_TEST_CONFIGURATION} = "-$hash";
 
     system( "scripts/config.py unset $hash" )
         and abort "Failed to disable $hash\n";

tests/scripts/depends-pkalgs.pl

@@ -59,6 +59,7 @@ while( my ($alg, $extras) = each %algs ) {
     print "\n******************************************\n";
     print "* Testing without alg: $alg\n";
     print "******************************************\n";
+    $ENV{MBEDTLS_TEST_CONFIGURATION} = "-$alg";
 
     system( "scripts/config.py unset $alg" )
         and abort "Failed to disable $alg\n";

tests/scripts/test-ref-configs.pl

@@ -66,6 +66,7 @@ while( my ($conf, $data) = each %configs ) {
     print "\n******************************************\n";
     print "* Testing configuration: $conf\n";
     print "******************************************\n";
+    $ENV{MBEDTLS_TEST_CONFIGURATION} = $conf;
 
     system( "cp configs/$conf $config_h" )
         and abort "Failed to activate $conf\n";

tests/suites/helpers.function

@@ -406,7 +406,7 @@ typedef enum
     TEST_RESULT_SKIPPED
 } test_result_t;
 
-static struct
+typedef struct
 {
     paramfail_test_state_t paramfail_test_state;
     test_result_t result;
@@ -415,7 +415,8 @@ static struct
     int line_no;
     unsigned long step;
 }
-test_info;
+test_info_t;
+static test_info_t test_info;
 
 #if defined(MBEDTLS_PLATFORM_C)
 mbedtls_platform_context platform_ctx;

tests/suites/host_test.function

@@ -368,6 +368,118 @@ static int run_test_snprintf( void )
             test_snprintf( 5, "123",         3 ) != 0 );
 }
 
+/** \brief Write the description of the test case to the outcome CSV file.
+ *
+ * \param outcome_file  The file to write to.
+ *                      If this is \c NULL, this function does nothing.
+ * \param argv0         The test suite name.
+ * \param test_case     The test case description.
+ */
+static void write_outcome_entry( FILE *outcome_file,
+                                 const char *argv0,
+                                 const char *test_case )
+{
+    /* The non-varying fields are initialized on first use. */
+    static const char *platform = NULL;
+    static const char *configuration = NULL;
+    static const char *test_suite = NULL;
+
+    if( outcome_file == NULL )
+        return;
+
+    if( platform == NULL )
+    {
+        platform = getenv( "MBEDTLS_TEST_PLATFORM" );
+        if( platform == NULL )
+            platform = "unknown";
+    }
+    if( configuration == NULL )
+    {
+        configuration = getenv( "MBEDTLS_TEST_CONFIGURATION" );
+        if( configuration == NULL )
+            configuration = "unknown";
+    }
+    if( test_suite == NULL )
+    {
+        test_suite = strrchr( argv0, '/' );
+        if( test_suite != NULL )
+            test_suite += 1; // skip the '/'
+        else
+            test_suite = argv0;
+    }
+
+    /* Write the beginning of the outcome line.
+     * Ignore errors: writing the outcome file is on a best-effort basis. */
+    mbedtls_fprintf( outcome_file, "%s;%s;%s;%s;",
+                     platform, configuration, test_suite, test_case );
+}
+
+/** \brief Write the result of the test case to the outcome CSV file.
+ *
+ * \param outcome_file  The file to write to.
+ *                      If this is \c NULL, this function does nothing.
+ * \param unmet_dep_count       The number of unmet dependencies.
+ * \param unmet_dependencies    The array of unmet dependencies.
+ * \param ret           The test dispatch status (DISPATCH_xxx).
+ * \param test_info     A pointer to the test info structure.
+ */
+static void write_outcome_result( FILE *outcome_file,
+                                  size_t unmet_dep_count,
+                                  char *unmet_dependencies[],
+                                  int ret,
+                                  const test_info_t *info )
+{
+    if( outcome_file == NULL )
+        return;
+
+    /* Write the end of the outcome line.
+     * Ignore errors: writing the outcome file is on a best-effort basis. */
+    switch( ret )
+    {
+        case DISPATCH_TEST_SUCCESS:
+            if( unmet_dep_count > 0 )
+            {
+                size_t i;
+                mbedtls_fprintf( outcome_file, "SKIP" );
+                for( i = 0; i < unmet_dep_count; i++ )
+                {
+                    mbedtls_fprintf( outcome_file, "%c%s",
+                                     i == 0 ? ';' : ':',
+                                     unmet_dependencies[i] );
+                }
+                break;
+            }
+            switch( info->result )
+            {
+                case TEST_RESULT_SUCCESS:
+                    mbedtls_fprintf( outcome_file, "PASS;" );
+                    break;
+                case TEST_RESULT_SKIPPED:
+                    mbedtls_fprintf( outcome_file, "SKIP;Runtime skip" );
+                    break;
+                default:
+                    mbedtls_fprintf( outcome_file, "FAIL;%s:%d:%s",
+                                     info->filename, info->line_no,
+                                     info->test );
+                    break;
+            }
+            break;
+        case DISPATCH_TEST_FN_NOT_FOUND:
+            mbedtls_fprintf( outcome_file, "FAIL;Test function not found" );
+            break;
+        case DISPATCH_INVALID_TEST_DATA:
+            mbedtls_fprintf( outcome_file, "FAIL;Invalid test data" );
+            break;
+        case DISPATCH_UNSUPPORTED_SUITE:
+            mbedtls_fprintf( outcome_file, "SKIP;Unsupported suite" );
+            break;
+        default:
+            mbedtls_fprintf( outcome_file, "FAIL;Unknown cause" );
+            break;
+    }
+    mbedtls_fprintf( outcome_file, "\n" );
+    fflush( outcome_file );
+}
 
 /**
  * \brief       Desktop implementation of execute_tests().
@@ -385,15 +497,16 @@ int execute_tests( int argc , const char ** argv )
     const char *default_filename = "DATA_FILE";
     const char *test_filename = NULL;
     const char **test_files = NULL;
-    int testfile_count = 0;
+    size_t testfile_count = 0;
     int option_verbose = 0;
     int function_id = 0;
 
     /* Other Local variables */
     int arg_index = 1;
     const char *next_arg;
-    int testfile_index, ret, i, cnt;
+    size_t testfile_index, i, cnt;
-    int total_errors = 0, total_tests = 0, total_skipped = 0;
+    int ret;
+    unsigned total_errors = 0, total_tests = 0, total_skipped = 0;
     FILE *file;
     char buf[5000];
     char *params[50];
@@ -403,6 +516,8 @@ int execute_tests( int argc , const char ** argv )
 #if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__))
     int stdout_fd = -1;
 #endif /* __unix__ || __APPLE__ __MACH__ */
+    const char *outcome_file_name = getenv( "MBEDTLS_TEST_OUTCOME_FILE" );
+    FILE *outcome_file = NULL;
 
 #if defined(MBEDTLS_MEMORY_BUFFER_ALLOC_C) && \
     !defined(TEST_SUITE_MEMORY_BUFFER_ALLOC)
@@ -410,6 +525,15 @@ int execute_tests( int argc , const char ** argv )
     mbedtls_memory_buffer_alloc_init( alloc_buf, sizeof( alloc_buf ) );
 #endif
 
+    if( outcome_file_name != NULL )
+    {
+        outcome_file = fopen( outcome_file_name, "a" );
+        if( outcome_file == NULL )
+        {
+            mbedtls_fprintf( stderr, "Unable to open outcome file. Continuing anyway.\n" );
+        }
+    }
+
     /*
      * The C standard doesn't guarantee that all-bits-0 is the representation
      * of a NULL pointer. We do however use that in our code for initializing
@@ -473,7 +597,7 @@ int execute_tests( int argc , const char ** argv )
           testfile_index < testfile_count;
           testfile_index++ )
     {
-        int unmet_dep_count = 0;
+        size_t unmet_dep_count = 0;
         char *unmet_dependencies[20];
 
         test_filename = test_files[ testfile_index ];
@@ -505,6 +629,7 @@ int execute_tests( int argc , const char ** argv )
                 mbedtls_fprintf( stdout, "." );
             mbedtls_fprintf( stdout, " " );
             fflush( stdout );
+            write_outcome_entry( outcome_file, argv[0], buf );
 
             total_tests++;
 
@@ -585,6 +710,9 @@ int execute_tests( int argc , const char ** argv )
 
             }
 
+            write_outcome_result( outcome_file,
+                                  unmet_dep_count, unmet_dependencies,
+                                  ret, &test_info );
             if( unmet_dep_count > 0 || ret == DISPATCH_UNSUPPORTED_SUITE )
             {
                 total_skipped++;
@@ -645,7 +773,7 @@ int execute_tests( int argc , const char ** argv )
             }
             else if( ret == DISPATCH_TEST_FN_NOT_FOUND )
             {
-                mbedtls_fprintf( stderr, "FAILED: FATAL TEST FUNCTION NOT FUND\n" );
+                mbedtls_fprintf( stderr, "FAILED: FATAL TEST FUNCTION NOT FOUND\n" );
                 fclose( file );
                 mbedtls_exit( 2 );
             }
@@ -659,14 +787,17 @@ int execute_tests( int argc , const char ** argv )
             free( unmet_dependencies[i] );
     }
 
+    if( outcome_file != NULL )
+        fclose( outcome_file );
+
     mbedtls_fprintf( stdout, "\n----------------------------------------------------------------------------\n\n");
     if( total_errors == 0 )
         mbedtls_fprintf( stdout, "PASSED" );
     else
         mbedtls_fprintf( stdout, "FAILED" );
 
-    mbedtls_fprintf( stdout, " (%d / %d tests (%d skipped))\n",
+    mbedtls_fprintf( stdout, " (%u / %u tests (%u skipped))\n",
-             total_tests - total_errors, total_tests, total_skipped );
+                     total_tests - total_errors, total_tests, total_skipped );
 
 #if defined(MBEDTLS_MEMORY_BUFFER_ALLOC_C) && \
     !defined(TEST_SUITE_MEMORY_BUFFER_ALLOC)

tests/suites/test_suite_version.data

@@ -1,8 +1,8 @@
 Check compiletime library version
-check_compiletime_version:"2.17.0"
+check_compiletime_version:"2.19.1"
 
 Check runtime library version
-check_runtime_version:"2.17.0"
+check_runtime_version:"2.19.1"
 
 Check for MBEDTLS_VERSION_C
 check_feature:"MBEDTLS_VERSION_C":0