Adds verbose mode to the test suites

Added a verbose option to the generated test suites which can list the
dependencies not met for skipped test cases.
Also clarifies internal interfaces between the main_test.function and test code,
and fixed a bug on calculating available tests in run-test-suites.pl.

tests/scripts/generate_code.pl

@@ -2,6 +2,8 @@
 
 # generate_code.pl
 #
+# This file is part of mbed TLS (https://tls.mbed.org)
+#
 # Copyright (c) 2009-2016, ARM Limited, All Rights Reserved
 #
 # Purpose
@@ -202,7 +204,7 @@ while($test_cases =~ /\/\* BEGIN_CASE *([\w:]*) \*\/\n(.*?)\n\/\* END_CASE \*\//
         if( substr($def, 0, 4) eq "int " )
         {
             $param_defs .= "    int param$i;\n";
-            $param_checks .= "    if( verify_int( params[$i], &param$i ) != 0 ) return( 2 );\n";
+            $param_checks .= "    if( verify_int( params[$i], &param$i ) != 0 ) return( DISPATCH_INVALID_TEST_DATA );\n";
             push @dispatch_params, "param$i";
 
             $mapping_regex .= ":([\\d\\w |\\+\\-\\(\\)]+)";
@@ -211,7 +213,7 @@ while($test_cases =~ /\/\* BEGIN_CASE *([\w:]*) \*\/\n(.*?)\n\/\* END_CASE \*\//
         elsif( substr($def, 0, 6) eq "char *" )
         {
             $param_defs .= "    char *param$i = params[$i];\n";
-            $param_checks .= "    if( verify_string( &param$i ) != 0 ) return( 2 );\n";
+            $param_checks .= "    if( verify_string( &param$i ) != 0 ) return( DISPATCH_INVALID_TEST_DATA );\n";
             push @dispatch_params, "param$i";
             $mapping_regex .= ":[^:\n]+";
         }
@@ -248,14 +250,14 @@ $param_defs
     if( cnt != $param_count )
     {
         mbedtls_fprintf( stderr, "\\nIncorrect argument count (%d != %d)\\n", cnt, $param_count );
-        return( 2 );
+        return( DISPATCH_INVALID_TEST_DATA );
     }
 
 $param_checks
     test_suite_$function_name( $call_params );
-    return ( 0 );
+    return ( DISPATCH_TEST_SUCCESS );
 $function_post_code
-    return ( 3 );
+    return ( DISPATCH_UNSUPPORTED_SUITE );
 }
 else
 END
@@ -283,9 +285,9 @@ while( my ($key, $value) = each(%case_deps) )
     if( strcmp( str, "$key" ) == 0 )
     {
 #if defined($key)
-        return( 0 );
+        return( DEPENDENCY_SUPPORTED );
 #else
-        return( 1 );
+        return( DEPENDENCY_NOT_SUPPORTED );
 #endif
     }
 END
@@ -298,7 +300,7 @@ while( my ($key, $value) = each(%mapping_values) )
     if( strcmp( str, "$key" ) == 0 )
     {
         *value = ( $key );
-        return( 0 );
+        return( KEY_VALUE_MAPPING_FOUND );
     }
 END
 
@@ -315,7 +317,7 @@ END
 
 $dispatch_code =~ s/^(.+)/    $1/mg;
 
-$test_main =~ s/TEST_FILENAME/$test_case_data/;
+$test_main =~ s/TEST_FILENAME/$test_case_data/g;
 $test_main =~ s/FUNCTION_CODE//;
 $test_main =~ s/DEP_CHECK_CODE/$dep_check_code/;
 $test_main =~ s/DISPATCH_FUNCTION/$dispatch_code/;

tests/scripts/run-test-suites.pl

@@ -2,6 +2,8 @@
 
 # run-test-suites.pl
 #
+# This file is part of mbed TLS (https://tls.mbed.org)
+#
 # Copyright (c) 2015-2016, ARM Limited, All Rights Reserved
 #
 # Purpose
@@ -66,7 +68,8 @@ for my $suite (@suites)
         print "(test cases passed:", $suite_cases_passed,
                 " failed:", $suite_cases_failed,
                 " skipped:", $suite_cases_skipped,
-                " of total:", ( $suite_cases_passed + $suite_cases_failed ),
+                " of total:", ($suite_cases_passed + $suite_cases_failed +
+                               $suite_cases_skipped),
                 ")\n"
     }
 

tests/suites/helpers.function

@@ -32,9 +32,18 @@ typedef UINT32 uint32_t;
 
 
 /*----------------------------------------------------------------------------*/
-/* Global variables */
+/* Constants */
 
-static int test_errors = 0;
+#define DEPENDENCY_SUPPORTED        0
+#define DEPENDENCY_NOT_SUPPORTED    1
+
+#define KEY_VALUE_MAPPING_FOUND     0
+#define KEY_VALUE_MAPPING_NOT_FOUND -1
+
+#define DISPATCH_TEST_SUCCESS       0
+#define DISPATCH_TEST_FN_NOT_FOUND  1
+#define DISPATCH_INVALID_TEST_DATA  2
+#define DISPATCH_UNSUPPORTED_SUITE  3
 
 
 /*----------------------------------------------------------------------------*/
@@ -80,6 +89,12 @@ static int test_errors = 0;
 #endif
 
 
+/*----------------------------------------------------------------------------*/
+/* Global variables */
+
+static int test_errors = 0;
+
+
 /*----------------------------------------------------------------------------*/
 /* Helper Functions */
 

tests/suites/main_test.function

@@ -60,7 +60,7 @@ int verify_int( char *str, int *value )
 MAPPING_CODE
 
     mbedtls_printf( "Expected integer for parameter and got: %s\n", str );
-    return( -1 );
+    return( KEY_VALUE_MAPPING_NOT_FOUND );
 }
 
 
@@ -81,7 +81,7 @@ int dep_check( char *str )
 
 DEP_CHECK_CODE
 
-    return( 1 );
+    return( DEPENDENCY_NOT_SUPPORTED );
 }
 
 int dispatch_test(int cnt, char *params[50])
@@ -91,14 +91,18 @@ int dispatch_test(int cnt, char *params[50])
     ((void) params);
 
 #if defined(TEST_SUITE_ACTIVE)
+    ret = DISPATCH_TEST_SUCCESS;
+
 DISPATCH_FUNCTION
     {
-        mbedtls_fprintf( stdout, "FAILED\nSkipping unknown test function '%s'\n", params[0] );
+        mbedtls_fprintf( stdout,
+                         "FAILED\nSkipping unknown test function '%s'\n",
+                         params[0] );
         fflush( stdout );
-        return( 1 );
+        ret = DISPATCH_TEST_FN_NOT_FOUND;
     }
 #else
-    return( 3 );
+    ret = DISPATCH_UNSUPPORTED_SUITE;
 #endif
     return( ret );
 }
@@ -107,6 +111,19 @@ DISPATCH_FUNCTION
 /*----------------------------------------------------------------------------*/
 /* Main Test code */
 
+#define USAGE \
+    "Usage: %s [OPTIONS] files...\n\n" \
+    "   Command line arguments:\n" \
+    "     files...          One or more test data file. If no file is specified\n" \
+    "                       the followimg default test case is used:\n" \
+    "                           %s\n\n" \
+    "   Options:\n" \
+    "     -v | --verbose    Display full information about each test\n" \
+    "     -h | --help       Display this information\n\n", \
+    argv[0], \
+    "TEST_FILENAME"
+
+
 int get_line( FILE *f, char *buf, size_t len )
 {
     char *ret;
@@ -216,11 +233,18 @@ static int run_test_snprintf( void )
 
 int main(int argc, const char *argv[])
 {
-    int testfile_index, testfile_count, ret, i, cnt;
+    /* Local Configurations and options */
-    int total_errors = 0, total_tests = 0, total_skipped = 0;
     const char *default_filename = "TEST_FILENAME";
     const char *test_filename = NULL;
     const char **test_files = NULL;
+    int testfile_count = 0;
+    int option_verbose = 0;
+
+    /* Other Local variables */
+    int arg_index = 1;
+    const char *next_arg;
+    int testfile_index, ret, i, cnt;
+    int total_errors = 0, total_tests = 0, total_skipped = 0;
     FILE *file;
     char buf[5000];
     char *params[50];
@@ -253,17 +277,41 @@ int main(int argc, const char *argv[])
         return( 0 );
     }
 
-    if ( argc <= 1 )
+    while( arg_index < argc)
+    {
+        next_arg = argv[ arg_index ];
+
+        if( strcmp(next_arg, "--verbose" ) == 0 ||
+                 strcmp(next_arg, "-v" ) == 0 )
+        {
+            option_verbose = 1;
+        }
+        else if( strcmp(next_arg, "--help" ) == 0 ||
+                 strcmp(next_arg, "-h" ) == 0 )
+        {
+            mbedtls_fprintf( stdout, USAGE );
+            mbedtls_exit( EXIT_SUCCESS );
+        }
+        else
+        {
+            /* Not an option, therefore treat all further arguments as the file
+             * list.
+             */
+            test_files = &argv[ arg_index ];
+            testfile_count = argc - arg_index;
+        }
+
+        arg_index++;
+    }
+
+    /* If no files were specified, assume a default */
+    if ( test_files == NULL || testfile_count == 0 )
     {
         test_files = &default_filename;
         testfile_count = 1;
     }
-    else
-    {
-        test_files = &argv[1];
-        testfile_count = argc - 1;
-    }
 
+    /* Now begin to execute the tests in the testfiles */
     for ( testfile_index = 0;
           testfile_index < testfile_count;
           testfile_index++ )
@@ -280,7 +328,8 @@ int main(int argc, const char *argv[])
 
         while( !feof( file ) )
         {
-            int skip = 0;
+            int unmet_dep_count = 0;
+            char *unmet_dependencies[20];
 
             if( ( ret = get_line( file, buf, sizeof(buf) ) ) != 0 )
                 break;
@@ -300,32 +349,61 @@ int main(int argc, const char *argv[])
             if( strcmp( params[0], "depends_on" ) == 0 )
             {
                 for( i = 1; i < cnt; i++ )
-                    if( dep_check( params[i] ) != 0 )
+                {
-                        skip = 1;
+                    if( dep_check( params[i] ) != DEPENDENCY_SUPPORTED )
+                    {
+                        unmet_dependencies[ i-1 ] = strdup(params[i]);
+                        if(  unmet_dependencies[ i-1 ] == NULL )
+                        {
+                            mbedtls_printf("FATAL: Out of memory\n");
+                            mbedtls_exit( MBEDTLS_PLATFORM_STD_EXIT_FAILURE );
+                        }
+                        unmet_dep_count++;
+                    }
+                }
 
                 if( ( ret = get_line( file, buf, sizeof(buf) ) ) != 0 )
                     break;
                 cnt = parse_arguments( buf, strlen(buf), params );
             }
-
+ 
-            if( skip == 0 )
+            // If there are no unmet dependencies execute the test
+            if( unmet_dep_count == 0 )
             {
                 test_errors = 0;
                 ret = dispatch_test( cnt, params );
             }
 
-            if( skip == 1 || ret == 3 )
+            if( unmet_dep_count > 0 || ret == DISPATCH_UNSUPPORTED_SUITE )
             {
                 total_skipped++;
                 mbedtls_fprintf( stdout, "----\n" );
+
+                if( 1 == option_verbose && ret == DISPATCH_UNSUPPORTED_SUITE )
+                {
+                    mbedtls_fprintf( stdout, "   Test Suite not enabled" );
+                }
+
+                if( 1 == option_verbose && unmet_dep_count > 0 )
+                {
+                    mbedtls_fprintf( stdout, "   Unmet dependencies: " );
+                    while( unmet_dep_count > 0)
+                    {
+                        mbedtls_fprintf(stdout, "%s  ",
+                                        unmet_dependencies[unmet_dep_count - 1]);
+                        free(unmet_dependencies[unmet_dep_count - 1]);
+                        unmet_dep_count--;
+                    }
+                    mbedtls_fprintf( stdout, "\n" );
+                }
                 fflush( stdout );
             }
-            else if( ret == 0 && test_errors == 0 )
+            else if( ret == DISPATCH_TEST_SUCCESS && test_errors == 0 )
             {
                 mbedtls_fprintf( stdout, "PASS\n" );
                 fflush( stdout );
             }
-            else if( ret == 2 )
+            else if( ret == DISPATCH_INVALID_TEST_DATA )
             {
                 mbedtls_fprintf( stderr, "FAILED: FATAL PARSE ERROR\n" );
                 fclose(file);