From 0626ebb4254fecbad689af7af5d92d0a3d515905 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 14 Dec 2018 18:23:13 +0100 Subject: [PATCH 1/2] run-test-suites: update the documentation Update the documentation to mention the optional verbosity level with -v. Print the documentation on --help. Die on an unsupported option. --- tests/scripts/run-test-suites.pl | 35 ++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/tests/scripts/run-test-suites.pl b/tests/scripts/run-test-suites.pl index d0d404621..7ffbd745b 100755 --- a/tests/scripts/run-test-suites.pl +++ b/tests/scripts/run-test-suites.pl @@ -4,19 +4,21 @@ # # This file is part of mbed TLS (https://tls.mbed.org) # -# Copyright (c) 2015-2016, ARM Limited, All Rights Reserved -# -# Purpose -# -# Executes all the available test suites, and provides a basic summary of the -# results. -# -# Usage: run-test-suites.pl [-v] -# -# Options : -# -v|--verbose - Provide a pass/fail/skip breakdown per test suite and -# in total -# +# Copyright (c) 2015-2018, ARM Limited, All Rights Reserved + +=head1 SYNOPSIS + +Execute all the test suites and print a summary of the results. + + run-test-suites.pl [[-v|--verbose] [VERBOSITY]] + +Options: + + -v|--verbose Print detailed failure information. + -v 2|--verbose=2 Print detailed failure information and summary messages. + -v 3|--verbose=3 Print detailed information about every test case. + +=cut use warnings; use strict; @@ -24,10 +26,13 @@ use strict; use utf8; use open qw(:std utf8); -use Getopt::Long; +use Getopt::Long qw(:config auto_help); +use Pod::Usage; my $verbose = 0; -GetOptions( "verbose|v:1" => \$verbose ); +GetOptions( + 'verbose|v:1' => \$verbose, + ) or die; # All test suites = executable files, excluding source files, debug # and profiling information, etc. We can't just grep {! /\./} because From bda9abf69641598fdc36e82169fb00c905d7f928 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 29 Nov 2018 10:15:06 +0000 Subject: [PATCH 2/2] Add a facility to skip running some test suites With the build option SKIP_TEST_SUITES=..., the specified test suites are built, but skipped when running tests. Usage: make check SKIP_TEST_SUITES=timing,gcm or cmake -D SKIP_TEST_SUITES=timing,gcm ... The list can be separated by any of space, comma or semicolon, and each element can be a regular expression in ERE syntax except that "." stands for itself. Skipping "foo" skips not only "foo" itself but also any "foo.bar", but does not skip "foobar". --- tests/CMakeLists.txt | 13 ++++++++++- tests/Makefile | 3 ++- tests/scripts/run-test-suites.pl | 37 ++++++++++++++++++++++++++++---- 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9fd4916bb..0f3c58f6a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -15,6 +15,13 @@ if(NOT PERL_FOUND) message(FATAL_ERROR "Cannot build test suites without Perl") endif() +# Test suites caught by SKIP_TEST_SUITES are built but not executed. +# "foo" as a skip pattern skips "test_suite_foo" and "test_suite_foo.bar" +# but not "test_suite_foobar". +string(REGEX REPLACE "[ ,;]" "|" SKIP_TEST_SUITES_REGEX "${SKIP_TEST_SUITES}") +string(REPLACE "." "\\." SKIP_TEST_SUITES_REGEX "${SKIP_TEST_SUITES_REGEX}") +set(SKIP_TEST_SUITES_REGEX "^(${SKIP_TEST_SUITES_REGEX})(\$|\\.)") + function(add_test_suite suite_name) if(ARGV1) set(data_name ${ARGV1}) @@ -31,7 +38,11 @@ function(add_test_suite suite_name) include_directories(${CMAKE_CURRENT_SOURCE_DIR}) add_executable(test_suite_${data_name} test_suite_${data_name}.c) target_link_libraries(test_suite_${data_name} ${libs}) - add_test(${data_name}-suite test_suite_${data_name} --verbose) + if(${data_name} MATCHES ${SKIP_TEST_SUITES_REGEX}) + message(STATUS "The test suite ${data_name} will not be executed.") + else() + add_test(${data_name}-suite test_suite_${data_name} --verbose) + endif() endfunction(add_test_suite) if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG) diff --git a/tests/Makefile b/tests/Makefile index d85617fdc..03f1a7f2f 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -444,7 +444,8 @@ else del /Q /F *.c *.exe endif +# Test suites caught by SKIP_TEST_SUITES are built but not executed. check: $(APPS) - perl scripts/run-test-suites.pl + perl scripts/run-test-suites.pl --skip=$(SKIP_TEST_SUITES) test: check diff --git a/tests/scripts/run-test-suites.pl b/tests/scripts/run-test-suites.pl index 7ffbd745b..1c9dc1dfc 100755 --- a/tests/scripts/run-test-suites.pl +++ b/tests/scripts/run-test-suites.pl @@ -10,13 +10,16 @@ Execute all the test suites and print a summary of the results. - run-test-suites.pl [[-v|--verbose] [VERBOSITY]] + run-test-suites.pl [[-v|--verbose] [VERBOSITY]] [--skip=SUITE[...]] Options: -v|--verbose Print detailed failure information. -v 2|--verbose=2 Print detailed failure information and summary messages. -v 3|--verbose=3 Print detailed information about every test case. + --skip=SUITE[,SUITE...] + Skip the specified SUITE(s). This option can be used + multiple times. =cut @@ -26,11 +29,13 @@ use strict; use utf8; use open qw(:std utf8); -use Getopt::Long qw(:config auto_help); +use Getopt::Long qw(:config auto_help gnu_compat); use Pod::Usage; my $verbose = 0; +my @skip_patterns = (); GetOptions( + 'skip=s' => \@skip_patterns, 'verbose|v:1' => \$verbose, ) or die; @@ -41,6 +46,17 @@ my @suites = grep { -x $_ || /\.exe$/ } glob 'test_suite_*'; @suites = grep { !/\.c$/ && !/\.data$/ && -f } @suites; die "$0: no test suite found\n" unless @suites; +# "foo" as a skip pattern skips "test_suite_foo" and "test_suite_foo.bar" +# but not "test_suite_foobar". +my $skip_re = + ( '\Atest_suite_(' . + join('|', map { + s/[ ,;]/|/g; # allow any of " ,;|" as separators + s/\./\./g; # "." in the input means ".", not "any character" + $_ + } @skip_patterns) . + ')(\z|\.)' ); + # in case test suites are linked dynamically $ENV{'LD_LIBRARY_PATH'} = '../library'; $ENV{'DYLD_LIBRARY_PATH'} = '../library'; @@ -50,6 +66,7 @@ my $prefix = $^O eq "MSWin32" ? '' : './'; my ($failed_suites, $total_tests_run, $failed, $suite_cases_passed, $suite_cases_failed, $suite_cases_skipped, $total_cases_passed, $total_cases_failed, $total_cases_skipped ); +my $suites_skipped = 0; sub pad_print_center { my( $width, $padchar, $string ) = @_; @@ -60,6 +77,12 @@ sub pad_print_center { for my $suite (@suites) { print "$suite ", "." x ( 72 - length($suite) - 2 - 4 ), " "; + if( $suite =~ /$skip_re/o ) { + print "SKIP\n"; + ++$suites_skipped; + next; + } + my $command = "$prefix$suite"; if( $verbose ) { $command .= ' -v'; @@ -106,7 +129,10 @@ for my $suite (@suites) print "-" x 72, "\n"; print $failed_suites ? "FAILED" : "PASSED"; -printf " (%d suites, %d tests run)\n", scalar @suites, $total_tests_run; +printf( " (%d suites, %d tests run%s)\n", + scalar(@suites) - $suites_skipped, + $total_tests_run, + $suites_skipped ? ", $suites_skipped suites skipped" : "" ); if( $verbose > 1 ) { print " test cases passed :", $total_cases_passed, "\n"; @@ -116,8 +142,11 @@ if( $verbose > 1 ) { "\n"; print " of available tests :", ( $total_cases_passed + $total_cases_failed + $total_cases_skipped ), - "\n" + "\n"; + if( $suites_skipped != 0 ) { + print "Note: $suites_skipped suites were skipped.\n"; } +} exit( $failed_suites ? 1 : 0 );