diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index aefe0295f..4fd66af75 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -2842,6 +2842,19 @@ component_test_zeroize () { unset gdb_disable_aslr } +component_test_psa_compliance () { + msg "build: make, default config (out-of-box), libmbedcrypto.a only" + make library/libmbedcrypto.a + + msg "unit test: test_psa_compliance.py" + ./tests/scripts/test_psa_compliance.py +} + +support_test_psa_compliance () { + local ver=($(cmake --version | sed 's/cmake version //; y/./ /; q')) + [ "${ver[0]}" -eq 3 ] && [ "${ver[1]}" -ge 10 ] +} + component_check_python_files () { msg "Lint: Python scripts" tests/scripts/check-python-files.sh diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py new file mode 100755 index 000000000..07fa76e60 --- /dev/null +++ b/tests/scripts/test_psa_compliance.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +#pylint: disable=missing-module-docstring +import os +import re +import shutil +import subprocess +import sys + +EXPECTED_FAILURES = { + 216, 221, 224, 225, 248, 249, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263 +} +PSA_ARCH_TESTS_REPO = 'https://github.com/ronald-cron-arm/psa-arch-tests.git' +PSA_ARCH_TESTS_REF = 'crypto1.0-3.0' + +#pylint: disable=too-many-statements +def main(): + mbedtls_dir = os.getcwd() + + mbedcrypto_lib = 'library/libmbedcrypto.a' + if not os.path.exists(mbedcrypto_lib): + subprocess.check_call(['make', mbedcrypto_lib]) + + psa_arch_tests_dir = 'psa-arch-tests' + try: + os.mkdir(psa_arch_tests_dir) + except FileExistsError: + pass + os.chdir(psa_arch_tests_dir) + + subprocess.check_call(['git', 'init']) + subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, PSA_ARCH_TESTS_REF]) + subprocess.check_call(['git', 'checkout', 'FETCH_HEAD']) + + build_dir = 'api-tests/build' + try: + shutil.rmtree(build_dir) + except FileNotFoundError: + pass + os.mkdir(build_dir) + os.chdir(build_dir) + + #pylint: disable=bad-continuation + subprocess.check_call([ + 'cmake', '..', '-GUnix Makefiles', + '-DTARGET=tgt_dev_apis_stdc', + '-DTOOLCHAIN=HOST_GCC', + '-DSUITE=CRYPTO', + '-DPSA_CRYPTO_LIB_FILENAME={}/library/libmbedcrypto.a'.format(mbedtls_dir), + '-DPSA_INCLUDE_PATHS={}/include'.format(mbedtls_dir) + ]) + subprocess.check_call(['cmake', '--build', '.']) + + proc = subprocess.Popen(['./psa-arch-tests-crypto'], + bufsize=1, stdout=subprocess.PIPE, universal_newlines=True) + + test_re = re.compile('^TEST(?:: ([0-9]*)| RESULT: FAILED)') + test = -1 + unexpected_successes = set(EXPECTED_FAILURES) + expected_failures = [] + unexpected_failures = [] + for line in proc.stdout: + print(line[:-1]) + match = test_re.match(line) + if match is not None: + if match.group(1) is not None: + test = int(match.group(1)) + else: + try: + unexpected_successes.remove(test) + expected_failures.append(test) + except KeyError: + unexpected_failures.append(test) + proc.wait() + + print() + print('***** test_psa_compliance.py report ******') + print() + print('Expected failures:', ', '.join(str(i) for i in expected_failures)) + print('Unexpected failures:', ', '.join(str(i) for i in unexpected_failures)) + print('Unexpected successes:', ', '.join(str(i) for i in sorted(unexpected_successes))) + print() + if unexpected_successes or unexpected_failures: + if unexpected_successes: + print('Unexpected successes encountered.') + #pylint: disable=line-too-long + print('Please remove the corresponding tests from EXPECTED_FAILURES in tests/scripts/compliance_test.py') + print() + print('FAILED') + sys.exit(1) + else: + os.chdir(mbedtls_dir) + shutil.rmtree(psa_arch_tests_dir) + print('SUCCESS') + +if __name__ == '__main__': + main()