Test script for config.py

Run config.py with various options and store the results in files.

This script also supports the now-removed config.pl.

This is a framework to run non-regression tests on config.py: run it
with the old version, run it with the new version, and compare the
output.

This is deliberately not a functional test suite so that we don't need
to maintain a set of known outputs. When something changes in
config.py (or config.h), run the script before, run it after, and
check manually whether any differences in the output are acceptable.

tests/scripts/test_config_script.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+
+"""Test helper for the Mbed TLS configuration file tool
+
+Run config.py with various parameters and write the results to files.
+
+This is a harness to help regression testing, not a functional tester.
+Sample usage:
+
+    test_config_script.py -d old
+    ## Modify config.py and/or config.h ##
+    test_config_script.py -d new
+    diff -ru old new
+"""
+
+## Copyright (C) 2019, ARM Limited, All Rights Reserved
+## SPDX-License-Identifier: Apache-2.0
+##
+## Licensed under the Apache License, Version 2.0 (the "License"); you may
+## not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+## WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+## This file is part of Mbed TLS (https://tls.mbed.org)
+
+import argparse
+import glob
+import os
+import re
+import shutil
+import subprocess
+
+OUTPUT_FILE_PREFIX = 'config-'
+
+def output_file_name(directory, stem, extension):
+    return os.path.join(directory,
+                        '{}{}.{}'.format(OUTPUT_FILE_PREFIX,
+                                         stem, extension))
+
+def cleanup_directory(directory):
+    """Remove old output files."""
+    for extension in []:
+        pattern = output_file_name(directory, '*', extension)
+        filenames = glob.glob(pattern)
+        for filename in filenames:
+            os.remove(filename)
+
+def prepare_directory(directory):
+    """Create the output directory if it doesn't exist yet.
+
+    If there are old output files, remove them.
+    """
+    if os.path.exists(directory):
+        cleanup_directory(directory)
+    else:
+        os.makedirs(directory)
+
+def guess_presets_from_help(help_text):
+    """Figure out what presets the script supports.
+
+    help_text should be the output from running the script with --help.
+    """
+    # Try the output format from config.py
+    hits = re.findall(r'\{([-\w,]+)\}', help_text)
+    for hit in hits:
+        words = set(hit.split(','))
+        if 'get' in words and 'set' in words and 'unset' in words:
+            words.remove('get')
+            words.remove('set')
+            words.remove('unset')
+            return words
+    # Try the output format from config.pl
+    hits = re.findall(r'\n +([-\w]+) +- ', help_text)
+    if hits:
+        return hits
+    raise Exception("Unable to figure out supported presets. Pass the '-p' option.")
+
+def list_presets(options):
+    """Return the list of presets to test.
+
+    The list is taken from the command line if present, otherwise it is
+    extracted from running the config script with --help.
+    """
+    if options.presets:
+        return re.split(r'[ ,]+', options.presets)
+    else:
+        help_text = subprocess.run([options.script, '--help'],
+                                   stdout=subprocess.PIPE,
+                                   stderr=subprocess.STDOUT).stdout
+        return guess_presets_from_help(help_text.decode('ascii'))
+
+def run_one(options, args):
+    """Run the config script with the given arguments.
+
+    Write the following files:
+    * config-xxx.h: modified file.
+    * config-xxx.out: standard output.
+    * config-xxx.err: standard output.
+    * config-xxx.status: exit code.
+    """
+    stem = '-'.join(args)
+    data_filename = output_file_name(options.output_directory, stem, 'h')
+    stdout_filename = output_file_name(options.output_directory, stem, 'out')
+    stderr_filename = output_file_name(options.output_directory, stem, 'err')
+    status_filename = output_file_name(options.output_directory, stem, 'status')
+    shutil.copy(options.input_file, data_filename)
+    # Pass only the file basename, not the full path, to avoid getting the
+    # directory name in error messages, which would make comparisons
+    # between output directories more difficult.
+    cmd = [os.path.abspath(options.script),
+           '-f', os.path.basename(data_filename)]
+    with open(stdout_filename, 'wb') as out:
+        with open(stderr_filename, 'wb') as err:
+            status = subprocess.call(cmd + args,
+                                     cwd=options.output_directory,
+                                     stdin=subprocess.DEVNULL,
+                                     stdout=out, stderr=err)
+    with open(status_filename, 'w') as status_file:
+        status_file.write('{}\n'.format(status))
+
+### A list of symbols to test with set and unset.
+TEST_SYMBOLS = [
+    'CUSTOM_OPTION',
+    'DOES_NOT_EXIST',
+    'MBEDTLS_AES_C',
+    'MBEDTLS_NO_UDBL_DIVISION',
+    'MBEDTLS_PLATFORM_ZEROIZE_ALT',
+]
+
+### A list of symbols to test with set with a value.
+TEST_SYMBOLS_WITH_VALUE = [
+    'CUSTOM_VALUE',
+    'MBEDTLS_MPI_MAX_SIZE',
+]
+
+def run_all(options):
+    """Run all the command lines to test."""
+    presets = list_presets(options)
+    for preset in presets:
+        run_one(options, [preset])
+    for symbol in TEST_SYMBOLS:
+        run_one(options, ['set', symbol])
+        run_one(options, ['--force', 'set', symbol])
+        run_one(options, ['unset', symbol])
+    for symbol in TEST_SYMBOLS_WITH_VALUE:
+        run_one(options, ['set', symbol, 'value'])
+        run_one(options, ['--force', 'set', symbol, 'value'])
+
+def main():
+    """Command line entry point."""
+    parser = argparse.ArgumentParser(description=__doc__,
+                                     formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument('-d', metavar='DIR',
+                        dest='output_directory', required=True,
+                        help="""Output directory.""")
+    parser.add_argument('-f', metavar='FILE',
+                        dest='input_file', default='include/mbedtls/config.h',
+                        help="""Config file (default: %(default)s).""")
+    parser.add_argument('-p', metavar='PRESET,...',
+                        dest='presets',
+                        help="""Presets to test (default: guessed from --help).""")
+    parser.add_argument('-s', metavar='FILE',
+                        dest='script', default='scripts/config.py',
+                        help="""Configuration script (default: %(default)s).""")
+    options = parser.parse_args()
+    prepare_directory(options.output_directory)
+    run_all(options)
+
+if __name__ == '__main__':
+    main()