Add script to generate release notes

The script uses assemble_changelog.py to generate the bulk of the content,
and adds some other headers, etc to create a standardised release note.

The output includes placeholders for the sha256 hashes of the zip and the
tarball which must be updated by hand.

The script displays a checklist, which, when followed, results in a note
that satisfies the release processes.

Signed-off-by: Dave Rodgman <dave.rodgman@arm.com>

scripts/assemble_changelog.py

@@ -218,13 +218,14 @@ class ChangeLog:
                                        category.name.decode('utf8'))
             self.categories[category.name] += category.body
 
-    def __init__(self, input_stream, changelog_format):
+    def __init__(self, input_stream, changelog_format, latest_only):
         """Create a changelog object.
 
         Populate the changelog object from the content of the file
         input_stream.
         """
         self.format = changelog_format
+        self.latest_only = latest_only
         whole_file = input_stream.read()
         (self.header,
          self.top_version_title, top_version_body,
@@ -247,12 +248,14 @@ class ChangeLog:
         """Write the changelog to the specified file.
         """
         with open(filename, 'wb') as out:
+            if not self.latest_only:
                 out.write(self.header)
                 out.write(self.top_version_title)
             for title, body in self.categories.items():
                 if not body:
                     continue
                 out.write(self.format.format_category(title, body))
+            if not self.latest_only:
                 out.write(self.trailer)
 
 
@@ -398,7 +401,7 @@ def check_output(generated_output_file, main_input_file, merged_files):
             if line not in generated_output:
                 raise LostContent(merged_file, line)
 
-def finish_output(changelog, output_file, input_file, merged_files):
+def finish_output(changelog, output_file, input_file, merged_files, latest_only):
     """Write the changelog to the output file.
 
     The input file and the list of merged files are used only for sanity
@@ -412,6 +415,7 @@ def finish_output(changelog, output_file, input_file, merged_files):
         # then move it into place atomically.
         output_temp = output_file + '.tmp'
     changelog.write(output_temp)
+    if not latest_only:
         check_output(output_temp, input_file, merged_files)
     if output_temp != output_file:
         os.rename(output_temp, output_file)
@@ -438,7 +442,7 @@ def merge_entries(options):
     Remove the merged entries if options.keep_entries is false.
     """
     with open(options.input, 'rb') as input_file:
-        changelog = ChangeLog(input_file, TextChangelogFormat)
+        changelog = ChangeLog(input_file, TextChangelogFormat, options.latest_only)
     files_to_merge = list_files_to_merge(options)
     if not files_to_merge:
         sys.stderr.write('There are no pending changelog entries.\n')
@@ -446,7 +450,7 @@ def merge_entries(options):
     for filename in files_to_merge:
         with open(filename, 'rb') as input_file:
             changelog.add_file(input_file)
-    finish_output(changelog, options.output, options.input, files_to_merge)
+    finish_output(changelog, options.output, options.input, files_to_merge, options.latest_only)
     if not options.keep_entries:
         remove_merged_entries(files_to_merge)
 
@@ -494,6 +498,9 @@ def main():
                         action='store_true',
                         help=('Only list the files that would be processed '
                               '(with some debugging information)'))
+    parser.add_argument('--latest-only',
+                        action='store_true',
+                        help=('Only generate the changes for the latest version'))
     options = parser.parse_args()
     set_defaults(options)
     if options.list_files_only:

scripts/generate_release_notes.py

@@ -0,0 +1,224 @@
+#!/usr/bin/env python3
+"""This script generates the MbedTLS release notes in markdown format.
+
+It does this by calling assemble_changelog.py to generate the bulk of
+content, and also inserting other content such as a brief description,
+hashes for the tar and zip files containing the release, etc.
+
+Returns 0 on success, 1 on failure.
+
+Note: must be run from Mbed TLS root."""
+
+# Copyright (c) 2020, Arm Limited, All Rights Reserved
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This file is part of Mbed TLS (https://tls.mbed.org)
+
+import re
+import sys
+import os.path
+import hashlib
+import argparse
+import tempfile
+import subprocess
+
+TEMPLATE = """## Description
+
+These are the release notes for MbedTLS version {version}.
+
+{description}
+
+{changelog}
+
+## Who should update
+
+{whoshouldupdate}
+
+## Checksum
+
+The SHA256 hashes for the archives are:
+
+```
+{tarhash}  mbedtls-{version}.tar.gz
+{ziphash}  mbedtls-{version}.zip
+```
+"""
+
+WHO_SHOULD_UPDATE_DEFAULT = 'We recommend all affected users should \
+update to take advantage of the bug fixes contained in this release at \
+an appropriate point in their development lifecycle.'
+
+
+CHECKLIST = '''Please review the release notes to ensure that all of the \
+following are documented (if needed):
+- Missing functionality
+- Changes in functionality
+- Known issues
+'''
+
+
+CUSTOM_WORDS = 'Hellman API APIs gz lifecycle Bugfix CMake inlined Crypto endian SHA xxx'
+
+
+def sha256_digest(filename):
+    """Read given file and return a SHA256 digest"""
+    h = hashlib.sha256()
+    with open(filename, 'rb') as f:
+        h.update(f.read())
+    return h.hexdigest()
+
+
+def error(text):
+    """Display error message and exit"""
+    print(f'ERROR: {text}')
+    sys.exit(1)
+
+
+def warn(text):
+    """Display warning message"""
+    print(f'WARNING: {text}')
+
+
+def generate_content(args):
+    """Return template populated with given content"""
+    for field in ('version', 'tarhash', 'ziphash', 'changelog',
+                  'description', 'whoshouldupdate'):
+        if not field in args:
+            error(f'{field} not specified')
+    return TEMPLATE.format(**args)
+
+
+def run_cmd(cmd, capture=True):
+    """Run given command in a shell and return the command output"""
+    # Note: [:-1] strips the trailing newline introduced by the shell.
+    if capture:
+        return subprocess.check_output(cmd, shell=True, input=None,
+                                   universal_newlines=True)[:-1]
+    else:
+        subprocess.call(cmd, shell=True)
+
+
+def parse_args(args):
+    """Parse command line arguments and return cleaned up args"""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('-o', '--output', default='ReleaseNotes.md',
+                        help='Output file (defaults to ReleaseNotes.md)')
+    parser.add_argument('-t', '--tar', action='store',
+                        help='Optional tar containing release (to generate hash)')
+    parser.add_argument('-z', '--zip', action='store',
+                        help='Optional zip containing release (to generate hash)')
+    parser.add_argument('-d', '--description', action='store', required=True,
+                        help='Short description of release (or name of file containing this)')
+    parser.add_argument('-w', '--who', action='store', default=WHO_SHOULD_UPDATE_DEFAULT,
+                        help='Optional short description of who should \
+                            update (or name of file containing this)')
+    args = parser.parse_args(args)
+
+    # If these exist as files, interpret as files containing
+    # desired content rather than literal content.
+    for field in ('description', 'who'):
+        if os.path.exists(getattr(args, field)):
+            with open(getattr(args, field), 'r') as f:
+                setattr(args, field, f.read())
+
+    return args
+
+
+def spellcheck(text):
+    with tempfile.NamedTemporaryFile() as temp_file:
+        with open(temp_file.name, 'w') as f:
+            f.write(text)
+        result = run_cmd(f'ispell -d american -w _- -a < {temp_file.name}')
+        input_lines = text.splitlines()
+        ispell_re = re.compile(r'& (\S+) \d+ \d+:.*')
+        bad_words = set()
+        bad_lines = set()
+        line_no = 1
+        for l in result.splitlines():
+            if l.strip() == '':
+                line_no += 1
+            elif l.startswith('&'):
+                m = ispell_re.fullmatch(l)
+                word = m.group(1)
+                if word.isupper():
+                    # ignore all-uppercase words
+                    pass
+                elif "_" in word:
+                    # part of a non-English 'word' like PSA_CRYPTO_ECC
+                    pass
+                elif word.startswith('-'):
+                    # ignore flags
+                    pass
+                elif word in CUSTOM_WORDS:
+                    # accept known-good words
+                    pass
+                else:
+                    bad_words.add(word)
+                    bad_lines.add(line_no)
+        if bad_words:
+            bad_lines = '\n'.join('    ' + input_lines[n] for n in sorted(bad_lines))
+            bad_words = ', '.join(bad_words)
+            warn('Release notes contain the following mis-spelled ' \
+                        f'words: {bad_words}:\n{bad_lines}\n')
+
+
+def gen_rel_notes(args):
+    """Return release note content from given command line args"""
+    # Get version by parsing version.h. Assumption is that bump_version
+    # has been run and this contains the correct version number.
+    version = run_cmd('cat include/mbedtls/version.h | \
+        clang -Iinclude -dM -E - | grep "MBEDTLS_VERSION_STRING "')
+    version = version.split()[-1][1:-1]
+
+    # Get main changelog content.
+    assemble_path = os.path.join(os.getcwd(), 'scripts', 'assemble_changelog.py')
+    with tempfile.NamedTemporaryFile() as temp_file:
+        run_cmd(f'{assemble_path} -o {temp_file.name} --latest-only')
+        with open(temp_file.name) as f:
+            changelog = f.read()
+
+    arg_hash = {
+        'version': version,
+        'tarhash': '',
+        'ziphash': '',
+        'changelog': changelog.strip(),
+        'description': args.description.strip(),
+        'whoshouldupdate': args.who.strip()
+    }
+
+    spellcheck(generate_content(arg_hash))
+
+    arg_hash['tarhash'] = sha256_digest(args.tar) if args.tar else "x" * 64
+    arg_hash['ziphash'] = sha256_digest(args.zip) if args.zip else "x" * 64
+    return generate_content(arg_hash)
+
+
+def main():
+    # Very basic check to see if we are in the root.
+    path = os.path.join(os.getcwd(), 'scripts', 'generate_release_notes.py')
+    if not os.path.exists(path):
+        error(f'{sys.argv[0]} must be run from the mbedtls root')
+
+    args = parse_args(sys.argv[1:])
+
+    content = gen_rel_notes(args)
+    with open(args.output, 'w') as f:
+        f.write(content)
+
+    print(CHECKLIST)
+
+
+if __name__ == '__main__':
+    main()