From 9005b1f513e035223483fed464d60c8bbf744246 Mon Sep 17 00:00:00 2001 From: Dave Rodgman Date: Tue, 25 Aug 2020 14:33:15 +0100 Subject: [PATCH] Add script to generate release notes The script uses assemble_changelog.py to generate the bulk of the content, and adds some other headers, etc to create a standardised release note. The output includes placeholders for the sha256 hashes of the zip and the tarball which must be updated by hand. The script displays a checklist, which, when followed, results in a note that satisfies the release processes. Signed-off-by: Dave Rodgman --- scripts/assemble_changelog.py | 23 +-- scripts/generate_release_notes.py | 224 ++++++++++++++++++++++++++++++ 2 files changed, 239 insertions(+), 8 deletions(-) create mode 100755 scripts/generate_release_notes.py diff --git a/scripts/assemble_changelog.py b/scripts/assemble_changelog.py index ffa3f161b..2dc421e15 100755 --- a/scripts/assemble_changelog.py +++ b/scripts/assemble_changelog.py @@ -218,13 +218,14 @@ class ChangeLog: category.name.decode('utf8')) self.categories[category.name] += category.body - def __init__(self, input_stream, changelog_format): + def __init__(self, input_stream, changelog_format, latest_only): """Create a changelog object. Populate the changelog object from the content of the file input_stream. """ self.format = changelog_format + self.latest_only = latest_only whole_file = input_stream.read() (self.header, self.top_version_title, top_version_body, @@ -247,13 +248,15 @@ class ChangeLog: """Write the changelog to the specified file. """ with open(filename, 'wb') as out: - out.write(self.header) - out.write(self.top_version_title) + if not self.latest_only: + out.write(self.header) + out.write(self.top_version_title) for title, body in self.categories.items(): if not body: continue out.write(self.format.format_category(title, body)) - out.write(self.trailer) + if not self.latest_only: + out.write(self.trailer) @functools.total_ordering @@ -398,7 +401,7 @@ def check_output(generated_output_file, main_input_file, merged_files): if line not in generated_output: raise LostContent(merged_file, line) -def finish_output(changelog, output_file, input_file, merged_files): +def finish_output(changelog, output_file, input_file, merged_files, latest_only): """Write the changelog to the output file. The input file and the list of merged files are used only for sanity @@ -412,7 +415,8 @@ def finish_output(changelog, output_file, input_file, merged_files): # then move it into place atomically. output_temp = output_file + '.tmp' changelog.write(output_temp) - check_output(output_temp, input_file, merged_files) + if not latest_only: + check_output(output_temp, input_file, merged_files) if output_temp != output_file: os.rename(output_temp, output_file) @@ -438,7 +442,7 @@ def merge_entries(options): Remove the merged entries if options.keep_entries is false. """ with open(options.input, 'rb') as input_file: - changelog = ChangeLog(input_file, TextChangelogFormat) + changelog = ChangeLog(input_file, TextChangelogFormat, options.latest_only) files_to_merge = list_files_to_merge(options) if not files_to_merge: sys.stderr.write('There are no pending changelog entries.\n') @@ -446,7 +450,7 @@ def merge_entries(options): for filename in files_to_merge: with open(filename, 'rb') as input_file: changelog.add_file(input_file) - finish_output(changelog, options.output, options.input, files_to_merge) + finish_output(changelog, options.output, options.input, files_to_merge, options.latest_only) if not options.keep_entries: remove_merged_entries(files_to_merge) @@ -494,6 +498,9 @@ def main(): action='store_true', help=('Only list the files that would be processed ' '(with some debugging information)')) + parser.add_argument('--latest-only', + action='store_true', + help=('Only generate the changes for the latest version')) options = parser.parse_args() set_defaults(options) if options.list_files_only: diff --git a/scripts/generate_release_notes.py b/scripts/generate_release_notes.py new file mode 100755 index 000000000..9e9129cb3 --- /dev/null +++ b/scripts/generate_release_notes.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +"""This script generates the MbedTLS release notes in markdown format. + +It does this by calling assemble_changelog.py to generate the bulk of +content, and also inserting other content such as a brief description, +hashes for the tar and zip files containing the release, etc. + +Returns 0 on success, 1 on failure. + +Note: must be run from Mbed TLS root.""" + +# Copyright (c) 2020, Arm Limited, All Rights Reserved +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file is part of Mbed TLS (https://tls.mbed.org) + +import re +import sys +import os.path +import hashlib +import argparse +import tempfile +import subprocess + +TEMPLATE = """## Description + +These are the release notes for MbedTLS version {version}. + +{description} + +{changelog} + +## Who should update + +{whoshouldupdate} + +## Checksum + +The SHA256 hashes for the archives are: + +``` +{tarhash} mbedtls-{version}.tar.gz +{ziphash} mbedtls-{version}.zip +``` +""" + +WHO_SHOULD_UPDATE_DEFAULT = 'We recommend all affected users should \ +update to take advantage of the bug fixes contained in this release at \ +an appropriate point in their development lifecycle.' + + +CHECKLIST = '''Please review the release notes to ensure that all of the \ +following are documented (if needed): +- Missing functionality +- Changes in functionality +- Known issues +''' + + +CUSTOM_WORDS = 'Hellman API APIs gz lifecycle Bugfix CMake inlined Crypto endian SHA xxx' + + +def sha256_digest(filename): + """Read given file and return a SHA256 digest""" + h = hashlib.sha256() + with open(filename, 'rb') as f: + h.update(f.read()) + return h.hexdigest() + + +def error(text): + """Display error message and exit""" + print(f'ERROR: {text}') + sys.exit(1) + + +def warn(text): + """Display warning message""" + print(f'WARNING: {text}') + + +def generate_content(args): + """Return template populated with given content""" + for field in ('version', 'tarhash', 'ziphash', 'changelog', + 'description', 'whoshouldupdate'): + if not field in args: + error(f'{field} not specified') + return TEMPLATE.format(**args) + + +def run_cmd(cmd, capture=True): + """Run given command in a shell and return the command output""" + # Note: [:-1] strips the trailing newline introduced by the shell. + if capture: + return subprocess.check_output(cmd, shell=True, input=None, + universal_newlines=True)[:-1] + else: + subprocess.call(cmd, shell=True) + + +def parse_args(args): + """Parse command line arguments and return cleaned up args""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('-o', '--output', default='ReleaseNotes.md', + help='Output file (defaults to ReleaseNotes.md)') + parser.add_argument('-t', '--tar', action='store', + help='Optional tar containing release (to generate hash)') + parser.add_argument('-z', '--zip', action='store', + help='Optional zip containing release (to generate hash)') + parser.add_argument('-d', '--description', action='store', required=True, + help='Short description of release (or name of file containing this)') + parser.add_argument('-w', '--who', action='store', default=WHO_SHOULD_UPDATE_DEFAULT, + help='Optional short description of who should \ + update (or name of file containing this)') + args = parser.parse_args(args) + + # If these exist as files, interpret as files containing + # desired content rather than literal content. + for field in ('description', 'who'): + if os.path.exists(getattr(args, field)): + with open(getattr(args, field), 'r') as f: + setattr(args, field, f.read()) + + return args + + +def spellcheck(text): + with tempfile.NamedTemporaryFile() as temp_file: + with open(temp_file.name, 'w') as f: + f.write(text) + result = run_cmd(f'ispell -d american -w _- -a < {temp_file.name}') + input_lines = text.splitlines() + ispell_re = re.compile(r'& (\S+) \d+ \d+:.*') + bad_words = set() + bad_lines = set() + line_no = 1 + for l in result.splitlines(): + if l.strip() == '': + line_no += 1 + elif l.startswith('&'): + m = ispell_re.fullmatch(l) + word = m.group(1) + if word.isupper(): + # ignore all-uppercase words + pass + elif "_" in word: + # part of a non-English 'word' like PSA_CRYPTO_ECC + pass + elif word.startswith('-'): + # ignore flags + pass + elif word in CUSTOM_WORDS: + # accept known-good words + pass + else: + bad_words.add(word) + bad_lines.add(line_no) + if bad_words: + bad_lines = '\n'.join(' ' + input_lines[n] for n in sorted(bad_lines)) + bad_words = ', '.join(bad_words) + warn('Release notes contain the following mis-spelled ' \ + f'words: {bad_words}:\n{bad_lines}\n') + + +def gen_rel_notes(args): + """Return release note content from given command line args""" + # Get version by parsing version.h. Assumption is that bump_version + # has been run and this contains the correct version number. + version = run_cmd('cat include/mbedtls/version.h | \ + clang -Iinclude -dM -E - | grep "MBEDTLS_VERSION_STRING "') + version = version.split()[-1][1:-1] + + # Get main changelog content. + assemble_path = os.path.join(os.getcwd(), 'scripts', 'assemble_changelog.py') + with tempfile.NamedTemporaryFile() as temp_file: + run_cmd(f'{assemble_path} -o {temp_file.name} --latest-only') + with open(temp_file.name) as f: + changelog = f.read() + + arg_hash = { + 'version': version, + 'tarhash': '', + 'ziphash': '', + 'changelog': changelog.strip(), + 'description': args.description.strip(), + 'whoshouldupdate': args.who.strip() + } + + spellcheck(generate_content(arg_hash)) + + arg_hash['tarhash'] = sha256_digest(args.tar) if args.tar else "x" * 64 + arg_hash['ziphash'] = sha256_digest(args.zip) if args.zip else "x" * 64 + return generate_content(arg_hash) + + +def main(): + # Very basic check to see if we are in the root. + path = os.path.join(os.getcwd(), 'scripts', 'generate_release_notes.py') + if not os.path.exists(path): + error(f'{sys.argv[0]} must be run from the mbedtls root') + + args = parse_args(sys.argv[1:]) + + content = gen_rel_notes(args) + with open(args.output, 'w') as f: + f.write(content) + + print(CHECKLIST) + + +if __name__ == '__main__': + main()