From 7f61575cba6ad7396d9d2cd3a28fcdf479ef71fb Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 25 Feb 2019 20:17:33 +0100 Subject: [PATCH 01/10] New, documented pylint configuration The pylint configuration in .pylint was a modified version of the output of `pylint --generate-rcfile` from an unknown version of pylint. Replace it with a file that only contains settings that are modified from the default, with an explanation of why each setting is modified. The new .pylintrc was written from scratch, based on the output of pylint on the current version of the files and on a judgement of what to silence generically, what to silence on a case-by-case basis and what to fix. --- .pylint | 425 ---------------------------- .pylintrc | 52 ++++ tests/scripts/check-python-files.sh | 6 +- 3 files changed, 55 insertions(+), 428 deletions(-) delete mode 100644 .pylint create mode 100644 .pylintrc diff --git a/.pylint b/.pylint deleted file mode 100644 index 934f30be5..000000000 --- a/.pylint +++ /dev/null @@ -1,425 +0,0 @@ -[MASTER] - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist= - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS - -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. -ignore-patterns= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Use multiple processes to speed up Pylint. -jobs=1 - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - -# Pickle collected data for later comparisons. -persistent=yes - -# Specify a configuration file. -#rcfile= - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable=print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,long-suffix,old-ne-operator,old-octal-literal,import-star-module-level,raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable= - - -[REPORTS] - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio).You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Tells whether to display a full report or only the messages -reports=no - -# Activate the evaluation score. -score=yes - - -[REFACTORING] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - - -[SIMILARITIES] - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - -# Minimum lines number of a similarity. -min-similarity-lines=4 - - -[FORMAT] - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Maximum number of characters on a single line. -max-line-length=79 - -# Maximum number of lines in a module -max-module-lines=2000 - -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma,dict-separator - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - - -[BASIC] - -# Naming hint for argument names -argument-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct argument names -argument-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Naming hint for attribute names -attr-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct attribute names -attr-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata - -# Naming hint for class attribute names -class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Regular expression matching correct class attribute names -class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Naming hint for class names -class-name-hint=[A-Z_][a-zA-Z0-9]+$ - -# Regular expression matching correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ - -# Naming hint for constant names -const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Regular expression matching correct constant names -const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - -# Naming hint for function names -function-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct function names -function-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - -# Naming hint for inline iteration names -inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ - -# Regular expression matching correct inline iteration names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Naming hint for method names -method-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct method names -method-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Naming hint for module names -module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Regular expression matching correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -property-classes=abc.abstractproperty - -# Naming hint for variable names -variable-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct variable names -variable-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 - - -[VARIABLES] - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_,_cb - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.*|^ignored_|^unused_ - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,future.builtins - - -[SPELLING] - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict,_fields,_replace,_source,_make - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs - - -[DESIGN] - -# Maximum number of arguments for function / method -max-args=5 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Maximum number of boolean expressions in a if statement -max-bool-expr=5 - -# Maximum number of branch for function / method body -max-branches=12 - -# Maximum number of locals for function / method body -max-locals=15 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of statements in function / method body -max-statements=50 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - - -[IMPORTS] - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub,TERMIOS,Bastion,rexec - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 000000000..168e0b759 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,52 @@ +[BASIC] +# We're ok with short funtion argument names. +# [invalid-name] +argument-rgx=[a-z_][a-z0-9_]*$ + +# Allow filter and map. +# [bad-builtin] +bad-functions=input + +# We prefer docstrings, but we don't require them on all functions. +# Require them only on long functions (for some value of long). +# [missing-docstring] +docstring-min-length=10 + +# Allow longer methods than the default. +# [invalid-name] +method-rgx=[a-z_][a-z0-9_]{2,35}$ + +# Allow module names containing a dash (but no underscore or uppercase letter). +# They are whole programs, not meant to be included by another module. +# [invalid-name] +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+)|[a-z][-0-9a-z]+)$ + +# Some functions don't need docstrings. +# [missing-docstring] +no-docstring-rgx=(run_)main$ + +# We're ok with short local or global variable names. +# [invalid-name] +variable-rgx=[a-z_][a-z0-9_]*$ + +[DESIGN] +# Allow more than the default 7 attributes. +# [too-many-instance-attributes] +max-attributes=15 + +[FORMAT] +# Allow longer modules than the default recommended maximum. +# [too-many-lines] +max-module-lines=2000 + +[MESSAGES CONTROL] +disable= + +[REPORTS] +# Don't diplay statistics. Just the facts. +reports=no + +[VARIABLES] +# Allow unused variables if their name starts with an underscore. +# [unused-argument] +dummy-variables-rgx=_.* diff --git a/tests/scripts/check-python-files.sh b/tests/scripts/check-python-files.sh index 009ba4cb0..a37d1d570 100755 --- a/tests/scripts/check-python-files.sh +++ b/tests/scripts/check-python-files.sh @@ -10,9 +10,9 @@ # PEP8 coding standards. if `hash pylint > /dev/null 2>&1`; then - pylint -j 2 tests/scripts/generate_test_code.py --rcfile .pylint - pylint -j 2 tests/scripts/test_generate_test_code.py --rcfile .pylint - pylint -j 2 tests/scripts/mbedtls_test.py --rcfile .pylint + pylint -j 2 tests/scripts/generate_test_code.py + pylint -j 2 tests/scripts/test_generate_test_code.py + pylint -j 2 tests/scripts/mbedtls_test.py else echo "$0: WARNING: 'pylint' not found! Skipping checks on Python files." fi From b2c269eeee8750048a5521df78434cc06510224f Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 25 Feb 2019 20:25:02 +0100 Subject: [PATCH 02/10] Call pylint3, not pylint We use Python 3, so call Pylint for Python 3, not for Python 2. --- tests/scripts/check-python-files.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/scripts/check-python-files.sh b/tests/scripts/check-python-files.sh index a37d1d570..e64d6b331 100755 --- a/tests/scripts/check-python-files.sh +++ b/tests/scripts/check-python-files.sh @@ -9,10 +9,10 @@ # Run 'pylint' on Python files for programming errors and helps enforcing # PEP8 coding standards. -if `hash pylint > /dev/null 2>&1`; then - pylint -j 2 tests/scripts/generate_test_code.py - pylint -j 2 tests/scripts/test_generate_test_code.py - pylint -j 2 tests/scripts/mbedtls_test.py +if `hash pylint3 > /dev/null 2>&1`; then + pylint3 -j 2 tests/scripts/generate_test_code.py + pylint3 -j 2 tests/scripts/test_generate_test_code.py + pylint3 -j 2 tests/scripts/mbedtls_test.py else - echo "$0: WARNING: 'pylint' not found! Skipping checks on Python files." + echo "$0: WARNING: 'pylint3' not found! Skipping checks on Python files." fi From aad2ebdf3074ea61cadae53b0f49475774edf0ac Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 25 Feb 2019 20:26:06 +0100 Subject: [PATCH 03/10] Fix pylint errors going uncaught Make check-python-files.sh run pylint on all *.py files (in directories where they are known to be present), rather than list files explicitly. Fix a bug whereby the return status of check-python-files.sh was only based on the last file passing, i.e. errors in other files were effectively ignored. Make check-python-files.sh run pylint unconditionally. Since pylint3 is not critical, make all.sh to skip running check-python-files.sh if pylint3 is not available. --- tests/scripts/all.sh | 3 +++ tests/scripts/check-python-files.sh | 8 +------- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 90f9632d9..fd9d664bb 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -1344,6 +1344,9 @@ component_test_zeroize () { unset gdb_disable_aslr } +support_check_python_files () { + type pylint3 >/dev/null 2>/dev/null +} component_check_python_files () { msg "Lint: Python scripts" record_status tests/scripts/check-python-files.sh diff --git a/tests/scripts/check-python-files.sh b/tests/scripts/check-python-files.sh index e64d6b331..929041822 100755 --- a/tests/scripts/check-python-files.sh +++ b/tests/scripts/check-python-files.sh @@ -9,10 +9,4 @@ # Run 'pylint' on Python files for programming errors and helps enforcing # PEP8 coding standards. -if `hash pylint3 > /dev/null 2>&1`; then - pylint3 -j 2 tests/scripts/generate_test_code.py - pylint3 -j 2 tests/scripts/test_generate_test_code.py - pylint3 -j 2 tests/scripts/mbedtls_test.py -else - echo "$0: WARNING: 'pylint3' not found! Skipping checks on Python files." -fi +pylint3 -j 2 scripts/*.py tests/scripts/*.py From 0d060ef328fc1ed8dc61a3e6229d6984dbb764a0 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 25 Feb 2019 20:35:31 +0100 Subject: [PATCH 04/10] check-files.py: document some classes and methods Document all classes and longer methods. Declare a static method as such. Pointed out by pylint. --- tests/scripts/check-files.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/scripts/check-files.py b/tests/scripts/check-files.py index 005a077c7..92cae1dc2 100755 --- a/tests/scripts/check-files.py +++ b/tests/scripts/check-files.py @@ -66,6 +66,9 @@ class IssueTracker(object): class PermissionIssueTracker(IssueTracker): + """Track files with bad permissions. + + Files that are not executable scripts must not be executable.""" def __init__(self): super().__init__() @@ -78,6 +81,8 @@ class PermissionIssueTracker(IssueTracker): class EndOfFileNewlineIssueTracker(IssueTracker): + """Track files that end with an incomplete line + (no newline character at the end of the last line).""" def __init__(self): super().__init__() @@ -90,6 +95,8 @@ class EndOfFileNewlineIssueTracker(IssueTracker): class Utf8BomIssueTracker(IssueTracker): + """Track files that start with a UTF-8 BOM. + Files should be ASCII or UTF-8. Valid UTF-8 does not start with a BOM.""" def __init__(self): super().__init__() @@ -102,6 +109,7 @@ class Utf8BomIssueTracker(IssueTracker): class LineEndingIssueTracker(IssueTracker): + """Track files with non-Unix line endings (i.e. files with CR).""" def __init__(self): super().__init__() @@ -112,6 +120,7 @@ class LineEndingIssueTracker(IssueTracker): class TrailingWhitespaceIssueTracker(IssueTracker): + """Track lines with trailing whitespace.""" def __init__(self): super().__init__() @@ -123,6 +132,7 @@ class TrailingWhitespaceIssueTracker(IssueTracker): class TabIssueTracker(IssueTracker): + """Track lines with tabs.""" def __init__(self): super().__init__() @@ -136,6 +146,8 @@ class TabIssueTracker(IssueTracker): class MergeArtifactIssueTracker(IssueTracker): + """Track lines with merge artifacts. + These are leftovers from a ``git merge`` that wasn't fully edited.""" def __init__(self): super().__init__() @@ -157,6 +169,7 @@ class MergeArtifactIssueTracker(IssueTracker): self.record_issue(filepath, line_number) class TodoIssueTracker(IssueTracker): + """Track lines containing ``TODO``.""" def __init__(self): super().__init__() @@ -172,8 +185,12 @@ class TodoIssueTracker(IssueTracker): class IntegrityChecker(object): + """Sanity-check files under the current directory.""" def __init__(self, log_file): + """Instantiate the sanity checker. + Check files under the current directory. + Write a report of issues to log_file.""" self.check_repo_path() self.logger = None self.setup_logger(log_file) @@ -197,7 +214,8 @@ class IntegrityChecker(object): TodoIssueTracker(), ] - def check_repo_path(self): + @staticmethod + def check_repo_path(): if not all(os.path.isdir(d) for d in ["include", "library", "tests"]): raise Exception("Must be run from Mbed TLS root") From 712afa74f43a5b335b43ec7c47ad0d66d792cb47 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 25 Feb 2019 20:36:52 +0100 Subject: [PATCH 05/10] abi_check.py: Document more methods --- scripts/abi_check.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/scripts/abi_check.py b/scripts/abi_check.py index fe5dd3f21..88435ef02 100755 --- a/scripts/abi_check.py +++ b/scripts/abi_check.py @@ -26,8 +26,16 @@ import tempfile class AbiChecker(object): + """API and ABI checker.""" def __init__(self, report_dir, old_rev, new_rev, keep_all_reports): + """Instantiate the API/ABI checker. + + report_dir: directory for output files + old_rev: reference git revision to compare against + new_rev: git revision to check + keep_all_reports: if false, delete old reports + """ self.repo_path = "." self.log = None self.setup_logger() @@ -42,7 +50,8 @@ class AbiChecker(object): self.git_command = "git" self.make_command = "make" - def check_repo_path(self): + @staticmethod + def check_repo_path(): current_dir = os.path.realpath('.') root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) if current_dir != root_dir: @@ -53,12 +62,15 @@ class AbiChecker(object): self.log.setLevel(logging.INFO) self.log.addHandler(logging.StreamHandler()) - def check_abi_tools_are_installed(self): + @staticmethod + def check_abi_tools_are_installed(): for command in ["abi-dumper", "abi-compliance-checker"]: if not shutil.which(command): raise Exception("{} not installed, aborting".format(command)) def get_clean_worktree_for_git_revision(self, git_rev): + """Make a separate worktree with git_rev checked out. + Do not modify the current worktree.""" self.log.info( "Checking out git worktree for revision {}".format(git_rev) ) @@ -88,6 +100,7 @@ class AbiChecker(object): raise Exception("git submodule update failed, aborting") def build_shared_libraries(self, git_worktree_path): + """Build the shared libraries in the specified worktree.""" my_environment = os.environ.copy() my_environment["CFLAGS"] = "-g -Og" my_environment["SHARED"] = "1" @@ -104,6 +117,9 @@ class AbiChecker(object): raise Exception("make failed, aborting") def get_abi_dumps_from_shared_libraries(self, git_ref, git_worktree_path): + """Generate the ABI dumps for the specified git revision. + It must be checked out in git_worktree_path and the shared libraries + must have been built.""" abi_dumps = {} for mbed_module in self.mbedtls_modules: output_path = os.path.join( @@ -129,6 +145,7 @@ class AbiChecker(object): return abi_dumps def cleanup_worktree(self, git_worktree_path): + """Remove the specified git worktree.""" shutil.rmtree(git_worktree_path) worktree_process = subprocess.Popen( [self.git_command, "worktree", "prune"], @@ -142,6 +159,7 @@ class AbiChecker(object): raise Exception("Worktree cleanup failed, aborting") def get_abi_dump_for_ref(self, git_rev): + """Generate the ABI dumps for the specified git revision.""" git_worktree_path = self.get_clean_worktree_for_git_revision(git_rev) self.update_git_submodules(git_worktree_path) self.build_shared_libraries(git_worktree_path) @@ -152,6 +170,9 @@ class AbiChecker(object): return abi_dumps def get_abi_compatibility_report(self): + """Generate a report of the differences between the reference ABI + and the new ABI. ABI dumps from self.old_rev and self.new_rev must + be available.""" compatibility_report = "" compliance_return_code = 0 for mbed_module in self.mbedtls_modules: @@ -201,6 +222,8 @@ class AbiChecker(object): return compliance_return_code def check_for_abi_changes(self): + """Generate a report of ABI differences + between self.old_rev and self.new_rev.""" self.check_repo_path() self.check_abi_tools_are_installed() self.old_dumps = self.get_abi_dump_for_ref(self.old_rev) From 6ee576e0b5dc8cccee57906ac186e3636714abef Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 25 Feb 2019 20:59:05 +0100 Subject: [PATCH 06/10] check-files.py: clean up class structure Line issue trackers are conceptually a subclass of file issue trackers: they're file issue trackers where issues arise from checking each line independently. So make it an actual subclass. Pylint pointed out the design smell: there was an abstract method that wasn't always overridden in concrete child classes. --- tests/scripts/check-files.py | 71 ++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 32 deletions(-) diff --git a/tests/scripts/check-files.py b/tests/scripts/check-files.py index 92cae1dc2..a6743bbfc 100755 --- a/tests/scripts/check-files.py +++ b/tests/scripts/check-files.py @@ -19,10 +19,12 @@ import codecs import sys -class IssueTracker(object): - """Base class for issue tracking. Issues should inherit from this and - overwrite either issue_with_line if they check the file line by line, or - overwrite check_file_for_issue if they check the file as a whole.""" +class FileIssueTracker(object): + """Base class for file-wide issue tracking. + + To implement a checker that processes a file as a whole, inherit from + this class and implement `check_file_for_issue`. + """ def __init__(self): self.heading = "" @@ -35,23 +37,14 @@ class IssueTracker(object): return False return True - def issue_with_line(self, line): - raise NotImplementedError - def check_file_for_issue(self, filepath): - with open(filepath, "rb") as f: - for i, line in enumerate(iter(f.readline, b"")): - self.check_file_line(filepath, line, i + 1) + raise NotImplementedError def record_issue(self, filepath, line_number): if filepath not in self.files_with_issues.keys(): self.files_with_issues[filepath] = [] self.files_with_issues[filepath].append(line_number) - def check_file_line(self, filepath, line, line_number): - if self.issue_with_line(line): - self.record_issue(filepath, line_number) - def output_file_issues(self, logger): if self.files_with_issues.values(): logger.info(self.heading) @@ -64,8 +57,26 @@ class IssueTracker(object): logger.info(filename) logger.info("") +class LineIssueTracker(FileIssueTracker): + """Base class for line-by-line issue tracking. -class PermissionIssueTracker(IssueTracker): + To implement a checker that processes files line by line, inherit from + this class and implement `line_with_issue`. + """ + + def issue_with_line(self, line, filepath): + raise NotImplementedError + + def check_file_line(self, filepath, line, line_number): + if self.issue_with_line(line, filepath): + self.record_issue(filepath, line_number) + + def check_file_for_issue(self, filepath): + with open(filepath, "rb") as f: + for i, line in enumerate(iter(f.readline, b"")): + self.check_file_line(filepath, line, i + 1) + +class PermissionIssueTracker(FileIssueTracker): """Track files with bad permissions. Files that are not executable scripts must not be executable.""" @@ -80,7 +91,7 @@ class PermissionIssueTracker(IssueTracker): self.files_with_issues[filepath] = None -class EndOfFileNewlineIssueTracker(IssueTracker): +class EndOfFileNewlineIssueTracker(FileIssueTracker): """Track files that end with an incomplete line (no newline character at the end of the last line).""" @@ -94,7 +105,7 @@ class EndOfFileNewlineIssueTracker(IssueTracker): self.files_with_issues[filepath] = None -class Utf8BomIssueTracker(IssueTracker): +class Utf8BomIssueTracker(FileIssueTracker): """Track files that start with a UTF-8 BOM. Files should be ASCII or UTF-8. Valid UTF-8 does not start with a BOM.""" @@ -108,18 +119,18 @@ class Utf8BomIssueTracker(IssueTracker): self.files_with_issues[filepath] = None -class LineEndingIssueTracker(IssueTracker): +class LineEndingIssueTracker(LineIssueTracker): """Track files with non-Unix line endings (i.e. files with CR).""" def __init__(self): super().__init__() self.heading = "Non Unix line endings:" - def issue_with_line(self, line): + def issue_with_line(self, line, _filepath): return b"\r" in line -class TrailingWhitespaceIssueTracker(IssueTracker): +class TrailingWhitespaceIssueTracker(LineIssueTracker): """Track lines with trailing whitespace.""" def __init__(self): @@ -127,11 +138,11 @@ class TrailingWhitespaceIssueTracker(IssueTracker): self.heading = "Trailing whitespace:" self.files_exemptions = [".md"] - def issue_with_line(self, line): + def issue_with_line(self, line, _filepath): return line.rstrip(b"\r\n") != line.rstrip() -class TabIssueTracker(IssueTracker): +class TabIssueTracker(LineIssueTracker): """Track lines with tabs.""" def __init__(self): @@ -141,11 +152,11 @@ class TabIssueTracker(IssueTracker): "Makefile", "generate_visualc_files.pl" ] - def issue_with_line(self, line): + def issue_with_line(self, line, _filepath): return b"\t" in line -class MergeArtifactIssueTracker(IssueTracker): +class MergeArtifactIssueTracker(LineIssueTracker): """Track lines with merge artifacts. These are leftovers from a ``git merge`` that wasn't fully edited.""" @@ -153,22 +164,18 @@ class MergeArtifactIssueTracker(IssueTracker): super().__init__() self.heading = "Merge artifact:" - def issue_with_line(self, filepath, line): + def issue_with_line(self, line, _filepath): # Detect leftover git conflict markers. if line.startswith(b'<<<<<<< ') or line.startswith(b'>>>>>>> '): return True if line.startswith(b'||||||| '): # from merge.conflictStyle=diff3 return True if line.rstrip(b'\r\n') == b'=======' and \ - not filepath.endswith('.md'): + not _filepath.endswith('.md'): return True return False - def check_file_line(self, filepath, line, line_number): - if self.issue_with_line(filepath, line): - self.record_issue(filepath, line_number) - -class TodoIssueTracker(IssueTracker): +class TodoIssueTracker(LineIssueTracker): """Track lines containing ``TODO``.""" def __init__(self): @@ -180,7 +187,7 @@ class TodoIssueTracker(IssueTracker): "pull_request_template.md", ] - def issue_with_line(self, line): + def issue_with_line(self, line, _filepath): return b"todo" in line.lower() From 1e9698af4b8b6d393f9537dfea0708cfe9306d42 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 25 Feb 2019 21:10:04 +0100 Subject: [PATCH 07/10] check-files.py: use class fields for class-wide constants In an issue tracker, heading and files_exemptions are class-wide constants, so make them so instead of being per-instance fields. --- tests/scripts/check-files.py | 64 ++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 36 deletions(-) diff --git a/tests/scripts/check-files.py b/tests/scripts/check-files.py index a6743bbfc..19fc528f7 100755 --- a/tests/scripts/check-files.py +++ b/tests/scripts/check-files.py @@ -23,12 +23,19 @@ class FileIssueTracker(object): """Base class for file-wide issue tracking. To implement a checker that processes a file as a whole, inherit from - this class and implement `check_file_for_issue`. + this class and implement `check_file_for_issue` and define ``heading``. + + ``files_exemptions``: files whose name ends with a string in this set + will not be checked. + + ``heading``: human-readable description of the issue """ + files_exemptions = frozenset() + # heading must be defined in derived classes. + # pylint: disable=no-member + def __init__(self): - self.heading = "" - self.files_exemptions = [] self.files_with_issues = {} def should_check_file(self, filepath): @@ -81,9 +88,7 @@ class PermissionIssueTracker(FileIssueTracker): Files that are not executable scripts must not be executable.""" - def __init__(self): - super().__init__() - self.heading = "Incorrect permissions:" + heading = "Incorrect permissions:" def check_file_for_issue(self, filepath): if not (os.access(filepath, os.X_OK) == @@ -95,9 +100,7 @@ class EndOfFileNewlineIssueTracker(FileIssueTracker): """Track files that end with an incomplete line (no newline character at the end of the last line).""" - def __init__(self): - super().__init__() - self.heading = "Missing newline at end of file:" + heading = "Missing newline at end of file:" def check_file_for_issue(self, filepath): with open(filepath, "rb") as f: @@ -109,9 +112,7 @@ class Utf8BomIssueTracker(FileIssueTracker): """Track files that start with a UTF-8 BOM. Files should be ASCII or UTF-8. Valid UTF-8 does not start with a BOM.""" - def __init__(self): - super().__init__() - self.heading = "UTF-8 BOM present:" + heading = "UTF-8 BOM present:" def check_file_for_issue(self, filepath): with open(filepath, "rb") as f: @@ -122,9 +123,7 @@ class Utf8BomIssueTracker(FileIssueTracker): class LineEndingIssueTracker(LineIssueTracker): """Track files with non-Unix line endings (i.e. files with CR).""" - def __init__(self): - super().__init__() - self.heading = "Non Unix line endings:" + heading = "Non Unix line endings:" def issue_with_line(self, line, _filepath): return b"\r" in line @@ -133,10 +132,8 @@ class LineEndingIssueTracker(LineIssueTracker): class TrailingWhitespaceIssueTracker(LineIssueTracker): """Track lines with trailing whitespace.""" - def __init__(self): - super().__init__() - self.heading = "Trailing whitespace:" - self.files_exemptions = [".md"] + heading = "Trailing whitespace:" + files_exemptions = frozenset(".md") def issue_with_line(self, line, _filepath): return line.rstrip(b"\r\n") != line.rstrip() @@ -145,12 +142,11 @@ class TrailingWhitespaceIssueTracker(LineIssueTracker): class TabIssueTracker(LineIssueTracker): """Track lines with tabs.""" - def __init__(self): - super().__init__() - self.heading = "Tabs present:" - self.files_exemptions = [ - "Makefile", "generate_visualc_files.pl" - ] + heading = "Tabs present:" + files_exemptions = frozenset([ + "Makefile", + "generate_visualc_files.pl", + ]) def issue_with_line(self, line, _filepath): return b"\t" in line @@ -160,9 +156,7 @@ class MergeArtifactIssueTracker(LineIssueTracker): """Track lines with merge artifacts. These are leftovers from a ``git merge`` that wasn't fully edited.""" - def __init__(self): - super().__init__() - self.heading = "Merge artifact:" + heading = "Merge artifact:" def issue_with_line(self, line, _filepath): # Detect leftover git conflict markers. @@ -178,14 +172,12 @@ class MergeArtifactIssueTracker(LineIssueTracker): class TodoIssueTracker(LineIssueTracker): """Track lines containing ``TODO``.""" - def __init__(self): - super().__init__() - self.heading = "TODO present:" - self.files_exemptions = [ - os.path.basename(__file__), - "benchmark.c", - "pull_request_template.md", - ] + heading = "TODO present:" + files_exemptions = frozenset([ + os.path.basename(__file__), + "benchmark.c", + "pull_request_template.md", + ]) def issue_with_line(self, line, _filepath): return b"todo" in line.lower() From 23e64f226bf01c6f913ea64dc333858bcc4d7dad Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 25 Feb 2019 21:24:27 +0100 Subject: [PATCH 08/10] check-files.py: readability improvement in permission check --- tests/scripts/check-files.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/scripts/check-files.py b/tests/scripts/check-files.py index 19fc528f7..00fd0edfb 100755 --- a/tests/scripts/check-files.py +++ b/tests/scripts/check-files.py @@ -91,8 +91,9 @@ class PermissionIssueTracker(FileIssueTracker): heading = "Incorrect permissions:" def check_file_for_issue(self, filepath): - if not (os.access(filepath, os.X_OK) == - filepath.endswith((".sh", ".pl", ".py"))): + is_executable = os.access(filepath, os.X_OK) + should_be_executable = filepath.endswith((".sh", ".pl", ".py")) + if is_executable != should_be_executable: self.files_with_issues[filepath] = None From e915d532a6b1a4c7ade1463124be74107c3a765c Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 25 Feb 2019 21:39:42 +0100 Subject: [PATCH 09/10] Silence pylint Silence pylint in specific places where we're doing slightly unusual or dodgy, but correct. --- scripts/abi_check.py | 4 +++- tests/scripts/generate_test_code.py | 2 +- tests/scripts/mbedtls_test.py | 3 ++- tests/scripts/test_generate_test_code.py | 4 +++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/abi_check.py b/scripts/abi_check.py index 88435ef02..7926bc8cd 100755 --- a/scripts/abi_check.py +++ b/scripts/abi_check.py @@ -268,7 +268,9 @@ def run_main(): ) return_code = abi_check.check_for_abi_changes() sys.exit(return_code) - except Exception: + except Exception: # pylint: disable=broad-except + # Print the backtrace and exit explicitly so as to exit with + # status 2, not 1. traceback.print_exc() sys.exit(2) diff --git a/tests/scripts/generate_test_code.py b/tests/scripts/generate_test_code.py index 125802442..1fff09992 100755 --- a/tests/scripts/generate_test_code.py +++ b/tests/scripts/generate_test_code.py @@ -238,7 +238,7 @@ class FileWrapper(io.FileIO, object): if hasattr(parent, '__next__'): line = parent.__next__() # Python 3 else: - line = parent.next() # Python 2 + line = parent.next() # Python 2 # pylint: disable=no-member if line is not None: self._line_no += 1 # Convert byte array to string with correct encoding and diff --git a/tests/scripts/mbedtls_test.py b/tests/scripts/mbedtls_test.py index c7027659f..ac2912d4c 100755 --- a/tests/scripts/mbedtls_test.py +++ b/tests/scripts/mbedtls_test.py @@ -37,7 +37,8 @@ https://github.com/ARMmbed/greentea import re import os import binascii -from mbed_host_tests import BaseHostTest, event_callback + +from mbed_host_tests import BaseHostTest, event_callback # pylint: disable=import-error class TestDataParserError(Exception): diff --git a/tests/scripts/test_generate_test_code.py b/tests/scripts/test_generate_test_code.py index 2ef12e18d..6d7113e18 100755 --- a/tests/scripts/test_generate_test_code.py +++ b/tests/scripts/test_generate_test_code.py @@ -22,7 +22,7 @@ Unit tests for generate_test_code.py """ - +# pylint: disable=wrong-import-order try: # Python 2 from StringIO import StringIO @@ -36,6 +36,7 @@ try: except ImportError: # Python 3 from unittest.mock import patch +# pylint: enable=wrong-import-order from generate_test_code import gen_dependencies, gen_dependencies_one_line from generate_test_code import gen_function_wrapper, gen_dispatch from generate_test_code import parse_until_pattern, GeneratorInputError @@ -336,6 +337,7 @@ class StringIOWrapper(StringIO, object): :param length: :return: """ + # pylint: disable=unused-argument line = super(StringIOWrapper, self).readline() if line is not None: self.line_no += 1 From a0c615ef42d69b8c600c354f67dafd95cba63831 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 27 Feb 2019 11:03:43 +0100 Subject: [PATCH 10/10] Allow main() to lack a docstring. --- .pylintrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index 168e0b759..037717e35 100644 --- a/.pylintrc +++ b/.pylintrc @@ -23,7 +23,7 @@ module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+)|[a-z][-0-9a-z]+)$ # Some functions don't need docstrings. # [missing-docstring] -no-docstring-rgx=(run_)main$ +no-docstring-rgx=(run_)?main$ # We're ok with short local or global variable names. # [invalid-name]