espressif/esp-idf
1
0
Fork 1
mirror of https://github.com/espressif/esp-idf.git synced 2025-08-05 13:44:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

check_kconfigs and check_deprecated_kconfigs now use files arguments

Browse Source
This commit is contained in:
Fu Hanxi
2020-10-30 17:28:24 +08:00
parent bcc8f2628c
commit 47a97d2b52
4 changed files with 151 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
.pre-commit-config.yaml
View File

@@ -34,16 +34,14 @@ repos:
always_run: true always_run: true
- id: check-kconfigs - id: check-kconfigs
name: Validate Kconfig files name: Validate Kconfig files
entry: tools/ci/check_kconfigs.py --exclude-submodules entry: tools/ci/check_kconfigs.py
language: python language: python
pass_filenames: false files: '^Kconfig$|Kconfig.*$'
always_run: true
- id: check-deprecated-kconfigs-options - id: check-deprecated-kconfigs-options
name: Check if any Kconfig Options Deprecated name: Check if any Kconfig Options Deprecated
entry: tools/ci/check_deprecated_kconfigs.py --exclude-submodules entry: tools/ci/check_deprecated_kconfigs.py
language: python language: python
pass_filenames: false files: 'sdkconfig\.ci$|sdkconfig\.rename$|sdkconfig.*$'
always_run: true
- id: cmake-lint - id: cmake-lint
name: Check CMake Files Format name: Check CMake Files Format
entry: cmakelint --linelength=120 --spaces=4 entry: cmakelint --linelength=120 --spaces=4

81
tools/ci/check_deprecated_kconfigs.py
View File

@@ -16,10 +16,13 @@
from __future__ import print_function from __future__ import print_function
from __future__ import unicode_literals from __future__ import unicode_literals
import argparse import argparse
import os import os
import sys import sys
from io import open from io import open
from check_kconfigs import valid_directory
from idf_ci_utils import get_submodule_dirs from idf_ci_utils import get_submodule_dirs
# FILES_TO_CHECK used as "startswith" pattern to match sdkconfig.defaults variants # FILES_TO_CHECK used as "startswith" pattern to match sdkconfig.defaults variants
@@ -49,54 +52,66 @@ def _valid_directory(path):
def main(): def main():
default_path = os.getenv('IDF_PATH', None)
parser = argparse.ArgumentParser(description='Kconfig options checker') parser = argparse.ArgumentParser(description='Kconfig options checker')
parser.add_argument('--directory', '-d', help='Path to directory to check recursively ' parser.add_argument('files', nargs='*',
'(for example $IDF_PATH)', help='Kconfig files')
type=_valid_directory, parser.add_argument('--includes', '-d', nargs='*',
required=default_path is None, help='Extra paths for recursively searching Kconfig files. (for example $IDF_PATH)',
default=default_path) type=valid_directory)
parser.add_argument('--exclude-submodules', action='store_true', help='Exclude submodules') parser.add_argument('--exclude-submodules', action='store_true',
help='Exclude submodules')
args = parser.parse_args() args = parser.parse_args()
# IGNORE_DIRS makes sense when the required directory is IDF_PATH success_counter = 0
check_ignore_dirs = default_path is not None and os.path.abspath(args.directory) == os.path.abspath(default_path) failure_counter = 0
ignore_counter = 0
ignores = 0
files_to_check = []
deprecated_options = set() deprecated_options = set()
ret = 0
ignore_dirs = IGNORE_DIRS ignore_dirs = IGNORE_DIRS
if args.exclude_submodules: if args.exclude_submodules:
for submodule in get_submodule_dirs(): for submodule in get_submodule_dirs(full_path=True):
ignore_dirs = ignore_dirs + tuple(submodule) ignore_dirs = ignore_dirs + tuple(submodule)
for root, dirnames, filenames in os.walk(args.directory): files = [os.path.abspath(file_path) for file_path in args.files]
for filename in filenames:
full_path = os.path.join(root, filename)
path_in_idf = os.path.relpath(full_path, args.directory)
if filename.startswith(FILES_TO_CHECK): if args.includes:
if check_ignore_dirs and path_in_idf.startswith(ignore_dirs): for directory in args.includes:
print('{}: Ignored'.format(path_in_idf)) for root, dirnames, filenames in os.walk(directory):
ignores += 1 for filename in filenames:
continue full_path = os.path.join(root, filename)
files_to_check.append(full_path) if filename.startswith(FILES_TO_CHECK):
elif filename == 'sdkconfig.rename': files.append(full_path)
deprecated_options |= _parse_path(full_path) elif filename == 'sdkconfig.rename':
deprecated_options |= _parse_path(full_path)
for path in files_to_check: for full_path in files:
used_options = _parse_path(path, '=') if full_path.startswith(ignore_dirs):
print('{}: Ignored'.format(full_path))
ignore_counter += 1
continue
used_options = _parse_path(full_path, '=')
used_deprecated_options = deprecated_options & used_options used_deprecated_options = deprecated_options & used_options
if len(used_deprecated_options) > 0: if len(used_deprecated_options) > 0:
print('{}: The following options are deprecated: {}'.format(path, ', '.join(used_deprecated_options))) print('{}: The following options are deprecated: {}'
ret = 1 .format(full_path, ', '.join(used_deprecated_options)))
failure_counter += 1
else:
print('{}: OK'.format(full_path))
success_counter += 1
if ignores > 0: if ignore_counter > 0:
print('{} files have been ignored.'.format(ignores)) print('{} files have been ignored.'.format(ignore_counter))
return ret if success_counter > 0:
print('{} files have been successfully checked.'.format(success_counter))
if failure_counter > 0:
print('{} files have errors. Please take a look at the log.'.format(failure_counter))
return 1
if not files:
print('WARNING: no files specified. Please specify files or use '
'"--includes" to search Kconfig files recursively')
return 0
if __name__ == "__main__": if __name__ == "__main__":

163
tools/ci/check_kconfigs.py
View File

@@ -16,12 +16,14 @@
from __future__ import print_function from __future__ import print_function
from __future__ import unicode_literals from __future__ import unicode_literals
import os
import sys
import re
import argparse import argparse
import os
import re
import sys
from io import open from io import open
from idf_ci_utils import get_submodule_dirs
from idf_ci_utils import get_submodule_dirs, IDF_PATH
# regular expression for matching Kconfig files # regular expression for matching Kconfig files
RE_KCONFIG = r'^Kconfig(\.projbuild)?(\.in)?$' RE_KCONFIG = r'^Kconfig(\.projbuild)?(\.in)?$'
@@ -34,10 +36,10 @@ OUTPUT_SUFFIX = '.new'
# accepts tuples but no lists. # accepts tuples but no lists.
IGNORE_DIRS = ( IGNORE_DIRS = (
# Kconfigs from submodules need to be ignored: # Kconfigs from submodules need to be ignored:
os.path.join('components', 'mqtt', 'esp-mqtt'), os.path.join(IDF_PATH, 'components', 'mqtt', 'esp-mqtt'),
# Test Kconfigs are also ignored # Test Kconfigs are also ignored
os.path.join('tools', 'ldgen', 'test', 'data'), os.path.join(IDF_PATH, 'tools', 'ldgen', 'test', 'data'),
os.path.join('tools', 'kconfig_new', 'test'), os.path.join(IDF_PATH, 'tools', 'kconfig_new', 'test'),
) )
SPACES_PER_INDENT = 4 SPACES_PER_INDENT = 4
@@ -283,8 +285,8 @@ class IndentAndNameChecker(BaseChecker):
common_prefix_len = len(common_prefix) common_prefix_len = len(common_prefix)
if common_prefix_len < self.min_prefix_length: if common_prefix_len < self.min_prefix_length:
raise InputError(self.path_in_idf, line_number, raise InputError(self.path_in_idf, line_number,
'The common prefix for the config names of the menu ending at this line is "{}". ' 'The common prefix for the config names of the menu ending at this line is "{}".\n'
'All config names in this menu should start with the same prefix of {} characters ' '\tAll config names in this menu should start with the same prefix of {} characters '
'or more.'.format(common_prefix, self.min_prefix_length), 'or more.'.format(common_prefix, self.min_prefix_length),
line) # no suggested correction for this line) # no suggested correction for this
if len(self.prefix_stack) > 0: if len(self.prefix_stack) > 0:
@@ -371,84 +373,105 @@ def valid_directory(path):
return path return path
def main(): def validate_kconfig_file(kconfig_full_path, verbose=False): # type: (str, bool) -> bool
default_path = os.getenv('IDF_PATH', None) suggestions_full_path = kconfig_full_path + OUTPUT_SUFFIX
fail = False
with open(kconfig_full_path, 'r', encoding='utf-8') as f, \
open(suggestions_full_path, 'w', encoding='utf-8', newline='\n') as f_o, \
LineRuleChecker(kconfig_full_path) as line_checker, \
SourceChecker(kconfig_full_path) as source_checker, \
IndentAndNameChecker(kconfig_full_path, debug=verbose) as indent_and_name_checker:
try:
for line_number, line in enumerate(f, start=1):
try:
for checker in [line_checker, indent_and_name_checker, source_checker]:
checker.process_line(line, line_number)
# The line is correct therefore we echo it to the output file
f_o.write(line)
except InputError as e:
print(e)
fail = True
f_o.write(e.suggested_line)
except UnicodeDecodeError:
raise ValueError("The encoding of {} is not Unicode.".format(kconfig_full_path))
if fail:
print('\t{} has been saved with suggestions for resolving the issues.\n'
'\tPlease note that the suggestions can be wrong and '
'you might need to re-run the checker several times '
'for solving all issues'.format(suggestions_full_path))
print('\tPlease fix the errors and run {} for checking the correctness of '
'Kconfig files.'.format(os.path.abspath(__file__)))
return False
else:
print('{}: OK'.format(kconfig_full_path))
try:
os.remove(suggestions_full_path)
except Exception:
# not a serious error is when the file cannot be deleted
print('{} cannot be deleted!'.format(suggestions_full_path))
finally:
return True
def main():
parser = argparse.ArgumentParser(description='Kconfig style checker') parser = argparse.ArgumentParser(description='Kconfig style checker')
parser.add_argument('--verbose', '-v', help='Print more information (useful for debugging)', parser.add_argument('files', nargs='*',
action='store_true', default=False) help='Kconfig files')
parser.add_argument('--directory', '-d', help='Path to directory where Kconfigs should be recursively checked ' parser.add_argument('--verbose', '-v', action='store_true',
'(for example $IDF_PATH)', help='Print more information (useful for debugging)')
type=valid_directory, parser.add_argument('--includes', '-d', nargs='*',
required=default_path is None, help='Extra paths for recursively searching Kconfig files. (for example $IDF_PATH)',
default=default_path) type=valid_directory)
parser.add_argument('--exclude-submodules', action='store_true', help='Exclude submodules') parser.add_argument('--exclude-submodules', action='store_true',
help='Exclude submodules')
args = parser.parse_args() args = parser.parse_args()
success_counter = 0 success_counter = 0
failure_counter = 0
ignore_counter = 0 ignore_counter = 0
failure = False
ignore_dirs = IGNORE_DIRS ignore_dirs = IGNORE_DIRS
if args.exclude_submodules: if args.exclude_submodules:
for submodule in get_submodule_dirs(): ignore_dirs = ignore_dirs + tuple(get_submodule_dirs(full_path=True))
ignore_dirs = ignore_dirs + tuple(submodule)
# IGNORE_DIRS makes sense when the required directory is IDF_PATH files = [os.path.abspath(file_path) for file_path in args.files]
check_ignore_dirs = default_path is not None and os.path.abspath(args.directory) == os.path.abspath(default_path)
for root, dirnames, filenames in os.walk(args.directory): if args.includes:
for filename in filenames: for directory in args.includes:
full_path = os.path.join(root, filename) for root, dirnames, filenames in os.walk(directory):
path_in_idf = os.path.relpath(full_path, args.directory) for filename in filenames:
if re.search(RE_KCONFIG, filename): full_path = os.path.join(root, filename)
if check_ignore_dirs and path_in_idf.startswith(ignore_dirs): if re.search(RE_KCONFIG, filename):
print('{}: Ignored'.format(path_in_idf)) files.append(full_path)
ignore_counter += 1 elif re.search(RE_KCONFIG, filename, re.IGNORECASE):
continue # On Windows Kconfig files are working with different cases!
suggestions_full_path = full_path + OUTPUT_SUFFIX print('{}: Incorrect filename. The case should be "Kconfig"!'.format(full_path))
with open(full_path, 'r', encoding='utf-8') as f, \ failure_counter += 1
open(suggestions_full_path, 'w', encoding='utf-8', newline='\n') as f_o, \
LineRuleChecker(path_in_idf) as line_checker, \
SourceChecker(path_in_idf) as source_checker, \
IndentAndNameChecker(path_in_idf, debug=args.verbose) as indent_and_name_checker:
try:
for line_number, line in enumerate(f, start=1):
try:
for checker in [line_checker, indent_and_name_checker, source_checker]:
checker.process_line(line, line_number)
# The line is correct therefore we echo it to the output file
f_o.write(line)
except InputError as e:
print(e)
failure = True
f_o.write(e.suggested_line)
except UnicodeDecodeError:
raise ValueError("The encoding of {} is not Unicode.".format(path_in_idf))
if failure: for full_path in files:
print('{} has been saved with suggestions for resolving the issues. Please note that the ' if full_path.startswith(ignore_dirs):
'suggestions can be wrong and you might need to re-run the checker several times ' print('{}: Ignored'.format(full_path))
'for solving all issues'.format(path_in_idf + OUTPUT_SUFFIX)) ignore_counter += 1
print('Please fix the errors and run {} for checking the correctness of ' continue
'Kconfigs.'.format(os.path.relpath(os.path.abspath(__file__), args.directory))) is_valid = validate_kconfig_file(full_path, args.verbose)
return 1 if is_valid:
else: success_counter += 1
success_counter += 1 else:
print('{}: OK'.format(path_in_idf)) failure_counter += 1
try:
os.remove(suggestions_full_path)
except Exception:
# not a serious error is when the file cannot be deleted
print('{} cannot be deleted!'.format(suggestions_full_path))
elif re.search(RE_KCONFIG, filename, re.IGNORECASE):
# On Windows Kconfig files are working with different cases!
raise ValueError('Incorrect filename of {}. The case should be "Kconfig"!'.format(path_in_idf))
if ignore_counter > 0: if ignore_counter > 0:
print('{} files have been ignored.'.format(ignore_counter)) print('{} files have been ignored.'.format(ignore_counter))
if success_counter > 0: if success_counter > 0:
print('{} files have been successfully checked.'.format(success_counter)) print('{} files have been successfully checked.'.format(success_counter))
if failure_counter > 0:
print('{} files have errors. Please take a look at the log.'.format(failure_counter))
return 1
if not files:
print('WARNING: no files specified. Please specify files or use '
'"--includes" to search Kconfig files recursively')
return 0 return 0

9
tools/ci/idf_ci_utils.py
View File

@@ -20,10 +20,10 @@ import logging
import os import os
import subprocess import subprocess
IDF_PATH = os.getenv('IDF_PATH', os.getcwd()) IDF_PATH = os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..'))
def get_submodule_dirs(): # type: () -> list def get_submodule_dirs(full_path=False): # type: (bool) -> list
""" """
To avoid issue could be introduced by multi-os or additional dependency, To avoid issue could be introduced by multi-os or additional dependency,
we use python and git to get this output we use python and git to get this output
@@ -36,7 +36,10 @@ def get_submodule_dirs(): # type: () -> list
'--get-regexp', 'path']).decode('utf8').strip().split('\n') '--get-regexp', 'path']).decode('utf8').strip().split('\n')
for line in lines: for line in lines:
_, path = line.split(' ') _, path = line.split(' ')
dirs.append(path) if full_path:
dirs.append(os.path.join(IDF_PATH, path))
else:
dirs.append(path)
except Exception as e: except Exception as e:
logging.warning(str(e)) logging.warning(str(e))