diff --git a/tools/idf_py_actions/diag/purge/README.md b/tools/idf_py_actions/diag/purge/README.md new file mode 100644 index 0000000000..a79ea05329 --- /dev/null +++ b/tools/idf_py_actions/diag/purge/README.md @@ -0,0 +1,16 @@ +# Purge format description for idf.py diag + +Once diagnostic information is collected, the purge file is utilized to remove +any sensitive data from the gathered files. By default, the purge file located +at `tools/idf_py_actions/diag/purge/purge.yml` is used unless it is specified +with the --purge argument, in which case the default file is not used. + +## Overview of Purge Structure + +It is a straightforward YAML file that includes a list of regular expressions +and the corresponding strings that should replace any matches. + + - regex: regular expression to look for + repl: substitute string for match + +The `regex.sub` function from Python is used internally. diff --git a/tools/idf_py_actions/diag/purge.yml b/tools/idf_py_actions/diag/purge/purge.yml similarity index 100% rename from tools/idf_py_actions/diag/purge.yml rename to tools/idf_py_actions/diag/purge/purge.yml diff --git a/tools/idf_py_actions/diag_ext.py b/tools/idf_py_actions/diag_ext.py index 2b0a15d63b..59f01c8cee 100644 --- a/tools/idf_py_actions/diag_ext.py +++ b/tools/idf_py_actions/diag_ext.py @@ -116,8 +116,6 @@ def log(level: int, msg: str, prefix: str) -> None: else: log_prefix = '' - msg = textwrap.indent(msg, prefix=log_prefix) - if LOG_FILE: try: log_msg = textwrap.indent(msg, prefix=f'{prefix} ') @@ -131,6 +129,8 @@ def log(level: int, msg: str, prefix: str) -> None: if level > LOG_LEVEL: return + msg = textwrap.indent(msg, prefix=log_prefix) + if not LOG_COLORS or level not in (LOG_FATAL, LOG_ERROR, LOG_WARNING): print(msg, file=sys.stderr) sys.stderr.flush() @@ -257,11 +257,8 @@ def diff_dirs(dir1: Path, dir2: Path) -> None: dbg(line.strip()) -def redact_files(dir1: Path, dir2: Path) -> None: +def redact_files(dir1: Path, dir2: Path, purge: list) -> None: """Show differences in files between two directories.""" - purge_path = Path(__file__).parent / 'diag' / 'purge.yml' - with open(purge_path, 'r') as f: - purge = yaml.safe_load(f.read()) regexes: List = [] for entry in purge: @@ -488,6 +485,47 @@ def validate_recipe(recipe: Dict) -> None: raise RuntimeError(f'Unknown command "{cmd}" in step "{step_name}"') +def validate_purge(purge: Any) -> None: + """Validate the loaded purge file. This is done manually to avoid any + dependencies and to provide more informative error messages. + """ + + if type(purge) is not list: + raise RuntimeError(f'Purge is not of type "list"') + + regex_keys = ['regex', 'repl'] + + for entry in purge: + if type(entry) is not dict: + raise RuntimeError(f'Purge entry "{entry}" is not of type "dict"') + + if 'regex' in entry: + for key in entry: + if key not in regex_keys: + raise RuntimeError((f'Unknown purge key "{key}" in "{entry}", ' + f'expecting "{regex_keys}"')) + + regex = entry.get('regex') + repl = entry.get('repl') + + # Required arguments + if type(regex) is not str: + raise RuntimeError(f'Argument "regex" for purge entry "{entry}" is not of type "str"') + try: + re.compile(regex) + except re.error as e: + raise RuntimeError((f'Argument "regex" for purge entry "{entry}" is not ' + f'a valid regular expression: {e}')) + + if not repl: + raise RuntimeError(f'Purge entry "{entry}" is missing "repl" argument') + if type(repl) is not str: + raise RuntimeError(f'Argument "repl" for purge entry "{entry}" is not of type "str"') + + else: + raise RuntimeError(f'Unknown purge entry "{entry}"') + + def get_output_path(src: Optional[str], dst: Optional[str], step: Dict, @@ -934,6 +972,7 @@ def create(action: str, check_recipes: bool, cmdl_recipes: Tuple, cmdl_tags: Tuple, + purge_file: str, append: bool, output: Optional[str]) -> None: @@ -1032,11 +1071,25 @@ def create(action: str, except Exception: die(f'File "{recipe_file}" is not a valid diagnostic file') + # Load purge file + dbg(f'Purge file: {purge_file}') + try: + with open(purge_file, 'r') as f: + purge = yaml.safe_load(f.read()) + except Exception: + die(f'Cannot load purge file "{purge_file}"') + + # Validate purge file + try: + validate_purge(purge) + except Exception: + die(f'File "{purge_file}" is not a valid purge file') + # Cook recipes try: for recipe_file, recipe in recipes.items(): desc = recipe.get('description') - dbg(f'Processing recipe "{desc} "file "{recipe_file}"') + dbg(f'Processing recipe "{desc}" file "{recipe_file}"') print(f'{desc}') process_recipe(recipe) except Exception: @@ -1050,9 +1103,9 @@ def create(action: str, LOG_FILE = None try: - redact_files(TMP_DIR_REPORT_PATH, TMP_DIR_REPORT_REDACTED_PATH) + redact_files(TMP_DIR_REPORT_PATH, TMP_DIR_REPORT_REDACTED_PATH, purge) except Exception: - err(f'The redaction was unsuccessful.') + err(f'The redaction was unsuccessful') try: shutil.move(TMP_DIR_REPORT_REDACTED_PATH, output_dir_path) @@ -1142,6 +1195,15 @@ def action_extensions(base_actions: Dict, project_path: str) -> Any: 'and the report directory specified with the --zip option with a zip ' 'extension is used for the zip file archive.') }, + { + 'names': ['-p', '--purge', 'purge_file'], + 'metavar': 'PATH', + 'type': str, + 'default': str(Path(__file__).parent / 'diag' / 'purge' / 'purge.yml'), + 'help': ('Purge file PATH containing a description of what information ' + 'should be redacted from the resulting report. ' + 'Default is "tools/idf_py_actions/diag/purge/purge.yml"') + }, ], }, },