From 4c1f574c3d495d292568a91dd4990d796f1cf3ee Mon Sep 17 00:00:00 2001 From: Nebojsa Cvetkovic Date: Thu, 6 Jul 2023 15:43:37 +0100 Subject: [PATCH 1/2] feat(idf.py): Allow adding arguments from file via @filename.txt --- tools/idf.py | 52 ++++++++++++++++++++++++++++-- tools/test_idf_py/args_a | 1 + tools/test_idf_py/args_b | 1 + tools/test_idf_py/args_circular_a | 1 + tools/test_idf_py/args_circular_b | 1 + tools/test_idf_py/args_recursive | 1 + tools/test_idf_py/test_idf_py.py | 53 +++++++++++++++++++++++++++++++ 7 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 tools/test_idf_py/args_a create mode 100644 tools/test_idf_py/args_b create mode 100644 tools/test_idf_py/args_circular_a create mode 100644 tools/test_idf_py/args_circular_b create mode 100644 tools/test_idf_py/args_recursive diff --git a/tools/idf.py b/tools/idf.py index 614b955083..2ab5085a9e 100755 --- a/tools/idf.py +++ b/tools/idf.py @@ -18,6 +18,7 @@ import json import locale import os import os.path +import shlex import subprocess import sys from collections import Counter, OrderedDict, _OrderedDictKeysView @@ -695,7 +696,7 @@ def init_cli(verbose_output: List=None) -> Any: return CLI(help=cli_help, verbose_output=verbose_output, all_actions=all_actions) -def main() -> None: +def main(argv=None) -> None: # Check the environment only when idf.py is invoked regularly from command line. checks_output = None if SHELL_COMPLETE_RUN else check_environment() @@ -713,7 +714,54 @@ def main() -> None: else: raise else: - cli(sys.argv[1:], prog_name=PROG, complete_var=SHELL_COMPLETE_VAR) + argv = expand_file_arguments(argv or sys.argv[1:]) + + cli(argv, prog_name=PROG, complete_var=SHELL_COMPLETE_VAR) + + +def expand_file_arguments(argv): + """ + Any argument starting with "@" gets replaced with all values read from a text file. + Text file arguments can be split by newline or by space. + Values are added "as-is", as if they were specified in this order + on the command line. + """ + visited = set() + expanded = False + + def expand_args(args, parent_path, file_stack): + expanded_args = [] + for arg in args: + if not arg.startswith("@"): + expanded_args.append(arg) + else: + nonlocal expanded, visited + expanded = True + + file_name = arg[1:] + rel_path = os.path.normpath(os.path.join(parent_path, file_name)) + + if rel_path in visited: + file_stack_str = ' -> '.join(['@' + f for f in file_stack + [file_name]]) + raise FatalError(f'Circular dependency in file argument expansion: {file_stack_str}') + visited.add(rel_path) + + try: + with open(rel_path, "r") as f: + for line in f: + expanded_args.extend(expand_args(shlex.split(line), os.path.dirname(rel_path), file_stack + [file_name])) + except IOError: + file_stack_str = ' -> '.join(['@' + f for f in file_stack + [file_name]]) + raise FatalError(f"File '{rel_path}' (expansion of {file_stack_str}) could not be opened. " + "Please ensure the file exists and you have the necessary permissions to read it.") + return expanded_args + + argv = expand_args(argv, os.getcwd(), []) + + if expanded: + print(f'Running: idf.py {" ".join(argv)}') + + return argv def _valid_unicode_config() -> Union[codecs.CodecInfo, bool]: diff --git a/tools/test_idf_py/args_a b/tools/test_idf_py/args_a new file mode 100644 index 0000000000..f0252ff863 --- /dev/null +++ b/tools/test_idf_py/args_a @@ -0,0 +1 @@ +-DAAA -DBBB \ No newline at end of file diff --git a/tools/test_idf_py/args_b b/tools/test_idf_py/args_b new file mode 100644 index 0000000000..1c01c9980a --- /dev/null +++ b/tools/test_idf_py/args_b @@ -0,0 +1 @@ +-DCCC -DDDD \ No newline at end of file diff --git a/tools/test_idf_py/args_circular_a b/tools/test_idf_py/args_circular_a new file mode 100644 index 0000000000..38cebf4772 --- /dev/null +++ b/tools/test_idf_py/args_circular_a @@ -0,0 +1 @@ +-DAAA @args_circular_b diff --git a/tools/test_idf_py/args_circular_b b/tools/test_idf_py/args_circular_b new file mode 100644 index 0000000000..f427223c9c --- /dev/null +++ b/tools/test_idf_py/args_circular_b @@ -0,0 +1 @@ +-DBBB @args_circular_a \ No newline at end of file diff --git a/tools/test_idf_py/args_recursive b/tools/test_idf_py/args_recursive new file mode 100644 index 0000000000..7d44f743e7 --- /dev/null +++ b/tools/test_idf_py/args_recursive @@ -0,0 +1 @@ +@args_a -DEEE -DFFF \ No newline at end of file diff --git a/tools/test_idf_py/test_idf_py.py b/tools/test_idf_py/test_idf_py.py index edc63bfec7..1e77047303 100755 --- a/tools/test_idf_py/test_idf_py.py +++ b/tools/test_idf_py/test_idf_py.py @@ -291,5 +291,58 @@ class TestROMs(TestWithoutExtensions): self.assertTrue(build_date_str == k['build_date_str']) +class TestFileArgumentExpansion(TestCase): + def test_file_expansion(self): + """Test @filename expansion functionality""" + try: + output = subprocess.check_output( + [sys.executable, idf_py_path, '--version', '@args_a'], + env=os.environ, + stderr=subprocess.STDOUT).decode('utf-8', 'ignore') + self.assertIn('Running: idf.py --version -DAAA -DBBB', output) + except subprocess.CalledProcessError as e: + self.fail(f'Process should have exited normally, but it exited with a return code of {e.returncode}') + + def test_multiple_file_arguments(self): + """Test multiple @filename arguments""" + try: + output = subprocess.check_output( + [sys.executable, idf_py_path, '--version', '@args_a', '@args_b'], + env=os.environ, + stderr=subprocess.STDOUT).decode('utf-8', 'ignore') + self.assertIn('Running: idf.py --version -DAAA -DBBB -DCCC -DDDD', output) + except subprocess.CalledProcessError as e: + self.fail(f'Process should have exited normally, but it exited with a return code of {e.returncode}') + + def test_recursive_expansion(self): + """Test recursive expansion of @filename arguments""" + try: + output = subprocess.check_output( + [sys.executable, idf_py_path, '--version', '@args_recursive'], + env=os.environ, + stderr=subprocess.STDOUT).decode('utf-8', 'ignore') + self.assertIn('Running: idf.py --version -DAAA -DBBB -DEEE -DFFF', output) + except subprocess.CalledProcessError as e: + self.fail(f'Process should have exited normally, but it exited with a return code of {e.returncode}') + + def test_circular_dependency(self): + """Test circular dependency detection in file argument expansion""" + with self.assertRaises(subprocess.CalledProcessError) as cm: + subprocess.check_output( + [sys.executable, idf_py_path, '--version', '@args_circular_a'], + env=os.environ, + stderr=subprocess.STDOUT).decode('utf-8', 'ignore') + self.assertIn('Circular dependency in file argument expansion', cm.exception.output.decode('utf-8', 'ignore')) + + def test_missing_file(self): + """Test missing file detection in file argument expansion""" + with self.assertRaises(subprocess.CalledProcessError) as cm: + subprocess.check_output( + [sys.executable, idf_py_path, '--version', '@args_non_existent'], + env=os.environ, + stderr=subprocess.STDOUT).decode('utf-8', 'ignore') + self.assertIn('(expansion of @args_non_existent) could not be opened', cm.exception.output.decode('utf-8', 'ignore')) + + if __name__ == '__main__': main() From 076779f41941f828b436c0b8e5b57374f4194620 Mon Sep 17 00:00:00 2001 From: Marek Fiala Date: Fri, 21 Jul 2023 14:46:26 +0200 Subject: [PATCH 2/2] feat(tools): idf.py adding arguments from file via @filename.txt - moved test inputs to one directory - removed `-` from test arguments Closes https://github.com/espressif/esp-idf/pull/11821 Closes https://github.com/espressif/esp-idf/issues/11783 --- tools/idf.py | 12 ++++++------ tools/test_idf_py/args_a | 1 - tools/test_idf_py/args_b | 1 - tools/test_idf_py/args_circular_a | 1 - tools/test_idf_py/args_circular_b | 1 - tools/test_idf_py/args_recursive | 1 - .../file_args_expansion_inputs/args_a | 1 + .../file_args_expansion_inputs/args_b | 1 + .../file_args_expansion_inputs/args_circular_a | 1 + .../file_args_expansion_inputs/args_circular_b | 1 + .../file_args_expansion_inputs/args_recursive | 1 + tools/test_idf_py/test_idf_py.py | 16 ++++++++-------- 12 files changed, 19 insertions(+), 19 deletions(-) delete mode 100644 tools/test_idf_py/args_a delete mode 100644 tools/test_idf_py/args_b delete mode 100644 tools/test_idf_py/args_circular_a delete mode 100644 tools/test_idf_py/args_circular_b delete mode 100644 tools/test_idf_py/args_recursive create mode 100644 tools/test_idf_py/file_args_expansion_inputs/args_a create mode 100644 tools/test_idf_py/file_args_expansion_inputs/args_b create mode 100644 tools/test_idf_py/file_args_expansion_inputs/args_circular_a create mode 100644 tools/test_idf_py/file_args_expansion_inputs/args_circular_b create mode 100644 tools/test_idf_py/file_args_expansion_inputs/args_recursive diff --git a/tools/idf.py b/tools/idf.py index 2ab5085a9e..126a5a4c90 100755 --- a/tools/idf.py +++ b/tools/idf.py @@ -696,7 +696,7 @@ def init_cli(verbose_output: List=None) -> Any: return CLI(help=cli_help, verbose_output=verbose_output, all_actions=all_actions) -def main(argv=None) -> None: +def main(argv: List[Any] = None) -> None: # Check the environment only when idf.py is invoked regularly from command line. checks_output = None if SHELL_COMPLETE_RUN else check_environment() @@ -719,7 +719,7 @@ def main(argv=None) -> None: cli(argv, prog_name=PROG, complete_var=SHELL_COMPLETE_VAR) -def expand_file_arguments(argv): +def expand_file_arguments(argv: List[Any]) -> List[Any]: """ Any argument starting with "@" gets replaced with all values read from a text file. Text file arguments can be split by newline or by space. @@ -729,10 +729,10 @@ def expand_file_arguments(argv): visited = set() expanded = False - def expand_args(args, parent_path, file_stack): + def expand_args(args: List[Any], parent_path: str, file_stack: List[str]) -> List[str]: expanded_args = [] for arg in args: - if not arg.startswith("@"): + if not arg.startswith('@'): expanded_args.append(arg) else: nonlocal expanded, visited @@ -747,13 +747,13 @@ def expand_file_arguments(argv): visited.add(rel_path) try: - with open(rel_path, "r") as f: + with open(rel_path, 'r') as f: for line in f: expanded_args.extend(expand_args(shlex.split(line), os.path.dirname(rel_path), file_stack + [file_name])) except IOError: file_stack_str = ' -> '.join(['@' + f for f in file_stack + [file_name]]) raise FatalError(f"File '{rel_path}' (expansion of {file_stack_str}) could not be opened. " - "Please ensure the file exists and you have the necessary permissions to read it.") + 'Please ensure the file exists and you have the necessary permissions to read it.') return expanded_args argv = expand_args(argv, os.getcwd(), []) diff --git a/tools/test_idf_py/args_a b/tools/test_idf_py/args_a deleted file mode 100644 index f0252ff863..0000000000 --- a/tools/test_idf_py/args_a +++ /dev/null @@ -1 +0,0 @@ --DAAA -DBBB \ No newline at end of file diff --git a/tools/test_idf_py/args_b b/tools/test_idf_py/args_b deleted file mode 100644 index 1c01c9980a..0000000000 --- a/tools/test_idf_py/args_b +++ /dev/null @@ -1 +0,0 @@ --DCCC -DDDD \ No newline at end of file diff --git a/tools/test_idf_py/args_circular_a b/tools/test_idf_py/args_circular_a deleted file mode 100644 index 38cebf4772..0000000000 --- a/tools/test_idf_py/args_circular_a +++ /dev/null @@ -1 +0,0 @@ --DAAA @args_circular_b diff --git a/tools/test_idf_py/args_circular_b b/tools/test_idf_py/args_circular_b deleted file mode 100644 index f427223c9c..0000000000 --- a/tools/test_idf_py/args_circular_b +++ /dev/null @@ -1 +0,0 @@ --DBBB @args_circular_a \ No newline at end of file diff --git a/tools/test_idf_py/args_recursive b/tools/test_idf_py/args_recursive deleted file mode 100644 index 7d44f743e7..0000000000 --- a/tools/test_idf_py/args_recursive +++ /dev/null @@ -1 +0,0 @@ -@args_a -DEEE -DFFF \ No newline at end of file diff --git a/tools/test_idf_py/file_args_expansion_inputs/args_a b/tools/test_idf_py/file_args_expansion_inputs/args_a new file mode 100644 index 0000000000..9d7fc88bab --- /dev/null +++ b/tools/test_idf_py/file_args_expansion_inputs/args_a @@ -0,0 +1 @@ +DAAA DBBB diff --git a/tools/test_idf_py/file_args_expansion_inputs/args_b b/tools/test_idf_py/file_args_expansion_inputs/args_b new file mode 100644 index 0000000000..cc007efe40 --- /dev/null +++ b/tools/test_idf_py/file_args_expansion_inputs/args_b @@ -0,0 +1 @@ +DCCC DDDD diff --git a/tools/test_idf_py/file_args_expansion_inputs/args_circular_a b/tools/test_idf_py/file_args_expansion_inputs/args_circular_a new file mode 100644 index 0000000000..8351f38759 --- /dev/null +++ b/tools/test_idf_py/file_args_expansion_inputs/args_circular_a @@ -0,0 +1 @@ +DAAA @args_circular_b diff --git a/tools/test_idf_py/file_args_expansion_inputs/args_circular_b b/tools/test_idf_py/file_args_expansion_inputs/args_circular_b new file mode 100644 index 0000000000..5b3db9769b --- /dev/null +++ b/tools/test_idf_py/file_args_expansion_inputs/args_circular_b @@ -0,0 +1 @@ +DBBB @args_circular_a diff --git a/tools/test_idf_py/file_args_expansion_inputs/args_recursive b/tools/test_idf_py/file_args_expansion_inputs/args_recursive new file mode 100644 index 0000000000..facd895eb0 --- /dev/null +++ b/tools/test_idf_py/file_args_expansion_inputs/args_recursive @@ -0,0 +1 @@ +@args_a DEEE DFFF diff --git a/tools/test_idf_py/test_idf_py.py b/tools/test_idf_py/test_idf_py.py index 1e77047303..678d93a09b 100755 --- a/tools/test_idf_py/test_idf_py.py +++ b/tools/test_idf_py/test_idf_py.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2019-2022 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2019-2023 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import json @@ -296,10 +296,10 @@ class TestFileArgumentExpansion(TestCase): """Test @filename expansion functionality""" try: output = subprocess.check_output( - [sys.executable, idf_py_path, '--version', '@args_a'], + [sys.executable, idf_py_path, '--version', '@file_args_expansion_inputs/args_a'], env=os.environ, stderr=subprocess.STDOUT).decode('utf-8', 'ignore') - self.assertIn('Running: idf.py --version -DAAA -DBBB', output) + self.assertIn('Running: idf.py --version DAAA DBBB', output) except subprocess.CalledProcessError as e: self.fail(f'Process should have exited normally, but it exited with a return code of {e.returncode}') @@ -307,10 +307,10 @@ class TestFileArgumentExpansion(TestCase): """Test multiple @filename arguments""" try: output = subprocess.check_output( - [sys.executable, idf_py_path, '--version', '@args_a', '@args_b'], + [sys.executable, idf_py_path, '--version', '@file_args_expansion_inputs/args_a', '@file_args_expansion_inputs/args_b'], env=os.environ, stderr=subprocess.STDOUT).decode('utf-8', 'ignore') - self.assertIn('Running: idf.py --version -DAAA -DBBB -DCCC -DDDD', output) + self.assertIn('Running: idf.py --version DAAA DBBB DCCC DDDD', output) except subprocess.CalledProcessError as e: self.fail(f'Process should have exited normally, but it exited with a return code of {e.returncode}') @@ -318,10 +318,10 @@ class TestFileArgumentExpansion(TestCase): """Test recursive expansion of @filename arguments""" try: output = subprocess.check_output( - [sys.executable, idf_py_path, '--version', '@args_recursive'], + [sys.executable, idf_py_path, '--version', '@file_args_expansion_inputs/args_recursive'], env=os.environ, stderr=subprocess.STDOUT).decode('utf-8', 'ignore') - self.assertIn('Running: idf.py --version -DAAA -DBBB -DEEE -DFFF', output) + self.assertIn('Running: idf.py --version DAAA DBBB DEEE DFFF', output) except subprocess.CalledProcessError as e: self.fail(f'Process should have exited normally, but it exited with a return code of {e.returncode}') @@ -329,7 +329,7 @@ class TestFileArgumentExpansion(TestCase): """Test circular dependency detection in file argument expansion""" with self.assertRaises(subprocess.CalledProcessError) as cm: subprocess.check_output( - [sys.executable, idf_py_path, '--version', '@args_circular_a'], + [sys.executable, idf_py_path, '--version', '@file_args_expansion_inputs/args_circular_a'], env=os.environ, stderr=subprocess.STDOUT).decode('utf-8', 'ignore') self.assertIn('Circular dependency in file argument expansion', cm.exception.output.decode('utf-8', 'ignore'))