From 9fd0943b7570445d412a5efefa6e3a52be1b8503 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 18 Feb 2020 00:03:23 +0200 Subject: [PATCH 01/61] Fixed an issue when quitting from PlatformIO IDE does not shutdown PIO Home server --- HISTORY.rst | 5 +++++ platformio/commands/home/web.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 5660956c..51dd797a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,6 +6,11 @@ Release Notes PlatformIO Core 4 ----------------- +4.2.2 (2020-??-??) +~~~~~~~~~~~~~~~~~~ + +* Fixed an issue when quitting from PlatformIO IDE does not shutdown PIO Home server + 4.2.1 (2020-02-17) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/home/web.py b/platformio/commands/home/web.py index 45b6c37d..32bf0692 100644 --- a/platformio/commands/home/web.py +++ b/platformio/commands/home/web.py @@ -18,7 +18,7 @@ from twisted.web import static # pylint: disable=import-error class WebRoot(static.File): def render_GET(self, request): - if request.args.get("__shutdown__", False): + if request.args.get(b"__shutdown__", False): reactor.stop() return "Server has been stopped" From ea5f2742f86358f583e9fc4e97945dcec63161bd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 18 Feb 2020 00:05:20 +0200 Subject: [PATCH 02/61] Bump version to 4.2.2a1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 120df28d..d5f2bcd5 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 2, 1) +VERSION = (4, 2, "2a1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From e15f227c483f4ef35f0021e01752494b5ceb72fa Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 18 Feb 2020 14:45:54 +0200 Subject: [PATCH 03/61] Docs: Sync Atmel SAM dev-platform --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 4b50528d..959644e7 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 4b50528d78ad3c5e36d149e275f20b5a7154e3a8 +Subproject commit 959644e711308af9c977025248d59acc60893252 From 1c87f8346323bbfa84c85da79c190232ef60b57b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 18 Feb 2020 21:55:01 +0200 Subject: [PATCH 04/61] Parse package dependencies declared as a list of strings --- platformio/package/manifest/parser.py | 16 +++++++++------- tests/package/test_manifest.py | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 35a00386..87d81f03 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -363,13 +363,15 @@ class LibraryJsonManifestParser(BaseManifestParser): return [dict(name=name, version=version) for name, version in raw.items()] if isinstance(raw, list): for i, dependency in enumerate(raw): - assert isinstance(dependency, dict) - for k, v in dependency.items(): - if k not in ("platforms", "frameworks", "authors"): - continue - if "*" in v: - del raw[i][k] - raw[i][k] = util.items_to_list(v) + if isinstance(dependency, dict): + for k, v in dependency.items(): + if k not in ("platforms", "frameworks", "authors"): + continue + if "*" in v: + del raw[i][k] + raw[i][k] = util.items_to_list(v) + else: + raw[i] = {"name": dependency} return raw raise ManifestParserError( "Invalid dependencies format, should be list or dictionary" diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 22f9691d..80de5e75 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -112,6 +112,21 @@ def test_library_json_parser(): }, ) + raw_data = parser.LibraryJsonManifestParser( + '{"dependencies": ["dep1", "dep2", "@owner/dep3"]}' + ).as_dict() + raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) + assert not jsondiff.diff( + raw_data, + { + "dependencies": [ + {"name": "@owner/dep3"}, + {"name": "dep1"}, + {"name": "dep2"}, + ], + }, + ) + # broken dependencies with pytest.raises(parser.ManifestParserError): parser.LibraryJsonManifestParser({"dependencies": ["deps1", "deps2"]}) From 81ba2a5a74fe362265894f4ae079a6dc04ef8b3f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 20 Feb 2020 18:22:12 +0200 Subject: [PATCH 05/61] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 959644e7..a6a059f8 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 959644e711308af9c977025248d59acc60893252 +Subproject commit a6a059f87e8903046809d8cb3ab6fbbf9cfe0d7e From 49aed34325e9089a245a18f29302d38cec5c3104 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 21 Feb 2020 15:44:24 +0200 Subject: [PATCH 06/61] Rename PIO Plus to Professional --- README.rst | 4 ++-- docs | 2 +- platformio/maintenance.py | 8 -------- platformio/managers/core.py | 2 +- 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index 992b7c0c..0122741d 100644 --- a/README.rst +++ b/README.rst @@ -58,8 +58,8 @@ Instruments * `Continuous Integration `_ * `Advanced Scripting API `_ -PIO Plus --------- +Professional +------------ * `PIO Check `_ * `PIO Remote `_ diff --git a/docs b/docs index a6a059f8..610d40f7 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit a6a059f87e8903046809d8cb3ab6fbbf9cfe0d7e +Subproject commit 610d40f7e7b23d165abb7a3ba0b20688ed3b7719 diff --git a/platformio/maintenance.py b/platformio/maintenance.py index c9b3d742..9a0c85e9 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -186,14 +186,6 @@ def after_upgrade(ctx): click.style("https://platformio.org/platformio-ide", fg="cyan"), ) ) - if not is_ci(): - click.echo( - "- %s us with PlatformIO Plus > %s" - % ( - click.style("support", fg="cyan"), - click.style("https://pioplus.com", fg="cyan"), - ) - ) click.echo("*" * terminal_width) click.echo("") diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 55a0ece4..64d48085 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -190,7 +190,7 @@ def get_contrib_pysite_deps(): def pioplus_call(args, **kwargs): if WINDOWS and sys.version_info < (2, 7, 6): raise exception.PlatformioException( - "PlatformIO Core Plus v%s does not run under Python version %s.\n" + "PlatformIO Remote v%s does not run under Python version %s.\n" "Minimum supported version is 2.7.6, please upgrade Python.\n" "Python 3 is not yet supported.\n" % (__version__, sys.version) ) From dd1fe749564509e83f9c2118272349eb5d6a9b25 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 21 Feb 2020 15:44:55 +0200 Subject: [PATCH 07/61] PyLint fix --- platformio/maintenance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 9a0c85e9..d2e7ea1c 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -28,7 +28,7 @@ from platformio.commands.upgrade import get_latest_version from platformio.managers.core import update_core_packages from platformio.managers.lib import LibraryManager from platformio.managers.platform import PlatformFactory, PlatformManager -from platformio.proc import is_ci, is_container +from platformio.proc import is_container def on_platformio_start(ctx, force, caller): From d3e151feebdb59a1f2e888441ae007a20e9c479c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 29 Feb 2020 18:44:37 +0200 Subject: [PATCH 08/61] Sync docs --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 610d40f7..08d3d48b 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 610d40f7e7b23d165abb7a3ba0b20688ed3b7719 +Subproject commit 08d3d48bc1dc182242075c1cb07790d1d2b33acf diff --git a/examples b/examples index e1d64112..5a66e52e 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit e1d641126d5fb8855da3070a029946fec25118b6 +Subproject commit 5a66e52e1fb59419842aa61fae08e97a710911ea From eee12b9b6684224ba0739e59634f7d6bc47e8069 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 29 Feb 2020 21:59:10 +0200 Subject: [PATCH 09/61] Fixed an issue "the JSON object must be str, not 'bytes'" when PIO Home is used with Python 3.5 // Resolve #3396 --- HISTORY.rst | 1 + platformio/commands/home/rpc/handlers/piocore.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 51dd797a..172f7bee 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,6 +10,7 @@ PlatformIO Core 4 ~~~~~~~~~~~~~~~~~~ * Fixed an issue when quitting from PlatformIO IDE does not shutdown PIO Home server +* Fixed an issue "the JSON object must be str, not 'bytes'" when PIO Home is used with Python 3.5 (`issue #3396 `_) 4.2.1 (2020-02-17) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index 9ef39a03..41009c4e 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -146,6 +146,8 @@ class PIOCoreRPC(object): raise Exception(text) if not to_json: return text + if is_bytes(out): + out = out.decode() try: return json.loads(out) except ValueError as e: From 7fba6f78d65aa9c69bb3d5c548269d83a3e55760 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 29 Feb 2020 21:59:58 +0200 Subject: [PATCH 10/61] Bump version to 4.2.2a2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index d5f2bcd5..974f992d 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 2, "2a1") +VERSION = (4, 2, "2a2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 2d4a3db250afbf102cb8dd7b366fd175846ff402 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 29 Feb 2020 23:08:08 +0200 Subject: [PATCH 11/61] Fixed an issue with expanding $WORKSPACE_DIR for library manager --- platformio/commands/lib.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index d7c42d4c..e3b415d5 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -21,7 +21,7 @@ import click import semantic_version from tabulate import tabulate -from platformio import exception, util +from platformio import exception, fs, util from platformio.commands import PlatformioCLI from platformio.compat import dump_json_to_unicode from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib @@ -106,17 +106,20 @@ def cli(ctx, **options): if not is_platformio_project(storage_dir): ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) continue - config = ProjectConfig.get_instance(os.path.join(storage_dir, "platformio.ini")) - config.validate(options["environment"], silent=in_silence) - libdeps_dir = config.get_optional_dir("libdeps") - for env in config.envs(): - if options["environment"] and env not in options["environment"]: - continue - storage_dir = os.path.join(libdeps_dir, env) - ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) - ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY][storage_dir] = config.get( - "env:" + env, "lib_deps", [] + with fs.cd(storage_dir): + config = ProjectConfig.get_instance( + os.path.join(storage_dir, "platformio.ini") ) + config.validate(options["environment"], silent=in_silence) + libdeps_dir = config.get_optional_dir("libdeps") + for env in config.envs(): + if options["environment"] and env not in options["environment"]: + continue + storage_dir = os.path.join(libdeps_dir, env) + ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) + ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY][storage_dir] = config.get( + "env:" + env, "lib_deps", [] + ) @cli.command("install", short_help="Install library") From a271143c520c0a80f1f08efd1be08fe35a60f8ea Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 2 Mar 2020 23:25:28 +0200 Subject: [PATCH 12/61] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 08d3d48b..b5a63783 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 08d3d48bc1dc182242075c1cb07790d1d2b33acf +Subproject commit b5a63783ec0d4009f4c261a2f88555c5c717c2d5 From 314fe7d309675b7cb1538f27b23434d959d3ce74 Mon Sep 17 00:00:00 2001 From: valeros Date: Tue, 3 Mar 2020 00:58:07 +0200 Subject: [PATCH 13/61] Initial support for menuconfig target --- platformio/managers/platform.py | 36 +++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 8f23f1bd..8e107c1f 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -436,16 +436,23 @@ class PlatformRunMixin(object): for key, value in variables.items(): args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value))) - def _write_and_flush(stream, data): - try: - stream.write(data) - stream.flush() - except IOError: - pass - proc.copy_pythonpath_to_osenv() + + if targets and "menuconfig" in targets: + return proc.exec_command( + args, stdout=sys.stdout, stderr=sys.stderr, stdin=sys.stdin + ) + if click._compat.isatty(sys.stdout): - result = proc.exec_command( + + def _write_and_flush(stream, data): + try: + stream.write(data) + stream.flush() + except IOError: + pass + + return proc.exec_command( args, stdout=proc.BuildAsyncPipe( line_callback=self._on_stdout_line, @@ -456,13 +463,12 @@ class PlatformRunMixin(object): data_callback=lambda data: _write_and_flush(sys.stderr, data), ), ) - else: - result = proc.exec_command( - args, - stdout=proc.LineBufferedAsyncPipe(line_callback=self._on_stdout_line), - stderr=proc.LineBufferedAsyncPipe(line_callback=self._on_stderr_line), - ) - return result + + return proc.exec_command( + args, + stdout=proc.LineBufferedAsyncPipe(line_callback=self._on_stdout_line), + stderr=proc.LineBufferedAsyncPipe(line_callback=self._on_stderr_line), + ) def _on_stdout_line(self, line): if "`buildprog' is up to date." in line: From de60f20c2146569617ad445d7cc4ca9200e44ed8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 3 Mar 2020 14:59:03 +0200 Subject: [PATCH 14/61] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index b5a63783..0764e124 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit b5a63783ec0d4009f4c261a2f88555c5c717c2d5 +Subproject commit 0764e1242957c60c2e92c3d65a06e9ac1059afa5 From 0c0ceb2caa794f698ee442dc2f6053a26d70ac5a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 3 Mar 2020 23:03:14 +0200 Subject: [PATCH 15/61] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 0764e124..ac32d956 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 0764e1242957c60c2e92c3d65a06e9ac1059afa5 +Subproject commit ac32d9569ef213b784c196f0e35ab38faab29176 From 261c46d4ef5e1b78344edc70cb23ef5f0051c51e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 3 Mar 2020 23:10:19 +0200 Subject: [PATCH 16/61] Add support for Arm Mbed "module.json" ``dependencies`` field // Resolve #3400 --- HISTORY.rst | 1 + platformio/package/manifest/parser.py | 11 +++++++++++ tests/package/test_manifest.py | 12 ++++++++++++ 3 files changed, 24 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 172f7bee..16145dfa 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,7 @@ PlatformIO Core 4 4.2.2 (2020-??-??) ~~~~~~~~~~~~~~~~~~ +* Added support for Arm Mbed "module.json" ``dependencies`` field (`issue #3400 `_) * Fixed an issue when quitting from PlatformIO IDE does not shutdown PIO Home server * Fixed an issue "the JSON object must be str, not 'bytes'" when PIO Home is used with Python 3.5 (`issue #3396 `_) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 87d81f03..8d9cf654 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -392,6 +392,8 @@ class ModuleJsonManifestParser(BaseManifestParser): if "licenses" in data: data["license"] = self._parse_license(data.get("licenses")) del data["licenses"] + if "dependencies" in data: + data["dependencies"] = self._parse_dependencies(data["dependencies"]) return data def _parse_authors(self, raw): @@ -411,6 +413,15 @@ class ModuleJsonManifestParser(BaseManifestParser): return None return raw[0].get("type") + @staticmethod + def _parse_dependencies(raw): + if isinstance(raw, dict): + return [ + dict(name=name, version=version, frameworks=["mbed"]) + for name, version in raw.items() + ] + raise ManifestParserError("Invalid dependencies format, should be a dictionary") + class LibraryPropertiesManifestParser(BaseManifestParser): manifest_type = ManifestFileType.LIBRARY_PROPERTIES diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 80de5e75..abf66ab3 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -154,6 +154,10 @@ def test_module_json_parser(): "url": "git@github.com:username/repo.git" }, "version": "1.2.3", + "dependencies": { + "usefulmodule": "^1.2.3", + "simplelog": "ARMmbed/simplelog#~0.0.1" + }, "customField": "Custom Value" } """ @@ -173,6 +177,14 @@ def test_module_json_parser(): "authors": [{"email": "name@surname.com", "name": "Name Surname"}], "version": "1.2.3", "repository": {"type": "git", "url": "git@github.com:username/repo.git"}, + "dependencies": [ + {"name": "usefulmodule", "version": "^1.2.3", "frameworks": ["mbed"]}, + { + "name": "simplelog", + "version": "ARMmbed/simplelog#~0.0.1", + "frameworks": ["mbed"], + }, + ], "customField": "Custom Value", }, ) From fbb62fa8a6b51467078312794ca35b1ca5f96db6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 3 Mar 2020 23:10:54 +0200 Subject: [PATCH 17/61] Bump version to 4.2.2a3 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 974f992d..859c5479 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 2, "2a2") +VERSION = (4, 2, "2a3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 3275bb59bffc997261862aec42b138d5eb380f7b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 4 Mar 2020 18:14:51 +0200 Subject: [PATCH 18/61] Fix test --- .style.yapf | 3 --- tests/package/test_manifest.py | 7 ++++--- 2 files changed, 4 insertions(+), 6 deletions(-) delete mode 100644 .style.yapf diff --git a/.style.yapf b/.style.yapf deleted file mode 100644 index 7bc85964..00000000 --- a/.style.yapf +++ /dev/null @@ -1,3 +0,0 @@ -[style] -blank_line_before_nested_class_or_def = true -allow_multiline_lambdas = true diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index abf66ab3..1ad66a75 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -162,9 +162,10 @@ def test_module_json_parser(): } """ - mp = parser.ModuleJsonManifestParser(contents) + raw_data = parser.ModuleJsonManifestParser(contents).as_dict() + raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) assert not jsondiff.diff( - mp.as_dict(), + raw_data, { "name": "YottaLibrary", "description": "This is Yotta library", @@ -178,12 +179,12 @@ def test_module_json_parser(): "version": "1.2.3", "repository": {"type": "git", "url": "git@github.com:username/repo.git"}, "dependencies": [ - {"name": "usefulmodule", "version": "^1.2.3", "frameworks": ["mbed"]}, { "name": "simplelog", "version": "ARMmbed/simplelog#~0.0.1", "frameworks": ["mbed"], }, + {"name": "usefulmodule", "version": "^1.2.3", "frameworks": ["mbed"]}, ], "customField": "Custom Value", }, From ce6b96ea843de39c3bb125cce1de039554b68c05 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 5 Mar 2020 23:52:13 +0200 Subject: [PATCH 19/61] Use native open/io.open for file contents reading/writing --- platformio/builder/tools/piolib.py | 3 +- platformio/builder/tools/piomaxlen.py | 4 +-- platformio/commands/debug/helpers.py | 10 +++--- platformio/commands/home/rpc/handlers/os.py | 4 ++- .../commands/home/rpc/handlers/project.py | 3 +- platformio/commands/project.py | 35 ++++++++++--------- platformio/commands/remote.py | 3 +- platformio/commands/run/helpers.py | 9 +++-- platformio/commands/test/processor.py | 5 +-- platformio/package/manifest/parser.py | 19 ++++++---- 10 files changed, 56 insertions(+), 39 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index b55c492e..1a659c6a 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -82,7 +82,8 @@ class LibBuilderFactory(object): fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT ): continue - content = fs.get_file_contents(join(root, fname)) + with open(join(root, fname)) as fp: + content = fp.read() if not content: continue if "Arduino.h" in content and include_re.search(content): diff --git a/platformio/builder/tools/piomaxlen.py b/platformio/builder/tools/piomaxlen.py index 3bf258db..04f1304f 100644 --- a/platformio/builder/tools/piomaxlen.py +++ b/platformio/builder/tools/piomaxlen.py @@ -18,7 +18,6 @@ from hashlib import md5 from os import makedirs from os.path import isdir, isfile, join -from platformio import fs from platformio.compat import WINDOWS, hashlib_encode_data # Windows CLI has limit with command length to 8192 @@ -67,7 +66,8 @@ def _file_long_data(env, data): ) if isfile(tmp_file): return tmp_file - fs.write_file_contents(tmp_file, data) + with open(tmp_file, "w") as fp: + fp.write(data) return tmp_file diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index 5edba11a..0ec7d447 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -252,12 +252,14 @@ def is_prog_obsolete(prog_path): break shasum.update(data) new_digest = shasum.hexdigest() - old_digest = ( - fs.get_file_contents(prog_hash_path) if isfile(prog_hash_path) else None - ) + old_digest = None + if isfile(prog_hash_path): + with open(prog_hash_path) as fp: + old_digest = fp.read() if new_digest == old_digest: return False - fs.write_file_contents(prog_hash_path, new_digest) + with open(prog_hash_path, "w") as fp: + fp.write(new_digest) return True diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index cefb0630..745ae817 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -15,6 +15,7 @@ from __future__ import absolute_import import glob +import io import os import shutil from functools import cmp_to_key @@ -66,7 +67,8 @@ class OSRPC(object): if uri.startswith("http"): return self.fetch_content(uri, data, headers, cache_valid) if os.path.isfile(uri): - return fs.get_file_contents(uri, encoding="utf8") + with io.open(uri, encoding="utf-8") as fp: + return fp.read() return None @staticmethod diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 51e47dcb..77a04646 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -244,7 +244,8 @@ class ProjectRPC(object): return project_dir if not os.path.isdir(src_dir): os.makedirs(src_dir) - fs.write_file_contents(main_path, main_content.strip()) + with open(main_path, "w") as fp: + fp.write(main_content.strip()) return project_dir def import_arduino(self, board, use_arduino_libs, arduino_project_dir): diff --git a/platformio/commands/project.py b/platformio/commands/project.py index 33d35006..3d73f4ff 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -187,9 +187,9 @@ def init_base_project(project_dir): def init_include_readme(include_dir): - fs.write_file_contents( - os.path.join(include_dir, "README"), - """ + with open(os.path.join(include_dir, "README"), "w") as fp: + fp.write( + """ This directory is intended for project header files. A header file is a file containing C declarations and macro definitions @@ -229,14 +229,14 @@ Read more about using header files in official GCC documentation: https://gcc.gnu.org/onlinedocs/cpp/Header-Files.html """, - ) + ) def init_lib_readme(lib_dir): # pylint: disable=line-too-long - fs.write_file_contents( - os.path.join(lib_dir, "README"), - """ + with open(os.path.join(lib_dir, "README"), "w") as fp: + fp.write( + """ This directory is intended for project specific (private) libraries. PlatformIO will compile them to static libraries and link into executable file. @@ -283,13 +283,13 @@ libraries scanning project source files. More information about PlatformIO Library Dependency Finder - https://docs.platformio.org/page/librarymanager/ldf.html """, - ) + ) def init_test_readme(test_dir): - fs.write_file_contents( - os.path.join(test_dir, "README"), - """ + with open(os.path.join(test_dir, "README"), "w") as fp: + fp.write( + """ This directory is intended for PIO Unit Testing and project tests. Unit Testing is a software testing method by which individual units of @@ -301,16 +301,16 @@ in the development cycle. More information about PIO Unit Testing: - https://docs.platformio.org/page/plus/unit-testing.html """, - ) + ) def init_ci_conf(project_dir): conf_path = os.path.join(project_dir, ".travis.yml") if os.path.isfile(conf_path): return - fs.write_file_contents( - conf_path, - """# Continuous Integration (CI) is the practice, in software + with open(conf_path, "w") as fp: + fp.write( + """# Continuous Integration (CI) is the practice, in software # engineering, of merging all developer working copies with a shared mainline # several times a day < https://docs.platformio.org/page/ci/index.html > # @@ -378,14 +378,15 @@ def init_ci_conf(project_dir): # script: # - platformio ci --lib="." --board=ID_1 --board=ID_2 --board=ID_N """, - ) + ) def init_cvs_ignore(project_dir): conf_path = os.path.join(project_dir, ".gitignore") if os.path.isfile(conf_path): return - fs.write_file_contents(conf_path, ".pio\n") + with open(conf_path, "w") as fp: + fp.write(".pio\n") def fill_project_envs( diff --git a/platformio/commands/remote.py b/platformio/commands/remote.py index 0ebbf856..ee950982 100644 --- a/platformio/commands/remote.py +++ b/platformio/commands/remote.py @@ -222,7 +222,8 @@ def device_monitor(ctx, **kwargs): sleep(0.1) if not t.is_alive(): return - kwargs["port"] = fs.get_file_contents(sock_file) + with open(sock_file) as fp: + kwargs["port"] = fp.read() ctx.invoke(device.device_monitor, **kwargs) t.join(2) finally: diff --git a/platformio/commands/run/helpers.py b/platformio/commands/run/helpers.py index ec038e84..ab362c24 100644 --- a/platformio/commands/run/helpers.py +++ b/platformio/commands/run/helpers.py @@ -53,9 +53,12 @@ def clean_build_dir(build_dir, config): if isdir(build_dir): # check project structure - if isfile(checksum_file) and fs.get_file_contents(checksum_file) == checksum: - return + if isfile(checksum_file): + with open(checksum_file) as fp: + if fp.read() == checksum: + return fs.rmtree(build_dir) makedirs(build_dir) - fs.write_file_contents(checksum_file, checksum) + with open(checksum_file, "w") as fp: + fp.write(checksum) diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index 7d9704b8..64a7d3dd 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -19,7 +19,7 @@ from string import Template import click -from platformio import exception, fs +from platformio import exception TRANSPORT_OPTIONS = { "arduino": { @@ -195,6 +195,7 @@ class TestProcessorBase(object): data = Template(tpl).substitute(baudrate=self.get_baudrate()) tmp_file = join(test_dir, "output_export.cpp") - fs.write_file_contents(tmp_file, data) + with open(tmp_file, "w") as fp: + fp.write(data) atexit.register(delete_tmptest_file, tmp_file) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 8d9cf654..60138609 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -13,6 +13,7 @@ # limitations under the License. import inspect +import io import json import os import re @@ -21,7 +22,6 @@ import requests from platformio import util from platformio.compat import get_class_attributes, string_types -from platformio.fs import get_file_contents from platformio.package.exception import ManifestParserError, UnknownManifestError from platformio.project.helpers import is_platformio_project @@ -59,24 +59,29 @@ class ManifestFileType(object): class ManifestParserFactory(object): @staticmethod - def new_from_file(path, remote_url=False): + def read_manifest_contents(path): + with io.open(path, encoding="utf-8") as fp: + return fp.read() + + @classmethod + def new_from_file(cls, path, remote_url=False): if not path or not os.path.isfile(path): raise UnknownManifestError("Manifest file does not exist %s" % path) type_from_uri = ManifestFileType.from_uri(path) if not type_from_uri: raise UnknownManifestError("Unknown manifest file type %s" % path) return ManifestParserFactory.new( - get_file_contents(path, encoding="utf8"), type_from_uri, remote_url + cls.read_manifest_contents(path), type_from_uri, remote_url ) - @staticmethod - def new_from_dir(path, remote_url=None): + @classmethod + def new_from_dir(cls, path, remote_url=None): assert os.path.isdir(path), "Invalid directory %s" % path type_from_uri = ManifestFileType.from_uri(remote_url) if remote_url else None if type_from_uri and os.path.isfile(os.path.join(path, type_from_uri)): return ManifestParserFactory.new( - get_file_contents(os.path.join(path, type_from_uri), encoding="utf8"), + cls.read_manifest_contents(os.path.join(path, type_from_uri)), type_from_uri, remote_url=remote_url, package_dir=path, @@ -88,7 +93,7 @@ class ManifestParserFactory(object): "Unknown manifest file type in %s directory" % path ) return ManifestParserFactory.new( - get_file_contents(os.path.join(path, type_from_dir), encoding="utf8"), + cls.read_manifest_contents(os.path.join(path, type_from_dir)), type_from_dir, remote_url=remote_url, package_dir=path, From 3a27fbc883cdcd12c241dafcf9d512c214c10813 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 5 Mar 2020 23:52:46 +0200 Subject: [PATCH 20/61] Fixed an issue when Python 2 does not keep encoding when converting .INO file // Resolve #3393 --- HISTORY.rst | 1 + platformio/builder/tools/piomisc.py | 48 ++++++++++++++++++++++++----- platformio/fs.py | 38 +++++------------------ 3 files changed, 50 insertions(+), 37 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 16145dfa..ea04061a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -12,6 +12,7 @@ PlatformIO Core 4 * Added support for Arm Mbed "module.json" ``dependencies`` field (`issue #3400 `_) * Fixed an issue when quitting from PlatformIO IDE does not shutdown PIO Home server * Fixed an issue "the JSON object must be str, not 'bytes'" when PIO Home is used with Python 3.5 (`issue #3396 `_) +* Fixed an issue when Python 2 does not keep encoding when converting .INO file (`issue #3393 `_) 4.2.1 (2020-02-17) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index ba019ec3..2df5303d 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -15,17 +15,19 @@ from __future__ import absolute_import import atexit +import io import re import sys from os import environ, remove, walk from os.path import basename, isdir, isfile, join, realpath, relpath, sep from tempfile import mkstemp +import click from SCons.Action import Action # pylint: disable=import-error from SCons.Script import ARGUMENTS # pylint: disable=import-error from platformio import fs, util -from platformio.compat import glob_escape +from platformio.compat import get_filesystem_encoding, get_locale_encoding, glob_escape from platformio.managers.core import get_core_package_dir from platformio.proc import exec_command @@ -48,6 +50,40 @@ class InoToCPPConverter(object): def __init__(self, env): self.env = env self._main_ino = None + self._safe_encoding = None + + def read_safe_contents(self, path): + last_exc = None + for encoding in ( + "utf-8", + None, + get_filesystem_encoding(), + get_locale_encoding(), + "latin-1", + ): + try: + with io.open(path, encoding=encoding) as fp: + contents = fp.read() + self._safe_encoding = encoding + return contents + except UnicodeDecodeError as e: + last_exc = e + click.secho( + "Unicode decode error has occurred, please remove invalid " + "(non-ASCII or non-UTF8) characters from %s file or convert it to UTF-8" + % path, + fg="yellow", + err=True, + ) + if last_exc: + raise last_exc + return None + + def write_safe_contents(self, path, contents): + with io.open( + path, "w", encoding=self._safe_encoding, errors="backslashreplace" + ) as fp: + return fp.write(contents) def is_main_node(self, contents): return self.DETECTMAIN_RE.search(contents) @@ -62,7 +98,7 @@ class InoToCPPConverter(object): assert nodes lines = [] for node in nodes: - contents = fs.get_file_contents(node.get_path()) + contents = self.read_safe_contents(node.get_path()) _lines = ['# 1 "%s"' % node.get_path().replace("\\", "/"), contents] if self.is_main_node(contents): lines = _lines + lines @@ -78,16 +114,14 @@ class InoToCPPConverter(object): def process(self, contents): out_file = self._main_ino + ".cpp" assert self._gcc_preprocess(contents, out_file) - contents = fs.get_file_contents(out_file) + contents = self.read_safe_contents(out_file) contents = self._join_multiline_strings(contents) - fs.write_file_contents( - out_file, self.append_prototypes(contents), errors="backslashreplace" - ) + self.write_safe_contents(out_file, self.append_prototypes(contents)) return out_file def _gcc_preprocess(self, contents, out_file): tmp_path = mkstemp()[1] - fs.write_file_contents(tmp_path, contents, errors="backslashreplace") + self.write_safe_contents(tmp_path, contents) self.env.Execute( self.env.VerboseAction( '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format( diff --git a/platformio/fs.py b/platformio/fs.py index ed0102cd..575a14e5 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import io import json import os import re @@ -49,30 +48,6 @@ def get_source_dir(): return os.path.dirname(curpath) -def get_file_contents(path, encoding=None): - try: - with io.open(path, encoding=encoding) as fp: - return fp.read() - except UnicodeDecodeError: - click.secho( - "Unicode decode error has occurred, please remove invalid " - "(non-ASCII or non-UTF8) characters from %s file" % path, - fg="yellow", - err=True, - ) - with io.open(path, encoding="latin-1") as fp: - return fp.read() - - -def write_file_contents(path, contents, errors=None): - try: - with open(path, "w") as fp: - return fp.write(contents) - except UnicodeEncodeError: - with io.open(path, "w", encoding="latin-1", errors=errors) as fp: - return fp.write(contents) - - def load_json(file_path): try: with open(file_path, "r") as f: @@ -102,11 +77,14 @@ def ensure_udev_rules(): from platformio.util import get_systype # pylint: disable=import-outside-toplevel def _rules_to_set(rules_path): - return set( - l.strip() - for l in get_file_contents(rules_path).split("\n") - if l.strip() and not l.startswith("#") - ) + result = set() + with open(rules_path) as fp: + for line in fp.readlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + result.add(line) + return result if "linux" not in get_systype(): return None From 59e1c8872680bd2c8af01471f3351707228f2c2c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 6 Mar 2020 00:37:48 +0200 Subject: [PATCH 21/61] Fixed an issue when ``"libArchive": false`` in "library.json" does not work // Resolve #3403 --- HISTORY.rst | 3 ++- platformio/builder/tools/piolib.py | 8 +++++--- platformio/builder/tools/pioproject.py | 4 ++-- platformio/project/config.py | 2 +- tests/test_projectconf.py | 2 ++ 5 files changed, 12 insertions(+), 7 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index ea04061a..1c444e3c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -12,7 +12,8 @@ PlatformIO Core 4 * Added support for Arm Mbed "module.json" ``dependencies`` field (`issue #3400 `_) * Fixed an issue when quitting from PlatformIO IDE does not shutdown PIO Home server * Fixed an issue "the JSON object must be str, not 'bytes'" when PIO Home is used with Python 3.5 (`issue #3396 `_) -* Fixed an issue when Python 2 does not keep encoding when converting .INO file (`issue #3393 `_) +* Fixed an issue when Python 2 does not keep encoding when converting ".ino" (`issue #3393 `_) +* Fixed an issue when ``"libArchive": false`` in "library.json" does not work (`issue #3403 `_) 4.2.1 (2020-02-17) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 1a659c6a..5cb50492 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -717,9 +717,11 @@ class PlatformIOLibBuilder(LibBuilderBase): @property def lib_archive(self): - unique_value = "_not_declared_%s" % id(self) - global_value = self.env.GetProjectOption("lib_archive", unique_value) - if global_value != unique_value: + missing = object() + global_value = self.env.GetProjectConfig().getraw( + "env:" + self.env["PIOENV"], "lib_archive", missing + ) + if global_value != missing: return global_value return self._manifest.get("build", {}).get( "libArchive", LibBuilderBase.lib_archive.fget(self) diff --git a/platformio/builder/tools/pioproject.py b/platformio/builder/tools/pioproject.py index 4a42b3d8..4bf848d9 100644 --- a/platformio/builder/tools/pioproject.py +++ b/platformio/builder/tools/pioproject.py @@ -14,7 +14,7 @@ from __future__ import absolute_import -from platformio.project.config import ProjectConfig, ProjectOptions +from platformio.project.config import MISSING, ProjectConfig, ProjectOptions def GetProjectConfig(env): @@ -25,7 +25,7 @@ def GetProjectOptions(env, as_dict=False): return env.GetProjectConfig().items(env=env["PIOENV"], as_dict=as_dict) -def GetProjectOption(env, option, default=None): +def GetProjectOption(env, option, default=MISSING): return env.GetProjectConfig().get("env:" + env["PIOENV"], option, default) diff --git a/platformio/project/config.py b/platformio/project/config.py index 4dc3c38c..23d089bf 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -280,7 +280,7 @@ class ProjectConfigBase(object): value = envvar_value if value == MISSING: - value = option_meta.default or default + value = default if default != MISSING else option_meta.default if value == MISSING: return None diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index 8172e692..0e68f2d3 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -123,6 +123,8 @@ def test_defaults(config): ) assert config.get("env:extra_2", "lib_compat_mode") == "soft" assert config.get("env:extra_2", "build_type") == "release" + assert config.get("env:extra_2", "build_type", None) is None + assert config.get("env:extra_2", "lib_archive", "no") is False def test_sections(config): From 3ef96cb215024709f463d5b1b5574ad66bca3ebf Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 6 Mar 2020 00:43:57 +0200 Subject: [PATCH 22/61] Minor fixes --- platformio/builder/tools/piolib.py | 3 ++- platformio/builder/tools/piomisc.py | 25 ++++++++++++------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 5cb50492..c9c97e28 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -18,6 +18,7 @@ from __future__ import absolute_import import hashlib +import io import os import re import sys @@ -82,7 +83,7 @@ class LibBuilderFactory(object): fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT ): continue - with open(join(root, fname)) as fp: + with io.open(join(root, fname), errors="ignore") as fp: content = fp.read() if not content: continue diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 2df5303d..1079f402 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -53,7 +53,7 @@ class InoToCPPConverter(object): self._safe_encoding = None def read_safe_contents(self, path): - last_exc = None + error_reported = False for encoding in ( "utf-8", None, @@ -66,18 +66,17 @@ class InoToCPPConverter(object): contents = fp.read() self._safe_encoding = encoding return contents - except UnicodeDecodeError as e: - last_exc = e - click.secho( - "Unicode decode error has occurred, please remove invalid " - "(non-ASCII or non-UTF8) characters from %s file or convert it to UTF-8" - % path, - fg="yellow", - err=True, - ) - if last_exc: - raise last_exc - return None + except UnicodeDecodeError: + if not error_reported: + error_reported = True + click.secho( + "Unicode decode error has occurred, please remove invalid " + "(non-ASCII or non-UTF8) characters from %s file or convert it to UTF-8" + % path, + fg="yellow", + err=True, + ) + return "" def write_safe_contents(self, path, contents): with io.open( From 620335631fe7ced33bb4c5d685cdf62ffe207e68 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 6 Mar 2020 22:08:38 +0200 Subject: [PATCH 23/61] Bump version to 4.2.2b1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 859c5479..b767891d 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 2, "2a3") +VERSION = (4, 2, "2b1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 0f02b3b653dd9fd6d18eb5ba73cc6212c29e6539 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 7 Mar 2020 17:44:28 +0200 Subject: [PATCH 24/61] Improved support for Arduino "library.properties" ``depends`` field --- HISTORY.rst | 1 + platformio/managers/lib.py | 10 ++++++++-- tests/commands/test_lib.py | 31 ++++++++++++++++--------------- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 1c444e3c..98551c98 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,6 +10,7 @@ PlatformIO Core 4 ~~~~~~~~~~~~~~~~~~ * Added support for Arm Mbed "module.json" ``dependencies`` field (`issue #3400 `_) +* Improved support for Arduino "library.properties" ``depends`` field * Fixed an issue when quitting from PlatformIO IDE does not shutdown PIO Home server * Fixed an issue "the JSON object must be str, not 'bytes'" when PIO Home is used with Python 3.5 (`issue #3396 `_) * Fixed an issue when Python 2 does not keep encoding when converting ".ino" (`issue #3393 `_) diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index b006d8af..d85f0fa0 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -26,6 +26,8 @@ from platformio import app, exception, util from platformio.compat import glob_escape from platformio.managers.package import BasePkgManager from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.package.exception import ManifestException +from platformio.package.manifest.parser import ManifestParserFactory from platformio.project.config import ProjectConfig @@ -281,8 +283,12 @@ class LibraryManager(BasePkgManager): if not pkg_dir: return None - manifest = self.load_manifest(pkg_dir) - if "dependencies" not in manifest: + manifest = None + try: + manifest = ManifestParserFactory.new_from_dir(pkg_dir).as_dict() + except ManifestException: + pass + if not manifest or not manifest.get("dependencies"): return pkg_dir if not silent: diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index dc5dc00a..752c2c30 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -44,7 +44,7 @@ def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_hom "ArduinoJson@~5.10.0", "547@2.2.4", "AsyncMqttClient@<=0.8.2", - "999@77d4eb3f8a", + "Adafruit PN532@1.2.0", ], ) validate_cliresult(result) @@ -62,7 +62,8 @@ def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_hom "AsyncMqttClient_ID346", "ESPAsyncTCP_ID305", "AsyncTCP_ID1826", - "RFcontrol_ID999", + "Adafruit PN532_ID29", + "Adafruit BusIO_ID6214", ] assert set(items1) == set(items2) @@ -135,7 +136,7 @@ def test_install_duplicates(clirunner, validate_cliresult, without_internet): assert "is already installed" in result.output # by ID - result = clirunner.invoke(cmd_lib, ["-g", "install", "999"]) + result = clirunner.invoke(cmd_lib, ["-g", "install", "29"]) validate_cliresult(result) assert "is already installed" in result.output @@ -202,7 +203,8 @@ def test_global_lib_list(clirunner, validate_cliresult): "PJON", "PJON", "PubSubClient", - "RFcontrol", + "Adafruit PN532", + "Adafruit BusIO", "platformio-libmirror", "rs485-nodeproto", ] @@ -219,7 +221,7 @@ def test_global_lib_list(clirunner, validate_cliresult): "PJON@07fe9aa", "PJON@1fb26fd", "PubSubClient@bef5814", - "RFcontrol@77d4eb3f8a", + "Adafruit PN532@1.2.0", ] assert set(versions1) >= set(versions2) @@ -230,9 +232,7 @@ def test_global_lib_update_check(clirunner, validate_cliresult): ) validate_cliresult(result) output = json.loads(result.output) - assert set(["RFcontrol", "ESPAsyncTCP", "NeoPixelBus"]) == set( - [l["name"] for l in output] - ) + assert set(["ESPAsyncTCP", "NeoPixelBus"]) == set([l["name"] for l in output]) def test_global_lib_update(clirunner, validate_cliresult): @@ -252,8 +252,7 @@ def test_global_lib_update(clirunner, validate_cliresult): result = clirunner.invoke(cmd_lib, ["-g", "update"]) validate_cliresult(result) assert result.output.count("[Detached]") == 5 - assert result.output.count("[Up-to-date]") == 10 - assert "Uninstalling RFcontrol @ 77d4eb3f8a" in result.output + assert result.output.count("[Up-to-date]") == 12 # update unknown library result = clirunner.invoke(cmd_lib, ["-g", "update", "Unknown"]) @@ -266,9 +265,10 @@ def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_home): result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) validate_cliresult(result) items = json.loads(result.output) - result = clirunner.invoke(cmd_lib, ["-g", "uninstall", items[5]["__pkg_dir"]]) + items = sorted(items, key=lambda item: item["__pkg_dir"]) + result = clirunner.invoke(cmd_lib, ["-g", "uninstall", items[0]["__pkg_dir"]]) validate_cliresult(result) - assert "Uninstalling AsyncTCP" in result.output + assert ("Uninstalling %s" % items[0]["name"]) in result.output # uninstall the rest libraries result = clirunner.invoke( @@ -279,7 +279,7 @@ def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_home): "1", "https://github.com/bblanchon/ArduinoJson.git", "ArduinoJson@!=5.6.7", - "RFcontrol", + "Adafruit PN532", ], ) validate_cliresult(result) @@ -291,13 +291,14 @@ def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_home): "PubSubClient", "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", "ESPAsyncTCP_ID305", - "SomeLib_ID54", + "ESP32WebServer", "NeoPixelBus_ID547", "PJON", "AsyncMqttClient_ID346", "ArduinoJson_ID64", + "SomeLib_ID54", "PJON@src-79de467ebe19de18287becff0a1fb42d", - "ESP32WebServer", + "AsyncTCP_ID1826", ] assert set(items1) == set(items2) From 60a7af6a8c88a8dc32fcb8b89b162ac13345f3b4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 9 Mar 2020 14:58:35 +0200 Subject: [PATCH 25/61] Docs: Update recent articles --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index ac32d956..1506c1bc 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit ac32d9569ef213b784c196f0e35ab38faab29176 +Subproject commit 1506c1bcbfc8a7a9564a225e07cafd766fb6e235 From b9fd97dae49d9afb1bf6566314c83026723907fb Mon Sep 17 00:00:00 2001 From: Ilia Motornyi Date: Mon, 9 Mar 2020 15:47:41 +0200 Subject: [PATCH 26/61] Changes required for CLion PlatformIO plugin (#3298) --- platformio/ide/tpls/clion/.gitignore.tpl | 1 + platformio/ide/tpls/clion/.idea/clion.iml.tpl | 8 - platformio/ide/tpls/clion/.idea/misc.xml.tpl | 16 -- .../ide/tpls/clion/.idea/modules.xml.tpl | 9 - .../ide/tpls/clion/.idea/platformio.iml.tpl | 8 - .../ide/tpls/clion/.idea/watcherTasks.xml.tpl | 30 -- .../ide/tpls/clion/.idea/workspace.xml.tpl | 259 ------------------ platformio/ide/tpls/clion/CMakeLists.txt.tpl | 75 +---- .../ide/tpls/clion/CMakeListsPrivate.txt.tpl | 7 +- 9 files changed, 13 insertions(+), 400 deletions(-) delete mode 100644 platformio/ide/tpls/clion/.idea/clion.iml.tpl delete mode 100644 platformio/ide/tpls/clion/.idea/misc.xml.tpl delete mode 100644 platformio/ide/tpls/clion/.idea/modules.xml.tpl delete mode 100644 platformio/ide/tpls/clion/.idea/platformio.iml.tpl delete mode 100644 platformio/ide/tpls/clion/.idea/watcherTasks.xml.tpl delete mode 100644 platformio/ide/tpls/clion/.idea/workspace.xml.tpl diff --git a/platformio/ide/tpls/clion/.gitignore.tpl b/platformio/ide/tpls/clion/.gitignore.tpl index ff1a5181..3fe18ad4 100644 --- a/platformio/ide/tpls/clion/.gitignore.tpl +++ b/platformio/ide/tpls/clion/.gitignore.tpl @@ -1,2 +1,3 @@ .pio CMakeListsPrivate.txt +cmake-build-*/ diff --git a/platformio/ide/tpls/clion/.idea/clion.iml.tpl b/platformio/ide/tpls/clion/.idea/clion.iml.tpl deleted file mode 100644 index bc2cd874..00000000 --- a/platformio/ide/tpls/clion/.idea/clion.iml.tpl +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/platformio/ide/tpls/clion/.idea/misc.xml.tpl b/platformio/ide/tpls/clion/.idea/misc.xml.tpl deleted file mode 100644 index 3463fba1..00000000 --- a/platformio/ide/tpls/clion/.idea/misc.xml.tpl +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/platformio/ide/tpls/clion/.idea/modules.xml.tpl b/platformio/ide/tpls/clion/.idea/modules.xml.tpl deleted file mode 100644 index 9ce81f04..00000000 --- a/platformio/ide/tpls/clion/.idea/modules.xml.tpl +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/platformio/ide/tpls/clion/.idea/platformio.iml.tpl b/platformio/ide/tpls/clion/.idea/platformio.iml.tpl deleted file mode 100644 index bc2cd874..00000000 --- a/platformio/ide/tpls/clion/.idea/platformio.iml.tpl +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/platformio/ide/tpls/clion/.idea/watcherTasks.xml.tpl b/platformio/ide/tpls/clion/.idea/watcherTasks.xml.tpl deleted file mode 100644 index fcf8b23d..00000000 --- a/platformio/ide/tpls/clion/.idea/watcherTasks.xml.tpl +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/platformio/ide/tpls/clion/.idea/workspace.xml.tpl b/platformio/ide/tpls/clion/.idea/workspace.xml.tpl deleted file mode 100644 index d9cf6a8f..00000000 --- a/platformio/ide/tpls/clion/.idea/workspace.xml.tpl +++ /dev/null @@ -1,259 +0,0 @@ - - - - - - - - - - - - - - - - - - - -% envs = config.envs() -% if len(envs) > 1: -% for env in envs: - -% end - -% else: - -% end - - - - - - - - - - - - - - - - - - - - % for file in src_files: - - - - % end - - - - - true - - - - - - - - - - - - - - - - - C/C++ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1435919971910 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/platformio/ide/tpls/clion/CMakeLists.txt.tpl b/platformio/ide/tpls/clion/CMakeLists.txt.tpl index 52848378..3153e616 100644 --- a/platformio/ide/tpls/clion/CMakeLists.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeLists.txt.tpl @@ -5,9 +5,12 @@ # please create `CMakeListsUser.txt` in the root of project. # The `CMakeListsUser.txt` will not be overwritten by PlatformIO. -cmake_minimum_required(VERSION 3.2) +cmake_minimum_required(VERSION 3.13) +set(CMAKE_SYSTEM_NAME Generic) +set(CMAKE_C_COMPILER_WORKS 1) +set(CMAKE_CXX_COMPILER_WORKS 1) -project("{{project_name}}") +project("{{project_name}}" C CXX) include(CMakeListsPrivate.txt) @@ -16,74 +19,14 @@ include(CMakeListsUser.txt) endif() add_custom_target( - PLATFORMIO_BUILD ALL - COMMAND ${PLATFORMIO_CMD} -f -c clion run "$<$>:-e${CMAKE_BUILD_TYPE}>" + Production ALL + COMMAND platformio -c clion run "$<$>:-e${CMAKE_BUILD_TYPE}>" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) add_custom_target( - PLATFORMIO_BUILD_VERBOSE ALL - COMMAND ${PLATFORMIO_CMD} -f -c clion run --verbose "$<$>:-e${CMAKE_BUILD_TYPE}>" - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) - -add_custom_target( - PLATFORMIO_UPLOAD ALL - COMMAND ${PLATFORMIO_CMD} -f -c clion run --target upload "$<$>:-e${CMAKE_BUILD_TYPE}>" - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) - -add_custom_target( - PLATFORMIO_CLEAN ALL - COMMAND ${PLATFORMIO_CMD} -f -c clion run --target clean "$<$>:-e${CMAKE_BUILD_TYPE}>" - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) - -add_custom_target( - PLATFORMIO_MONITOR ALL - COMMAND ${PLATFORMIO_CMD} -f -c clion device monitor "$<$>:-e${CMAKE_BUILD_TYPE}>" - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) - -add_custom_target( - PLATFORMIO_TEST ALL - COMMAND ${PLATFORMIO_CMD} -f -c clion test "$<$>:-e${CMAKE_BUILD_TYPE}>" - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) - -add_custom_target( - PLATFORMIO_PROGRAM ALL - COMMAND ${PLATFORMIO_CMD} -f -c clion run --target program "$<$>:-e${CMAKE_BUILD_TYPE}>" - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) - -add_custom_target( - PLATFORMIO_UPLOADFS ALL - COMMAND ${PLATFORMIO_CMD} -f -c clion run --target uploadfs "$<$>:-e${CMAKE_BUILD_TYPE}>" - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) - -add_custom_target( - PLATFORMIO_BUILD_DEBUG ALL - COMMAND ${PLATFORMIO_CMD} -f -c clion run --target debug "$<$>:-e${CMAKE_BUILD_TYPE}>" - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) - -add_custom_target( - PLATFORMIO_UPDATE_ALL ALL - COMMAND ${PLATFORMIO_CMD} -f -c clion update - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) - -add_custom_target( - PLATFORMIO_REBUILD_PROJECT_INDEX ALL - COMMAND ${PLATFORMIO_CMD} -f -c clion init --ide clion - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) - -add_custom_target( - PLATFORMIO_DEVICE_LIST ALL - COMMAND ${PLATFORMIO_CMD} -f -c clion device list + Debug ALL + COMMAND platformio -c clion run --target debug "$<$>:-e${CMAKE_BUILD_TYPE}>" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) diff --git a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl index 0c6eda0c..df8171fa 100644 --- a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl @@ -29,14 +29,13 @@ % envs = config.envs() % if len(envs) > 1: -set(CMAKE_CONFIGURATION_TYPES "{{ ";".join(envs) }};" CACHE STRING "" FORCE) +set(CMAKE_CONFIGURATION_TYPES "{{ ";".join(envs) }};" CACHE STRING "Build Types reflect PlatformIO Environments" FORCE) % else: -set(CMAKE_CONFIGURATION_TYPES "{{ env_name }}" CACHE STRING "" FORCE) +set(CMAKE_CONFIGURATION_TYPES "{{ env_name }}" CACHE STRING "Build Types reflect PlatformIO Environments" FORCE) % end -set(PLATFORMIO_CMD "{{ _normalize_path(platformio_path) }}") % if svd_path: -set(SVD_PATH "{{ _normalize_path(svd_path) }}") +set(CLION_SVD_FILE_PATH "{{ _normalize_path(svd_path) }}" CACHE FILEPATH "Peripheral Registers Definitions File" FORCE) % end SET(CMAKE_C_COMPILER "{{ _normalize_path(cc_path) }}") From 073efef2a1e794c31ab690a41bad1a6ff41d00a5 Mon Sep 17 00:00:00 2001 From: valeros Date: Tue, 10 Mar 2020 15:54:01 +0200 Subject: [PATCH 27/61] Explicitly use Python-x64 with Appveyor CI --- .appveyor.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index 12d10edf..955b9aca 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -7,13 +7,15 @@ environment: matrix: - TOXENV: "py27" PLATFORMIO_BUILD_CACHE_DIR: C:\Temp\PIO_Build_Cache_P2_{build} + PYTHON_DIRS: C:\Python27-x64;C:\Python27-x64\Scripts - TOXENV: "py36" PLATFORMIO_BUILD_CACHE_DIR: C:\Temp\PIO_Build_Cache_P3_{build} + PYTHON_DIRS: C:\Python36-x64;C:\Python36-x64\Scripts install: - cmd: git submodule update --init --recursive - - cmd: SET PATH=C:\MinGW\bin;%PATH% + - cmd: SET PATH=%PYTHON_DIRS%;C:\MinGW\bin;%PATH% - cmd: SET PLATFORMIO_CORE_DIR=C:\.pio - cmd: pip install --force-reinstall tox From 8c3de609ab0f302e57d0973964bee82955f9f6be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vojt=C4=9Bch=20Bo=C4=8Dek?= Date: Wed, 11 Mar 2020 12:22:01 +0100 Subject: [PATCH 28/61] Add ESP crash trace decoding to monitor (#3383) * Implement mechanism for adding platform filters into miniterm Updates platformio/platform-espressif8266#31 * DeviceMonitorFilter: fixes for Windows and Python2 --- platformio/commands/device.py | 61 +++++++++++++++++++++++++++++++---- platformio/compat.py | 2 +- 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/platformio/commands/device.py b/platformio/commands/device.py index 24e7d3ca..245ae743 100644 --- a/platformio/commands/device.py +++ b/platformio/commands/device.py @@ -12,15 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import sys from fnmatch import fnmatch -from os import getcwd import click from serial.tools import miniterm from platformio import exception, fs, util -from platformio.compat import dump_json_to_unicode +from platformio.compat import dump_json_to_unicode, load_python_module from platformio.managers.platform import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.exception import NotPlatformIOProjectError @@ -165,7 +165,7 @@ def device_list( # pylint: disable=too-many-branches @click.option( "-d", "--project-dir", - default=getcwd, + default=os.getcwd, type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), ) @click.option( @@ -186,6 +186,12 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches except NotPlatformIOProjectError: pass + platform = None + if "platform" in project_options: + with fs.cd(kwargs["project_dir"]): + platform = PlatformFactory.newPlatform(project_options["platform"]) + register_platform_filters(platform, kwargs["project_dir"], kwargs["environment"]) + if not kwargs["port"]: ports = util.get_serial_ports(filter_hwid=True) if len(ports) == 1: @@ -193,7 +199,7 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches elif "platform" in project_options and "board" in project_options: board_hwids = get_board_hwids( kwargs["project_dir"], - project_options["platform"], + platform, project_options["board"], ) for item in ports: @@ -227,7 +233,6 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches except Exception as e: raise exception.MinitermException(e) - def apply_project_monitor_options(cli_options, project_options): for k in ("port", "speed", "rts", "dtr"): k2 = "monitor_%s" % k @@ -274,7 +279,51 @@ def get_project_options(environment=None): def get_board_hwids(project_dir, platform, board): with fs.cd(project_dir): return ( - PlatformFactory.newPlatform(platform) + platform .board_config(board) .get("build.hwids", []) ) + +class DeviceMonitorFilter(miniterm.Transform): + # NAME = "esp_exception_decoder" - all filters must have one + + # Called by PlatformIO to pass context + def __init__(self, project_dir, environment): + super(DeviceMonitorFilter, self).__init__() + + self.project_dir = project_dir + self.environment = environment + + self.config = ProjectConfig.get_instance() + if not self.environment: + default_envs = self.config.default_envs() + if default_envs: + self.environment = default_envs[0] + else: + self.environment = self.config.envs()[0] + + # Called by the miniterm library when the filter is acutally used + def __call__(self): + return self + +def register_platform_filters(platform, project_dir, environment): + monitor_dir = os.path.join(platform.get_dir(), "monitor") + if not os.path.isdir(monitor_dir): + return + + for fn in os.listdir(monitor_dir): + if not fn.startswith("filter_") or not fn.endswith(".py"): + continue + path = os.path.join(monitor_dir, fn) + if not os.path.isfile(path): + continue + + dot = fn.find(".") + module = load_python_module("platformio.commands.device.%s" % fn[:dot], path) + for key in dir(module): + member = getattr(module, key) + try: + if issubclass(member, DeviceMonitorFilter) and hasattr(member, "NAME"): + miniterm.TRANSFORMATIONS[member.NAME] = member(project_dir, environment) + except TypeError: + pass diff --git a/platformio/compat.py b/platformio/compat.py index 9107f8b1..0f4dd3c3 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -58,7 +58,7 @@ if PY2: def path_to_unicode(path): if isinstance(path, unicode): return path - return path.decode(get_filesystem_encoding()).encode("utf-8") + return path.decode(get_filesystem_encoding()) def hashlib_encode_data(data): if is_bytes(data): From 0ff37c9999b2f47c470919185ffaa3ba9c2c31c8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 12 Mar 2020 14:24:20 +0200 Subject: [PATCH 29/61] Implement universal "get_object_members" helper --- platformio/compat.py | 12 +++++++----- platformio/package/manifest/parser.py | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/platformio/compat.py b/platformio/compat.py index 0f4dd3c3..c812e98d 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -38,12 +38,14 @@ def get_locale_encoding(): return None -def get_class_attributes(cls): - attributes = inspect.getmembers(cls, lambda a: not inspect.isroutine(a)) +def get_object_members(obj, ignore_private=True): + members = inspect.getmembers(obj, lambda a: not inspect.isroutine(a)) + if not ignore_private: + return members return { - a[0]: a[1] - for a in attributes - if not (a[0].startswith("__") and a[0].endswith("__")) + item[0]: item[1] + for item in members + if not (item[0].startswith("__") and item[0].endswith("__")) } diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 60138609..e8ec5929 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -21,7 +21,7 @@ import re import requests from platformio import util -from platformio.compat import get_class_attributes, string_types +from platformio.compat import get_object_members, string_types from platformio.package.exception import ManifestParserError, UnknownManifestError from platformio.project.helpers import is_platformio_project @@ -40,7 +40,7 @@ class ManifestFileType(object): @classmethod def items(cls): - return get_class_attributes(ManifestFileType) + return get_object_members(ManifestFileType) @classmethod def from_uri(cls, uri): From 108b892e30e599bd22a547384c544363ed75fae5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 12 Mar 2020 14:28:54 +0200 Subject: [PATCH 30/61] Control device monitor output with filters and text transformations --- HISTORY.rst | 1 + docs | 2 +- platformio/commands/device/__init__.py | 15 +++ .../commands/{device.py => device/command.py} | 127 +++--------------- .../commands/device/filters/__init__.py | 13 ++ platformio/commands/device/filters/base.py | 42 ++++++ platformio/commands/device/helpers.py | 102 ++++++++++++++ platformio/commands/remote.py | 11 +- platformio/commands/run/command.py | 2 +- platformio/project/options.py | 8 ++ 10 files changed, 206 insertions(+), 117 deletions(-) create mode 100644 platformio/commands/device/__init__.py rename platformio/commands/{device.py => device/command.py} (64%) create mode 100644 platformio/commands/device/filters/__init__.py create mode 100644 platformio/commands/device/filters/base.py create mode 100644 platformio/commands/device/helpers.py diff --git a/HISTORY.rst b/HISTORY.rst index 98551c98..fba1bf03 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,7 @@ PlatformIO Core 4 4.2.2 (2020-??-??) ~~~~~~~~~~~~~~~~~~ +* Control device monitor output with `filters and text transformations `__ * Added support for Arm Mbed "module.json" ``dependencies`` field (`issue #3400 `_) * Improved support for Arduino "library.properties" ``depends`` field * Fixed an issue when quitting from PlatformIO IDE does not shutdown PIO Home server diff --git a/docs b/docs index 1506c1bc..d6a2968e 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 1506c1bcbfc8a7a9564a225e07cafd766fb6e235 +Subproject commit d6a2968edce17a75db7c8847c8e2fa925e192b1d diff --git a/platformio/commands/device/__init__.py b/platformio/commands/device/__init__.py new file mode 100644 index 00000000..bcee03cc --- /dev/null +++ b/platformio/commands/device/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.commands.device.filters.base import DeviceMonitorFilter diff --git a/platformio/commands/device.py b/platformio/commands/device/command.py similarity index 64% rename from platformio/commands/device.py rename to platformio/commands/device/command.py index 245ae743..5c90f205 100644 --- a/platformio/commands/device.py +++ b/platformio/commands/device/command.py @@ -20,9 +20,9 @@ import click from serial.tools import miniterm from platformio import exception, fs, util -from platformio.compat import dump_json_to_unicode, load_python_module +from platformio.commands.device import helpers as device_helpers +from platformio.compat import dump_json_to_unicode from platformio.managers.platform import PlatformFactory -from platformio.project.config import ProjectConfig from platformio.project.exception import NotPlatformIOProjectError @@ -135,7 +135,7 @@ def device_list( # pylint: disable=too-many-branches help="Set the encoding for the serial port (e.g. hexlify, " "Latin1, UTF-8), default: UTF-8", ) -@click.option("--filter", "-f", multiple=True, help="Add text transformation") +@click.option("--filter", "-f", multiple=True, help="Add filters/text transformations") @click.option( "--eol", default="CRLF", @@ -174,15 +174,11 @@ def device_list( # pylint: disable=too-many-branches help="Load configuration from `platformio.ini` and specified environment", ) def device_monitor(**kwargs): # pylint: disable=too-many-branches - click.echo( - "Looking for advanced Serial Monitor with UI? " - "Check http://bit.ly/pio-advanced-monitor" - ) project_options = {} try: with fs.cd(kwargs["project_dir"]): - project_options = get_project_options(kwargs["environment"]) - kwargs = apply_project_monitor_options(kwargs, project_options) + project_options = device_helpers.get_project_options(kwargs["environment"]) + kwargs = device_helpers.apply_project_monitor_options(kwargs, project_options) except NotPlatformIOProjectError: pass @@ -190,17 +186,17 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches if "platform" in project_options: with fs.cd(kwargs["project_dir"]): platform = PlatformFactory.newPlatform(project_options["platform"]) - register_platform_filters(platform, kwargs["project_dir"], kwargs["environment"]) + device_helpers.register_platform_filters( + platform, kwargs["project_dir"], kwargs["environment"] + ) if not kwargs["port"]: ports = util.get_serial_ports(filter_hwid=True) if len(ports) == 1: kwargs["port"] = ports[0]["port"] elif "platform" in project_options and "board" in project_options: - board_hwids = get_board_hwids( - kwargs["project_dir"], - platform, - project_options["board"], + board_hwids = device_helpers.get_board_hwids( + kwargs["project_dir"], platform, project_options["board"], ) for item in ports: for hwid in board_hwids: @@ -217,12 +213,18 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches break # override system argv with patched options - sys.argv = ["monitor"] + options_to_argv( + sys.argv = ["monitor"] + device_helpers.options_to_argv( kwargs, project_options, ignore=("port", "baud", "rts", "dtr", "environment", "project_dir"), ) + if not kwargs["quiet"]: + click.echo( + "Available filters and text transformations: %s" + % ", ".join(sorted(miniterm.TRANSFORMATIONS.keys())) + ) + click.echo("More details at http://bit.ly/pio-monitor-filters") try: miniterm.main( default_port=kwargs["port"], @@ -232,98 +234,3 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches ) except Exception as e: raise exception.MinitermException(e) - -def apply_project_monitor_options(cli_options, project_options): - for k in ("port", "speed", "rts", "dtr"): - k2 = "monitor_%s" % k - if k == "speed": - k = "baud" - if cli_options[k] is None and k2 in project_options: - cli_options[k] = project_options[k2] - if k != "port": - cli_options[k] = int(cli_options[k]) - return cli_options - - -def options_to_argv(cli_options, project_options, ignore=None): - result = project_options.get("monitor_flags", []) - for k, v in cli_options.items(): - if v is None or (ignore and k in ignore): - continue - k = "--" + k.replace("_", "-") - if k in project_options.get("monitor_flags", []): - continue - if isinstance(v, bool): - if v: - result.append(k) - elif isinstance(v, tuple): - for i in v: - result.extend([k, i]) - else: - result.extend([k, str(v)]) - return result - - -def get_project_options(environment=None): - config = ProjectConfig.get_instance() - config.validate(envs=[environment] if environment else None) - if not environment: - default_envs = config.default_envs() - if default_envs: - environment = default_envs[0] - else: - environment = config.envs()[0] - return config.items(env=environment, as_dict=True) - - -def get_board_hwids(project_dir, platform, board): - with fs.cd(project_dir): - return ( - platform - .board_config(board) - .get("build.hwids", []) - ) - -class DeviceMonitorFilter(miniterm.Transform): - # NAME = "esp_exception_decoder" - all filters must have one - - # Called by PlatformIO to pass context - def __init__(self, project_dir, environment): - super(DeviceMonitorFilter, self).__init__() - - self.project_dir = project_dir - self.environment = environment - - self.config = ProjectConfig.get_instance() - if not self.environment: - default_envs = self.config.default_envs() - if default_envs: - self.environment = default_envs[0] - else: - self.environment = self.config.envs()[0] - - # Called by the miniterm library when the filter is acutally used - def __call__(self): - return self - -def register_platform_filters(platform, project_dir, environment): - monitor_dir = os.path.join(platform.get_dir(), "monitor") - if not os.path.isdir(monitor_dir): - return - - for fn in os.listdir(monitor_dir): - if not fn.startswith("filter_") or not fn.endswith(".py"): - continue - path = os.path.join(monitor_dir, fn) - if not os.path.isfile(path): - continue - - dot = fn.find(".") - module = load_python_module("platformio.commands.device.%s" % fn[:dot], path) - for key in dir(module): - member = getattr(module, key) - try: - if issubclass(member, DeviceMonitorFilter) and hasattr(member, "NAME"): - miniterm.TRANSFORMATIONS[member.NAME] = member(project_dir, environment) - except TypeError: - pass diff --git a/platformio/commands/device/filters/__init__.py b/platformio/commands/device/filters/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/commands/device/filters/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/commands/device/filters/base.py b/platformio/commands/device/filters/base.py new file mode 100644 index 00000000..c532eb18 --- /dev/null +++ b/platformio/commands/device/filters/base.py @@ -0,0 +1,42 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from serial.tools import miniterm + +from platformio.project.config import ProjectConfig + + +class DeviceMonitorFilter(miniterm.Transform): + def __init__(self, project_dir, environment): + """ Called by PlatformIO to pass context """ + super(DeviceMonitorFilter, self).__init__() + + self.project_dir = project_dir + self.environment = environment + + self.config = ProjectConfig.get_instance() + if not self.environment: + default_envs = self.config.default_envs() + if default_envs: + self.environment = default_envs[0] + else: + self.environment = self.config.envs()[0] + + def __call__(self): + """ Called by the miniterm library when the filter is actually used """ + return self + + @property + def NAME(self): + raise NotImplementedError("Please declare NAME attribute for the filter class") diff --git a/platformio/commands/device/helpers.py b/platformio/commands/device/helpers.py new file mode 100644 index 00000000..17b72967 --- /dev/null +++ b/platformio/commands/device/helpers.py @@ -0,0 +1,102 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import os + +from serial.tools import miniterm + +from platformio import fs +from platformio.commands.device import DeviceMonitorFilter +from platformio.compat import get_object_members, load_python_module +from platformio.project.config import ProjectConfig + + +def apply_project_monitor_options(cli_options, project_options): + for k in ("port", "speed", "rts", "dtr"): + k2 = "monitor_%s" % k + if k == "speed": + k = "baud" + if cli_options[k] is None and k2 in project_options: + cli_options[k] = project_options[k2] + if k != "port": + cli_options[k] = int(cli_options[k]) + return cli_options + + +def options_to_argv(cli_options, project_options, ignore=None): + confmon_flags = project_options.get("monitor_flags", []) + result = confmon_flags[::] + + for f in project_options.get("monitor_filters", []): + result.extend(["--filter", f]) + + for k, v in cli_options.items(): + if v is None or (ignore and k in ignore): + continue + k = "--" + k.replace("_", "-") + if k in confmon_flags: + continue + if isinstance(v, bool): + if v: + result.append(k) + elif isinstance(v, tuple): + for i in v: + result.extend([k, i]) + else: + result.extend([k, str(v)]) + return result + + +def get_project_options(environment=None): + config = ProjectConfig.get_instance() + config.validate(envs=[environment] if environment else None) + if not environment: + default_envs = config.default_envs() + if default_envs: + environment = default_envs[0] + else: + environment = config.envs()[0] + return config.items(env=environment, as_dict=True) + + +def get_board_hwids(project_dir, platform, board): + with fs.cd(project_dir): + return platform.board_config(board).get("build.hwids", []) + + +def register_platform_filters(platform, project_dir, environment): + monitor_dir = os.path.join(platform.get_dir(), "monitor") + if not os.path.isdir(monitor_dir): + return + + for fn in os.listdir(monitor_dir): + if not fn.startswith("filter_") or not fn.endswith(".py"): + continue + path = os.path.join(monitor_dir, fn) + if not os.path.isfile(path): + continue + + module = load_python_module( + "platformio.commands.device.filters.%s" % fn[: fn.find(".")], path + ) + for cls in get_object_members(module).values(): + if ( + not inspect.isclass(cls) + or not issubclass(cls, DeviceMonitorFilter) + or cls == DeviceMonitorFilter + ): + continue + obj = cls(project_dir, environment) + miniterm.TRANSFORMATIONS[obj.NAME] = obj diff --git a/platformio/commands/remote.py b/platformio/commands/remote.py index ee950982..ca296b69 100644 --- a/platformio/commands/remote.py +++ b/platformio/commands/remote.py @@ -21,7 +21,8 @@ from time import sleep import click from platformio import exception, fs -from platformio.commands import device +from platformio.commands.device import helpers as device_helpers +from platformio.commands.device.command import device_monitor as cmd_device_monitor from platformio.managers.core import pioplus_call from platformio.project.exception import NotPlatformIOProjectError @@ -197,8 +198,8 @@ def device_monitor(ctx, **kwargs): project_options = {} try: with fs.cd(kwargs["project_dir"]): - project_options = device.get_project_options(kwargs["environment"]) - kwargs = device.apply_project_monitor_options(kwargs, project_options) + project_options = device_helpers.get_project_options(kwargs["environment"]) + kwargs = device_helpers.apply_project_monitor_options(kwargs, project_options) except NotPlatformIOProjectError: pass @@ -206,7 +207,7 @@ def device_monitor(ctx, **kwargs): def _tx_target(sock_dir): pioplus_argv = ["remote", "device", "monitor"] - pioplus_argv.extend(device.options_to_argv(kwargs, project_options)) + pioplus_argv.extend(device_helpers.options_to_argv(kwargs, project_options)) pioplus_argv.extend(["--sock", sock_dir]) try: pioplus_call(pioplus_argv) @@ -224,7 +225,7 @@ def device_monitor(ctx, **kwargs): return with open(sock_file) as fp: kwargs["port"] = fp.read() - ctx.invoke(device.device_monitor, **kwargs) + ctx.invoke(cmd_device_monitor, **kwargs) t.join(2) finally: fs.rmtree(sock_dir) diff --git a/platformio/commands/run/command.py b/platformio/commands/run/command.py index 5058c159..378eaf0d 100644 --- a/platformio/commands/run/command.py +++ b/platformio/commands/run/command.py @@ -21,7 +21,7 @@ import click from tabulate import tabulate from platformio import app, exception, fs, util -from platformio.commands.device import device_monitor as cmd_device_monitor +from platformio.commands.device.command import device_monitor as cmd_device_monitor from platformio.commands.run.helpers import clean_build_dir, handle_legacy_libdeps from platformio.commands.run.processor import EnvironmentProcessor from platformio.commands.test.processor import CTX_META_TEST_IS_RUNNING diff --git a/platformio/project/options.py b/platformio/project/options.py index d38ed566..ecf030b8 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -445,6 +445,14 @@ ProjectOptions = OrderedDict( oldnames=["monitor_baud"], default=9600, ), + ConfigEnvOption( + group="monitor", + name="monitor_filters", + description=( + "Apply the filters and text transformations to monitor output" + ), + multiple=True, + ), ConfigEnvOption( group="monitor", name="monitor_rts", From 5d0faaa5a81ae552a2c9895d77e9137ffd52b8f1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 12 Mar 2020 15:09:20 +0200 Subject: [PATCH 31/61] Refactor docs structure --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index d6a2968e..6642f073 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit d6a2968edce17a75db7c8847c8e2fa925e192b1d +Subproject commit 6642f073cf450888d5ed4b233fd1e4c49a32fba3 From 9a5ebfb642dd983b1881e85a64fbb075da224e8b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 12 Mar 2020 15:10:25 +0200 Subject: [PATCH 32/61] Bump version to 4.3.0b1 --- HISTORY.rst | 2 +- platformio/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index fba1bf03..ebacf134 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,7 +6,7 @@ Release Notes PlatformIO Core 4 ----------------- -4.2.2 (2020-??-??) +4.3.0 (2020-??-??) ~~~~~~~~~~~~~~~~~~ * Control device monitor output with `filters and text transformations `__ diff --git a/platformio/__init__.py b/platformio/__init__.py index b767891d..2488ec09 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 2, "2b1") +VERSION = (4, 3, "0b1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 58a4ff8246df240a85b52c44e3a424fbbed36b28 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 14 Mar 2020 12:18:00 +0200 Subject: [PATCH 33/61] Skip broken Click 7.1 & 7.1.1, see Click's issue #1501 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 06a915a6..2d3cd72f 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ from platformio.compat import PY2 install_requires = [ "bottle<0.13", - "click>=5,<8", + "click>=5,<8,!=7.1,!=7.1.1", "colorama", "pyserial>=3,<4,!=3.3", "requests>=2.4.0,<3", From a22ed40256f9559018ec261464b8c3383c412b7c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 14 Mar 2020 19:31:00 +0200 Subject: [PATCH 34/61] Added initial support for an official "PlatformIO for CLion IDE" plugin // Resolve #2201 --- HISTORY.rst | 9 +++++++++ docs | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index ebacf134..4fe65cbe 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,15 @@ PlatformIO Core 4 4.3.0 (2020-??-??) ~~~~~~~~~~~~~~~~~~ +* Added initial support for an official `PlatformIO for CLion IDE `__ plugin: + + - Smart C and C++ editor + - Code refactoring + - On-the-fly code analysis + - "New PlatformIO Project" wizard + - Building, Uploading, Testing + - Integrated debugger (inline variable view, conditional breakpoints, expressions, watchpoints, peripheral registers, multi-thread support, etc.) + * Control device monitor output with `filters and text transformations `__ * Added support for Arm Mbed "module.json" ``dependencies`` field (`issue #3400 `_) * Improved support for Arduino "library.properties" ``depends`` field diff --git a/docs b/docs index 6642f073..fe9f6afa 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 6642f073cf450888d5ed4b233fd1e4c49a32fba3 +Subproject commit fe9f6afa8cc5d46b32c4ab4febff5fb48207c83e From ba040ba2ba0488f15495783b32af20d1b3e07bf8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 14 Mar 2020 20:32:42 +0200 Subject: [PATCH 35/61] Docs: Workaround for ReadTheDocs bug --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index fe9f6afa..84da8178 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit fe9f6afa8cc5d46b32c4ab4febff5fb48207c83e +Subproject commit 84da8178824381422fde455caf23624a1848da3a From 314f634e1694c6037a2e98c728266aae301d685c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 15 Mar 2020 00:41:16 +0200 Subject: [PATCH 36/61] Docs: Improvements for CLion docs. --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 84da8178..f3056c3a 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 84da8178824381422fde455caf23624a1848da3a +Subproject commit f3056c3a84e140f999b1a2766a479837beceb141 From f81b0b2a847c26bb8c180447f1cb792d03a1d5ba Mon Sep 17 00:00:00 2001 From: Matthew Mirvish Date: Tue, 17 Mar 2020 10:30:28 -0400 Subject: [PATCH 37/61] Ensure all commands in compilation_commands.json use absolute paths. (#3415) * Fix resolving of absolute path for toolchain By placing the `where_is_program` call into this function, all references to the compiler will be made absolute, instead of just ones in the top environment. Previously, all references to the compiler for user source code would not use the full path in the compilation database, which broke `clangd`'s detection of system includes. * Linting issue --- platformio/builder/tools/compilation_db.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/platformio/builder/tools/compilation_db.py b/platformio/builder/tools/compilation_db.py index 2226f825..52463e83 100644 --- a/platformio/builder/tools/compilation_db.py +++ b/platformio/builder/tools/compilation_db.py @@ -76,6 +76,16 @@ def makeEmitCompilationDbEntry(comstr): :return: target(s), source(s) """ + # Resolve absolute path of toolchain + for cmd in ("CC", "CXX", "AS"): + if cmd not in env: + continue + if os.path.isabs(env[cmd]): + continue + env[cmd] = where_is_program( + env.subst("$%s" % cmd), env.subst("${ENV['PATH']}") + ) + dbtarget = __CompilationDbNode(source) entry = env.__COMPILATIONDB_Entry( @@ -195,14 +205,6 @@ def generate(env, **kwargs): ) def CompilationDatabase(env, target): - # Resolve absolute path of toolchain - for cmd in ("CC", "CXX", "AS"): - if cmd not in env: - continue - env[cmd] = where_is_program( - env.subst("$%s" % cmd), env.subst("${ENV['PATH']}") - ) - result = env.__COMPILATIONDB_Database(target=target, source=[]) env.AlwaysBuild(result) From 4e6095ca134ba917ade0bc07987177d6284236c6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 17 Mar 2020 17:39:11 +0200 Subject: [PATCH 38/61] Update docs and history --- HISTORY.rst | 7 ++++--- docs | 2 +- examples | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 4fe65cbe..47e3fe17 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,7 +9,7 @@ PlatformIO Core 4 4.3.0 (2020-??-??) ~~~~~~~~~~~~~~~~~~ -* Added initial support for an official `PlatformIO for CLion IDE `__ plugin: +* Added initial support for an official `PlatformIO for CLion IDE `__ plugin: - Smart C and C++ editor - Code refactoring @@ -18,13 +18,14 @@ PlatformIO Core 4 - Building, Uploading, Testing - Integrated debugger (inline variable view, conditional breakpoints, expressions, watchpoints, peripheral registers, multi-thread support, etc.) -* Control device monitor output with `filters and text transformations `__ +* Control device monitor output with `filters and text transformations `__ (`pull #3383 `_) * Added support for Arm Mbed "module.json" ``dependencies`` field (`issue #3400 `_) * Improved support for Arduino "library.properties" ``depends`` field * Fixed an issue when quitting from PlatformIO IDE does not shutdown PIO Home server * Fixed an issue "the JSON object must be str, not 'bytes'" when PIO Home is used with Python 3.5 (`issue #3396 `_) * Fixed an issue when Python 2 does not keep encoding when converting ".ino" (`issue #3393 `_) * Fixed an issue when ``"libArchive": false`` in "library.json" does not work (`issue #3403 `_) +* Fixed an issue when not all commands in `compilation database "compile_commands.json" `__ use absolute paths (`pull #3415 `_) 4.2.1 (2020-02-17) ~~~~~~~~~~~~~~~~~~ @@ -54,7 +55,7 @@ PlatformIO Core 4 - Show computed project configuration with a new `platformio project config `_ command or dump to JSON with ``platformio project config --json-output`` (`issue #3335 `_) - Moved ``platformio init`` command to `platformio project init `_ -* Generate `compilation database "compile_commands.json" `_ (`issue #2990 `_) +* Generate `compilation database "compile_commands.json" `__ (`issue #2990 `_) * Control debug flags and optimization level with a new `debug_build_flags `__ option * Install a dev-platform with ALL declared packages using a new ``--with-all-packages`` option for `pio platform install `__ command (`issue #3345 `_) * Added support for "pythonPackages" in `platform.json `__ manifest (PlatformIO Package Manager will install dependent Python packages from PyPi registry automatically when dev-platform is installed) diff --git a/docs b/docs index f3056c3a..05a2a8ff 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit f3056c3a84e140f999b1a2766a479837beceb141 +Subproject commit 05a2a8ff4c1bb4121151cf8b32f4f8f05a709be5 diff --git a/examples b/examples index 5a66e52e..370c2c41 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 5a66e52e1fb59419842aa61fae08e97a710911ea +Subproject commit 370c2c41a10fe02aafe0f4910f827a00b0f24ad8 From 5a720336227d17f273735c2afe0a3c6862b982ba Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 17 Mar 2020 17:42:54 +0200 Subject: [PATCH 39/61] Fixed an issue when unknown transport is used for PIO Unit Testing // Resolve #3422 --- HISTORY.rst | 1 + docs | 2 +- platformio/commands/test/processor.py | 5 ++++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 47e3fe17..622254cb 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -26,6 +26,7 @@ PlatformIO Core 4 * Fixed an issue when Python 2 does not keep encoding when converting ".ino" (`issue #3393 `_) * Fixed an issue when ``"libArchive": false`` in "library.json" does not work (`issue #3403 `_) * Fixed an issue when not all commands in `compilation database "compile_commands.json" `__ use absolute paths (`pull #3415 `_) +* Fixed an issue when unknown transport is used for `PIO Unit Testing `__ engine (`issue #3422 `_) 4.2.1 (2020-02-17) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index 05a2a8ff..ebf066f7 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 05a2a8ff4c1bb4121151cf8b32f4f8f05a709be5 +Subproject commit ebf066f7911e9b39ead1d0278c2b530e27c3c8be diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index 64a7d3dd..5291b9f7 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -83,6 +83,7 @@ class TestProcessorBase(object): self._outputcpp_generated = False def get_transport(self): + transport = None if self.env_options.get("platform") == "native": transport = "native" elif "framework" in self.env_options: @@ -91,7 +92,9 @@ class TestProcessorBase(object): transport = self.env_options["test_transport"] if transport not in TRANSPORT_OPTIONS: raise exception.PlatformioException( - "Unknown Unit Test transport `%s`" % transport + "Unknown Unit Test transport `%s`. Please check a documentation how " + "to create an own 'Test Transport':\n" + "- https://docs.platformio.org/page/plus/unit-testing.html" % transport ) return transport.lower() From 0df72411a09ef3421906c88c85e8d8b5e69d999f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 17 Mar 2020 23:08:57 +0200 Subject: [PATCH 40/61] Device Monitor Filter API, implement "time" and "log2file" filters // Resolve #981 Resolve #670 --- HISTORY.rst | 10 ++++- docs | 2 +- platformio/commands/device/command.py | 11 ++++- platformio/commands/device/filters/base.py | 4 +- .../commands/device/filters/log2file.py | 44 +++++++++++++++++++ platformio/commands/device/filters/time.py | 34 ++++++++++++++ platformio/commands/device/helpers.py | 36 ++++++++------- 7 files changed, 118 insertions(+), 23 deletions(-) create mode 100644 platformio/commands/device/filters/log2file.py create mode 100644 platformio/commands/device/filters/time.py diff --git a/HISTORY.rst b/HISTORY.rst index 622254cb..9ddfca12 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,7 +9,7 @@ PlatformIO Core 4 4.3.0 (2020-??-??) ~~~~~~~~~~~~~~~~~~ -* Added initial support for an official `PlatformIO for CLion IDE `__ plugin: +* Initial support for an official `PlatformIO for CLion IDE `__ plugin: - Smart C and C++ editor - Code refactoring @@ -18,7 +18,13 @@ PlatformIO Core 4 - Building, Uploading, Testing - Integrated debugger (inline variable view, conditional breakpoints, expressions, watchpoints, peripheral registers, multi-thread support, etc.) -* Control device monitor output with `filters and text transformations `__ (`pull #3383 `_) +* `Device Monitor 2.0 `__ + + - Added **PlatformIO Device Monitor Filter API** (dev-platforms can extend base device monitor with a custom functionality, such as exception decoding) (`pull #3383 `_) + - Configure project device monitor with `monitor_filters `__ option + - Show a timestamp for each new line with ``time`` filter (`issue #981 `_) + - `Capture device monitor output to a file `__ with ``log2file`` filter (`issue #670 `_) + * Added support for Arm Mbed "module.json" ``dependencies`` field (`issue #3400 `_) * Improved support for Arduino "library.properties" ``depends`` field * Fixed an issue when quitting from PlatformIO IDE does not shutdown PIO Home server diff --git a/docs b/docs index ebf066f7..873114bd 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit ebf066f7911e9b39ead1d0278c2b530e27c3c8be +Subproject commit 873114bdbb091a26e25b61d9536b76439d5f9c3a diff --git a/platformio/commands/device/command.py b/platformio/commands/device/command.py index 5c90f205..e93b1214 100644 --- a/platformio/commands/device/command.py +++ b/platformio/commands/device/command.py @@ -174,6 +174,13 @@ def device_list( # pylint: disable=too-many-branches help="Load configuration from `platformio.ini` and specified environment", ) def device_monitor(**kwargs): # pylint: disable=too-many-branches + # load default monitor filters + filters_dir = os.path.join(fs.get_source_dir(), "commands", "device", "filters") + for name in os.listdir(filters_dir): + if not name.endswith(".py"): + continue + device_helpers.load_monitor_filter(os.path.join(filters_dir, name)) + project_options = {} try: with fs.cd(kwargs["project_dir"]): @@ -221,10 +228,10 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches if not kwargs["quiet"]: click.echo( - "Available filters and text transformations: %s" + "--- Available filters and text transformations: %s" % ", ".join(sorted(miniterm.TRANSFORMATIONS.keys())) ) - click.echo("More details at http://bit.ly/pio-monitor-filters") + click.echo("--- More details at http://bit.ly/pio-monitor-filters") try: miniterm.main( default_port=kwargs["port"], diff --git a/platformio/commands/device/filters/base.py b/platformio/commands/device/filters/base.py index c532eb18..bc0880b3 100644 --- a/platformio/commands/device/filters/base.py +++ b/platformio/commands/device/filters/base.py @@ -18,7 +18,7 @@ from platformio.project.config import ProjectConfig class DeviceMonitorFilter(miniterm.Transform): - def __init__(self, project_dir, environment): + def __init__(self, project_dir=None, environment=None): """ Called by PlatformIO to pass context """ super(DeviceMonitorFilter, self).__init__() @@ -30,7 +30,7 @@ class DeviceMonitorFilter(miniterm.Transform): default_envs = self.config.default_envs() if default_envs: self.environment = default_envs[0] - else: + elif self.config.envs(): self.environment = self.config.envs()[0] def __call__(self): diff --git a/platformio/commands/device/filters/log2file.py b/platformio/commands/device/filters/log2file.py new file mode 100644 index 00000000..69118510 --- /dev/null +++ b/platformio/commands/device/filters/log2file.py @@ -0,0 +1,44 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import io +import os.path +from datetime import datetime + +from platformio.commands.device import DeviceMonitorFilter + + +class LogToFile(DeviceMonitorFilter): + NAME = "log2file" + + def __init__(self, *args, **kwargs): + super(LogToFile, self).__init__(*args, **kwargs) + self._log_fp = None + + def __call__(self): + log_file_name = "platformio-device-monitor-%s.log" % datetime.now().strftime( + "%y%m%d-%H%M%S" + ) + print("--- Logging an output to %s" % os.path.abspath(log_file_name)) + self._log_fp = io.open(log_file_name, "w", encoding="utf-8") + return self + + def __del__(self): + if self._log_fp: + self._log_fp.close() + + def rx(self, text): + self._log_fp.write(text) + self._log_fp.flush() + return text diff --git a/platformio/commands/device/filters/time.py b/platformio/commands/device/filters/time.py new file mode 100644 index 00000000..203e6aa9 --- /dev/null +++ b/platformio/commands/device/filters/time.py @@ -0,0 +1,34 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime + +from platformio.commands.device import DeviceMonitorFilter + + +class Timestamp(DeviceMonitorFilter): + NAME = "time" + + def __init__(self, *args, **kwargs): + super(Timestamp, self).__init__(*args, **kwargs) + self._first_text_received = False + + def rx(self, text): + if self._first_text_received and "\n" not in text: + return text + timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3] + if not self._first_text_received: + self._first_text_received = True + return "%s > %s" % (timestamp, text) + return text.replace("\n", "\n%s > " % timestamp) diff --git a/platformio/commands/device/helpers.py b/platformio/commands/device/helpers.py index 17b72967..3bfe8fc6 100644 --- a/platformio/commands/device/helpers.py +++ b/platformio/commands/device/helpers.py @@ -76,27 +76,31 @@ def get_board_hwids(project_dir, platform, board): return platform.board_config(board).get("build.hwids", []) +def load_monitor_filter(path, project_dir=None, environment=None): + name = os.path.basename(path) + name = name[: name.find(".")] + module = load_python_module("platformio.commands.device.filters.%s" % name, path) + for cls in get_object_members(module).values(): + if ( + not inspect.isclass(cls) + or not issubclass(cls, DeviceMonitorFilter) + or cls == DeviceMonitorFilter + ): + continue + obj = cls(project_dir, environment) + miniterm.TRANSFORMATIONS[obj.NAME] = obj + return True + + def register_platform_filters(platform, project_dir, environment): monitor_dir = os.path.join(platform.get_dir(), "monitor") if not os.path.isdir(monitor_dir): return - for fn in os.listdir(monitor_dir): - if not fn.startswith("filter_") or not fn.endswith(".py"): + for name in os.listdir(monitor_dir): + if not name.startswith("filter_") or not name.endswith(".py"): continue - path = os.path.join(monitor_dir, fn) + path = os.path.join(monitor_dir, name) if not os.path.isfile(path): continue - - module = load_python_module( - "platformio.commands.device.filters.%s" % fn[: fn.find(".")], path - ) - for cls in get_object_members(module).values(): - if ( - not inspect.isclass(cls) - or not issubclass(cls, DeviceMonitorFilter) - or cls == DeviceMonitorFilter - ): - continue - obj = cls(project_dir, environment) - miniterm.TRANSFORMATIONS[obj.NAME] = obj + load_monitor_filter(path, project_dir, environment) From 24a23b67dd56c0aa878f4e7b61550c24f61bced0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 17 Mar 2020 23:10:06 +0200 Subject: [PATCH 41/61] Fix formatting issue --- HISTORY.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 9ddfca12..17d63084 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -20,10 +20,10 @@ PlatformIO Core 4 * `Device Monitor 2.0 `__ - - Added **PlatformIO Device Monitor Filter API** (dev-platforms can extend base device monitor with a custom functionality, such as exception decoding) (`pull #3383 `_) - - Configure project device monitor with `monitor_filters `__ option - - Show a timestamp for each new line with ``time`` filter (`issue #981 `_) - - `Capture device monitor output to a file `__ with ``log2file`` filter (`issue #670 `_) + - Added **PlatformIO Device Monitor Filter API** (dev-platforms can extend base device monitor with a custom functionality, such as exception decoding) (`pull #3383 `_) + - Configure project device monitor with `monitor_filters `__ option + - Show a timestamp for each new line with ``time`` filter (`issue #981 `_) + - `Capture device monitor output to a file `__ with ``log2file`` filter (`issue #670 `_) * Added support for Arm Mbed "module.json" ``dependencies`` field (`issue #3400 `_) * Improved support for Arduino "library.properties" ``depends`` field From 605cd36e2783733d57e4ad2a29b17c6f67806a2e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 18 Mar 2020 00:09:40 +0200 Subject: [PATCH 42/61] Send a text to device on ENTER with ``send_on_enter`` filter // Resolve #926 --- HISTORY.rst | 3 +- docs | 2 +- .../commands/device/filters/send_on_enter.py | 31 +++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 platformio/commands/device/filters/send_on_enter.py diff --git a/HISTORY.rst b/HISTORY.rst index 17d63084..f5806fa1 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -22,8 +22,9 @@ PlatformIO Core 4 - Added **PlatformIO Device Monitor Filter API** (dev-platforms can extend base device monitor with a custom functionality, such as exception decoding) (`pull #3383 `_) - Configure project device monitor with `monitor_filters `__ option - - Show a timestamp for each new line with ``time`` filter (`issue #981 `_) - `Capture device monitor output to a file `__ with ``log2file`` filter (`issue #670 `_) + - Show a timestamp for each new line with ``time`` filter (`issue #981 `_) + - Send a text to device on ENTER with ``send_on_enter`` filter (`issue #926 `_) * Added support for Arm Mbed "module.json" ``dependencies`` field (`issue #3400 `_) * Improved support for Arduino "library.properties" ``depends`` field diff --git a/docs b/docs index 873114bd..4fc291ef 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 873114bdbb091a26e25b61d9536b76439d5f9c3a +Subproject commit 4fc291ef035ef4464afef8f8c53d18f545155512 diff --git a/platformio/commands/device/filters/send_on_enter.py b/platformio/commands/device/filters/send_on_enter.py new file mode 100644 index 00000000..102348d1 --- /dev/null +++ b/platformio/commands/device/filters/send_on_enter.py @@ -0,0 +1,31 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.commands.device import DeviceMonitorFilter + + +class SendOnEnter(DeviceMonitorFilter): + NAME = "send_on_enter" + + def __init__(self, *args, **kwargs): + super(SendOnEnter, self).__init__(*args, **kwargs) + self._buffer = "" + + def tx(self, text): + self._buffer += text + if "\r\n" in self._buffer: + text = self._buffer + self._buffer = "" + return text + return "" From 9a1b5d869d13b2f5372b54772e149a13f274d977 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 18 Mar 2020 00:13:03 +0200 Subject: [PATCH 43/61] Bump version to 4.3.0b2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 2488ec09..0fac99a0 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 3, "0b1") +VERSION = (4, 3, "0b2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From db9829a11e5a89337dddc7a7eb972c22a18966a4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 18 Mar 2020 00:36:07 +0200 Subject: [PATCH 44/61] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 4fc291ef..4e1c19c7 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 4fc291ef035ef4464afef8f8c53d18f545155512 +Subproject commit 4e1c19c77a8438b8cd6027307142b851127720db From fff33d8c29527de132ea2c2f06bf16b1f2524f79 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 18 Mar 2020 17:25:40 +0200 Subject: [PATCH 45/61] Do not send CR+NL for "send_on_enter" device monitor filter --- platformio/commands/device/filters/send_on_enter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/commands/device/filters/send_on_enter.py b/platformio/commands/device/filters/send_on_enter.py index 102348d1..10ca2103 100644 --- a/platformio/commands/device/filters/send_on_enter.py +++ b/platformio/commands/device/filters/send_on_enter.py @@ -24,8 +24,8 @@ class SendOnEnter(DeviceMonitorFilter): def tx(self, text): self._buffer += text - if "\r\n" in self._buffer: - text = self._buffer + if self._buffer.endswith("\r\n"): + text = self._buffer[:-2] self._buffer = "" return text return "" From 3dd3ea1c355894e4038fe57ce9c4cae129ba6bc5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 18 Mar 2020 18:55:54 +0200 Subject: [PATCH 46/61] Show a hexadecimal representation of the data (code point of each character) with ``hexlify`` filter --- HISTORY.rst | 1 + docs | 2 +- platformio/commands/device/filters/hexlify.py | 38 +++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 platformio/commands/device/filters/hexlify.py diff --git a/HISTORY.rst b/HISTORY.rst index f5806fa1..1ad463f8 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -25,6 +25,7 @@ PlatformIO Core 4 - `Capture device monitor output to a file `__ with ``log2file`` filter (`issue #670 `_) - Show a timestamp for each new line with ``time`` filter (`issue #981 `_) - Send a text to device on ENTER with ``send_on_enter`` filter (`issue #926 `_) + - Show a hexadecimal representation of the data (code point of each character) with ``hexlify`` filter * Added support for Arm Mbed "module.json" ``dependencies`` field (`issue #3400 `_) * Improved support for Arduino "library.properties" ``depends`` field diff --git a/docs b/docs index 4e1c19c7..03c5eb48 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 4e1c19c77a8438b8cd6027307142b851127720db +Subproject commit 03c5eb487fe7abd85745a6810bab06eaf29b5a81 diff --git a/platformio/commands/device/filters/hexlify.py b/platformio/commands/device/filters/hexlify.py new file mode 100644 index 00000000..1023b573 --- /dev/null +++ b/platformio/commands/device/filters/hexlify.py @@ -0,0 +1,38 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import serial + +from platformio.commands.device import DeviceMonitorFilter + + +class Hexlify(DeviceMonitorFilter): + NAME = "hexlify" + + def __init__(self, *args, **kwargs): + super(Hexlify, self).__init__(*args, **kwargs) + self._counter = 0 + + def rx(self, text): + result = "" + for b in serial.iterbytes(text): + if (self._counter % 16) == 0: + result += "\n{:04X} | ".format(self._counter) + asciicode = ord(b) + if asciicode <= 255: + result += "{:02X} ".format(asciicode) + else: + result += "?? " + self._counter += 1 + return result From 52689bc5e8eba9c51ba7610436af00b57a74450e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 19 Mar 2020 00:19:59 +0200 Subject: [PATCH 47/61] Wait until debug server is ready --- platformio/commands/debug/client.py | 27 +++----- platformio/commands/debug/process.py | 30 ++++++--- platformio/commands/debug/server.py | 97 +++++++++++++++++----------- 3 files changed, 90 insertions(+), 64 deletions(-) diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py index fa468bba..610f598d 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/client.py @@ -20,6 +20,7 @@ from hashlib import sha1 from os.path import basename, dirname, isdir, join, realpath, splitext from tempfile import mkdtemp +from twisted.internet import defer # pylint: disable=import-error from twisted.internet import protocol # pylint: disable=import-error from twisted.internet import reactor # pylint: disable=import-error from twisted.internet import stdio # pylint: disable=import-error @@ -33,8 +34,6 @@ from platformio.commands.debug.server import DebugServer from platformio.compat import hashlib_encode_data, is_bytes from platformio.project.helpers import get_project_cache_dir -LOG_FILE = None - class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes @@ -42,6 +41,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes INIT_COMPLETED_BANNER = "PlatformIO: Initialization completed" def __init__(self, project_dir, args, debug_options, env_options): + super(GDBClient, self).__init__() self.project_dir = project_dir self.args = list(args) self.debug_options = debug_options @@ -55,10 +55,10 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes self._gdbsrc_dir = mkdtemp(dir=get_project_cache_dir(), prefix=".piodebug-") self._target_is_run = False - self._last_server_activity = 0 self._auto_continue_timer = None self._errors_buffer = b"" + @defer.inlineCallbacks def spawn(self, gdb_path, prog_path): session_hash = gdb_path + prog_path self._session_id = sha1(hashlib_encode_data(session_hash)).hexdigest() @@ -75,10 +75,10 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes "LOAD_CMDS": "\n".join(self.debug_options["load_cmds"] or []), } - self._debug_server.spawn(patterns) - + yield self._debug_server.spawn(patterns) if not patterns["DEBUG_PORT"]: patterns["DEBUG_PORT"] = self._debug_server.get_debug_port() + self.generate_pioinit(self._gdbsrc_dir, patterns) # start GDB client @@ -100,9 +100,10 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes args.extend(["--data-directory", gdb_data_dir]) args.append(patterns["PROG_PATH"]) - return reactor.spawnProcess( + transport = reactor.spawnProcess( self, gdb_path, args, path=self.project_dir, env=os.environ ) + defer.returnValue(transport) @staticmethod def _get_data_dir(gdb_path): @@ -175,12 +176,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes stdio.StandardIO(p) def onStdInData(self, data): - if LOG_FILE: - with open(LOG_FILE, "ab") as fp: - fp.write(data) - - self._last_server_activity = time.time() - + super(GDBClient, self).onStdInData(data) if b"-exec-run" in data: if self._target_is_run: token, _ = data.split(b"-", 1) @@ -206,11 +202,6 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes reactor.stop() def outReceived(self, data): - if LOG_FILE: - with open(LOG_FILE, "ab") as fp: - fp.write(data) - - self._last_server_activity = time.time() super(GDBClient, self).outReceived(data) self._handle_error(data) # go to init break automatically @@ -232,7 +223,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes def _auto_exec_continue(self): auto_exec_delay = 0.5 # in seconds - if self._last_server_activity > (time.time() - auto_exec_delay): + if self._last_activity > (time.time() - auto_exec_delay): return if self._auto_continue_timer: self._auto_continue_timer.stop() diff --git a/platformio/commands/debug/process.py b/platformio/commands/debug/process.py index d23f26b3..67557f3d 100644 --- a/platformio/commands/debug/process.py +++ b/platformio/commands/debug/process.py @@ -13,6 +13,7 @@ # limitations under the License. import signal +import time import click from twisted.internet import protocol # pylint: disable=import-error @@ -22,12 +23,11 @@ from platformio.compat import string_types from platformio.proc import get_pythonexe_path from platformio.project.helpers import get_project_core_dir -LOG_FILE = None - class BaseProcess(protocol.ProcessProtocol, object): STDOUT_CHUNK_SIZE = 2048 + LOG_FILE = None COMMON_PATTERNS = { "PLATFORMIO_HOME_DIR": get_project_core_dir(), @@ -35,6 +35,9 @@ class BaseProcess(protocol.ProcessProtocol, object): "PYTHONEXE": get_pythonexe_path(), } + def __init__(self): + self._last_activity = 0 + def apply_patterns(self, source, patterns=None): _patterns = self.COMMON_PATTERNS.copy() _patterns.update(patterns or {}) @@ -61,23 +64,30 @@ class BaseProcess(protocol.ProcessProtocol, object): return source + def onStdInData(self, data): + self._last_activity = time.time() + if self.LOG_FILE: + with open(self.LOG_FILE, "ab") as fp: + fp.write(data) + def outReceived(self, data): - if LOG_FILE: - with open(LOG_FILE, "ab") as fp: + self._last_activity = time.time() + if self.LOG_FILE: + with open(self.LOG_FILE, "ab") as fp: fp.write(data) while data: chunk = data[: self.STDOUT_CHUNK_SIZE] click.echo(chunk, nl=False) data = data[self.STDOUT_CHUNK_SIZE :] - @staticmethod - def errReceived(data): - if LOG_FILE: - with open(LOG_FILE, "ab") as fp: + def errReceived(self, data): + self._last_activity = time.time() + if self.LOG_FILE: + with open(self.LOG_FILE, "ab") as fp: fp.write(data) click.echo(data, nl=False, err=True) - @staticmethod - def processEnded(_): + def processEnded(self, _): + self._last_activity = time.time() # Allow terminating via SIGINT/CTRL+C signal.signal(signal.SIGINT, signal.default_int_handler) diff --git a/platformio/commands/debug/server.py b/platformio/commands/debug/server.py index 855628c3..1d2b2a11 100644 --- a/platformio/commands/debug/server.py +++ b/platformio/commands/debug/server.py @@ -13,8 +13,10 @@ # limitations under the License. import os +import time from os.path import isdir, isfile, join +from twisted.internet import defer # pylint: disable=import-error from twisted.internet import reactor # pylint: disable=import-error from platformio import fs, util @@ -26,13 +28,15 @@ from platformio.proc import where_is_program class DebugServer(BaseProcess): def __init__(self, debug_options, env_options): + super(DebugServer, self).__init__() self.debug_options = debug_options self.env_options = env_options - self._debug_port = None + self._debug_port = ":3333" self._transport = None self._process_ended = False + @defer.inlineCallbacks def spawn(self, patterns): # pylint: disable=too-many-branches systype = util.get_systype() server = self.debug_options.get("server") @@ -62,10 +66,10 @@ class DebugServer(BaseProcess): % server_executable ) - self._debug_port = ":3333" openocd_pipe_allowed = all( [not self.debug_options["port"], "openocd" in server_executable] ) + openocd_pipe_allowed = False if openocd_pipe_allowed: args = [] if server["cwd"]: @@ -79,43 +83,64 @@ class DebugServer(BaseProcess): ) self._debug_port = '| "%s" %s' % (server_executable, str_args) self._debug_port = fs.to_unix_path(self._debug_port) - else: - env = os.environ.copy() - # prepend server "lib" folder to LD path - if ( - "windows" not in systype - and server["cwd"] - and isdir(join(server["cwd"], "lib")) - ): - ld_key = ( - "DYLD_LIBRARY_PATH" if "darwin" in systype else "LD_LIBRARY_PATH" - ) - env[ld_key] = join(server["cwd"], "lib") - if os.environ.get(ld_key): - env[ld_key] = "%s:%s" % (env[ld_key], os.environ.get(ld_key)) - # prepend BIN to PATH - if server["cwd"] and isdir(join(server["cwd"], "bin")): - env["PATH"] = "%s%s%s" % ( - join(server["cwd"], "bin"), - os.pathsep, - os.environ.get("PATH", os.environ.get("Path", "")), - ) + return self._debug_port - self._transport = reactor.spawnProcess( - self, - server_executable, - [server_executable] + server["arguments"], - path=server["cwd"], - env=env, + env = os.environ.copy() + # prepend server "lib" folder to LD path + if ( + "windows" not in systype + and server["cwd"] + and isdir(join(server["cwd"], "lib")) + ): + ld_key = "DYLD_LIBRARY_PATH" if "darwin" in systype else "LD_LIBRARY_PATH" + env[ld_key] = join(server["cwd"], "lib") + if os.environ.get(ld_key): + env[ld_key] = "%s:%s" % (env[ld_key], os.environ.get(ld_key)) + # prepend BIN to PATH + if server["cwd"] and isdir(join(server["cwd"], "bin")): + env["PATH"] = "%s%s%s" % ( + join(server["cwd"], "bin"), + os.pathsep, + os.environ.get("PATH", os.environ.get("Path", "")), ) - if "mspdebug" in server_executable.lower(): - self._debug_port = ":2000" - elif "jlink" in server_executable.lower(): - self._debug_port = ":2331" - elif "qemu" in server_executable.lower(): - self._debug_port = ":1234" - return self._transport + self._transport = reactor.spawnProcess( + self, + server_executable, + [server_executable] + server["arguments"], + path=server["cwd"], + env=env, + ) + if "mspdebug" in server_executable.lower(): + self._debug_port = ":2000" + elif "jlink" in server_executable.lower(): + self._debug_port = ":2331" + elif "qemu" in server_executable.lower(): + self._debug_port = ":1234" + + yield self._wait_until_ready() + + return self._debug_port + + @defer.inlineCallbacks + def _wait_until_ready(self): + timeout = 10 + elapsed = 0 + delay = 1 + ready_delay = 0.5 + while ( + not self._process_ended + and elapsed < timeout + and not (self._last_activity < (time.time() - ready_delay)) + ): + yield self.async_sleep(delay) + elapsed += delay + + @staticmethod + def async_sleep(secs): + d = defer.Deferred() + reactor.callLater(secs, d.callback, None) + return d def get_debug_port(self): return self._debug_port From 352a0b73778589bd45d0da54dcf8df2b0dc06913 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 19 Mar 2020 00:46:23 +0200 Subject: [PATCH 48/61] Wait for an output from debug server --- platformio/commands/debug/server.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/platformio/commands/debug/server.py b/platformio/commands/debug/server.py index 1d2b2a11..a129a6aa 100644 --- a/platformio/commands/debug/server.py +++ b/platformio/commands/debug/server.py @@ -131,7 +131,10 @@ class DebugServer(BaseProcess): while ( not self._process_ended and elapsed < timeout - and not (self._last_activity < (time.time() - ready_delay)) + and ( + not self._last_activity + or not (self._last_activity < (time.time() - ready_delay)) + ) ): yield self.async_sleep(delay) elapsed += delay From 69f5fdf8e1be08b4c9f76795d7d211a1f80ece3e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 19 Mar 2020 01:05:12 +0200 Subject: [PATCH 49/61] Remove debug code --- platformio/commands/debug/server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/platformio/commands/debug/server.py b/platformio/commands/debug/server.py index a129a6aa..a74b1684 100644 --- a/platformio/commands/debug/server.py +++ b/platformio/commands/debug/server.py @@ -69,7 +69,6 @@ class DebugServer(BaseProcess): openocd_pipe_allowed = all( [not self.debug_options["port"], "openocd" in server_executable] ) - openocd_pipe_allowed = False if openocd_pipe_allowed: args = [] if server["cwd"]: From 475f89822223d5f79d7466154f78f3376520bf2a Mon Sep 17 00:00:00 2001 From: ShahRustam Date: Thu, 19 Mar 2020 13:26:51 +0200 Subject: [PATCH 50/61] Replace installer script with a new one // Resolve #3420 (#3428) * Replace installer script with a new one. Resolve #3420 * temp file name fix * get-platformio.py script update. * small fix --- scripts/get-platformio.py | 217 ++++++++++++-------------------------- 1 file changed, 69 insertions(+), 148 deletions(-) diff --git a/scripts/get-platformio.py b/scripts/get-platformio.py index 988f6054..192421b1 100644 --- a/scripts/get-platformio.py +++ b/scripts/get-platformio.py @@ -12,173 +12,94 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -import subprocess -import site +import tempfile +import io import sys -from platform import system -from tempfile import NamedTemporaryFile +import subprocess -CURINTERPRETER_PATH = os.path.normpath(sys.executable) -IS_WINDOWS = system().lower() == "windows" +NEW_SCRIPT_URL = "https://raw.githubusercontent.com/platformio/platformio-core-installer/develop/get-platformio.py" -def fix_winpython_pathenv(): - """ - Add Python & Python Scripts to the search path on Windows - """ - try: - import _winreg as winreg - except ImportError: - import winreg +def download_with_requests(url, dst): + import requests - # took these lines from the native "win_add2path.py" - pythonpath = os.path.dirname(os.path.normpath(sys.executable)) - scripts = os.path.join(pythonpath, "Scripts") - appdata = os.environ["APPDATA"] - if hasattr(site, "USER_SITE"): - userpath = site.USER_SITE.replace(appdata, "%APPDATA%") - userscripts = os.path.join(userpath, "Scripts") - else: - userscripts = None - - with winreg.CreateKey(winreg.HKEY_CURRENT_USER, "Environment") as key: - try: - envpath = winreg.QueryValueEx(key, "PATH")[0] - except WindowsError: - envpath = u"%PATH%" - - paths = [envpath] - for path in (pythonpath, scripts, userscripts): - if path and path not in envpath and os.path.isdir(path): - paths.append(path) - - envpath = os.pathsep.join(paths) - winreg.SetValueEx(key, "PATH", 0, winreg.REG_EXPAND_SZ, envpath) - return True + resp = requests.get(url, stream=True) + itercontent = resp.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE) + with open(dst, "wb") as fp: + for chunk in itercontent: + fp.write(chunk) + return dst -def exec_command(*args, **kwargs): - result = {"out": None, "err": None, "returncode": None} +def download_with_urllib3(url, dst): + import urllib3 + http = urllib3.PoolManager() + r = http.request('GET', url, preload_content=False) - kwargs['stdout'] = subprocess.PIPE - kwargs['stderr'] = subprocess.PIPE - kwargs['shell'] = IS_WINDOWS + with open(dst, 'wb') as out: + while True: + data = r.read(io.DEFAULT_BUFFER_SIZE) + if not data: + break + out.write(data) - p = subprocess.Popen(*args, **kwargs) - result['out'], result['err'] = p.communicate() - result['returncode'] = p.returncode - - for k, v in result.items(): - if v and isinstance(v, str): - result[k].strip() - - return result + r.release_conn() + return dst -def print_exec_result(result): - if result['returncode'] == 0: - print(result['out']) - else: - raise Exception("\n".join([result['out'], result['err']])) - - -def exec_python_cmd(args): - return exec_command([CURINTERPRETER_PATH] + args) - - -def install_pip(): - r = exec_python_cmd(["-m", "pip", "--version"]) - if r['returncode'] == 0: - print(r['out']) - return - try: - from urllib2 import urlopen - except ImportError: +def download_with_urllib(url, dst): + if sys.version_info[0] == 3: from urllib.request import urlopen + else: + from urllib import urlopen - f = NamedTemporaryFile(delete=False) - response = urlopen("https://bootstrap.pypa.io/get-pip.py") - f.write(response.read()) - f.close() + response = urlopen(url) + CHUNK = 16 * 1024 + with open(dst, 'wb') as f: + while True: + chunk = response.read(CHUNK) + if not chunk: + break + f.write(chunk) - try: - r = exec_python_cmd([f.name]) - finally: - os.unlink(f.name) - - print_exec_result(r) + return dst -def install_platformio(): - r = None - cmd = ["-m", "pip", "install", "-U", "platformio"] - # cmd = [ - # "-m", "pip", "install", "-U", - # "https://github.com/platformio/platformio-core/archive/develop.zip" - # ] - try: - r = exec_python_cmd(cmd) - assert r['returncode'] == 0 - except AssertionError: - cmd.insert(2, "--no-cache-dir") - r = exec_python_cmd(cmd) - if r: - print_exec_result(r) +def download_with_curl(url, dst): + subprocess.check_output(["curl", "-o", dst, url]) + return dst + + +def download_with_wget(url, dst): + subprocess.check_output(["wget", "-O", dst, url]) + return dst + + +def download_file(url, dst): + methods = [ + download_with_requests, + download_with_urllib3, + download_with_urllib, + download_with_curl, + download_with_wget + ] + for method in methods: + try: + method(url, dst) + return dst + except: + pass + raise Exception("Could not download file '%s' to '%s' "%(url, dst)) def main(): - steps = [("Fixing Windows %PATH% Environment", fix_winpython_pathenv), - ("Installing Python Package Manager", install_pip), - ("Installing PlatformIO and dependencies", install_platformio)] - - if not IS_WINDOWS: - del steps[0] - - is_error = False - for s in steps: - if is_error: - break - print("\n==> %s ..." % s[0]) - try: - s[1]() - print("[SUCCESS]") - except Exception as e: - is_error = True - print(str(e)) - print("[FAILURE]") - - permission_errors = ("permission denied", "not permitted") - if (any([m in str(e).lower() for m in permission_errors]) and - not IS_WINDOWS): - print(""" ------------------ -Permission denied ------------------ - -You need the `sudo` permission to install Python packages. Try - -$ sudo python -c "$(curl -fsSL -https://raw.githubusercontent.com/platformio/platformio/develop/scripts/get-platformio.py)" -""") - - if is_error: - print("The installation process has been FAILED!\n" - "Please report about this problem here\n" - "< https://github.com/platformio/platformio-core/issues >") - return - else: - print("\n ==> Installation process has been " - "successfully FINISHED! <==\n") - print(""" - ----------------------------------------- -Please RESTART your Terminal Application ----------------------------------------- - -Then run `platformio --help` command. - -""") + print("This installer script is deprecated and will be removed in the next release. Please use %s" % + NEW_SCRIPT_URL) + with tempfile.NamedTemporaryFile() as tmp_file: + dst = download_file(NEW_SCRIPT_URL, str(tmp_file.name)) + command = [sys.executable, dst] + command.extend(sys.argv[1:]) + subprocess.check_call(command) if __name__ == "__main__": From a8606f4efa0486cc63e781efedf4db05b6437a25 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 19 Mar 2020 14:49:25 +0200 Subject: [PATCH 51/61] Refactor debug GDB initial configurations --- docs | 2 +- platformio/commands/debug/client.py | 21 ++++----------------- platformio/commands/debug/initcfgs.py | 18 ++++++++++++++++++ 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/docs b/docs index 03c5eb48..e366c5d3 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 03c5eb487fe7abd85745a6810bab06eaf29b5a81 +Subproject commit e366c5d3642137d1326635cebafa666f198131db diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py index 610f598d..4e2298f9 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/client.py @@ -27,8 +27,9 @@ from twisted.internet import stdio # pylint: disable=import-error from twisted.internet import task # pylint: disable=import-error from platformio import app, fs, proc, telemetry, util -from platformio.commands.debug import helpers, initcfgs +from platformio.commands.debug import helpers from platformio.commands.debug.exception import DebugInvalidOptionsError +from platformio.commands.debug.initcfgs import get_gdb_init_config from platformio.commands.debug.process import BaseProcess from platformio.commands.debug.server import DebugServer from platformio.compat import hashlib_encode_data, is_bytes @@ -113,22 +114,8 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes return gdb_data_dir if isdir(gdb_data_dir) else None def generate_pioinit(self, dst_dir, patterns): - server_exe = ( - (self.debug_options.get("server") or {}).get("executable", "").lower() - ) - if "jlink" in server_exe: - cfg = initcfgs.GDB_JLINK_INIT_CONFIG - elif "st-util" in server_exe: - cfg = initcfgs.GDB_STUTIL_INIT_CONFIG - elif "mspdebug" in server_exe: - cfg = initcfgs.GDB_MSPDEBUG_INIT_CONFIG - elif "qemu" in server_exe: - cfg = initcfgs.GDB_QEMU_INIT_CONFIG - elif self.debug_options["require_debug_port"]: - cfg = initcfgs.GDB_BLACKMAGIC_INIT_CONFIG - else: - cfg = initcfgs.GDB_DEFAULT_INIT_CONFIG - commands = cfg.split("\n") + # default GDB init commands depending on debug tool + commands = get_gdb_init_config(self.debug_options).split("\n") if self.debug_options["init_cmds"]: commands = self.debug_options["init_cmds"] diff --git a/platformio/commands/debug/initcfgs.py b/platformio/commands/debug/initcfgs.py index 50da1779..d39629d5 100644 --- a/platformio/commands/debug/initcfgs.py +++ b/platformio/commands/debug/initcfgs.py @@ -123,3 +123,21 @@ $LOAD_CMDS pio_reset_halt_target $INIT_BREAK """ + + +TOOL_TO_CONFIG = { + "jlink": GDB_JLINK_INIT_CONFIG, + "mspdebug": GDB_MSPDEBUG_INIT_CONFIG, + "qemu": GDB_QEMU_INIT_CONFIG, + "blackmagic": GDB_BLACKMAGIC_INIT_CONFIG, +} + + +def get_gdb_init_config(debug_options): + tool = debug_options.get("tool") + if tool and tool in TOOL_TO_CONFIG: + return TOOL_TO_CONFIG[tool] + server_exe = (debug_options.get("server") or {}).get("executable", "").lower() + if "st-util" in server_exe: + return GDB_STUTIL_INIT_CONFIG + return GDB_DEFAULT_INIT_CONFIG From 0e3c3abf736d3bb4de607251164a10514f49acdb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 19 Mar 2020 15:16:55 +0200 Subject: [PATCH 52/61] GDB init commands for Renode simulation framework // Issue #3401 --- platformio/commands/debug/initcfgs.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/platformio/commands/debug/initcfgs.py b/platformio/commands/debug/initcfgs.py index d39629d5..a392b9eb 100644 --- a/platformio/commands/debug/initcfgs.py +++ b/platformio/commands/debug/initcfgs.py @@ -124,12 +124,29 @@ pio_reset_halt_target $INIT_BREAK """ +GDB_RENODE_INIT_CONFIG = """ +define pio_reset_halt_target + monitor reset +end + +define pio_reset_run_target + monitor reset +end + +target extended-remote $DEBUG_PORT +$LOAD_CMDS +pio_reset_halt_target +$INIT_BREAK +monitor start +""" + TOOL_TO_CONFIG = { "jlink": GDB_JLINK_INIT_CONFIG, "mspdebug": GDB_MSPDEBUG_INIT_CONFIG, "qemu": GDB_QEMU_INIT_CONFIG, "blackmagic": GDB_BLACKMAGIC_INIT_CONFIG, + "renode": GDB_RENODE_INIT_CONFIG, } From aa939b07b10021ef538312921752c04081f0b893 Mon Sep 17 00:00:00 2001 From: valeros Date: Thu, 19 Mar 2020 16:17:51 +0200 Subject: [PATCH 53/61] Update default init config for Renode --- platformio/commands/debug/initcfgs.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/platformio/commands/debug/initcfgs.py b/platformio/commands/debug/initcfgs.py index a392b9eb..55e7c34f 100644 --- a/platformio/commands/debug/initcfgs.py +++ b/platformio/commands/debug/initcfgs.py @@ -126,16 +126,17 @@ $INIT_BREAK GDB_RENODE_INIT_CONFIG = """ define pio_reset_halt_target - monitor reset + monitor machine Reset + $LOAD_CMDS + monitor start end define pio_reset_run_target - monitor reset + pio_reset_halt_target end target extended-remote $DEBUG_PORT $LOAD_CMDS -pio_reset_halt_target $INIT_BREAK monitor start """ From 720c29350db03f62ab320575252620809f807d2d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 19 Mar 2020 16:58:18 +0200 Subject: [PATCH 54/61] Add docs for Renode debugging tool // Issue #3401 --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index e366c5d3..1f615138 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit e366c5d3642137d1326635cebafa666f198131db +Subproject commit 1f6151389291d3c7babec8db0b50154b00666761 From 24a0d9123e2f4074486c551a404b41fcb6fa0dbb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 19 Mar 2020 17:04:05 +0200 Subject: [PATCH 55/61] Update history with initial support for Renode --- HISTORY.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/HISTORY.rst b/HISTORY.rst index 1ad463f8..04aafc9d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -27,6 +27,7 @@ PlatformIO Core 4 - Send a text to device on ENTER with ``send_on_enter`` filter (`issue #926 `_) - Show a hexadecimal representation of the data (code point of each character) with ``hexlify`` filter +* Initial support for `Renode `__ simulation framework (`issue #3401 `_) * Added support for Arm Mbed "module.json" ``dependencies`` field (`issue #3400 `_) * Improved support for Arduino "library.properties" ``depends`` field * Fixed an issue when quitting from PlatformIO IDE does not shutdown PIO Home server From 15647c81f07e9df36e25de76dded301f1d91563b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 19 Mar 2020 18:26:30 +0200 Subject: [PATCH 56/61] New standalone (1-script) PlatformIO Core Installer --- HISTORY.rst | 1 + docs | 2 +- scripts/get-platformio.py | 19 +++++++++---------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 04aafc9d..ecd4dd04 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -27,6 +27,7 @@ PlatformIO Core 4 - Send a text to device on ENTER with ``send_on_enter`` filter (`issue #926 `_) - Show a hexadecimal representation of the data (code point of each character) with ``hexlify`` filter +* New standalone (1-script) `PlatformIO Core Installer `_ * Initial support for `Renode `__ simulation framework (`issue #3401 `_) * Added support for Arm Mbed "module.json" ``dependencies`` field (`issue #3400 `_) * Improved support for Arduino "library.properties" ``depends`` field diff --git a/docs b/docs index 1f615138..0af3f530 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 1f6151389291d3c7babec8db0b50154b00666761 +Subproject commit 0af3f53030196d1e9dec83e380814048ab9948ce diff --git a/scripts/get-platformio.py b/scripts/get-platformio.py index 192421b1..9d99ea97 100644 --- a/scripts/get-platformio.py +++ b/scripts/get-platformio.py @@ -17,7 +17,7 @@ import io import sys import subprocess -NEW_SCRIPT_URL = "https://raw.githubusercontent.com/platformio/platformio-core-installer/develop/get-platformio.py" +MAIN_SCRIPT_URL = "https://raw.githubusercontent.com/platformio/platformio-core-installer/master/get-platformio.py" def download_with_requests(url, dst): @@ -33,10 +33,11 @@ def download_with_requests(url, dst): def download_with_urllib3(url, dst): import urllib3 - http = urllib3.PoolManager() - r = http.request('GET', url, preload_content=False) - with open(dst, 'wb') as out: + http = urllib3.PoolManager() + r = http.request("GET", url, preload_content=False) + + with open(dst, "wb") as out: while True: data = r.read(io.DEFAULT_BUFFER_SIZE) if not data: @@ -55,7 +56,7 @@ def download_with_urllib(url, dst): response = urlopen(url) CHUNK = 16 * 1024 - with open(dst, 'wb') as f: + with open(dst, "wb") as f: while True: chunk = response.read(CHUNK) if not chunk: @@ -81,7 +82,7 @@ def download_file(url, dst): download_with_urllib3, download_with_urllib, download_with_curl, - download_with_wget + download_with_wget, ] for method in methods: try: @@ -89,14 +90,12 @@ def download_file(url, dst): return dst except: pass - raise Exception("Could not download file '%s' to '%s' "%(url, dst)) + raise Exception("Could not download file '%s' to '%s' " % (url, dst)) def main(): - print("This installer script is deprecated and will be removed in the next release. Please use %s" % - NEW_SCRIPT_URL) with tempfile.NamedTemporaryFile() as tmp_file: - dst = download_file(NEW_SCRIPT_URL, str(tmp_file.name)) + dst = download_file(MAIN_SCRIPT_URL, str(tmp_file.name)) command = [sys.executable, dst] command.extend(sys.argv[1:]) subprocess.check_call(command) From d9647dec950a77a95719eedaaecef494c8cc8041 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 19 Mar 2020 19:17:54 +0200 Subject: [PATCH 57/61] Add support for debugging server "ready_pattern" --- platformio/commands/debug/helpers.py | 23 +++++++++++++---------- platformio/commands/debug/server.py | 19 ++++++++++--------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index 0ec7d447..4604a861 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -125,7 +125,8 @@ def validate_debug_options(cmd_ctx, env_options): server_options["executable"] = server_options["arguments"][0] server_options["arguments"] = server_options["arguments"][1:] elif "server" in tool_settings: - server_package = tool_settings["server"].get("package") + server_options = tool_settings["server"] + server_package = server_options.get("package") server_package_dir = ( platform.get_package_dir(server_package) if server_package else None ) @@ -134,15 +135,17 @@ def validate_debug_options(cmd_ctx, env_options): with_packages=[server_package], skip_default_package=True, silent=True ) server_package_dir = platform.get_package_dir(server_package) - server_options = dict( - cwd=server_package_dir if server_package else None, - executable=tool_settings["server"].get("executable"), - arguments=[ - a.replace("$PACKAGE_DIR", server_package_dir) - if server_package_dir - else a - for a in tool_settings["server"].get("arguments", []) - ], + server_options.update( + dict( + cwd=server_package_dir if server_package else None, + executable=server_options.get("executable"), + arguments=[ + a.replace("$PACKAGE_DIR", server_package_dir) + if server_package_dir + else a + for a in server_options.get("arguments", []) + ], + ) ) extra_cmds = _cleanup_cmds(env_options.get("debug_extra_cmds")) diff --git a/platformio/commands/debug/server.py b/platformio/commands/debug/server.py index a74b1684..d50ac9d0 100644 --- a/platformio/commands/debug/server.py +++ b/platformio/commands/debug/server.py @@ -35,6 +35,7 @@ class DebugServer(BaseProcess): self._debug_port = ":3333" self._transport = None self._process_ended = False + self._ready = False @defer.inlineCallbacks def spawn(self, patterns): # pylint: disable=too-many-branches @@ -126,15 +127,7 @@ class DebugServer(BaseProcess): timeout = 10 elapsed = 0 delay = 1 - ready_delay = 0.5 - while ( - not self._process_ended - and elapsed < timeout - and ( - not self._last_activity - or not (self._last_activity < (time.time() - ready_delay)) - ) - ): + while not self._ready and not self._process_ended and elapsed < timeout: yield self.async_sleep(delay) elapsed += delay @@ -151,6 +144,14 @@ class DebugServer(BaseProcess): super(DebugServer, self).outReceived( escape_gdbmi_stream("@", data) if is_gdbmi_mode() else data ) + if self._ready: + return + ready_pattern = self.debug_options.get("server", {}).get("ready_pattern") + if ready_pattern: + self._ready = ready_pattern.encode() in data + else: + ready_delay = 0.5 + self._ready = self._last_activity < (time.time() - ready_delay) def processEnded(self, reason): self._process_ended = True From 4c2a157dced3be9dd81dcc4425b4bfab891bfcbe Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 19 Mar 2020 19:28:13 +0200 Subject: [PATCH 58/61] Bump version to 4.3.0rc1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 0fac99a0..7df83f59 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 3, "0b2") +VERSION = (4, 3, "0rc1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 2d68e28a703d590a68e8f62c64481c30f7af4d10 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 19 Mar 2020 21:33:23 +0200 Subject: [PATCH 59/61] Fix auto-ready logic for debugging server --- platformio/commands/debug/server.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/platformio/commands/debug/server.py b/platformio/commands/debug/server.py index d50ac9d0..29a00678 100644 --- a/platformio/commands/debug/server.py +++ b/platformio/commands/debug/server.py @@ -126,9 +126,12 @@ class DebugServer(BaseProcess): def _wait_until_ready(self): timeout = 10 elapsed = 0 - delay = 1 + delay = 0.5 + auto_ready_delay = 0.5 while not self._ready and not self._process_ended and elapsed < timeout: yield self.async_sleep(delay) + if not self.debug_options.get("server", {}).get("ready_pattern"): + self._ready = self._last_activity < (time.time() - auto_ready_delay) elapsed += delay @staticmethod @@ -149,9 +152,6 @@ class DebugServer(BaseProcess): ready_pattern = self.debug_options.get("server", {}).get("ready_pattern") if ready_pattern: self._ready = ready_pattern.encode() in data - else: - ready_delay = 0.5 - self._ready = self._last_activity < (time.time() - ready_delay) def processEnded(self, reason): self._process_ended = True From a55f04dc288466c3d07c088f907f7be7bf52178d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 19 Mar 2020 22:36:55 +0200 Subject: [PATCH 60/61] Warn that can't allocate socket for PIO Home --- platformio/commands/home/command.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py index 208354bf..229c875e 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home/command.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=too-many-locals +# pylint: disable=too-many-locals,too-many-statements import mimetypes import socket @@ -63,6 +63,7 @@ def cli(port, host, no_open, shutdown_timeout): from twisted.internet import reactor from twisted.web import server + from twisted.internet.error import CannotListenError from platformio.commands.home.rpc.handlers.app import AppRPC from platformio.commands.home.rpc.handlers.ide import IDERPC @@ -121,6 +122,12 @@ def cli(port, host, no_open, shutdown_timeout): click.echo("") click.echo("Open PlatformIO Home in your browser by this URL => %s" % home_url) + try: + reactor.listenTCP(port, site, interface=host) + except CannotListenError as e: + click.secho(str(e), fg="red", err=True) + already_started = True + if already_started: click.secho( "PlatformIO Home server is already started in another process.", fg="yellow" @@ -129,7 +136,6 @@ def cli(port, host, no_open, shutdown_timeout): click.echo("PIO Home has been started. Press Ctrl+C to shutdown.") - reactor.listenTCP(port, site, interface=host) reactor.run() From 6414e1d9e3bfb7b0fb3e0386db180ff198482c8e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 19 Mar 2020 22:37:16 +0200 Subject: [PATCH 61/61] Bump version to 4.3.0 --- HISTORY.rst | 2 +- docs | 2 +- platformio/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index ecd4dd04..4771a3c0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,7 +6,7 @@ Release Notes PlatformIO Core 4 ----------------- -4.3.0 (2020-??-??) +4.3.0 (2020-03-19) ~~~~~~~~~~~~~~~~~~ * Initial support for an official `PlatformIO for CLion IDE `__ plugin: diff --git a/docs b/docs index 0af3f530..51b7dd49 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 0af3f53030196d1e9dec83e380814048ab9948ce +Subproject commit 51b7dd49b703fd6156375a93d4bbb6e0890ecb09 diff --git a/platformio/__init__.py b/platformio/__init__.py index 7df83f59..c5dd8dcc 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 3, "0rc1") +VERSION = (4, 3, 0) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio"