From 34176f974b637262298cce12e10e838510b1ae8e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 30 Aug 2019 15:45:21 +0300 Subject: [PATCH 001/221] Fix generator for CLion when project is empty // Issue #2824 --- platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl | 1 + 1 file changed, 1 insertion(+) diff --git a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl index 7967881e..f96163fb 100644 --- a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl @@ -59,6 +59,7 @@ endif() % leftover_envs = set(envs) ^ set([env_name]) % +% ide_data = {} % if leftover_envs: % ide_data = load_project_ide_data(project_dir, leftover_envs) % end From f966eeb60465aa9fd9acf2e2aa0bb34e087d512f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 30 Aug 2019 16:40:44 +0300 Subject: [PATCH 002/221] Fixed an issue with project generator for CLion IDE when 2 environments were used // Resolve #2824 --- HISTORY.rst | 5 +++++ platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl | 2 +- platformio/project/helpers.py | 11 ++++++----- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index f9f1ccc3..5ac2772b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,6 +6,11 @@ Release Notes PlatformIO 4.0 -------------- +4.0.4 (2019-??-??) +~~~~~~~~~~~~~~~~~~ + +* Fixed an issue with project generator for `CLion IDE `__ when 2 environments were used (`issue #2824 `_) + 4.0.3 (2019-08-30) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl index f96163fb..40f31e09 100644 --- a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl @@ -57,7 +57,7 @@ if (CMAKE_BUILD_TYPE MATCHES "{{ env_name }}") %end endif() -% leftover_envs = set(envs) ^ set([env_name]) +% leftover_envs = list(set(envs) ^ set([env_name])) % % ide_data = {} % if leftover_envs: diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 7ead95eb..9de1dbe2 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -194,10 +194,11 @@ def compute_project_checksum(config): return checksum.hexdigest() -def load_project_ide_data(project_dir, envs): +def load_project_ide_data(project_dir, env_or_envs): from platformio.commands.run import cli as cmd_run - assert envs - if not isinstance(envs, (list, tuple, set)): + assert env_or_envs + envs = env_or_envs + if not isinstance(envs, list): envs = [envs] args = ["--project-dir", project_dir, "--target", "idedata"] for env in envs: @@ -217,6 +218,6 @@ def load_project_ide_data(project_dir, envs): _data = json.loads(line) if "env_name" in _data: data[_data['env_name']] = _data - if len(envs) == 1 and envs[0] in data: - return data[envs[0]] + if not isinstance(env_or_envs, list) and env_or_envs in data: + return data[env_or_envs] return data or None From e7da3d7f5f7eb2a6316dbdfcd8ec12b1733f7bc2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 30 Aug 2019 16:41:17 +0300 Subject: [PATCH 003/221] Bump version to 4.0.4a1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 1b010521..0fd3f680 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, 3) +VERSION = (4, 0, "4a1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 92cd03cf2af3a9e0d3d78436d54ac01978c0e19e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 30 Aug 2019 18:12:26 +0300 Subject: [PATCH 004/221] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 704ff85c..7d221858 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 704ff85c7de13079a28a7bd9a7fb2adefb071eee +Subproject commit 7d22185894299bdcb19341765e01ac7a761feedf From f26e3c42dd594c1a591d176658b87169321ada6a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 31 Aug 2019 11:40:16 +0300 Subject: [PATCH 005/221] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 7d221858..9325e44f 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 7d22185894299bdcb19341765e01ac7a761feedf +Subproject commit 9325e44fc288136ac6895f9a4e689710718abe42 From bdce78ba6fbe96ff3d1bee5c0266e56ea49cad0d Mon Sep 17 00:00:00 2001 From: Peter Date: Sat, 31 Aug 2019 18:47:32 +1000 Subject: [PATCH 006/221] Stop ModemManager corrupting Arduino uploads (#2966) On boards like the Arduino Micro, when in bootloader mode it appears ModemManager interferes with the programming process and result in a catastrophic failure with no end of different errors including, but not limited to: ``` error: programmer did not respond to command: write block error: butterfly programmer uses avr_write_page() but does not provide a cmd() method. error: programmer did not respond to command: set addr ``` After this, the device could appear to be completely non-functional, refusing to enumerate or appear for programming, but thankfully a double-reset will usually recover it, but the underlying ModemManager issue will still prevent successful programming. Hence the additional rules. This affects not only PlatformIO, but also the Arduino IDE (on linux). --- scripts/99-platformio-udev.rules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/99-platformio-udev.rules b/scripts/99-platformio-udev.rules index be7c3e85..e3f479af 100644 --- a/scripts/99-platformio-udev.rules +++ b/scripts/99-platformio-udev.rules @@ -40,8 +40,8 @@ SUBSYSTEMS=="usb", ATTRS{idVendor}=="067b", ATTRS{idProduct}=="2303", MODE:="066 SUBSYSTEMS=="usb", ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="7523", MODE:="0666" # Arduino boards -SUBSYSTEMS=="usb", ATTRS{idVendor}=="2341", ATTRS{idProduct}=="[08][02]*", MODE:="0666" -SUBSYSTEMS=="usb", ATTRS{idVendor}=="2a03", ATTRS{idProduct}=="[08][02]*", MODE:="0666" +SUBSYSTEMS=="usb", ATTRS{idVendor}=="2341", ATTRS{idProduct}=="[08][02]*", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" +SUBSYSTEMS=="usb", ATTRS{idVendor}=="2a03", ATTRS{idProduct}=="[08][02]*", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Arduino SAM-BA ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="6124", ENV{ID_MM_DEVICE_IGNORE}="1" From fe237f15aafcf6d5c9a775e205ea0259c230c513 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 31 Aug 2019 23:39:41 +0300 Subject: [PATCH 007/221] Implement "extends" for project configuration // Resolve #2953 --- HISTORY.rst | 23 ++-- docs | 2 +- platformio/project/config.py | 57 ++++++--- platformio/project/options.py | 1 + tests/test_projectconf.py | 212 ++++++++++++++++++++-------------- 5 files changed, 180 insertions(+), 115 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 5ac2772b..88975168 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,12 +3,13 @@ Release Notes .. _release_notes_4_0: -PlatformIO 4.0 --------------- +PlatformIO Core 4.0 +------------------- -4.0.4 (2019-??-??) +4.1.0 (2019-??-??) ~~~~~~~~~~~~~~~~~~ +* Extend project environment configuration in "platformio.ini" with other sections using a new `extends `__ option (`issue #2953 `_) * Fixed an issue with project generator for `CLion IDE `__ when 2 environments were used (`issue #2824 `_) 4.0.3 (2019-08-30) @@ -109,8 +110,8 @@ PlatformIO 4.0 - Fixed "systemd-udevd" warnings in `99-platformio-udev.rules `__ (`issue #2442 `_) - Fixed an issue when package cache (Library Manager) expires too fast (`issue #2559 `_) -PlatformIO 3.0 --------------- +PlatformIO Core 3.0 +------------------- 3.6.7 (2019-04-23) ~~~~~~~~~~~~~~~~~~ @@ -710,8 +711,8 @@ PlatformIO 3.0 (`issue #742 `_) * Stopped supporting Python 2.6 -PlatformIO 2.0 --------------- +PlatformIO Core 2.0 +-------------------- 2.11.2 (2016-08-02) ~~~~~~~~~~~~~~~~~~~ @@ -1496,8 +1497,8 @@ PlatformIO 2.0 * Fixed bug with creating copies of source files (`issue #177 `_) -PlatformIO 1.0 --------------- +PlatformIO Core 1.0 +------------------- 1.5.0 (2015-05-15) ~~~~~~~~~~~~~~~~~~ @@ -1687,8 +1688,8 @@ PlatformIO 1.0 error (`issue #81 `_) * Several bug fixes, increased stability and performance improvements -PlatformIO 0.0 --------------- +PlatformIO Core 0.0 +------------------- 0.10.2 (2015-01-06) ~~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index 9325e44f..90af5fea 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 9325e44fc288136ac6895f9a4e689710718abe42 +Subproject commit 90af5feac507b7c593570149e0aa789d389df498 diff --git a/platformio/project/config.py b/platformio/project/config.py index 6ca36a14..63ac5eec 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -90,6 +90,9 @@ class ProjectConfig(object): if isfile(path): self.read(path, parse_extra) + def __repr__(self): + return "" % (self.path or "in-memory") + def __getattr__(self, name): return getattr(self._parser, name) @@ -163,34 +166,46 @@ class ProjectConfig(object): "in section [%s]" % (option, section)) return True + def walk_options(self, root_section): + extends_queue = (["env", root_section] if + root_section.startswith("env:") else [root_section]) + extends_done = [] + while extends_queue: + section = extends_queue.pop() + extends_done.append(section) + if not self._parser.has_section(section): + continue + for option in self._parser.options(section): + yield (section, option) + if self._parser.has_option(section, "extends"): + extends_queue.extend( + self.parse_multi_values( + self._parser.get(section, "extends"))[::-1]) + def options(self, section=None, env=None): + result = [] assert section or env if not section: section = "env:" + env - options = self._parser.options(section) - # handle global options from [env] - if ((env or section.startswith("env:")) - and self._parser.has_section("env")): - for option in self._parser.options("env"): - if option not in options: - options.append(option) + for _, option in self.walk_options(section): + if option not in result: + result.append(option) # handle system environment variables scope = section.split(":", 1)[0] for option_meta in ProjectOptions.values(): - if option_meta.scope != scope or option_meta.name in options: + if option_meta.scope != scope or option_meta.name in result: continue if option_meta.sysenvvar and option_meta.sysenvvar in os.environ: - options.append(option_meta.name) + result.append(option_meta.name) - return options + return result def has_option(self, section, option): if self._parser.has_option(section, option): return True - return (section.startswith("env:") and self._parser.has_section("env") - and self._parser.has_option("env", option)) + return option in self.options(section) def items(self, section=None, env=None, as_dict=False): assert section or env @@ -215,12 +230,16 @@ class ProjectConfig(object): if not self.expand_interpolations: return self._parser.get(section, option) - try: + value = None + found = False + for sec, opt in self.walk_options(section): + if opt == option: + value = self._parser.get(sec, option) + found = True + break + + if not found: value = self._parser.get(section, option) - except ConfigParser.NoOptionError as e: - if not section.startswith("env:"): - raise e - value = self._parser.get("env", option) if "${" not in value or "}" not in value: return value @@ -267,13 +286,13 @@ class ProjectConfig(object): return default try: - return self._covert_value(value, option_meta.type) + return self._cast_to(value, option_meta.type) except click.BadParameter as e: raise exception.ProjectOptionValueError(e.format_message(), option, section) @staticmethod - def _covert_value(value, to_type): + def _cast_to(value, to_type): items = value if not isinstance(value, (list, tuple)): items = [value] diff --git a/platformio/project/options.py b/platformio/project/options.py index bc2f5c3d..1ff6e76c 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -90,6 +90,7 @@ ProjectOptions = OrderedDict([ # # [env] # + ConfigEnvOption(name="extends", multiple=True), # Generic ConfigEnvOption(name="platform", buildenvvar="PIOPLATFORM"), diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index 1ad8b2cd..10bf4b2f 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -34,6 +34,17 @@ lib_deps = Lib2 lib_ignore = ${custom.lib_ignore} +[strict_ldf] +lib_ldf_mode = chain+ +lib_compat_mode = strict + +[monitor_custom] +monitor_speed = 9600 + +[strict_settings] +extends = strict_ldf, monitor_custom +build_flags = -D RELEASE + [custom] debug_flags = -D RELEASE lib_flags = -lc -lm @@ -43,6 +54,10 @@ lib_ignore = LibIgnoreCustom [env:base] build_flags = ${custom.debug_flags} ${custom.extra_flags} targets = + +[env:test_extends] +extends = strict_settings + """ EXTRA_ENVS_CONFIG = """ @@ -66,52 +81,70 @@ build_flags = -Og """ -def test_real_config(tmpdir): +@pytest.fixture(scope="module") +def config(tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("project") tmpdir.join("platformio.ini").write(BASE_CONFIG) tmpdir.join("extra_envs.ini").write(EXTRA_ENVS_CONFIG) tmpdir.join("extra_debug.ini").write(EXTRA_DEBUG_CONFIG) - - config = None with tmpdir.as_cwd(): - config = ProjectConfig(tmpdir.join("platformio.ini").strpath) - assert config - assert len(config.warnings) == 2 - assert "lib_install" in config.warnings[1] + return ProjectConfig(tmpdir.join("platformio.ini").strpath) - config.validate(["extra_2", "base"], silent=True) - with pytest.raises(UnknownEnvNames): - config.validate(["non-existing-env"]) + +def test_empty_config(): + config = ProjectConfig("/non/existing/platformio.ini") # unknown section with pytest.raises(ConfigParser.NoSectionError): config.getraw("unknown_section", "unknown_option") - # unknown option - with pytest.raises(ConfigParser.NoOptionError): - config.getraw("custom", "unknown_option") - # unknown option even if exists in [env] - with pytest.raises(ConfigParser.NoOptionError): - config.getraw("platformio", "monitor_speed") - # sections + assert config.sections() == [] + assert config.get("section", "option") is None + assert config.get("section", "option", 13) == 13 + + +def test_warnings(config): + config.validate(["extra_2", "base"], silent=True) + assert len(config.warnings) == 2 + assert "lib_install" in config.warnings[1] + + with pytest.raises(UnknownEnvNames): + config.validate(["non-existing-env"]) + + +def test_sections(config): + with pytest.raises(ConfigParser.NoSectionError): + config.getraw("unknown_section", "unknown_option") + assert config.sections() == [ - "platformio", "env", "custom", "env:base", "env:extra_1", "env:extra_2" + "platformio", "env", "strict_ldf", "monitor_custom", "strict_settings", + "custom", "env:base", "env:test_extends", "env:extra_1", "env:extra_2" ] - # envs - assert config.envs() == ["base", "extra_1", "extra_2"] + +def test_envs(config): + assert config.envs() == ["base", "test_extends", "extra_1", "extra_2"] assert config.default_envs() == ["base", "extra_2"] - # options + +def test_options(config): assert config.options(env="base") == [ "build_flags", "targets", "monitor_speed", "lib_deps", "lib_ignore" ] + assert config.options(env="test_extends") == [ + "extends", "build_flags", "lib_ldf_mode", "lib_compat_mode", + "monitor_speed", "lib_deps", "lib_ignore" + ] - # has_option + +def test_has_option(config): assert config.has_option("env:base", "monitor_speed") assert not config.has_option("custom", "monitor_speed") assert not config.has_option("env:extra_1", "lib_install") + assert config.has_option("env:test_extends", "lib_compat_mode") - # sysenv + +def test_sysenv_options(config): assert config.get("custom", "extra_flags") is None assert config.get("env:base", "build_flags") == ["-D DEBUG=1"] assert config.get("env:base", "upload_port") is None @@ -126,72 +159,83 @@ def test_real_config(tmpdir): assert config.get("env:base", "upload_port") == "/dev/sysenv/port" assert config.get("env:extra_2", "upload_port") == "/dev/extra_2/port" - # getraw - assert config.getraw("env:base", "targets") == "" - assert config.getraw("env:extra_1", "lib_deps") == "574" - assert config.getraw("env:extra_1", "build_flags") == "-lc -lm -D DEBUG=1" - - # get - assert config.get("custom", "debug_flags") == "-D DEBUG=1" - assert config.get("env:extra_1", "build_flags") == [ - "-lc -lm -D DEBUG=1", "-DSYSENVDEPS1 -DSYSENVDEPS2" + # env var as option + assert config.options(env="test_extends") == [ + "extends", "build_flags", "lib_ldf_mode", "lib_compat_mode", + "monitor_speed", "lib_deps", "lib_ignore", "upload_port" ] - assert config.get("env:extra_2", "build_flags") == [ - "-Og", "-DSYSENVDEPS1 -DSYSENVDEPS2"] - assert config.get("env:extra_2", "monitor_speed") == "115200" - assert config.get("env:base", "build_flags") == ([ - "-D DEBUG=1 -L /usr/local/lib", "-DSYSENVDEPS1 -DSYSENVDEPS2" - ]) - # items - assert config.items("custom") == [ - ("debug_flags", "-D DEBUG=1"), - ("lib_flags", "-lc -lm"), - ("extra_flags", "-L /usr/local/lib"), - ("lib_ignore", "LibIgnoreCustom") - ] # yapf: disable - assert config.items(env="base") == [ - ("build_flags", [ - "-D DEBUG=1 -L /usr/local/lib", "-DSYSENVDEPS1 -DSYSENVDEPS2"]), - ("targets", []), - ("monitor_speed", "115200"), - ("lib_deps", ["Lib1", "Lib2"]), - ("lib_ignore", ["LibIgnoreCustom"]), - ("upload_port", "/dev/sysenv/port") - ] # yapf: disable - assert config.items(env="extra_1") == [ - ("build_flags", ["-lc -lm -D DEBUG=1", "-DSYSENVDEPS1 -DSYSENVDEPS2"]), - ("lib_deps", ["574"]), - ("monitor_speed", "115200"), - ("lib_ignore", ["LibIgnoreCustom"]), - ("upload_port", "/dev/sysenv/port") - ] # yapf: disable - assert config.items(env="extra_2") == [ - ("build_flags", ["-Og", "-DSYSENVDEPS1 -DSYSENVDEPS2"]), - ("lib_ignore", ["LibIgnoreCustom", "Lib3"]), - ("upload_port", "/dev/extra_2/port"), - ("monitor_speed", "115200"), - ("lib_deps", ["Lib1", "Lib2"]) - ] # yapf: disable + # sysenv + os.environ["PLATFORMIO_HOME_DIR"] = "/custom/core/dir" + assert config.get("platformio", "core_dir") == "/custom/core/dir" # cleanup system environment variables del os.environ["PLATFORMIO_BUILD_FLAGS"] del os.environ["PLATFORMIO_UPLOAD_PORT"] del os.environ["__PIO_TEST_CNF_EXTRA_FLAGS"] - - -def test_empty_config(): - config = ProjectConfig("/non/existing/platformio.ini") - - # unknown section - with pytest.raises(ConfigParser.NoSectionError): - config.getraw("unknown_section", "unknown_option") - - assert config.sections() == [] - assert config.get("section", "option") is None - assert config.get("section", "option", 13) == 13 - - # sysenv - os.environ["PLATFORMIO_HOME_DIR"] = "/custom/core/dir" - assert config.get("platformio", "core_dir") == "/custom/core/dir" del os.environ["PLATFORMIO_HOME_DIR"] + + +def test_getraw_value(config): + # unknown option + with pytest.raises(ConfigParser.NoOptionError): + config.getraw("custom", "unknown_option") + # unknown option even if exists in [env] + with pytest.raises(ConfigParser.NoOptionError): + config.getraw("platformio", "monitor_speed") + + # known + assert config.getraw("env:base", "targets") == "" + assert config.getraw("env:extra_1", "lib_deps") == "574" + assert config.getraw("env:extra_1", "build_flags") == "-lc -lm -D DEBUG=1" + + # extended + assert config.getraw("env:test_extends", "lib_ldf_mode") == "chain+" + assert config.getraw("env", "monitor_speed") == "115200" + assert config.getraw("env:test_extends", "monitor_speed") == "9600" + + +def test_get_value(config): + assert config.get("custom", "debug_flags") == "-D DEBUG=1" + assert config.get("env:extra_1", "build_flags") == ["-lc -lm -D DEBUG=1"] + assert config.get("env:extra_2", "build_flags") == ["-Og"] + assert config.get("env:extra_2", "monitor_speed") == "115200" + assert config.get("env:base", "build_flags") == ["-D DEBUG=1"] + + +def test_items(config): + assert config.items("custom") == [ + ("debug_flags", "-D DEBUG=1"), + ("lib_flags", "-lc -lm"), + ("extra_flags", None), + ("lib_ignore", "LibIgnoreCustom") + ] # yapf: disable + assert config.items(env="base") == [ + ("build_flags", ["-D DEBUG=1"]), + ("targets", []), + ("monitor_speed", "115200"), + ("lib_deps", ["Lib1", "Lib2"]), + ("lib_ignore", ["LibIgnoreCustom"]), + ] # yapf: disable + assert config.items(env="extra_1") == [ + ("build_flags", ["-lc -lm -D DEBUG=1"]), + ("lib_deps", ["574"]), + ("monitor_speed", "115200"), + ("lib_ignore", ["LibIgnoreCustom"]), + ] # yapf: disable + assert config.items(env="extra_2") == [ + ("build_flags", ["-Og"]), + ("lib_ignore", ["LibIgnoreCustom", "Lib3"]), + ("upload_port", "/dev/extra_2/port"), + ("monitor_speed", "115200"), + ("lib_deps", ["Lib1", "Lib2"]) + ] # yapf: disable + assert config.items(env="test_extends") == [ + ("extends", ["strict_settings"]), + ("build_flags", ["-D RELEASE"]), + ("lib_ldf_mode", "chain+"), + ("lib_compat_mode", "strict"), + ("monitor_speed", "9600"), + ("lib_deps", ["Lib1", "Lib2"]), + ("lib_ignore", ["LibIgnoreCustom"]) + ] # yapf: disable From af049eecc9b1455c866f537f1cb0ffe3c6c5ec9c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 31 Aug 2019 23:40:28 +0300 Subject: [PATCH 008/221] Bump version to 4.1.0a1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 0fd3f680..32b79b8b 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "4a1") +VERSION = (4, 1, "0a1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 1be2e510da1e20d37c4ac13a7259ee01fe4c9e2e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 2 Sep 2019 12:50:55 +0300 Subject: [PATCH 009/221] Sync nRF52 dev/platform --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 90af5fea..083a75db 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 90af5feac507b7c593570149e0aa789d389df498 +Subproject commit 083a75dbe345d1d7ebdb70462bc45747e7f84b36 From 9f76293684702655e988641c858595d828a5ccf6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 2 Sep 2019 14:13:58 +0300 Subject: [PATCH 010/221] Cleanup Segger UDEV rules --- scripts/99-platformio-udev.rules | 66 +------------------------------- 1 file changed, 1 insertion(+), 65 deletions(-) diff --git a/scripts/99-platformio-udev.rules b/scripts/99-platformio-udev.rules index e3f479af..0264e217 100644 --- a/scripts/99-platformio-udev.rules +++ b/scripts/99-platformio-udev.rules @@ -159,22 +159,7 @@ ATTRS{idVendor}=="09fb", ATTRS{idProduct}=="6001", MODE="0666" ATTRS{idVendor}=="0fbb", ATTRS{idProduct}=="1000", MODE="0666" # SEGGER J-Link -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0101", MODE="0666" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0102", MODE="0666" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0103", MODE="0666" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0104", MODE="0666" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0105", MODE="0666" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0107", MODE="0666" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0108", MODE="0666" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1010", MODE="0666" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1011", MODE="0666" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1012", MODE="0666" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1013", MODE="0666" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1014", MODE="0666" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1015", MODE="0666" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1016", MODE="0666" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1017", MODE="0666" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1018", MODE="0666" +ATTRS{idVendor}=="1366", MODE="0666" # Raisonance RLink ATTRS{idVendor}=="138e", ATTRS{idProduct}=="9000", MODE="0666" @@ -211,52 +196,3 @@ ATTRS{idVendor}=="c251", ATTRS{idProduct}=="2710", MODE="0666" # CMSIS-DAP compatible adapters ATTRS{product}=="*CMSIS-DAP*", MODE="0666" - -#SEGGER J-LIK -ATTR{idProduct}=="1001", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1002", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1003", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1004", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1005", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1006", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1007", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1008", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1009", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="100a", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="100b", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="100c", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="100d", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="100e", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="100f", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1010", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1011", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1012", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1013", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1014", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1015", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1016", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1017", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1018", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1019", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="101a", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="101b", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="101c", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="101d", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="101e", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="101f", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1020", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1021", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1022", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1023", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1024", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1025", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1026", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1027", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1028", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="1029", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="102a", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="102b", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="102c", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="102d", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="102e", ATTR{idVendor}=="1366", MODE="0666" -ATTR{idProduct}=="102f", ATTR{idVendor}=="1366", MODE="0666" From be3e26c202f9531db844b5703966e3bd50155f42 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 2 Sep 2019 14:24:35 +0300 Subject: [PATCH 011/221] Cleanup UDEV rules --- scripts/99-platformio-udev.rules | 39 +++++--------------------------- 1 file changed, 6 insertions(+), 33 deletions(-) diff --git a/scripts/99-platformio-udev.rules b/scripts/99-platformio-udev.rules index 0264e217..97a5caa5 100644 --- a/scripts/99-platformio-udev.rules +++ b/scripts/99-platformio-udev.rules @@ -27,9 +27,6 @@ # CP210X USB UART SUBSYSTEMS=="usb", ATTRS{idVendor}=="10c4", ATTRS{idProduct}=="ea60", MODE:="0666" -# FT232R USB UART -SUBSYSTEMS=="usb", ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6001", MODE:="0666" - # FT231XS USB UART SUBSYSTEMS=="usb", ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6015", MODE:="0666" @@ -53,9 +50,6 @@ KERNEL=="ttyACM*", ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="6124", MODE:="066 SUBSYSTEMS=="usb", ATTRS{idVendor}=="16d0", ATTRS{idProduct}=="0753", MODE:="0666" KERNEL=="ttyACM*", ATTRS{idVendor}=="16d0", ATTRS{idProduct}=="0753", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1" -# STM32 discovery boards, with onboard st/linkv2 -SUBSYSTEMS=="usb", ATTRS{idVendor}=="0483", ATTRS{idProduct}=="374?", MODE:="0666" - # Maple with DFU SUBSYSTEMS=="usb", ATTRS{idVendor}=="1eaf", ATTRS{idProduct}=="000[34]", MODE:="0666" @@ -134,23 +128,14 @@ ATTRS{idVendor}=="0403", ATTRS{idProduct}=="cff8", MODE="0666" # TI ICDI ATTRS{idVendor}=="0451", ATTRS{idProduct}=="c32a", MODE="0666" -# STLink v1 -ATTRS{idVendor}=="0483", ATTRS{idProduct}=="3744", MODE="0666" - -# STLink v2 -ATTRS{idVendor}=="0483", ATTRS{idProduct}=="3748", MODE="0666" - -# STLink v2-1 -ATTRS{idVendor}=="0483", ATTRS{idProduct}=="374b", MODE="0666" +# STLink probes +ATTRS{idVendor}=="0483", MODE="0666" # Hilscher NXHX Boards ATTRS{idVendor}=="0640", ATTRS{idProduct}=="0028", MODE="0666" -# Hitex STR9-comStick -ATTRS{idVendor}=="0640", ATTRS{idProduct}=="002c", MODE="0666" - -# Hitex STM32-PerformanceStick -ATTRS{idVendor}=="0640", ATTRS{idProduct}=="002d", MODE="0666" +# Hitex probes +ATTRS{idVendor}=="0640", MODE="0666" # Altera USB Blaster ATTRS{idVendor}=="09fb", ATTRS{idProduct}=="6001", MODE="0666" @@ -167,20 +152,8 @@ ATTRS{idVendor}=="138e", ATTRS{idProduct}=="9000", MODE="0666" # Debug Board for Neo1973 ATTRS{idVendor}=="1457", ATTRS{idProduct}=="5118", MODE="0666" -# Olimex ARM-USB-OCD -ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="0003", MODE="0666" - -# Olimex ARM-USB-OCD-TINY -ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="0004", MODE="0666" - -# Olimex ARM-JTAG-EW -ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="001e", MODE="0666" - -# Olimex ARM-USB-OCD-TINY-H -ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="002a", MODE="0666" - -# Olimex ARM-USB-OCD-H -ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="002b", MODE="0666" +# Olimex probes +ATTRS{idVendor}=="15ba", MODE="0666" # USBprog with OpenOCD firmware ATTRS{idVendor}=="1781", ATTRS{idProduct}=="0c63", MODE="0666" From 6265233903b433a09960d235c49566c617f6d0d7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 2 Sep 2019 16:01:15 +0300 Subject: [PATCH 012/221] Optimize udev rules --- scripts/99-platformio-udev.rules | 98 +++++++++++++++----------------- 1 file changed, 47 insertions(+), 51 deletions(-) diff --git a/scripts/99-platformio-udev.rules b/scripts/99-platformio-udev.rules index 97a5caa5..41ca21fa 100644 --- a/scripts/99-platformio-udev.rules +++ b/scripts/99-platformio-udev.rules @@ -25,39 +25,35 @@ # # CP210X USB UART -SUBSYSTEMS=="usb", ATTRS{idVendor}=="10c4", ATTRS{idProduct}=="ea60", MODE:="0666" +ATTRS{idVendor}=="10c4", ATTRS{idProduct}=="ea60", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # FT231XS USB UART -SUBSYSTEMS=="usb", ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6015", MODE:="0666" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6015", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Prolific Technology, Inc. PL2303 Serial Port -SUBSYSTEMS=="usb", ATTRS{idVendor}=="067b", ATTRS{idProduct}=="2303", MODE:="0666" +ATTRS{idVendor}=="067b", ATTRS{idProduct}=="2303", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # QinHeng Electronics HL-340 USB-Serial adapter -SUBSYSTEMS=="usb", ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="7523", MODE:="0666" +ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="7523", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Arduino boards -SUBSYSTEMS=="usb", ATTRS{idVendor}=="2341", ATTRS{idProduct}=="[08][02]*", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" -SUBSYSTEMS=="usb", ATTRS{idVendor}=="2a03", ATTRS{idProduct}=="[08][02]*", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" +ATTRS{idVendor}=="2341", ATTRS{idProduct}=="[08][02]*", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" +ATTRS{idVendor}=="2a03", ATTRS{idProduct}=="[08][02]*", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Arduino SAM-BA -ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="6124", ENV{ID_MM_DEVICE_IGNORE}="1" -ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="6124", ENV{MTP_NO_PROBE}="1" -SUBSYSTEMS=="usb", ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="6124", MODE:="0666" -KERNEL=="ttyACM*", ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="6124", MODE:="0666" +ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="6124", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{MTP_NO_PROBE}="1" # Digistump boards -SUBSYSTEMS=="usb", ATTRS{idVendor}=="16d0", ATTRS{idProduct}=="0753", MODE:="0666" -KERNEL=="ttyACM*", ATTRS{idVendor}=="16d0", ATTRS{idProduct}=="0753", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1" +ATTRS{idVendor}=="16d0", ATTRS{idProduct}=="0753", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Maple with DFU -SUBSYSTEMS=="usb", ATTRS{idVendor}=="1eaf", ATTRS{idProduct}=="000[34]", MODE:="0666" +ATTRS{idVendor}=="1eaf", ATTRS{idProduct}=="000[34]", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # USBtiny -SUBSYSTEMS=="usb", ATTRS{idProduct}=="0c9f", ATTRS{idVendor}=="1781", MODE="0666" +ATTRS{idProduct}=="0c9f", ATTRS{idVendor}=="1781", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # USBasp V2.0 -SUBSYSTEMS=="usb", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="05dc", MODE:="0666" +ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="05dc", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Teensy boards ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789B]?", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" @@ -66,10 +62,10 @@ SUBSYSTEMS=="usb", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789ABCD]?", MO KERNEL=="ttyACM*", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789B]?", MODE:="0666" #TI Stellaris Launchpad -SUBSYSTEMS=="usb", ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="0666" +ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" #TI MSP430 Launchpad -SUBSYSTEMS=="usb", ATTRS{idVendor}=="0451", ATTRS{idProduct}=="f432", MODE="0666" +ATTRS{idVendor}=="0451", ATTRS{idProduct}=="f432", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # @@ -77,95 +73,95 @@ SUBSYSTEMS=="usb", ATTRS{idVendor}=="0451", ATTRS{idProduct}=="f432", MODE="0666 # # Black Magic Probe -SUBSYSTEM=="tty", ATTRS{interface}=="Black Magic GDB Server" -SUBSYSTEM=="tty", ATTRS{interface}=="Black Magic UART Port" +SUBSYSTEM=="tty", ATTRS{interface}=="Black Magic GDB Server", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" +SUBSYSTEM=="tty", ATTRS{interface}=="Black Magic UART Port", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # opendous and estick -ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="204f", MODE="0666" +ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="204f", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Original FT232/FT245 VID:PID -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6001", MODE="0666" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6001", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Original FT2232 VID:PID -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6010", MODE="0666" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6010", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Original FT4232 VID:PID -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6011", MODE="0666" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6011", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Original FT232H VID:PID -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6014", MODE="0666" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6014", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # DISTORTEC JTAG-lock-pick Tiny 2 -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8220", MODE="0666" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8220", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # TUMPA, TUMPA Lite -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a98", MODE="0666" -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a99", MODE="0666" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a98", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a99", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # XDS100v2 -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="a6d0", MODE="0666" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="a6d0", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Xverve Signalyzer Tool (DT-USB-ST), Signalyzer LITE (DT-USB-SLITE) -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca0", MODE="0666" -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca1", MODE="0666" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca0", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca1", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # TI/Luminary Stellaris Evaluation Board FTDI (several) -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcd9", MODE="0666" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcd9", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # TI/Luminary Stellaris In-Circuit Debug Interface FTDI (ICDI) Board -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcda", MODE="0666" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcda", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # egnite Turtelizer 2 -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bdc8", MODE="0666" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bdc8", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Section5 ICEbear -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c140", MODE="0666" -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c141", MODE="0666" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c140", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c141", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Amontec JTAGkey and JTAGkey-tiny -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="cff8", MODE="0666" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="cff8", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # TI ICDI -ATTRS{idVendor}=="0451", ATTRS{idProduct}=="c32a", MODE="0666" +ATTRS{idVendor}=="0451", ATTRS{idProduct}=="c32a", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # STLink probes -ATTRS{idVendor}=="0483", MODE="0666" +ATTRS{idVendor}=="0483", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Hilscher NXHX Boards -ATTRS{idVendor}=="0640", ATTRS{idProduct}=="0028", MODE="0666" +ATTRS{idVendor}=="0640", ATTRS{idProduct}=="0028", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Hitex probes -ATTRS{idVendor}=="0640", MODE="0666" +ATTRS{idVendor}=="0640", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Altera USB Blaster -ATTRS{idVendor}=="09fb", ATTRS{idProduct}=="6001", MODE="0666" +ATTRS{idVendor}=="09fb", ATTRS{idProduct}=="6001", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Amontec JTAGkey-HiSpeed -ATTRS{idVendor}=="0fbb", ATTRS{idProduct}=="1000", MODE="0666" +ATTRS{idVendor}=="0fbb", ATTRS{idProduct}=="1000", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # SEGGER J-Link -ATTRS{idVendor}=="1366", MODE="0666" +ATTRS{idVendor}=="1366", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Raisonance RLink -ATTRS{idVendor}=="138e", ATTRS{idProduct}=="9000", MODE="0666" +ATTRS{idVendor}=="138e", ATTRS{idProduct}=="9000", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Debug Board for Neo1973 -ATTRS{idVendor}=="1457", ATTRS{idProduct}=="5118", MODE="0666" +ATTRS{idVendor}=="1457", ATTRS{idProduct}=="5118", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Olimex probes -ATTRS{idVendor}=="15ba", MODE="0666" +ATTRS{idVendor}=="15ba", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # USBprog with OpenOCD firmware -ATTRS{idVendor}=="1781", ATTRS{idProduct}=="0c63", MODE="0666" +ATTRS{idVendor}=="1781", ATTRS{idProduct}=="0c63", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # TI/Luminary Stellaris In-Circuit Debug Interface (ICDI) Board -ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="0666" +ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Marvell Sheevaplug -ATTRS{idVendor}=="9e88", ATTRS{idProduct}=="9e8f", MODE="0666" +ATTRS{idVendor}=="9e88", ATTRS{idProduct}=="9e8f", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Keil Software, Inc. ULink -ATTRS{idVendor}=="c251", ATTRS{idProduct}=="2710", MODE="0666" +ATTRS{idVendor}=="c251", ATTRS{idProduct}=="2710", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # CMSIS-DAP compatible adapters -ATTRS{product}=="*CMSIS-DAP*", MODE="0666" +ATTRS{product}=="*CMSIS-DAP*", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" From 2c242944c7021112db9ad974c96df3e42454d216 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 2 Sep 2019 16:48:33 +0300 Subject: [PATCH 013/221] Fixed default PIO Unified Debugger configuration for J-Link probe --- HISTORY.rst | 1 + platformio/commands/debug/initcfgs.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 88975168..ead20705 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,6 +11,7 @@ PlatformIO Core 4.0 * Extend project environment configuration in "platformio.ini" with other sections using a new `extends `__ option (`issue #2953 `_) * Fixed an issue with project generator for `CLion IDE `__ when 2 environments were used (`issue #2824 `_) +* Fixed default PIO Unified Debugger configuration for `J-Link probe `__ 4.0.3 (2019-08-30) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/debug/initcfgs.py b/platformio/commands/debug/initcfgs.py index a9d71d32..9b70babe 100644 --- a/platformio/commands/debug/initcfgs.py +++ b/platformio/commands/debug/initcfgs.py @@ -60,7 +60,6 @@ target extended-remote $DEBUG_PORT $INIT_BREAK pio_reset_halt_target $LOAD_CMDS -pio_reset_halt_target """ GDB_BLACKMAGIC_INIT_CONFIG = """ From 7a07a2e63eb51f8e403ecaf0fb2e4029d7b54744 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 3 Sep 2019 15:31:33 +0300 Subject: [PATCH 014/221] Generate ``.ccls`` LSP file for Emacs --- HISTORY.rst | 3 ++- platformio/ide/tpls/emacs/.ccls.tpl | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 platformio/ide/tpls/emacs/.ccls.tpl diff --git a/HISTORY.rst b/HISTORY.rst index ead20705..f8283452 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,8 +10,9 @@ PlatformIO Core 4.0 ~~~~~~~~~~~~~~~~~~ * Extend project environment configuration in "platformio.ini" with other sections using a new `extends `__ option (`issue #2953 `_) +* Generate ``.ccls`` LSP file for `Emacs `__ cross references, hierarchies, completion and semantic highlighting * Fixed an issue with project generator for `CLion IDE `__ when 2 environments were used (`issue #2824 `_) -* Fixed default PIO Unified Debugger configuration for `J-Link probe `__ +* Fixed default PIO Unified Debugger configuration for `J-Link probe `__ 4.0.3 (2019-08-30) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/ide/tpls/emacs/.ccls.tpl b/platformio/ide/tpls/emacs/.ccls.tpl new file mode 100644 index 00000000..ad22fce7 --- /dev/null +++ b/platformio/ide/tpls/emacs/.ccls.tpl @@ -0,0 +1,22 @@ +% import re +% STD_RE = re.compile(r"\-std=[a-z\+]+(\d+)") +% cc_stds = STD_RE.findall(cc_flags) +% cxx_stds = STD_RE.findall(cxx_flags) +% +% +clang + +% if cc_stds: +{{"%c"}} -std=c{{ cc_stds[-1] }} +% end +% if cxx_stds: +{{"%cpp"}} -std=c++{{ cxx_stds[-1] }} +% end + +% for include in includes: +-I{{ include }} +% end + +% for define in defines: +-D{{ define }} +% end From b7bc4401eb9fc4cf3ac7813ee2b4526d979b5168 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 8 Sep 2019 14:01:41 +0300 Subject: [PATCH 015/221] Use isolated SCons DB per build environment --- platformio/builder/main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index f29f2f85..d1a8763d 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -117,8 +117,7 @@ env.LoadPioPlatform() env.SConscriptChdir(0) env.SConsignFile( - join("$PROJECTBUILD_DIR", - ".sconsign.dblite" if PY2 else ".sconsign3.dblite")) + join("$BUILD_DIR", ".sconsign.dblite" if PY2 else ".sconsign3.dblite")) for item in env.GetExtraScripts("pre"): env.SConscript(item, exports="env") From f61d03ec8fb57cf906000f039ceef2bef5469cd7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 8 Sep 2019 18:04:41 +0300 Subject: [PATCH 016/221] PIO Check (#2982) --- docs | 2 +- examples | 2 +- platformio/commands/check/__init__.py | 15 + platformio/commands/check/command.py | 247 +++++++++++++++++ platformio/commands/check/tools.py | 385 ++++++++++++++++++++++++++ platformio/managers/core.py | 4 +- platformio/project/options.py | 5 + tests/commands/test_check.py | 294 ++++++++++++++++++++ 8 files changed, 951 insertions(+), 3 deletions(-) create mode 100644 platformio/commands/check/__init__.py create mode 100644 platformio/commands/check/command.py create mode 100644 platformio/commands/check/tools.py create mode 100644 tests/commands/test_check.py diff --git a/docs b/docs index 083a75db..29f80d45 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 083a75dbe345d1d7ebdb70462bc45747e7f84b36 +Subproject commit 29f80d45f2d7fe14918b507d84ec8badc54fe087 diff --git a/examples b/examples index 6859117a..a71564ab 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 6859117a8c0b5d293a00e29b339250d0587f31de +Subproject commit a71564ab46d27c387f17814056b659f826b7db24 diff --git a/platformio/commands/check/__init__.py b/platformio/commands/check/__init__.py new file mode 100644 index 00000000..76307f21 --- /dev/null +++ b/platformio/commands/check/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.commands.check.command import cli diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py new file mode 100644 index 00000000..00b6c274 --- /dev/null +++ b/platformio/commands/check/command.py @@ -0,0 +1,247 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=too-many-arguments,too-many-locals,too-many-branches +# pylint: disable=redefined-builtin,too-many-statements + +import os +from collections import Counter +from os.path import basename, dirname, isfile, join +from time import time + +import click +from tabulate import tabulate + +from platformio import exception, fs, util +from platformio.commands.check.tools import CheckToolFactory, DefectItem +from platformio.compat import dump_json_to_unicode +from platformio.project.config import ProjectConfig +from platformio.project.helpers import (find_project_dir_above, + get_project_dir, + get_project_include_dir, + get_project_src_dir) + + +@click.command("check", short_help="Run a static analysis tool on code") +@click.option("-e", "--environment", multiple=True) +@click.option("-d", + "--project-dir", + default=os.getcwd, + type=click.Path(exists=True, + file_okay=True, + dir_okay=True, + writable=True, + resolve_path=True)) +@click.option("-c", + "--project-conf", + type=click.Path(exists=True, + file_okay=True, + dir_okay=False, + readable=True, + resolve_path=True)) +@click.option("--filter", multiple=True, help="Pattern: + -") +@click.option("--flags", multiple=True) +@click.option("--severity", + type=click.Choice(DefectItem.SEVERITY_LABELS.values()), + default=DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_LOW]) +@click.option("-s", "--silent", is_flag=True) +@click.option("-v", "--verbose", is_flag=True) +@click.option("--json-output", is_flag=True) +def cli(environment, project_dir, project_conf, filter, flags, severity, + silent, verbose, json_output): + # find project directory on upper level + if isfile(project_dir): + project_dir = find_project_dir_above(project_dir) + + results = [] + with fs.cd(project_dir): + config = ProjectConfig.get_instance( + project_conf or join(project_dir, "platformio.ini")) + config.validate(environment) + + default_envs = config.default_envs() + for envname in config.envs(): + skipenv = any([ + environment and envname not in environment, not environment + and default_envs and envname not in default_envs + ]) + + env_options = config.items(env=envname, as_dict=True) + env_dump = [] + for k, v in env_options.items(): + if k not in ("platform", "framework", "board"): + continue + env_dump.append( + "%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v)) + + default_filter = [ + "+<%s/>" % basename(d) + for d in (get_project_src_dir(), get_project_include_dir()) + ] + + tool_options = dict( + verbose=verbose, + silent=silent, + filter=filter + or env_options.get("check_filter", default_filter), + flags=flags or env_options.get("check_flags", ()), + severity=severity if not silent else "high") + + for tool in env_options.get("check_tool", ["cppcheck"]): + if skipenv: + results.append({"env": envname, "tool": tool}) + continue + if not silent and not json_output: + print_processing_header(tool, envname, env_dump) + + ct = CheckToolFactory.new(tool, project_dir, config, envname, + tool_options) + + result = {"env": envname, "tool": tool, "duration": time()} + rc = ct.check(on_defect_callback=None if ( + json_output or verbose + ) else lambda defect: click.echo(repr(defect))) + + result['defects'] = ct.get_defects() + result['duration'] = time() - result['duration'] + result['succeeded'] = ( + rc == 0 and not any(d.severity == DefectItem.SEVERITY_HIGH + for d in result['defects'])) + results.append(result) + + if verbose: + click.echo("\n".join(repr(d) for d in result['defects'])) + + if not json_output and not silent: + print_processing_footer(result) + + if json_output: + click.echo(dump_json_to_unicode(results_to_json(results))) + elif not silent: + print_check_summary(results) + + command_failed = any(r.get("succeeded") is False for r in results) + if command_failed: + raise exception.ReturnErrorCode(1) + + +def results_to_json(raw): + results = [] + for item in raw: + item.update({ + "ignored": item.get("succeeded") is None, + "succeeded": bool(item.get("succeeded")), + "defects": [d.to_json() for d in item.get("defects", [])] + }) + results.append(item) + + return results + + +def print_processing_header(tool, envname, envdump): + click.echo( + "Checking %s > %s (%s)" % + (click.style(envname, fg="cyan", bold=True), tool, "; ".join(envdump))) + terminal_width, _ = click.get_terminal_size() + click.secho("-" * terminal_width, bold=True) + + +def print_processing_footer(result): + is_failed = not result.get("succeeded") + util.print_labeled_bar( + "[%s] Took %.2f seconds" % + ((click.style("FAILED", fg="red", bold=True) if is_failed else + click.style("PASSED", fg="green", bold=True)), result['duration']), + is_error=is_failed) + + +def print_defects_stats(results): + components = dict() + + def _append_defect(component, defect): + if not components.get(component): + components[component] = Counter() + components[component].update( + {DefectItem.SEVERITY_LABELS[defect.severity]: 1}) + + for result in results: + for defect in result.get("defects", []): + component = dirname(defect.file) or defect.file + _append_defect(component, defect) + + if component.startswith(get_project_dir()): + while os.sep in component: + component = dirname(component) + _append_defect(component, defect) + + if not components: + return + + severity_labels = list(DefectItem.SEVERITY_LABELS.values()) + severity_labels.reverse() + tabular_data = list() + for k, v in components.items(): + tool_defect = [v.get(s, 0) for s in severity_labels] + tabular_data.append([k] + tool_defect) + + total = ["Total"] + [sum(d) for d in list(zip(*tabular_data))[1:]] + tabular_data.sort() + tabular_data.append([]) # Empty line as delimeter + tabular_data.append(total) + + headers = ["Component"] + headers.extend([l.upper() for l in severity_labels]) + headers = [click.style(h, bold=True) for h in headers] + click.echo(tabulate(tabular_data, headers=headers, numalign="center")) + click.echo() + + +def print_check_summary(results): + click.echo() + + tabular_data = [] + succeeded_nums = 0 + failed_nums = 0 + duration = 0 + + print_defects_stats(results) + for result in results: + duration += result.get("duration", 0) + if result.get("succeeded") is False: + failed_nums += 1 + status_str = click.style("FAILED", fg="red") + elif result.get("succeeded") is None: + status_str = "IGNORED" + else: + succeeded_nums += 1 + status_str = click.style("PASSED", fg="green") + + tabular_data.append( + (click.style(result['env'], fg="cyan"), result['tool'], status_str, + util.humanize_duration_time(result.get("duration")))) + + click.echo(tabulate(tabular_data, + headers=[ + click.style(s, bold=True) + for s in ("Environment", "Tool", "Status", + "Duration") + ]), + err=failed_nums) + + util.print_labeled_bar( + "%s%d succeeded in %s" % + ("%d failed, " % failed_nums if failed_nums else "", succeeded_nums, + util.humanize_duration_time(duration)), + is_error=failed_nums, + fg="red" if failed_nums else "green") diff --git a/platformio/commands/check/tools.py b/platformio/commands/check/tools.py new file mode 100644 index 00000000..c2b682db --- /dev/null +++ b/platformio/commands/check/tools.py @@ -0,0 +1,385 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=too-many-arguments,too-many-instance-attributes +# pylint: disable=redefined-builtin + +import re +import sys +from os import remove +from os.path import isfile, join, relpath +from tempfile import NamedTemporaryFile + +import click + +from platformio import exception, fs, proc +from platformio.managers.core import get_core_package_dir +from platformio.project.helpers import (get_project_core_dir, get_project_dir, + load_project_ide_data) + + +class DefectItem(object): + + SEVERITY_HIGH = 1 + SEVERITY_MEDIUM = 2 + SEVERITY_LOW = 4 + SEVERITY_LABELS = {4: "low", 2: "medium", 1: "high"} + + def __init__(self, + severity, + category, + message, + file="unknown", + line=0, + column=0, + id=None, + callstack=None, + cwe=None): + assert severity in (self.SEVERITY_HIGH, self.SEVERITY_MEDIUM, + self.SEVERITY_LOW) + self.severity = severity + self.category = category + self.message = message + self.line = line + self.column = column + self.callstack = callstack + self.cwe = cwe + self.id = id + self.file = file + if file.startswith(get_project_dir()): + self.file = relpath(file, get_project_dir()) + + def __repr__(self): + defect_color = None + if self.severity == self.SEVERITY_HIGH: + defect_color = "red" + elif self.severity == self.SEVERITY_MEDIUM: + defect_color = "yellow" + + format_str = "{file}:{line}: [{severity}:{category}] {message} {id}" + return format_str.format(severity=click.style( + self.SEVERITY_LABELS[self.severity], fg=defect_color), + category=click.style(self.category.lower(), + fg=defect_color), + file=click.style(self.file, bold=True), + message=self.message, + line=self.line, + id="%s" % "[%s]" % self.id if self.id else "") + + def __or__(self, defect): + return self.severity | defect.severity + + @staticmethod + def severity_to_int(label): + for key, value in DefectItem.SEVERITY_LABELS.items(): + if label == value: + return key + raise Exception("Unknown severity label -> %s" % label) + + def to_json(self): + return { + "severity": self.SEVERITY_LABELS[self.severity], + "category": self.category, + "message": self.message, + "file": self.file, + "line": self.line, + "column": self.column, + "callstack": self.callstack, + "id": self.id, + "cwe": self.cwe + } + + +class CheckToolFactory(object): + + @staticmethod + def new(tool, project_dir, config, envname, options): + clsname = "%sCheckTool" % tool.title() + try: + obj = getattr(sys.modules[__name__], clsname)(project_dir, config, + envname, options) + except AttributeError: + raise exception.PlatformioException("Unknown check tool `%s`" % + tool) + assert isinstance(obj, CheckToolBase) + return obj + + +class CheckToolBase(object): + + def __init__(self, project_dir, config, envname, options): + self.config = config + self.envname = envname + self.options = options + self.cpp_defines = [] + self.cpp_includes = [] + self._bad_input = False + self._load_cpp_data(project_dir, envname) + + self._defects = [] + self._on_defect_callback = None + + def _load_cpp_data(self, project_dir, envname): + data = load_project_ide_data(project_dir, envname) + if not data: + return + self.cpp_includes = data.get("includes", []) + self.cpp_defines = data.get("defines", []) + self.cpp_defines.extend( + self._get_toolchain_defines(data.get("cc_path"))) + + def get_flags(self, tool): + result = [] + flags = self.options.get("flags", []) + for flag in flags: + if ":" not in flag: + result.extend([f for f in flag.split(" ") if f]) + elif flag.startswith("%s:" % tool): + result.extend( + [f for f in flag.split(":", 1)[1].split(" ") if f]) + + return result + + @staticmethod + def _get_toolchain_defines(cc_path): + defines = [] + result = proc.exec_command( + "echo | %s -dM -E -x c++ -" % cc_path, shell=True) + + for line in result['out'].split("\n"): + tokens = line.strip().split(" ", 2) + if not tokens or tokens[0] != "#define": + continue + if len(tokens) > 2: + defines.append("%s=%s" % (tokens[1], tokens[2])) + else: + defines.append(tokens[1]) + + return defines + + @staticmethod + def is_flag_set(flag, flags): + return any(flag in f for f in flags) + + def get_defects(self): + return self._defects + + def configure_command(self): + raise NotImplementedError + + def on_tool_output(self, line): + line = self.tool_output_filter(line) + if not line: + return + + defect = self.parse_defect(line) + if isinstance(defect, DefectItem): + self._defects.append(defect) + if self._on_defect_callback: + self._on_defect_callback(defect) + elif self.options.get("verbose"): + click.echo(line) + + @staticmethod + def tool_output_filter(line): + return line + + @staticmethod + def parse_defect(raw_line): + return raw_line + + def clean_up(self): + pass + + def exceeds_severity_threshold(self, severity): + return severity <= DefectItem.severity_to_int( + self.options.get("severity")) + + def get_project_src_files(self): + file_extensions = ["h", "hpp", "c", "cc", "cpp", "ino"] + return fs.match_src_files(get_project_dir(), + self.options.get("filter"), file_extensions) + + def check(self, on_defect_callback=None): + self._on_defect_callback = on_defect_callback + cmd = self.configure_command() + if self.options.get("verbose"): + click.echo(" ".join(cmd)) + + proc.exec_command( + cmd, + stdout=proc.LineBufferedAsyncPipe(self.on_tool_output), + stderr=proc.LineBufferedAsyncPipe(self.on_tool_output)) + + self.clean_up() + + return self._bad_input + + +class CppcheckCheckTool(CheckToolBase): + + def __init__(self, *args, **kwargs): + self._tmp_files = [] + self.defect_fields = [ + "severity", "message", "file", "line", "column", "callstack", + "cwe", "id" + ] + super(CppcheckCheckTool, self).__init__(*args, **kwargs) + + def tool_output_filter(self, line): + if not self.options.get( + "verbose") and "--suppress=unmatchedSuppression:" in line: + return "" + + if any(msg in line for msg in ("No C or C++ source files found", + "unrecognized command line option")): + self._bad_input = True + + return line + + def parse_defect(self, raw_line): + if "<&PIO&>" not in raw_line or any(f not in raw_line + for f in self.defect_fields): + return None + + args = dict() + for field in raw_line.split("<&PIO&>"): + field = field.strip().replace('"', "") + name, value = field.split("=", 1) + args[name] = value + + args['category'] = args['severity'] + if args['severity'] == "error": + args['severity'] = DefectItem.SEVERITY_HIGH + elif args['severity'] == "warning": + args['severity'] = DefectItem.SEVERITY_MEDIUM + else: + args['severity'] = DefectItem.SEVERITY_LOW + + if self.exceeds_severity_threshold(args['severity']): + return DefectItem(**args) + + return None + + def configure_command(self): + tool_path = join(get_core_package_dir("tool-cppcheck"), "cppcheck") + + cmd = [ + tool_path, "--error-exitcode=1", + "--verbose" if self.options.get("verbose") else "--quiet" + ] + + cmd.append('--template="%s"' % "<&PIO&>".join( + ["{0}={{{0}}}".format(f) for f in self.defect_fields])) + + flags = self.get_flags("cppcheck") + if not self.is_flag_set("--platform", flags): + cmd.append("--platform=unspecified") + if not self.is_flag_set("--enable", flags): + enabled_checks = [ + "warning", "style", "performance", "portability", + "unusedFunction" + ] + cmd.append("--enable=%s" % ",".join(enabled_checks)) + + cmd.extend(["-D%s" % d for d in self.cpp_defines]) + cmd.extend(flags) + + cmd.append("--file-list=%s" % self._generate_src_file()) + cmd.append("--includes-file=%s" % self._generate_inc_file()) + + core_dir = get_project_core_dir() + cmd.append("--suppress=*:%s*" % core_dir) + cmd.append("--suppress=unmatchedSuppression:%s*" % core_dir) + + return cmd + + def _create_tmp_file(self, data): + with NamedTemporaryFile("w", delete=False) as fp: + fp.write(data) + self._tmp_files.append(fp.name) + return fp.name + + def _generate_src_file(self): + return self._create_tmp_file("\n".join(self.get_project_src_files())) + + def _generate_inc_file(self): + return self._create_tmp_file("\n".join(self.cpp_includes)) + + def clean_up(self): + for f in self._tmp_files: + if isfile(f): + remove(f) + + # delete temporary dump files generated by addons + if not self.is_flag_set("--addon", self.get_flags("cppcheck")): + return + for f in self.get_project_src_files(): + dump_file = f + ".dump" + if isfile(dump_file): + remove(dump_file) + + +class ClangtidyCheckTool(CheckToolBase): + + def tool_output_filter(self, line): + if not self.options.get( + "verbose") and "[clang-diagnostic-error]" in line: + return "" + + if "[CommonOptionsParser]" in line: + self._bad_input = True + return line + + if any(d in line for d in ("note: ", "error: ", "warning: ")): + return line + + return "" + + def parse_defect(self, raw_line): + match = re.match(r"^(.*):(\d+):(\d+):\s+([^:]+):\s(.+)\[([^]]+)\]$", + raw_line) + if not match: + return raw_line + + file, line, column, category, message, defect_id = match.groups() + + severity = DefectItem.SEVERITY_LOW + if category == "error": + severity = DefectItem.SEVERITY_HIGH + elif category == "warning": + severity = DefectItem.SEVERITY_MEDIUM + + if self.exceeds_severity_threshold(severity): + return DefectItem(severity, category, message, file, line, column, + defect_id) + + return None + + def configure_command(self): + tool_path = join(get_core_package_dir("tool-clangtidy"), "clang-tidy") + + cmd = [tool_path, "--quiet"] + flags = self.get_flags("clangtidy") + if not self.is_flag_set("--checks", flags): + cmd.append("--checks=*") + + cmd.extend(flags) + cmd.extend(self.get_project_src_files()) + cmd.append("--") + + cmd.extend(["-D%s" % d for d in self.cpp_defines]) + cmd.extend(["-I%s" % inc for inc in self.cpp_includes]) + + return cmd diff --git a/platformio/managers/core.py b/platformio/managers/core.py index b9d5a648..0d386326 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -29,7 +29,9 @@ CORE_PACKAGES = { "~2.%d%d.190418" % (sys.version_info[0], sys.version_info[1]), "tool-pioplus": "^2.5.2", "tool-unity": "~1.20403.0", - "tool-scons": "~2.20501.7" if PY2 else "~3.30101.0" + "tool-scons": "~2.20501.7" if PY2 else "~3.30101.0", + "tool-cppcheck": "~1.189.0", + "tool-clangtidy": "^1.80000.0" } PIOPLUS_AUTO_UPDATES_MAX = 100 diff --git a/platformio/project/options.py b/platformio/project/options.py index 1ff6e76c..96733c2a 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -195,6 +195,11 @@ ProjectOptions = OrderedDict([ type=click.Path( exists=True, file_okay=True, dir_okay=False)), + # Check + ConfigEnvOption(name="check_tool", multiple=True), + ConfigEnvOption(name="check_filter", multiple=True), + ConfigEnvOption(name="check_flags", multiple=True), + # Other ConfigEnvOption(name="extra_scripts", oldnames=["extra_script"], diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py new file mode 100644 index 00000000..72352429 --- /dev/null +++ b/tests/commands/test_check.py @@ -0,0 +1,294 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import json +from os.path import isfile, join + +from platformio.commands.check import cli as cmd_check + +DEFAULT_CONFIG = """ +[env:native] +platform = native +""" + +TEST_CODE = """ +#include + +void run_defects() { + /* Freeing a pointer twice */ + int* doubleFreePi = (int*)malloc(sizeof(int)); + *doubleFreePi=2; + free(doubleFreePi); + free(doubleFreePi); + + /* Reading uninitialized memory */ + int* uninitializedPi = (int*)malloc(sizeof(int)); + *uninitializedPi++; + free(uninitializedPi); + + /* Delete instead of delete [] */ + int* wrongDeletePi = new int[10]; + wrongDeletePi++; + delete wrongDeletePi; + + /* Index out of bounds */ + int arr[10]; + for(int i=0; i < 11; i++) { + arr[i] = 0; + } +} + +void unusedFuntion(){ +} + +int main() { + run_defects(); +} +""" + +EXPECTED_ERRORS = 4 +EXPECTED_WARNINGS = 1 +EXPECTED_STYLE = 1 +EXPECTED_DEFECTS = EXPECTED_ERRORS + EXPECTED_WARNINGS + EXPECTED_STYLE + + +def count_defects(output): + error, warning, style = 0, 0, 0 + for l in output.split("\n"): + if "[high:error]" in l: + error += 1 + elif "[medium:warning]" in l: + warning += 1 + elif "[low:style]" in l: + style += 1 + return error, warning, style + + +def prepare_project(tmpdir): + tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) + tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) + + +def test_check_cli_output(clirunner, tmpdir): + prepare_project(tmpdir) + + result = clirunner.invoke( + cmd_check, ["--project-dir", str(tmpdir)]) + + errors, warnings, style = count_defects(result.output) + + assert (result.exit_code != 0) + assert (errors + warnings + style == EXPECTED_DEFECTS) + + +def test_check_json_output(clirunner, tmpdir): + prepare_project(tmpdir) + + result = clirunner.invoke( + cmd_check, ["--project-dir", str(tmpdir), "--json-output"]) + output = json.loads(result.stdout.strip()) + + assert isinstance(output, list) + assert (len(output[0].get("defects", [])) == EXPECTED_DEFECTS) + + +def test_check_tool_defines_passed(clirunner, tmpdir): + prepare_project(tmpdir) + + result = clirunner.invoke( + cmd_check, ["--project-dir", str(tmpdir), "--verbose"]) + output = result.output + + assert ("PLATFORMIO=" in output) + assert ("__GNUC__" in output) + + +def test_check_severity_threshold(clirunner, tmpdir): + prepare_project(tmpdir) + + result = clirunner.invoke( + cmd_check, ["--project-dir", str(tmpdir), "--severity=high"]) + + errors, warnings, style = count_defects(result.output) + + assert (result.exit_code != 0) + assert (errors == EXPECTED_ERRORS) + assert (warnings == 0) + assert (style == 0) + + +def test_check_includes_passed(clirunner, tmpdir): + prepare_project(tmpdir) + + result = clirunner.invoke( + cmd_check, ["--project-dir", str(tmpdir), "--verbose"]) + output = result.output + + inc_count = 0 + for l in output.split("\n"): + if l.startswith("Includes:"): + inc_count = l.count("-I") + + # at least 1 include path for default mode + assert (inc_count > 1) + + +def test_check_silent_mode(clirunner, tmpdir): + prepare_project(tmpdir) + + result = clirunner.invoke( + cmd_check, ["--project-dir", str(tmpdir), "--silent"]) + + errors, warnings, style = count_defects(result.output) + + assert result.exit_code != 0 + assert errors == EXPECTED_ERRORS + assert warnings == 0 + assert style == 0 + + +def test_check_filter_sources(clirunner, tmpdir): + prepare_project(tmpdir) + + tmpdir.mkdir(join("src", "app")).join("additional.cpp").write(TEST_CODE) + + result = clirunner.invoke( + cmd_check, + ["--project-dir", + str(tmpdir), "--filter=-<*> +"]) + + errors, warnings, style = count_defects(result.output) + + assert result.exit_code != 0 + assert errors == EXPECTED_ERRORS + assert warnings == EXPECTED_WARNINGS + assert style == EXPECTED_STYLE + + +def test_check_failed_if_no_source_files(clirunner, tmpdir): + tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) + tmpdir.mkdir("src") + + result = clirunner.invoke( + cmd_check, ["--project-dir", str(tmpdir)]) + + errors, warnings, style = count_defects(result.output) + + assert result.exit_code != 0 + assert errors == 0 + assert warnings == 0 + assert style == 0 + + +def test_check_failed_if_bad_flag_passed(clirunner, tmpdir): + prepare_project(tmpdir) + + result = clirunner.invoke( + cmd_check, ["--project-dir", str(tmpdir), '"--flags=--UNKNOWN"']) + + errors, warnings, style = count_defects(result.output) + + assert result.exit_code != 0 + assert errors == 0 + assert warnings == 0 + assert style == 0 + + +def test_check_success_if_no_errors(clirunner, tmpdir): + tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) + tmpdir.mkdir("src").join("main.c").write(""" +#include + +void unused_functin(){ + int unusedVar = 0; + int* iP = &unusedVar; + *iP++; +} + +int main() { +} +""") + + result = clirunner.invoke( + cmd_check, ["--project-dir", str(tmpdir)]) + + errors, warnings, style = count_defects(result.output) + + assert "[PASSED]" in result.output + assert result.exit_code == 0 + assert errors == 0 + assert warnings == 1 + assert style == 1 + + +def test_check_individual_flags_passed(clirunner, tmpdir): + config = DEFAULT_CONFIG + "\ncheck_tool = cppcheck, clangtidy" + config += "\ncheck_flags = cppcheck: --std=c++11 \n\tclangtidy: --fix-errors" + tmpdir.join("platformio.ini").write(config) + tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) + result = clirunner.invoke( + cmd_check, ["--project-dir", str(tmpdir), "-v"]) + + clang_flags_found = cppcheck_flags_found = False + for l in result.output.split("\n"): + if "--fix" in l and "clang-tidy" in l and "--std=c++11" not in l: + clang_flags_found = True + elif "--std=c++11" in l and "cppcheck" in l and "--fix" not in l: + cppcheck_flags_found = True + + assert clang_flags_found + assert cppcheck_flags_found + + +def test_check_cppcheck_misra_addon(clirunner, tmpdir): + prepare_project(tmpdir) + + tmpdir.join("misra.json").write(""" +{ + "script": "addons/misra.py", + "args": ["--rule-texts=rules.txt"] +} +""") + + tmpdir.join("rules.txt").write(""" +Appendix A Summary of guidelines +Rule 3.1 Required +R3.1 text. +Rule 4.1 Required +R4.1 text. +Rule 10.4 Mandatory +R10.4 text. +Rule 11.5 Advisory +R11.5 text. +Rule 15.5 Advisory +R15.5 text. +Rule 15.6 Required +R15.6 text. +Rule 17.7 Required +R17.7 text. +Rule 20.1 Advisory +R20.1 text. +Rule 21.3 Required +R21.3 Found MISRA defect +Rule 21.4 +R21.4 text. +""") + + result = clirunner.invoke(cmd_check, [ + "--project-dir", str(tmpdir), "--flags=--addon=misra.json"]) + + assert result.exit_code != 0 + assert "R21.3 Found MISRA defect" in result.output + assert not isfile(join(str(tmpdir), "src", "main.cpp.dump")) From c720933d34553f71caac427b8ddc912a3825131c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 8 Sep 2019 23:33:25 +0300 Subject: [PATCH 017/221] Refactor PIO Check --- docs | 2 +- examples | 2 +- platformio/commands/check/command.py | 17 +- platformio/commands/check/defect.py | 94 +++++ platformio/commands/check/tools.py | 385 ------------------- platformio/commands/check/tools/__init__.py | 32 ++ platformio/commands/check/tools/base.py | 144 +++++++ platformio/commands/check/tools/clangtidy.py | 72 ++++ platformio/commands/check/tools/cppcheck.py | 124 ++++++ platformio/project/options.py | 3 + tests/commands/test_check.py | 98 +++-- 11 files changed, 527 insertions(+), 446 deletions(-) create mode 100644 platformio/commands/check/defect.py delete mode 100644 platformio/commands/check/tools.py create mode 100644 platformio/commands/check/tools/__init__.py create mode 100644 platformio/commands/check/tools/base.py create mode 100644 platformio/commands/check/tools/clangtidy.py create mode 100644 platformio/commands/check/tools/cppcheck.py diff --git a/docs b/docs index 29f80d45..98017a5f 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 29f80d45f2d7fe14918b507d84ec8badc54fe087 +Subproject commit 98017a5fffec8603af4358bd988dce5e98cad52a diff --git a/examples b/examples index a71564ab..6859117a 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit a71564ab46d27c387f17814056b659f826b7db24 +Subproject commit 6859117a8c0b5d293a00e29b339250d0587f31de diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py index 00b6c274..52b7c339 100644 --- a/platformio/commands/check/command.py +++ b/platformio/commands/check/command.py @@ -24,7 +24,8 @@ import click from tabulate import tabulate from platformio import exception, fs, util -from platformio.commands.check.tools import CheckToolFactory, DefectItem +from platformio.commands.check.tools import CheckToolFactory +from platformio.commands.check.defect import DefectItem from platformio.compat import dump_json_to_unicode from platformio.project.config import ProjectConfig from platformio.project.helpers import (find_project_dir_above, @@ -53,8 +54,8 @@ from platformio.project.helpers import (find_project_dir_above, @click.option("--filter", multiple=True, help="Pattern: + -") @click.option("--flags", multiple=True) @click.option("--severity", - type=click.Choice(DefectItem.SEVERITY_LABELS.values()), - default=DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_LOW]) + multiple=True, + type=click.Choice(DefectItem.SEVERITY_LABELS.values())) @click.option("-s", "--silent", is_flag=True) @click.option("-v", "--verbose", is_flag=True) @click.option("--json-output", is_flag=True) @@ -95,8 +96,11 @@ def cli(environment, project_dir, project_conf, filter, flags, severity, silent=silent, filter=filter or env_options.get("check_filter", default_filter), - flags=flags or env_options.get("check_flags", ()), - severity=severity if not silent else "high") + flags=flags or env_options.get("check_flags"), + severity=[ + DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_HIGH] + ] if silent else + (severity or env_options.get("check_severity"))) for tool in env_options.get("check_tool", ["cppcheck"]): if skipenv: @@ -124,6 +128,8 @@ def cli(environment, project_dir, project_conf, filter, flags, severity, click.echo("\n".join(repr(d) for d in result['defects'])) if not json_output and not silent: + if not result['defects']: + click.echo("No defects found") print_processing_footer(result) if json_output: @@ -216,6 +222,7 @@ def print_check_summary(results): duration = 0 print_defects_stats(results) + for result in results: duration += result.get("duration", 0) if result.get("succeeded") is False: diff --git a/platformio/commands/check/defect.py b/platformio/commands/check/defect.py new file mode 100644 index 00000000..7c2fbcff --- /dev/null +++ b/platformio/commands/check/defect.py @@ -0,0 +1,94 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os.path import relpath + +import click + +from platformio.project.helpers import get_project_dir + +# pylint: disable=too-many-instance-attributes, redefined-builtin +# pylint: disable=too-many-arguments + + +class DefectItem(object): + + SEVERITY_HIGH = 1 + SEVERITY_MEDIUM = 2 + SEVERITY_LOW = 4 + SEVERITY_LABELS = {4: "low", 2: "medium", 1: "high"} + + def __init__(self, + severity, + category, + message, + file="unknown", + line=0, + column=0, + id=None, + callstack=None, + cwe=None): + assert severity in (self.SEVERITY_HIGH, self.SEVERITY_MEDIUM, + self.SEVERITY_LOW) + self.severity = severity + self.category = category + self.message = message + self.line = line + self.column = column + self.callstack = callstack + self.cwe = cwe + self.id = id + self.file = file + if file.startswith(get_project_dir()): + self.file = relpath(file, get_project_dir()) + + def __repr__(self): + defect_color = None + if self.severity == self.SEVERITY_HIGH: + defect_color = "red" + elif self.severity == self.SEVERITY_MEDIUM: + defect_color = "yellow" + + format_str = "{file}:{line}: [{severity}:{category}] {message} {id}" + return format_str.format(severity=click.style( + self.SEVERITY_LABELS[self.severity], fg=defect_color), + category=click.style(self.category.lower(), + fg=defect_color), + file=click.style(self.file, bold=True), + message=self.message, + line=self.line, + id="%s" % "[%s]" % self.id if self.id else "") + + def __or__(self, defect): + return self.severity | defect.severity + + @staticmethod + def severity_to_int(label): + for key, value in DefectItem.SEVERITY_LABELS.items(): + if label == value: + return key + raise Exception("Unknown severity label -> %s" % label) + + def to_json(self): + return { + "severity": self.SEVERITY_LABELS[self.severity], + "category": self.category, + "message": self.message, + "file": self.file, + "line": self.line, + "column": self.column, + "callstack": self.callstack, + "id": self.id, + "cwe": self.cwe + } diff --git a/platformio/commands/check/tools.py b/platformio/commands/check/tools.py deleted file mode 100644 index c2b682db..00000000 --- a/platformio/commands/check/tools.py +++ /dev/null @@ -1,385 +0,0 @@ -# Copyright (c) 2019-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pylint: disable=too-many-arguments,too-many-instance-attributes -# pylint: disable=redefined-builtin - -import re -import sys -from os import remove -from os.path import isfile, join, relpath -from tempfile import NamedTemporaryFile - -import click - -from platformio import exception, fs, proc -from platformio.managers.core import get_core_package_dir -from platformio.project.helpers import (get_project_core_dir, get_project_dir, - load_project_ide_data) - - -class DefectItem(object): - - SEVERITY_HIGH = 1 - SEVERITY_MEDIUM = 2 - SEVERITY_LOW = 4 - SEVERITY_LABELS = {4: "low", 2: "medium", 1: "high"} - - def __init__(self, - severity, - category, - message, - file="unknown", - line=0, - column=0, - id=None, - callstack=None, - cwe=None): - assert severity in (self.SEVERITY_HIGH, self.SEVERITY_MEDIUM, - self.SEVERITY_LOW) - self.severity = severity - self.category = category - self.message = message - self.line = line - self.column = column - self.callstack = callstack - self.cwe = cwe - self.id = id - self.file = file - if file.startswith(get_project_dir()): - self.file = relpath(file, get_project_dir()) - - def __repr__(self): - defect_color = None - if self.severity == self.SEVERITY_HIGH: - defect_color = "red" - elif self.severity == self.SEVERITY_MEDIUM: - defect_color = "yellow" - - format_str = "{file}:{line}: [{severity}:{category}] {message} {id}" - return format_str.format(severity=click.style( - self.SEVERITY_LABELS[self.severity], fg=defect_color), - category=click.style(self.category.lower(), - fg=defect_color), - file=click.style(self.file, bold=True), - message=self.message, - line=self.line, - id="%s" % "[%s]" % self.id if self.id else "") - - def __or__(self, defect): - return self.severity | defect.severity - - @staticmethod - def severity_to_int(label): - for key, value in DefectItem.SEVERITY_LABELS.items(): - if label == value: - return key - raise Exception("Unknown severity label -> %s" % label) - - def to_json(self): - return { - "severity": self.SEVERITY_LABELS[self.severity], - "category": self.category, - "message": self.message, - "file": self.file, - "line": self.line, - "column": self.column, - "callstack": self.callstack, - "id": self.id, - "cwe": self.cwe - } - - -class CheckToolFactory(object): - - @staticmethod - def new(tool, project_dir, config, envname, options): - clsname = "%sCheckTool" % tool.title() - try: - obj = getattr(sys.modules[__name__], clsname)(project_dir, config, - envname, options) - except AttributeError: - raise exception.PlatformioException("Unknown check tool `%s`" % - tool) - assert isinstance(obj, CheckToolBase) - return obj - - -class CheckToolBase(object): - - def __init__(self, project_dir, config, envname, options): - self.config = config - self.envname = envname - self.options = options - self.cpp_defines = [] - self.cpp_includes = [] - self._bad_input = False - self._load_cpp_data(project_dir, envname) - - self._defects = [] - self._on_defect_callback = None - - def _load_cpp_data(self, project_dir, envname): - data = load_project_ide_data(project_dir, envname) - if not data: - return - self.cpp_includes = data.get("includes", []) - self.cpp_defines = data.get("defines", []) - self.cpp_defines.extend( - self._get_toolchain_defines(data.get("cc_path"))) - - def get_flags(self, tool): - result = [] - flags = self.options.get("flags", []) - for flag in flags: - if ":" not in flag: - result.extend([f for f in flag.split(" ") if f]) - elif flag.startswith("%s:" % tool): - result.extend( - [f for f in flag.split(":", 1)[1].split(" ") if f]) - - return result - - @staticmethod - def _get_toolchain_defines(cc_path): - defines = [] - result = proc.exec_command( - "echo | %s -dM -E -x c++ -" % cc_path, shell=True) - - for line in result['out'].split("\n"): - tokens = line.strip().split(" ", 2) - if not tokens or tokens[0] != "#define": - continue - if len(tokens) > 2: - defines.append("%s=%s" % (tokens[1], tokens[2])) - else: - defines.append(tokens[1]) - - return defines - - @staticmethod - def is_flag_set(flag, flags): - return any(flag in f for f in flags) - - def get_defects(self): - return self._defects - - def configure_command(self): - raise NotImplementedError - - def on_tool_output(self, line): - line = self.tool_output_filter(line) - if not line: - return - - defect = self.parse_defect(line) - if isinstance(defect, DefectItem): - self._defects.append(defect) - if self._on_defect_callback: - self._on_defect_callback(defect) - elif self.options.get("verbose"): - click.echo(line) - - @staticmethod - def tool_output_filter(line): - return line - - @staticmethod - def parse_defect(raw_line): - return raw_line - - def clean_up(self): - pass - - def exceeds_severity_threshold(self, severity): - return severity <= DefectItem.severity_to_int( - self.options.get("severity")) - - def get_project_src_files(self): - file_extensions = ["h", "hpp", "c", "cc", "cpp", "ino"] - return fs.match_src_files(get_project_dir(), - self.options.get("filter"), file_extensions) - - def check(self, on_defect_callback=None): - self._on_defect_callback = on_defect_callback - cmd = self.configure_command() - if self.options.get("verbose"): - click.echo(" ".join(cmd)) - - proc.exec_command( - cmd, - stdout=proc.LineBufferedAsyncPipe(self.on_tool_output), - stderr=proc.LineBufferedAsyncPipe(self.on_tool_output)) - - self.clean_up() - - return self._bad_input - - -class CppcheckCheckTool(CheckToolBase): - - def __init__(self, *args, **kwargs): - self._tmp_files = [] - self.defect_fields = [ - "severity", "message", "file", "line", "column", "callstack", - "cwe", "id" - ] - super(CppcheckCheckTool, self).__init__(*args, **kwargs) - - def tool_output_filter(self, line): - if not self.options.get( - "verbose") and "--suppress=unmatchedSuppression:" in line: - return "" - - if any(msg in line for msg in ("No C or C++ source files found", - "unrecognized command line option")): - self._bad_input = True - - return line - - def parse_defect(self, raw_line): - if "<&PIO&>" not in raw_line or any(f not in raw_line - for f in self.defect_fields): - return None - - args = dict() - for field in raw_line.split("<&PIO&>"): - field = field.strip().replace('"', "") - name, value = field.split("=", 1) - args[name] = value - - args['category'] = args['severity'] - if args['severity'] == "error": - args['severity'] = DefectItem.SEVERITY_HIGH - elif args['severity'] == "warning": - args['severity'] = DefectItem.SEVERITY_MEDIUM - else: - args['severity'] = DefectItem.SEVERITY_LOW - - if self.exceeds_severity_threshold(args['severity']): - return DefectItem(**args) - - return None - - def configure_command(self): - tool_path = join(get_core_package_dir("tool-cppcheck"), "cppcheck") - - cmd = [ - tool_path, "--error-exitcode=1", - "--verbose" if self.options.get("verbose") else "--quiet" - ] - - cmd.append('--template="%s"' % "<&PIO&>".join( - ["{0}={{{0}}}".format(f) for f in self.defect_fields])) - - flags = self.get_flags("cppcheck") - if not self.is_flag_set("--platform", flags): - cmd.append("--platform=unspecified") - if not self.is_flag_set("--enable", flags): - enabled_checks = [ - "warning", "style", "performance", "portability", - "unusedFunction" - ] - cmd.append("--enable=%s" % ",".join(enabled_checks)) - - cmd.extend(["-D%s" % d for d in self.cpp_defines]) - cmd.extend(flags) - - cmd.append("--file-list=%s" % self._generate_src_file()) - cmd.append("--includes-file=%s" % self._generate_inc_file()) - - core_dir = get_project_core_dir() - cmd.append("--suppress=*:%s*" % core_dir) - cmd.append("--suppress=unmatchedSuppression:%s*" % core_dir) - - return cmd - - def _create_tmp_file(self, data): - with NamedTemporaryFile("w", delete=False) as fp: - fp.write(data) - self._tmp_files.append(fp.name) - return fp.name - - def _generate_src_file(self): - return self._create_tmp_file("\n".join(self.get_project_src_files())) - - def _generate_inc_file(self): - return self._create_tmp_file("\n".join(self.cpp_includes)) - - def clean_up(self): - for f in self._tmp_files: - if isfile(f): - remove(f) - - # delete temporary dump files generated by addons - if not self.is_flag_set("--addon", self.get_flags("cppcheck")): - return - for f in self.get_project_src_files(): - dump_file = f + ".dump" - if isfile(dump_file): - remove(dump_file) - - -class ClangtidyCheckTool(CheckToolBase): - - def tool_output_filter(self, line): - if not self.options.get( - "verbose") and "[clang-diagnostic-error]" in line: - return "" - - if "[CommonOptionsParser]" in line: - self._bad_input = True - return line - - if any(d in line for d in ("note: ", "error: ", "warning: ")): - return line - - return "" - - def parse_defect(self, raw_line): - match = re.match(r"^(.*):(\d+):(\d+):\s+([^:]+):\s(.+)\[([^]]+)\]$", - raw_line) - if not match: - return raw_line - - file, line, column, category, message, defect_id = match.groups() - - severity = DefectItem.SEVERITY_LOW - if category == "error": - severity = DefectItem.SEVERITY_HIGH - elif category == "warning": - severity = DefectItem.SEVERITY_MEDIUM - - if self.exceeds_severity_threshold(severity): - return DefectItem(severity, category, message, file, line, column, - defect_id) - - return None - - def configure_command(self): - tool_path = join(get_core_package_dir("tool-clangtidy"), "clang-tidy") - - cmd = [tool_path, "--quiet"] - flags = self.get_flags("clangtidy") - if not self.is_flag_set("--checks", flags): - cmd.append("--checks=*") - - cmd.extend(flags) - cmd.extend(self.get_project_src_files()) - cmd.append("--") - - cmd.extend(["-D%s" % d for d in self.cpp_defines]) - cmd.extend(["-I%s" % inc for inc in self.cpp_includes]) - - return cmd diff --git a/platformio/commands/check/tools/__init__.py b/platformio/commands/check/tools/__init__.py new file mode 100644 index 00000000..9853b595 --- /dev/null +++ b/platformio/commands/check/tools/__init__.py @@ -0,0 +1,32 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio import exception +from platformio.commands.check.tools.cppcheck import CppcheckCheckTool +from platformio.commands.check.tools.clangtidy import ClangtidyCheckTool + + +class CheckToolFactory(object): + + @staticmethod + def new(tool, project_dir, config, envname, options): + cls = None + if tool == "cppcheck": + cls = CppcheckCheckTool + elif tool == "clangtidy": + cls = ClangtidyCheckTool + else: + raise exception.PlatformioException("Unknown check tool `%s`" % + tool) + return cls(project_dir, config, envname, options) diff --git a/platformio/commands/check/tools/base.py b/platformio/commands/check/tools/base.py new file mode 100644 index 00000000..e2a9ccb9 --- /dev/null +++ b/platformio/commands/check/tools/base.py @@ -0,0 +1,144 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio import fs, proc +from platformio.commands.check.defect import DefectItem +from platformio.project.helpers import (get_project_dir, load_project_ide_data) + + +class CheckToolBase(object): # pylint: disable=too-many-instance-attributes + + def __init__(self, project_dir, config, envname, options): + self.config = config + self.envname = envname + self.options = options + self.cpp_defines = [] + self.cpp_includes = [] + + self._defects = [] + self._on_defect_callback = None + self._bad_input = False + self._load_cpp_data(project_dir, envname) + + # detect all defects by default + if not self.options.get("severity"): + self.options['severity'] = [ + DefectItem.SEVERITY_LOW, DefectItem.SEVERITY_MEDIUM, + DefectItem.SEVERITY_HIGH + ] + # cast to severity by ids + self.options['severity'] = [ + s if isinstance(s, int) else DefectItem.severity_to_int(s) + for s in self.options['severity'] + ] + + def _load_cpp_data(self, project_dir, envname): + data = load_project_ide_data(project_dir, envname) + if not data: + return + self.cpp_includes = data.get("includes", []) + self.cpp_defines = data.get("defines", []) + self.cpp_defines.extend( + self._get_toolchain_defines(data.get("cc_path"))) + + def get_flags(self, tool): + result = [] + flags = self.options.get("flags") or [] + for flag in flags: + if ":" not in flag: + result.extend([f for f in flag.split(" ") if f]) + elif flag.startswith("%s:" % tool): + result.extend( + [f for f in flag.split(":", 1)[1].split(" ") if f]) + + return result + + @staticmethod + def _get_toolchain_defines(cc_path): + defines = [] + result = proc.exec_command("echo | %s -dM -E -x c++ -" % cc_path, + shell=True) + + for line in result['out'].split("\n"): + tokens = line.strip().split(" ", 2) + if not tokens or tokens[0] != "#define": + continue + if len(tokens) > 2: + defines.append("%s=%s" % (tokens[1], tokens[2])) + else: + defines.append(tokens[1]) + + return defines + + @staticmethod + def is_flag_set(flag, flags): + return any(flag in f for f in flags) + + def get_defects(self): + return self._defects + + def configure_command(self): + raise NotImplementedError + + def on_tool_output(self, line): + line = self.tool_output_filter(line) + if not line: + return + + defect = self.parse_defect(line) + + if not isinstance(defect, DefectItem): + if self.options.get("verbose"): + click.echo(line) + return + + if defect.severity not in self.options['severity']: + return + + self._defects.append(defect) + if self._on_defect_callback: + self._on_defect_callback(defect) + + @staticmethod + def tool_output_filter(line): + return line + + @staticmethod + def parse_defect(raw_line): + return raw_line + + def clean_up(self): + pass + + def get_project_src_files(self): + file_extensions = ["h", "hpp", "c", "cc", "cpp", "ino"] + return fs.match_src_files(get_project_dir(), + self.options.get("filter"), file_extensions) + + def check(self, on_defect_callback=None): + self._on_defect_callback = on_defect_callback + cmd = self.configure_command() + if self.options.get("verbose"): + click.echo(" ".join(cmd)) + + proc.exec_command( + cmd, + stdout=proc.LineBufferedAsyncPipe(self.on_tool_output), + stderr=proc.LineBufferedAsyncPipe(self.on_tool_output)) + + self.clean_up() + + return self._bad_input diff --git a/platformio/commands/check/tools/clangtidy.py b/platformio/commands/check/tools/clangtidy.py new file mode 100644 index 00000000..83eec7a2 --- /dev/null +++ b/platformio/commands/check/tools/clangtidy.py @@ -0,0 +1,72 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from os.path import join + + +from platformio.commands.check.tools.base import CheckToolBase +from platformio.commands.check.defect import DefectItem +from platformio.managers.core import get_core_package_dir + + +class ClangtidyCheckTool(CheckToolBase): + + def tool_output_filter(self, line): + if not self.options.get( + "verbose") and "[clang-diagnostic-error]" in line: + return "" + + if "[CommonOptionsParser]" in line: + self._bad_input = True + return line + + if any(d in line for d in ("note: ", "error: ", "warning: ")): + return line + + return "" + + def parse_defect(self, raw_line): + match = re.match(r"^(.*):(\d+):(\d+):\s+([^:]+):\s(.+)\[([^]]+)\]$", + raw_line) + if not match: + return raw_line + + file, line, column, category, message, defect_id = match.groups() + + severity = DefectItem.SEVERITY_LOW + if category == "error": + severity = DefectItem.SEVERITY_HIGH + elif category == "warning": + severity = DefectItem.SEVERITY_MEDIUM + + return DefectItem(severity, category, message, file, line, column, + defect_id) + + def configure_command(self): + tool_path = join(get_core_package_dir("tool-clangtidy"), "clang-tidy") + + cmd = [tool_path, "--quiet"] + flags = self.get_flags("clangtidy") + if not self.is_flag_set("--checks", flags): + cmd.append("--checks=*") + + cmd.extend(flags) + cmd.extend(self.get_project_src_files()) + cmd.append("--") + + cmd.extend(["-D%s" % d for d in self.cpp_defines]) + cmd.extend(["-I%s" % inc for inc in self.cpp_includes]) + + return cmd diff --git a/platformio/commands/check/tools/cppcheck.py b/platformio/commands/check/tools/cppcheck.py new file mode 100644 index 00000000..b7e3a3e0 --- /dev/null +++ b/platformio/commands/check/tools/cppcheck.py @@ -0,0 +1,124 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os import remove +from os.path import isfile, join +from tempfile import NamedTemporaryFile + + +from platformio.commands.check.tools.base import CheckToolBase +from platformio.commands.check.defect import DefectItem +from platformio.managers.core import get_core_package_dir +from platformio.project.helpers import get_project_core_dir + + +class CppcheckCheckTool(CheckToolBase): + + def __init__(self, *args, **kwargs): + self._tmp_files = [] + self.defect_fields = [ + "severity", "message", "file", "line", "column", "callstack", + "cwe", "id" + ] + super(CppcheckCheckTool, self).__init__(*args, **kwargs) + + def tool_output_filter(self, line): + if not self.options.get( + "verbose") and "--suppress=unmatchedSuppression:" in line: + return "" + + if any(msg in line for msg in ("No C or C++ source files found", + "unrecognized command line option")): + self._bad_input = True + + return line + + def parse_defect(self, raw_line): + if "<&PIO&>" not in raw_line or any(f not in raw_line + for f in self.defect_fields): + return None + + args = dict() + for field in raw_line.split("<&PIO&>"): + field = field.strip().replace('"', "") + name, value = field.split("=", 1) + args[name] = value + + args['category'] = args['severity'] + if args['severity'] == "error": + args['severity'] = DefectItem.SEVERITY_HIGH + elif args['severity'] == "warning": + args['severity'] = DefectItem.SEVERITY_MEDIUM + else: + args['severity'] = DefectItem.SEVERITY_LOW + + return DefectItem(**args) + + def configure_command(self): + tool_path = join(get_core_package_dir("tool-cppcheck"), "cppcheck") + + cmd = [ + tool_path, "--error-exitcode=1", + "--verbose" if self.options.get("verbose") else "--quiet" + ] + + cmd.append('--template="%s"' % "<&PIO&>".join( + ["{0}={{{0}}}".format(f) for f in self.defect_fields])) + + flags = self.get_flags("cppcheck") + if not self.is_flag_set("--platform", flags): + cmd.append("--platform=unspecified") + if not self.is_flag_set("--enable", flags): + enabled_checks = [ + "warning", "style", "performance", "portability", + "unusedFunction" + ] + cmd.append("--enable=%s" % ",".join(enabled_checks)) + + cmd.extend(["-D%s" % d for d in self.cpp_defines]) + cmd.extend(flags) + + cmd.append("--file-list=%s" % self._generate_src_file()) + cmd.append("--includes-file=%s" % self._generate_inc_file()) + + core_dir = get_project_core_dir() + cmd.append("--suppress=*:%s*" % core_dir) + cmd.append("--suppress=unmatchedSuppression:%s*" % core_dir) + + return cmd + + def _create_tmp_file(self, data): + with NamedTemporaryFile("w", delete=False) as fp: + fp.write(data) + self._tmp_files.append(fp.name) + return fp.name + + def _generate_src_file(self): + return self._create_tmp_file("\n".join(self.get_project_src_files())) + + def _generate_inc_file(self): + return self._create_tmp_file("\n".join(self.cpp_includes)) + + def clean_up(self): + for f in self._tmp_files: + if isfile(f): + remove(f) + + # delete temporary dump files generated by addons + if not self.is_flag_set("--addon", self.get_flags("cppcheck")): + return + for f in self.get_project_src_files(): + dump_file = f + ".dump" + if isfile(dump_file): + remove(dump_file) diff --git a/platformio/project/options.py b/platformio/project/options.py index 96733c2a..59a11081 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -199,6 +199,9 @@ ProjectOptions = OrderedDict([ ConfigEnvOption(name="check_tool", multiple=True), ConfigEnvOption(name="check_filter", multiple=True), ConfigEnvOption(name="check_flags", multiple=True), + ConfigEnvOption(name="check_severity", + multiple=True, + type=click.Choice(["low", "medium", "high"])), # Other ConfigEnvOption(name="extra_scripts", diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index 72352429..29a377ab 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. - import json from os.path import isfile, join +import pytest + from platformio.commands.check import cli as cmd_check DEFAULT_CONFIG = """ @@ -64,6 +65,14 @@ EXPECTED_STYLE = 1 EXPECTED_DEFECTS = EXPECTED_ERRORS + EXPECTED_WARNINGS + EXPECTED_STYLE +@pytest.fixture(scope="module") +def check_dir(tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("project") + tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) + tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) + return tmpdir + + def count_defects(output): error, warning, style = 0, 0, 0 for l in output.split("\n"): @@ -76,16 +85,8 @@ def count_defects(output): return error, warning, style -def prepare_project(tmpdir): - tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) - tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) - - -def test_check_cli_output(clirunner, tmpdir): - prepare_project(tmpdir) - - result = clirunner.invoke( - cmd_check, ["--project-dir", str(tmpdir)]) +def test_check_cli_output(clirunner, check_dir): + result = clirunner.invoke(cmd_check, ["--project-dir", str(check_dir)]) errors, warnings, style = count_defects(result.output) @@ -93,33 +94,30 @@ def test_check_cli_output(clirunner, tmpdir): assert (errors + warnings + style == EXPECTED_DEFECTS) -def test_check_json_output(clirunner, tmpdir): - prepare_project(tmpdir) - +def test_check_json_output(clirunner, check_dir): result = clirunner.invoke( - cmd_check, ["--project-dir", str(tmpdir), "--json-output"]) + cmd_check, + ["--project-dir", str(check_dir), "--json-output"]) output = json.loads(result.stdout.strip()) assert isinstance(output, list) assert (len(output[0].get("defects", [])) == EXPECTED_DEFECTS) -def test_check_tool_defines_passed(clirunner, tmpdir): - prepare_project(tmpdir) - +def test_check_tool_defines_passed(clirunner, check_dir): result = clirunner.invoke( - cmd_check, ["--project-dir", str(tmpdir), "--verbose"]) + cmd_check, + ["--project-dir", str(check_dir), "--verbose"]) output = result.output assert ("PLATFORMIO=" in output) assert ("__GNUC__" in output) -def test_check_severity_threshold(clirunner, tmpdir): - prepare_project(tmpdir) - +def test_check_severity_threshold(clirunner, check_dir): result = clirunner.invoke( - cmd_check, ["--project-dir", str(tmpdir), "--severity=high"]) + cmd_check, + ["--project-dir", str(check_dir), "--severity=high"]) errors, warnings, style = count_defects(result.output) @@ -129,11 +127,10 @@ def test_check_severity_threshold(clirunner, tmpdir): assert (style == 0) -def test_check_includes_passed(clirunner, tmpdir): - prepare_project(tmpdir) - +def test_check_includes_passed(clirunner, check_dir): result = clirunner.invoke( - cmd_check, ["--project-dir", str(tmpdir), "--verbose"]) + cmd_check, + ["--project-dir", str(check_dir), "--verbose"]) output = result.output inc_count = 0 @@ -145,11 +142,10 @@ def test_check_includes_passed(clirunner, tmpdir): assert (inc_count > 1) -def test_check_silent_mode(clirunner, tmpdir): - prepare_project(tmpdir) - +def test_check_silent_mode(clirunner, check_dir): result = clirunner.invoke( - cmd_check, ["--project-dir", str(tmpdir), "--silent"]) + cmd_check, + ["--project-dir", str(check_dir), "--silent"]) errors, warnings, style = count_defects(result.output) @@ -159,15 +155,13 @@ def test_check_silent_mode(clirunner, tmpdir): assert style == 0 -def test_check_filter_sources(clirunner, tmpdir): - prepare_project(tmpdir) - - tmpdir.mkdir(join("src", "app")).join("additional.cpp").write(TEST_CODE) +def test_check_filter_sources(clirunner, check_dir): + check_dir.mkdir(join("src", "app")).join("additional.cpp").write(TEST_CODE) result = clirunner.invoke( cmd_check, ["--project-dir", - str(tmpdir), "--filter=-<*> +"]) + str(check_dir), "--filter=-<*> +"]) errors, warnings, style = count_defects(result.output) @@ -181,8 +175,7 @@ def test_check_failed_if_no_source_files(clirunner, tmpdir): tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) tmpdir.mkdir("src") - result = clirunner.invoke( - cmd_check, ["--project-dir", str(tmpdir)]) + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) errors, warnings, style = count_defects(result.output) @@ -192,11 +185,10 @@ def test_check_failed_if_no_source_files(clirunner, tmpdir): assert style == 0 -def test_check_failed_if_bad_flag_passed(clirunner, tmpdir): - prepare_project(tmpdir) - +def test_check_failed_if_bad_flag_passed(clirunner, check_dir): result = clirunner.invoke( - cmd_check, ["--project-dir", str(tmpdir), '"--flags=--UNKNOWN"']) + cmd_check, ["--project-dir", + str(check_dir), '"--flags=--UNKNOWN"']) errors, warnings, style = count_defects(result.output) @@ -221,8 +213,7 @@ int main() { } """) - result = clirunner.invoke( - cmd_check, ["--project-dir", str(tmpdir)]) + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) errors, warnings, style = count_defects(result.output) @@ -238,8 +229,7 @@ def test_check_individual_flags_passed(clirunner, tmpdir): config += "\ncheck_flags = cppcheck: --std=c++11 \n\tclangtidy: --fix-errors" tmpdir.join("platformio.ini").write(config) tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) - result = clirunner.invoke( - cmd_check, ["--project-dir", str(tmpdir), "-v"]) + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) clang_flags_found = cppcheck_flags_found = False for l in result.output.split("\n"): @@ -252,17 +242,15 @@ def test_check_individual_flags_passed(clirunner, tmpdir): assert cppcheck_flags_found -def test_check_cppcheck_misra_addon(clirunner, tmpdir): - prepare_project(tmpdir) - - tmpdir.join("misra.json").write(""" +def test_check_cppcheck_misra_addon(clirunner, check_dir): + check_dir.join("misra.json").write(""" { "script": "addons/misra.py", "args": ["--rule-texts=rules.txt"] } """) - tmpdir.join("rules.txt").write(""" + check_dir.join("rules.txt").write(""" Appendix A Summary of guidelines Rule 3.1 Required R3.1 text. @@ -286,9 +274,11 @@ Rule 21.4 R21.4 text. """) - result = clirunner.invoke(cmd_check, [ - "--project-dir", str(tmpdir), "--flags=--addon=misra.json"]) + result = clirunner.invoke( + cmd_check, + ["--project-dir", + str(check_dir), "--flags=--addon=misra.json"]) assert result.exit_code != 0 assert "R21.3 Found MISRA defect" in result.output - assert not isfile(join(str(tmpdir), "src", "main.cpp.dump")) + assert not isfile(join(str(check_dir), "src", "main.cpp.dump")) From 96567dea4d2444e3d243ae872a251935e0111b60 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 8 Sep 2019 23:44:18 +0300 Subject: [PATCH 018/221] PyLint fix --- platformio/commands/check/tools/clangtidy.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/platformio/commands/check/tools/clangtidy.py b/platformio/commands/check/tools/clangtidy.py index 83eec7a2..94baacc3 100644 --- a/platformio/commands/check/tools/clangtidy.py +++ b/platformio/commands/check/tools/clangtidy.py @@ -15,7 +15,6 @@ import re from os.path import join - from platformio.commands.check.tools.base import CheckToolBase from platformio.commands.check.defect import DefectItem from platformio.managers.core import get_core_package_dir @@ -43,7 +42,7 @@ class ClangtidyCheckTool(CheckToolBase): if not match: return raw_line - file, line, column, category, message, defect_id = match.groups() + file_, line, column, category, message, defect_id = match.groups() severity = DefectItem.SEVERITY_LOW if category == "error": @@ -51,7 +50,7 @@ class ClangtidyCheckTool(CheckToolBase): elif category == "warning": severity = DefectItem.SEVERITY_MEDIUM - return DefectItem(severity, category, message, file, line, column, + return DefectItem(severity, category, message, file_, line, column, defect_id) def configure_command(self): From 1e26feb56659035ec681bc1127ee707043329d11 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 9 Sep 2019 23:34:44 +0300 Subject: [PATCH 019/221] Bump version to 4.1.0b1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 32b79b8b..c2ca4c96 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, "0a1") +VERSION = (4, 1, "0b1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From e08dc5f0d725f3881b20ed40d43fc6650cdba5e8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 10 Sep 2019 17:48:47 +0300 Subject: [PATCH 020/221] Docs: Sync Microchip PIC32 dev/platform --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 98017a5f..ae6b8559 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 98017a5fffec8603af4358bd988dce5e98cad52a +Subproject commit ae6b855904711c8f7988ed9808452f05d55d725e From 43ae62afd83268886796943d5e2810d39ef6ffb6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 13 Sep 2019 16:01:42 +0300 Subject: [PATCH 021/221] Sync Aceinna and GD32V dev/platforms. --- docs | 2 +- examples | 2 +- scripts/docspregen.py | 128 +++++++++++++++++++++++------------------- 3 files changed, 72 insertions(+), 60 deletions(-) diff --git a/docs b/docs index ae6b8559..5a1fd478 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit ae6b855904711c8f7988ed9808452f05d55d725e +Subproject commit 5a1fd4781f0a91f07b817307ebf5e5a0b3011c09 diff --git a/examples b/examples index 6859117a..7e80325b 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 6859117a8c0b5d293a00e29b339250d0587f31de +Subproject commit 7e80325b974382034f1247372974bdfd2a6194ae diff --git a/scripts/docspregen.py b/scripts/docspregen.py index dc18c5a7..2ed1b2d5 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -13,16 +13,22 @@ # limitations under the License. import os -import urlparse from os.path import dirname, isdir, isfile, join, realpath from sys import exit as sys_exit from sys import path path.append("..") +import click + from platformio import fs, util from platformio.managers.platform import PlatformFactory, PlatformManager +try: + from urlparse import ParseResult, urlparse, urlunparse +except ImportError: + from urllib.parse import ParseResult, urlparse, urlunparse + RST_COPYRIGHT = """.. Copyright (c) 2014-present PlatformIO Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -48,14 +54,14 @@ def is_compat_platform_and_framework(platform, framework): def campaign_url(url, source="platformio", medium="docs"): - data = urlparse.urlparse(url) + data = urlparse(url) query = data.query if query: query += "&" query += "utm_source=%s&utm_medium=%s" % (source, medium) - return urlparse.urlunparse( - urlparse.ParseResult(data.scheme, data.netloc, data.path, data.params, - query, data.fragment)) + return urlunparse( + ParseResult(data.scheme, data.netloc, data.path, data.params, query, + data.fragment)) def generate_boards_table(boards, skip_columns=None): @@ -64,7 +70,7 @@ def generate_boards_table(boards, skip_columns=None): ("Platform", ":ref:`platform_{platform}`"), ("Debug", "{debug}"), ("MCU", "{mcu}"), - ("Frequency", "{f_cpu:d}MHz"), + ("Frequency", "{f_cpu}MHz"), ("Flash", "{rom}"), ("RAM", "{ram}"), ] @@ -90,15 +96,14 @@ def generate_boards_table(boards, skip_columns=None): elif data['debug']: debug = "External" - variables = dict( - id=data['id'], - name=data['name'], - platform=data['platform'], - debug=debug, - mcu=data['mcu'].upper(), - f_cpu=int(data['fcpu']) / 1000000, - ram=fs.format_filesize(data['ram']), - rom=fs.format_filesize(data['rom'])) + variables = dict(id=data['id'], + name=data['name'], + platform=data['platform'], + debug=debug, + mcu=data['mcu'].upper(), + f_cpu=int(data['fcpu'] / 1000000.0), + ram=fs.format_filesize(data['ram']), + rom=fs.format_filesize(data['rom'])) for (name, template) in columns: if skip_columns and name in skip_columns: @@ -132,8 +137,9 @@ Frameworks lines.append(""" * - :ref:`framework_{name}` - {description}""".format(**framework)) - assert known >= set(frameworks), "Unknown frameworks %s " % ( - set(frameworks) - known) + if set(frameworks) - known: + click.secho("Unknown frameworks %s " % ( + set(frameworks) - known), fg="red") return lines @@ -208,8 +214,8 @@ Boards listed below have on-board debug probe and **ARE READY** for debugging! You do not need to use/buy external debug probe. """) lines.extend( - generate_boards_table( - onboard_debug, skip_columns=skip_board_columns)) + generate_boards_table(onboard_debug, + skip_columns=skip_board_columns)) if external_debug: lines.append(""" External Debug Tools @@ -220,8 +226,8 @@ external debug probe. They **ARE NOT READY** for debugging. Please click on board name for the further details. """) lines.extend( - generate_boards_table( - external_debug, skip_columns=skip_board_columns)) + generate_boards_table(external_debug, + skip_columns=skip_board_columns)) return lines @@ -239,13 +245,18 @@ Packages * - Name - Description""") for name in sorted(packagenames): - assert name in API_PACKAGES, name - lines.append(""" + if name not in API_PACKAGES: + click.secho("Unknown package `%s`" % name, fg="red") + lines.append(""" + * - {name} + - + """.format(name=name)) + else: + lines.append(""" * - `{name} <{url}>`__ - - {description}""".format( - name=name, - url=campaign_url(API_PACKAGES[name]['url']), - description=API_PACKAGES[name]['description'])) + - {description}""".format(name=name, + url=campaign_url(API_PACKAGES[name]['url']), + description=API_PACKAGES[name]['description'])) if is_embedded: lines.append(""" @@ -344,8 +355,9 @@ Examples are listed from `%s development platform repository <%s>`_: generate_debug_contents( compatible_boards, skip_board_columns=["Platform"], - extra_rst="%s_debug.rst" % name if isfile( - join(rst_dir, "%s_debug.rst" % name)) else None)) + extra_rst="%s_debug.rst" % + name if isfile(join(rst_dir, "%s_debug.rst" % + name)) else None)) # # Development version of dev/platform @@ -483,8 +495,9 @@ For more detailed information please visit `vendor site <%s>`_. lines.extend( generate_debug_contents( compatible_boards, - extra_rst="%s_debug.rst" % type_ if isfile( - join(rst_dir, "%s_debug.rst" % type_)) else None)) + extra_rst="%s_debug.rst" % + type_ if isfile(join(rst_dir, "%s_debug.rst" % + type_)) else None)) if compatible_platforms: # examples @@ -494,11 +507,10 @@ Examples """) for manifest in compatible_platforms: p = PlatformFactory.newPlatform(manifest['name']) - lines.append( - "* `%s for %s <%s>`_" % - (data['title'], manifest['title'], - campaign_url( - "%s/tree/master/examples" % p.repository_url[:-4]))) + lines.append("* `%s for %s <%s>`_" % + (data['title'], manifest['title'], + campaign_url("%s/tree/master/examples" % + p.repository_url[:-4]))) # Platforms lines.extend( @@ -568,7 +580,7 @@ popular embedded boards and IDE. else: platforms[platform] = [data] - for platform, boards in sorted(platforms.iteritems()): + for platform, boards in sorted(platforms.items()): p = PlatformFactory.newPlatform(platform) lines.append(p.title) lines.append("-" * len(p.title)) @@ -605,21 +617,20 @@ def update_embedded_board(rst_path, board): board_manifest_url = board_manifest_url[:-4] board_manifest_url += "/blob/master/boards/%s.json" % board['id'] - variables = dict( - id=board['id'], - name=board['name'], - platform=board['platform'], - platform_description=platform.description, - url=campaign_url(board['url']), - mcu=board_config.get("build", {}).get("mcu", ""), - mcu_upper=board['mcu'].upper(), - f_cpu=board['fcpu'], - f_cpu_mhz=int(board['fcpu']) / 1000000, - ram=fs.format_filesize(board['ram']), - rom=fs.format_filesize(board['rom']), - vendor=board['vendor'], - board_manifest_url=board_manifest_url, - upload_protocol=board_config.get("upload.protocol", "")) + variables = dict(id=board['id'], + name=board['name'], + platform=board['platform'], + platform_description=platform.description, + url=campaign_url(board['url']), + mcu=board_config.get("build", {}).get("mcu", ""), + mcu_upper=board['mcu'].upper(), + f_cpu=board['fcpu'], + f_cpu_mhz=int(int(board['fcpu']) / 1000000), + ram=fs.format_filesize(board['ram']), + rom=fs.format_filesize(board['rom']), + vendor=board['vendor'], + board_manifest_url=board_manifest_url, + upload_protocol=board_config.get("upload.protocol", "")) lines = [RST_COPYRIGHT] lines.append(".. _board_{platform}_{id}:".format(**variables)) @@ -639,7 +650,7 @@ Platform :ref:`platform_{platform}`: {platform_description} * - **Microcontroller** - {mcu_upper} * - **Frequency** - - {f_cpu_mhz}MHz + - {f_cpu_mhz:d}MHz * - **Flash** - {rom} * - **RAM** @@ -805,15 +816,14 @@ Boards .. note:: For more detailed ``board`` information please scroll tables below by horizontal. """) - for vendor, boards in sorted(vendors.iteritems()): + for vendor, boards in sorted(vendors.items()): lines.append(str(vendor)) lines.append("~" * len(vendor)) lines.extend(generate_boards_table(boards)) # save - with open( - join(fs.get_source_dir(), "..", "docs", "plus", "debugging.rst"), - "r+") as fp: + with open(join(fs.get_source_dir(), "..", "docs", "plus", "debugging.rst"), + "r+") as fp: content = fp.read() fp.seek(0) fp.truncate() @@ -823,7 +833,9 @@ Boards # Debug tools for tool, platforms in tool_to_platforms.items(): tool_path = join(DOCS_ROOT_DIR, "plus", "debug-tools", "%s.rst" % tool) - assert isfile(tool_path), tool + if not isfile(tool_path): + click.secho("Unknown debug tool `%s`" % tool, fg="red") + continue platforms = sorted(set(platforms)) lines = [".. begin_platforms"] From 08a94b6f7c973d346de659efc8f89f9b17f71f7d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 16 Sep 2019 18:58:29 +0300 Subject: [PATCH 022/221] New article "Arduino In-circuit Debugging with PlatformIO" --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 5a1fd478..38cb2742 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 5a1fd4781f0a91f07b817307ebf5e5a0b3011c09 +Subproject commit 38cb27428fd08da289c474b0cd0e7ada699e9203 From 123963f760cdb69915e380aba11d86209fad31c3 Mon Sep 17 00:00:00 2001 From: Thomas Bleijendaal Date: Mon, 16 Sep 2019 20:02:07 +0200 Subject: [PATCH 023/221] UTF8 decoding should ignore invalid characters (#3026) Some boards, like ESP32 based boards, give some unintelligible data when connecting to them via Serial. This is sometimes data that is send with the wrong baud rate (hard baked into the boot loader), or something else. It's hard to prevent this from happening. When a build is uploaded to the ESP board for unit testing, the decoding of the incoming stream should not fail the test due to some garbled content. Since the read data is validated on line 95, any garbage is automatically ignored and only outputted to the console. --- platformio/commands/test/embedded.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/commands/test/embedded.py b/platformio/commands/test/embedded.py index 6c1c57c4..6870dcd5 100644 --- a/platformio/commands/test/embedded.py +++ b/platformio/commands/test/embedded.py @@ -90,7 +90,7 @@ class EmbeddedTestProcessor(TestProcessorBase): if not line: continue if isinstance(line, bytes): - line = line.decode("utf8") + line = line.decode("utf8", "ignore") self.on_run_out(line) if all([l in line for l in ("Tests", "Failures", "Ignored")]): break From 6531dcbc78e0cd287c871f2fd49ba9c74dd5f7c0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 16 Sep 2019 21:38:47 +0300 Subject: [PATCH 024/221] Allow to skip checking of unpacked data --- platformio/unpacker.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/platformio/unpacker.py b/platformio/unpacker.py index 271b4911..4d018072 100644 --- a/platformio/unpacker.py +++ b/platformio/unpacker.py @@ -110,7 +110,7 @@ class FileUnpacker(object): if self._unpacker: self._unpacker.close() - def unpack(self, dest_dir=".", with_progress=True): + def unpack(self, dest_dir=".", with_progress=True, check_unpacked=True): assert self._unpacker if not with_progress: click.echo("Unpacking...") @@ -122,6 +122,9 @@ class FileUnpacker(object): for item in pb: self._unpacker.extract_item(item, dest_dir) + if not check_unpacked: + return True + # check on disk for item in self._unpacker.get_items(): filename = self._unpacker.get_item_filename(item) From cd8dc24454176d05ab1360cb51a32b40f1fa7e7f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 18 Sep 2019 18:47:55 +0300 Subject: [PATCH 025/221] Docs: Sync Espressif32 dev/platform --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 38cb2742..10d5ce9d 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 38cb27428fd08da289c474b0cd0e7ada699e9203 +Subproject commit 10d5ce9d8e35d5556aa2189a87bb3958561c2ad7 From 61b6eea52cac98426703fd2c176f03445199e4ae Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 23 Sep 2019 20:51:02 +0300 Subject: [PATCH 026/221] New "--no-ansi" flag for PIO Core --- HISTORY.rst | 1 + docs | 2 +- platformio/__main__.py | 40 +++++++++++++++++++++++----------------- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index f8283452..b4fd0ac6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,6 +10,7 @@ PlatformIO Core 4.0 ~~~~~~~~~~~~~~~~~~ * Extend project environment configuration in "platformio.ini" with other sections using a new `extends `__ option (`issue #2953 `_) +* New ``--no-ansi`` flag for `PIO Core `__ to disable ANSI control characters * Generate ``.ccls`` LSP file for `Emacs `__ cross references, hierarchies, completion and semantic highlighting * Fixed an issue with project generator for `CLion IDE `__ when 2 environments were used (`issue #2824 `_) * Fixed default PIO Unified Debugger configuration for `J-Link probe `__ diff --git a/docs b/docs index 10d5ce9d..52c3dc5a 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 10d5ce9d8e35d5556aa2189a87bb3958561c2ad7 +Subproject commit 52c3dc5a884d81f85d79b912739f4300d1c7d80d diff --git a/platformio/__main__.py b/platformio/__main__.py index d88f69a8..ced8e50a 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -26,19 +26,35 @@ from platformio.compat import CYGWIN @click.command(cls=PlatformioCLI, context_settings=dict(help_option_names=["-h", "--help"])) @click.version_option(__version__, prog_name="PlatformIO") -@click.option("--force", - "-f", +@click.option("--force", "-f", is_flag=True, help="DEPRECATE") +@click.option("--caller", "-c", help="Caller ID (service)") +@click.option("--no-ansi", is_flag=True, - help="Force to accept any confirmation prompts.") -@click.option("--caller", "-c", help="Caller ID (service).") + help="Do not print ANSI control characters") @click.pass_context -def cli(ctx, force, caller): +def cli(ctx, force, caller, no_ansi): + try: + if (no_ansi or str( + os.getenv( + "PLATFORMIO_NO_ANSI", + os.getenv("PLATFORMIO_DISABLE_COLOR"))).lower() == "true"): + # pylint: disable=protected-access + click._compat.isatty = lambda stream: False + elif str( + os.getenv( + "PLATFORMIO_FORCE_ANSI", + os.getenv("PLATFORMIO_FORCE_COLOR"))).lower() == "true": + # pylint: disable=protected-access + click._compat.isatty = lambda stream: True + except: # pylint: disable=bare-except + pass + maintenance.on_platformio_start(ctx, force, caller) @cli.resultcallback() @click.pass_context -def process_result(ctx, result, force, caller): # pylint: disable=W0613 +def process_result(ctx, result, *_, **__): maintenance.on_platformio_end(ctx, result) @@ -55,16 +71,6 @@ def configure(): except (AttributeError, ImportError): pass - try: - if str(os.getenv("PLATFORMIO_DISABLE_COLOR", "")).lower() == "true": - # pylint: disable=protected-access - click._compat.isatty = lambda stream: False - elif str(os.getenv("PLATFORMIO_FORCE_COLOR", "")).lower() == "true": - # pylint: disable=protected-access - click._compat.isatty = lambda stream: True - except: # pylint: disable=bare-except - pass - # Handle IOError issue with VSCode's Terminal (Windows) click_echo_origin = [click.echo, click.secho] @@ -87,7 +93,7 @@ def main(argv=None): sys.argv = argv try: configure() - cli(None, None, None) + cli() # pylint: disable=no-value-for-parameter except SystemExit: pass except Exception as e: # pylint: disable=broad-except From 5e144a2c98c6c63ec4ec4fc25b8dc054af355fa8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 23 Sep 2019 21:57:31 +0300 Subject: [PATCH 027/221] Add PIO Check to changelog --- HISTORY.rst | 11 ++++++++++- platformio/commands/check/command.py | 2 +- platformio/commands/check/tools/__init__.py | 2 +- platformio/commands/check/tools/base.py | 2 +- platformio/commands/check/tools/clangtidy.py | 2 +- platformio/commands/check/tools/cppcheck.py | 3 +-- 6 files changed, 15 insertions(+), 7 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index b4fd0ac6..d381bdd3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,9 +9,18 @@ PlatformIO Core 4.0 4.1.0 (2019-??-??) ~~~~~~~~~~~~~~~~~~ +* `PIO Check `__ – automated code analysis without hassle: + + - Potential NULL pointer dereferences + - Possible indexing beyond array bounds + - Suspicious assignments + - Reads of potentially uninitialized objects + - Unused variables or functions + - Out of scope memory usage. + * Extend project environment configuration in "platformio.ini" with other sections using a new `extends `__ option (`issue #2953 `_) -* New ``--no-ansi`` flag for `PIO Core `__ to disable ANSI control characters * Generate ``.ccls`` LSP file for `Emacs `__ cross references, hierarchies, completion and semantic highlighting +* Added ``--no-ansi`` flag for `PIO Core `__ to disable ANSI control characters * Fixed an issue with project generator for `CLion IDE `__ when 2 environments were used (`issue #2824 `_) * Fixed default PIO Unified Debugger configuration for `J-Link probe `__ diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py index 52b7c339..28c4431d 100644 --- a/platformio/commands/check/command.py +++ b/platformio/commands/check/command.py @@ -24,8 +24,8 @@ import click from tabulate import tabulate from platformio import exception, fs, util -from platformio.commands.check.tools import CheckToolFactory from platformio.commands.check.defect import DefectItem +from platformio.commands.check.tools import CheckToolFactory from platformio.compat import dump_json_to_unicode from platformio.project.config import ProjectConfig from platformio.project.helpers import (find_project_dir_above, diff --git a/platformio/commands/check/tools/__init__.py b/platformio/commands/check/tools/__init__.py index 9853b595..e76cf48b 100644 --- a/platformio/commands/check/tools/__init__.py +++ b/platformio/commands/check/tools/__init__.py @@ -13,8 +13,8 @@ # limitations under the License. from platformio import exception -from platformio.commands.check.tools.cppcheck import CppcheckCheckTool from platformio.commands.check.tools.clangtidy import ClangtidyCheckTool +from platformio.commands.check.tools.cppcheck import CppcheckCheckTool class CheckToolFactory(object): diff --git a/platformio/commands/check/tools/base.py b/platformio/commands/check/tools/base.py index e2a9ccb9..7d6224b2 100644 --- a/platformio/commands/check/tools/base.py +++ b/platformio/commands/check/tools/base.py @@ -16,7 +16,7 @@ import click from platformio import fs, proc from platformio.commands.check.defect import DefectItem -from platformio.project.helpers import (get_project_dir, load_project_ide_data) +from platformio.project.helpers import get_project_dir, load_project_ide_data class CheckToolBase(object): # pylint: disable=too-many-instance-attributes diff --git a/platformio/commands/check/tools/clangtidy.py b/platformio/commands/check/tools/clangtidy.py index 94baacc3..843d02dc 100644 --- a/platformio/commands/check/tools/clangtidy.py +++ b/platformio/commands/check/tools/clangtidy.py @@ -15,8 +15,8 @@ import re from os.path import join -from platformio.commands.check.tools.base import CheckToolBase from platformio.commands.check.defect import DefectItem +from platformio.commands.check.tools.base import CheckToolBase from platformio.managers.core import get_core_package_dir diff --git a/platformio/commands/check/tools/cppcheck.py b/platformio/commands/check/tools/cppcheck.py index b7e3a3e0..e74031bd 100644 --- a/platformio/commands/check/tools/cppcheck.py +++ b/platformio/commands/check/tools/cppcheck.py @@ -16,9 +16,8 @@ from os import remove from os.path import isfile, join from tempfile import NamedTemporaryFile - -from platformio.commands.check.tools.base import CheckToolBase from platformio.commands.check.defect import DefectItem +from platformio.commands.check.tools.base import CheckToolBase from platformio.managers.core import get_core_package_dir from platformio.project.helpers import get_project_core_dir From 7c41c7c2f321c3d6a8e4beedba9ee63b1d9f8043 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 23 Sep 2019 23:13:48 +0300 Subject: [PATCH 028/221] Introduce Black to automate code formatting --- .isort.cfg | 2 +- .pylintrc | 2 + Makefile | 7 +- platformio/__init__.py | 3 +- platformio/__main__.py | 34 +- platformio/app.py | 93 ++-- platformio/builder/main.py | 64 ++- platformio/builder/tools/pioide.py | 77 ++-- platformio/builder/tools/piolib.py | 288 +++++++------ platformio/builder/tools/piomisc.py | 92 ++-- platformio/builder/tools/pioplatform.py | 85 ++-- platformio/builder/tools/pioproject.py | 13 +- platformio/builder/tools/pioupload.py | 108 ++--- platformio/builder/tools/piowinhooks.py | 11 +- platformio/builder/tools/platformio.py | 97 +++-- platformio/commands/__init__.py | 17 +- platformio/commands/boards.py | 28 +- platformio/commands/check/command.py | 186 ++++---- platformio/commands/check/defect.py | 43 +- platformio/commands/check/tools/__init__.py | 4 +- platformio/commands/check/tools/base.py | 33 +- platformio/commands/check/tools/clangtidy.py | 10 +- platformio/commands/check/tools/cppcheck.py | 61 ++- platformio/commands/ci.py | 71 ++- platformio/commands/debug/client.py | 95 ++-- platformio/commands/debug/command.py | 103 +++-- platformio/commands/debug/helpers.py | 141 +++--- platformio/commands/debug/process.py | 9 +- platformio/commands/debug/server.py | 76 ++-- platformio/commands/device.py | 184 ++++---- platformio/commands/home/command.py | 35 +- platformio/commands/home/helpers.py | 8 +- platformio/commands/home/rpc/handlers/app.py | 41 +- platformio/commands/home/rpc/handlers/ide.py | 12 +- platformio/commands/home/rpc/handlers/misc.py | 23 +- platformio/commands/home/rpc/handlers/os.py | 27 +- .../commands/home/rpc/handlers/piocore.py | 28 +- .../commands/home/rpc/handlers/project.py | 229 +++++----- platformio/commands/home/rpc/server.py | 18 +- platformio/commands/home/web.py | 4 +- platformio/commands/init.py | 161 +++---- platformio/commands/lib.py | 408 ++++++++++-------- platformio/commands/platform.py | 299 +++++++------ platformio/commands/remote.py | 167 +++---- platformio/commands/run/command.py | 202 +++++---- platformio/commands/run/helpers.py | 14 +- platformio/commands/run/processor.py | 32 +- platformio/commands/settings.py | 22 +- platformio/commands/test/command.py | 214 +++++---- platformio/commands/test/embedded.py | 39 +- platformio/commands/test/native.py | 12 +- platformio/commands/test/processor.py | 108 ++--- platformio/commands/update.py | 25 +- platformio/commands/upgrade.py | 64 +-- platformio/compat.py | 23 +- platformio/downloader.py | 30 +- platformio/exception.py | 91 ++-- platformio/fs.py | 32 +- platformio/ide/projectgenerator.py | 50 +-- platformio/lockfile.py | 3 +- platformio/maintenance.py | 171 +++++--- platformio/managers/core.py | 51 +-- platformio/managers/lib.py | 308 ++++++------- platformio/managers/package.py | 285 ++++++------ platformio/managers/platform.py | 343 ++++++++------- platformio/proc.py | 21 +- platformio/project/config.py | 61 +-- platformio/project/helpers.py | 83 ++-- platformio/project/options.py | 393 +++++++++-------- platformio/telemetry.py | 142 +++--- platformio/unpacker.py | 10 +- platformio/util.py | 136 +++--- platformio/vcsclient.py | 37 +- tests/commands/test_boards.py | 9 +- tests/commands/test_check.py | 72 ++-- tests/commands/test_ci.py | 94 ++-- tests/commands/test_init.py | 53 +-- tests/commands/test_lib.py | 257 +++++++---- tests/commands/test_platform.py | 52 +-- tests/commands/test_test.py | 21 +- tests/conftest.py | 11 +- tests/test_builder.py | 59 +-- tests/test_examples.py | 20 +- tests/test_ino2cpp.py | 14 +- tests/test_maintenance.py | 37 +- tests/test_managers.py | 165 +++---- tests/test_misc.py | 6 +- tests/test_pkgmanifest.py | 14 +- tests/test_projectconf.py | 46 +- tox.ini | 2 +- 90 files changed, 4064 insertions(+), 3367 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index eba1b0f3..7e5d6e51 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,3 +1,3 @@ [settings] -line_length=79 +line_length=88 known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial,twisted,autobahn,jsonrpc,tabulate diff --git a/.pylintrc b/.pylintrc index 180a05b8..e4203273 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,5 +1,7 @@ [MESSAGES CONTROL] disable= + bad-continuation, + bad-whitespace, missing-docstring, ungrouped-imports, invalid-name, diff --git a/Makefile b/Makefile index efbb4d28..ff246abd 100644 --- a/Makefile +++ b/Makefile @@ -5,13 +5,14 @@ isort: isort -rc ./platformio isort -rc ./tests -yapf: - yapf --recursive --in-place platformio/ +black: + black --target-version py27 ./platformio + black --target-version py27 ./tests test: py.test --verbose --capture=no --exitfirst -n 3 --dist=loadscope tests --ignore tests/test_examples.py --ignore tests/test_pkgmanifest.py -before-commit: isort yapf lint test +before-commit: isort black lint test clean-docs: rm -rf docs/_build diff --git a/platformio/__init__.py b/platformio/__init__.py index c2ca4c96..e4672201 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -21,7 +21,8 @@ __description__ = ( "Cross-platform IDE and unified debugger. " "Remote unit testing and firmware updates. " "Arduino, ARM mbed, Espressif (ESP8266/ESP32), STM32, PIC32, nRF51/nRF52, " - "FPGA, CMSIS, SPL, AVR, Samsung ARTIK, libOpenCM3") + "FPGA, CMSIS, SPL, AVR, Samsung ARTIK, libOpenCM3" +) __url__ = "https://platformio.org" __author__ = "PlatformIO" diff --git a/platformio/__main__.py b/platformio/__main__.py index ced8e50a..dbf66524 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -23,27 +23,31 @@ from platformio.commands import PlatformioCLI from platformio.compat import CYGWIN -@click.command(cls=PlatformioCLI, - context_settings=dict(help_option_names=["-h", "--help"])) +@click.command( + cls=PlatformioCLI, context_settings=dict(help_option_names=["-h", "--help"]) +) @click.version_option(__version__, prog_name="PlatformIO") @click.option("--force", "-f", is_flag=True, help="DEPRECATE") @click.option("--caller", "-c", help="Caller ID (service)") -@click.option("--no-ansi", - is_flag=True, - help="Do not print ANSI control characters") +@click.option("--no-ansi", is_flag=True, help="Do not print ANSI control characters") @click.pass_context def cli(ctx, force, caller, no_ansi): try: - if (no_ansi or str( - os.getenv( - "PLATFORMIO_NO_ANSI", - os.getenv("PLATFORMIO_DISABLE_COLOR"))).lower() == "true"): + if ( + no_ansi + or str( + os.getenv("PLATFORMIO_NO_ANSI", os.getenv("PLATFORMIO_DISABLE_COLOR")) + ).lower() + == "true" + ): # pylint: disable=protected-access click._compat.isatty = lambda stream: False - elif str( - os.getenv( - "PLATFORMIO_FORCE_ANSI", - os.getenv("PLATFORMIO_FORCE_COLOR"))).lower() == "true": + elif ( + str( + os.getenv("PLATFORMIO_FORCE_ANSI", os.getenv("PLATFORMIO_FORCE_COLOR")) + ).lower() + == "true" + ): # pylint: disable=protected-access click._compat.isatty = lambda stream: True except: # pylint: disable=bare-except @@ -67,6 +71,7 @@ def configure(): # /en/latest/security.html#insecureplatformwarning try: import urllib3 + urllib3.disable_warnings() except (AttributeError, ImportError): pass @@ -79,7 +84,8 @@ def configure(): click_echo_origin[origin](*args, **kwargs) except IOError: (sys.stderr.write if kwargs.get("err") else sys.stdout.write)( - "%s\n" % (args[0] if args else "")) + "%s\n" % (args[0] if args else "") + ) click.echo = lambda *args, **kwargs: _safe_echo(0, *args, **kwargs) click.secho = lambda *args, **kwargs: _safe_echo(1, *args, **kwargs) diff --git a/platformio/app.py b/platformio/app.py index 66a6f126..2858b37a 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -23,11 +23,9 @@ from time import time import requests from platformio import exception, fs, lockfile -from platformio.compat import (WINDOWS, dump_json_to_unicode, - hashlib_encode_data) +from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data from platformio.proc import is_ci -from platformio.project.helpers import (get_project_cache_dir, - get_project_core_dir) +from platformio.project.helpers import get_project_cache_dir, get_project_core_dir def get_default_projects_dir(): @@ -35,6 +33,7 @@ def get_default_projects_dir(): try: assert WINDOWS import ctypes.wintypes + buf = ctypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH) ctypes.windll.shell32.SHGetFolderPathW(None, 5, None, 0, buf) docs_dir = buf.value @@ -51,45 +50,41 @@ def projects_dir_validate(projects_dir): DEFAULT_SETTINGS = { "auto_update_libraries": { "description": "Automatically update libraries (Yes/No)", - "value": False + "value": False, }, "auto_update_platforms": { "description": "Automatically update platforms (Yes/No)", - "value": False + "value": False, }, "check_libraries_interval": { "description": "Check for the library updates interval (days)", - "value": 7 + "value": 7, }, "check_platformio_interval": { "description": "Check for the new PlatformIO interval (days)", - "value": 3 + "value": 3, }, "check_platforms_interval": { "description": "Check for the platform updates interval (days)", - "value": 7 + "value": 7, }, "enable_cache": { "description": "Enable caching for API requests and Library Manager", - "value": True - }, - "strict_ssl": { - "description": "Strict SSL for PlatformIO Services", - "value": False + "value": True, }, + "strict_ssl": {"description": "Strict SSL for PlatformIO Services", "value": False}, "enable_telemetry": { - "description": - ("Telemetry service (Yes/No)"), - "value": True + "description": ("Telemetry service (Yes/No)"), + "value": True, }, "force_verbose": { "description": "Force verbose output when processing environments", - "value": False + "value": False, }, "projects_dir": { "description": "Default location for PlatformIO projects (PIO Home)", "value": get_default_projects_dir(), - "validator": projects_dir_validate + "validator": projects_dir_validate, }, } @@ -97,7 +92,6 @@ SESSION_VARS = {"command_ctx": None, "force_option": False, "caller_id": None} class State(object): - def __init__(self, path=None, lock=False): self.path = path self.lock = lock @@ -113,8 +107,12 @@ class State(object): if isfile(self.path): self._storage = fs.load_json(self.path) assert isinstance(self._storage, dict) - except (AssertionError, ValueError, UnicodeDecodeError, - exception.InvalidJSONFile): + except ( + AssertionError, + ValueError, + UnicodeDecodeError, + exception.InvalidJSONFile, + ): self._storage = {} return self @@ -174,7 +172,6 @@ class State(object): class ContentCache(object): - def __init__(self, cache_dir=None): self.cache_dir = None self._db_path = None @@ -277,8 +274,11 @@ class ContentCache(object): continue expire, path = line.split("=") try: - if time() < int(expire) and isfile(path) and \ - path not in paths_for_delete: + if ( + time() < int(expire) + and isfile(path) + and path not in paths_for_delete + ): newlines.append(line) continue except ValueError: @@ -317,11 +317,11 @@ def sanitize_setting(name, value): defdata = DEFAULT_SETTINGS[name] try: if "validator" in defdata: - value = defdata['validator'](value) - elif isinstance(defdata['value'], bool): + value = defdata["validator"](value) + elif isinstance(defdata["value"], bool): if not isinstance(value, bool): value = str(value).lower() in ("true", "yes", "y", "1") - elif isinstance(defdata['value'], int): + elif isinstance(defdata["value"], int): value = int(value) except Exception: raise exception.InvalidSettingValue(value, name) @@ -351,24 +351,24 @@ def get_setting(name): return sanitize_setting(name, getenv(_env_name)) with State() as state: - if "settings" in state and name in state['settings']: - return state['settings'][name] + if "settings" in state and name in state["settings"]: + return state["settings"][name] - return DEFAULT_SETTINGS[name]['value'] + return DEFAULT_SETTINGS[name]["value"] def set_setting(name, value): with State(lock=True) as state: if "settings" not in state: - state['settings'] = {} - state['settings'][name] = sanitize_setting(name, value) + state["settings"] = {} + state["settings"][name] = sanitize_setting(name, value) state.modified = True def reset_settings(): with State(lock=True) as state: if "settings" in state: - del state['settings'] + del state["settings"] def get_session_var(name, default=None): @@ -381,11 +381,13 @@ def set_session_var(name, value): def is_disabled_progressbar(): - return any([ - get_session_var("force_option"), - is_ci(), - getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true" - ]) + return any( + [ + get_session_var("force_option"), + is_ci(), + getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true", + ] + ) def get_cid(): @@ -397,9 +399,16 @@ def get_cid(): uid = getenv("C9_UID") elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")): try: - uid = requests.get("{api}/user?token={token}".format( - api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")), - token=getenv("USER_TOKEN"))).json().get("id") + uid = ( + requests.get( + "{api}/user?token={token}".format( + api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")), + token=getenv("USER_TOKEN"), + ) + ) + .json() + .get("id") + ) except: # pylint: disable=bare-except pass if not uid: diff --git a/platformio/builder/main.py b/platformio/builder/main.py index d1a8763d..52adb5a3 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -43,17 +43,27 @@ clivars.AddVariables( ("PROJECT_CONFIG",), ("PIOENV",), ("PIOTEST_RUNNING_NAME",), - ("UPLOAD_PORT",) + ("UPLOAD_PORT",), ) # yapf: disable DEFAULT_ENV_OPTIONS = dict( tools=[ - "ar", "gas", "gcc", "g++", "gnulink", "platformio", "pioplatform", - "pioproject", "piowinhooks", "piolib", "pioupload", "piomisc", "pioide" + "ar", + "gas", + "gcc", + "g++", + "gnulink", + "platformio", + "pioplatform", + "pioproject", + "piowinhooks", + "piolib", + "pioupload", + "piomisc", + "pioide", ], toolpath=[join(fs.get_source_dir(), "builder", "tools")], variables=clivars, - # Propagating External Environment ENV=environ, UNIX_TIME=int(time()), @@ -75,16 +85,17 @@ DEFAULT_ENV_OPTIONS = dict( LIBSOURCE_DIRS=[ project_helpers.get_project_lib_dir(), join("$PROJECTLIBDEPS_DIR", "$PIOENV"), - project_helpers.get_project_global_lib_dir() + project_helpers.get_project_global_lib_dir(), ], PROGNAME="program", PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"), - PYTHONEXE=get_pythonexe_path()) + PYTHONEXE=get_pythonexe_path(), +) if not int(ARGUMENTS.get("PIOVERBOSE", 0)): - DEFAULT_ENV_OPTIONS['ARCOMSTR'] = "Archiving $TARGET" - DEFAULT_ENV_OPTIONS['LINKCOMSTR'] = "Linking $TARGET" - DEFAULT_ENV_OPTIONS['RANLIBCOMSTR'] = "Indexing $TARGET" + DEFAULT_ENV_OPTIONS["ARCOMSTR"] = "Archiving $TARGET" + DEFAULT_ENV_OPTIONS["LINKCOMSTR"] = "Linking $TARGET" + DEFAULT_ENV_OPTIONS["RANLIBCOMSTR"] = "Indexing $TARGET" for k in ("ASCOMSTR", "ASPPCOMSTR", "CCCOMSTR", "CXXCOMSTR"): DEFAULT_ENV_OPTIONS[k] = "Compiling $TARGET" @@ -94,8 +105,10 @@ env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS) env.Replace( **{ key: PlatformBase.decode_scons_arg(env[key]) - for key in list(clivars.keys()) if key in env - }) + for key in list(clivars.keys()) + if key in env + } +) if env.subst("$BUILDCACHE_DIR"): if not isdir(env.subst("$BUILDCACHE_DIR")): @@ -106,18 +119,17 @@ if int(ARGUMENTS.get("ISATTY", 0)): # pylint: disable=protected-access click._compat.isatty = lambda stream: True -if env.GetOption('clean'): +if env.GetOption("clean"): env.PioClean(env.subst("$BUILD_DIR")) env.Exit(0) elif not int(ARGUMENTS.get("PIOVERBOSE", 0)): - print("Verbose mode can be enabled via `-v, --verbose` option") + print ("Verbose mode can be enabled via `-v, --verbose` option") env.LoadProjectOptions() env.LoadPioPlatform() env.SConscriptChdir(0) -env.SConsignFile( - join("$BUILD_DIR", ".sconsign.dblite" if PY2 else ".sconsign3.dblite")) +env.SConsignFile(join("$BUILD_DIR", ".sconsign.dblite" if PY2 else ".sconsign3.dblite")) for item in env.GetExtraScripts("pre"): env.SConscript(item, exports="env") @@ -144,10 +156,13 @@ if env.get("SIZETOOL") and "nobuild" not in COMMAND_LINE_TARGETS: Default("checkprogsize") # Print configured protocols -env.AddPreAction(["upload", "program"], - env.VerboseAction( - lambda source, target, env: env.PrintUploadInfo(), - "Configuring upload protocol...")) +env.AddPreAction( + ["upload", "program"], + env.VerboseAction( + lambda source, target, env: env.PrintUploadInfo(), + "Configuring upload protocol...", + ), +) AlwaysBuild(env.Alias("debug", DEFAULT_TARGETS)) AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS)) @@ -155,12 +170,15 @@ AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS)) ############################################################################## if "envdump" in COMMAND_LINE_TARGETS: - print(env.Dump()) + print (env.Dump()) env.Exit(0) if "idedata" in COMMAND_LINE_TARGETS: Import("projenv") - print("\n%s\n" % dump_json_to_unicode( - env.DumpIDEData(projenv) # pylint: disable=undefined-variable - )) + print ( + "\n%s\n" + % dump_json_to_unicode( + env.DumpIDEData(projenv) # pylint: disable=undefined-variable + ) + ) env.Exit(0) diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 43d32404..39d388a4 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -45,7 +45,7 @@ def _dump_includes(env, projenv): join(toolchain_dir, "*", "include*"), join(toolchain_dir, "*", "include", "c++", "*"), join(toolchain_dir, "*", "include", "c++", "*", "*-*-*"), - join(toolchain_dir, "lib", "gcc", "*", "*", "include*") + join(toolchain_dir, "lib", "gcc", "*", "*", "include*"), ] for g in toolchain_incglobs: includes.extend(glob(g)) @@ -54,9 +54,7 @@ def _dump_includes(env, projenv): if unity_dir: includes.append(unity_dir) - includes.extend( - [env.subst("$PROJECTINCLUDE_DIR"), - env.subst("$PROJECTSRC_DIR")]) + includes.extend([env.subst("$PROJECTINCLUDE_DIR"), env.subst("$PROJECTSRC_DIR")]) # remove duplicates result = [] @@ -71,15 +69,15 @@ def _get_gcc_defines(env): items = [] try: sysenv = environ.copy() - sysenv['PATH'] = str(env['ENV']['PATH']) - result = exec_command("echo | %s -dM -E -" % env.subst("$CC"), - env=sysenv, - shell=True) + sysenv["PATH"] = str(env["ENV"]["PATH"]) + result = exec_command( + "echo | %s -dM -E -" % env.subst("$CC"), env=sysenv, shell=True + ) except OSError: return items - if result['returncode'] != 0: + if result["returncode"] != 0: return items - for line in result['out'].split("\n"): + for line in result["out"].split("\n"): tokens = line.strip().split(" ", 2) if not tokens or tokens[0] != "#define": continue @@ -94,17 +92,22 @@ def _dump_defines(env): defines = [] # global symbols for item in processDefines(env.get("CPPDEFINES", [])): - defines.append(env.subst(item).replace('\\', '')) + defines.append(env.subst(item).replace("\\", "")) # special symbol for Atmel AVR MCU - if env['PIOPLATFORM'] == "atmelavr": + if env["PIOPLATFORM"] == "atmelavr": board_mcu = env.get("BOARD_MCU") if not board_mcu and "BOARD" in env: board_mcu = env.BoardConfig().get("build.mcu") if board_mcu: defines.append( - str("__AVR_%s__" % board_mcu.upper().replace( - "ATMEGA", "ATmega").replace("ATTINY", "ATtiny"))) + str( + "__AVR_%s__" + % board_mcu.upper() + .replace("ATMEGA", "ATmega") + .replace("ATTINY", "ATtiny") + ) + ) # built-in GCC marcos # if env.GetCompilerType() == "gcc": @@ -140,33 +143,22 @@ def DumpIDEData(env, projenv): LINTCXXCOM = "$CXXFLAGS $CCFLAGS $CPPFLAGS" data = { - "env_name": - env['PIOENV'], + "env_name": env["PIOENV"], "libsource_dirs": [env.subst(l) for l in env.GetLibSourceDirs()], - "defines": - _dump_defines(env), - "includes": - _dump_includes(env, projenv), - "cc_flags": - env.subst(LINTCCOM), - "cxx_flags": - env.subst(LINTCXXCOM), - "cc_path": - where_is_program(env.subst("$CC"), env.subst("${ENV['PATH']}")), - "cxx_path": - where_is_program(env.subst("$CXX"), env.subst("${ENV['PATH']}")), - "gdb_path": - where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")), - "prog_path": - env.subst("$PROG_PATH"), - "flash_extra_images": [{ - "offset": item[0], - "path": env.subst(item[1]) - } for item in env.get("FLASH_EXTRA_IMAGES", [])], - "svd_path": - _get_svd_path(env), - "compiler_type": - env.GetCompilerType() + "defines": _dump_defines(env), + "includes": _dump_includes(env, projenv), + "cc_flags": env.subst(LINTCCOM), + "cxx_flags": env.subst(LINTCXXCOM), + "cc_path": where_is_program(env.subst("$CC"), env.subst("${ENV['PATH']}")), + "cxx_path": where_is_program(env.subst("$CXX"), env.subst("${ENV['PATH']}")), + "gdb_path": where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")), + "prog_path": env.subst("$PROG_PATH"), + "flash_extra_images": [ + {"offset": item[0], "path": env.subst(item[1])} + for item in env.get("FLASH_EXTRA_IMAGES", []) + ], + "svd_path": _get_svd_path(env), + "compiler_type": env.GetCompilerType(), } env_ = env.Clone() @@ -180,10 +172,7 @@ def DumpIDEData(env, projenv): _new_defines.append(item) env_.Replace(CPPDEFINES=_new_defines) - data.update({ - "cc_flags": env_.subst(LINTCCOM), - "cxx_flags": env_.subst(LINTCXXCOM) - }) + data.update({"cc_flags": env_.subst(LINTCCOM), "cxx_flags": env_.subst(LINTCXXCOM)}) return data diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 185b2985..e67b9f3b 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -22,8 +22,16 @@ import hashlib import os import re import sys -from os.path import (basename, commonprefix, expanduser, isdir, isfile, join, - realpath, sep) +from os.path import ( + basename, + commonprefix, + expanduser, + isdir, + isfile, + join, + realpath, + sep, +) import click import SCons.Scanner # pylint: disable=import-error @@ -33,13 +41,16 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error from platformio import exception, fs, util from platformio.builder.tools import platformio as piotool -from platformio.compat import (WINDOWS, get_file_contents, hashlib_encode_data, - string_types) +from platformio.compat import ( + WINDOWS, + get_file_contents, + hashlib_encode_data, + string_types, +) from platformio.managers.lib import LibraryManager class LibBuilderFactory(object): - @staticmethod def new(env, path, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))): clsname = "UnknownLibBuilder" @@ -47,31 +58,30 @@ class LibBuilderFactory(object): clsname = "PlatformIOLibBuilder" else: used_frameworks = LibBuilderFactory.get_used_frameworks(env, path) - common_frameworks = (set(env.get("PIOFRAMEWORK", [])) - & set(used_frameworks)) + common_frameworks = set(env.get("PIOFRAMEWORK", [])) & set(used_frameworks) if common_frameworks: clsname = "%sLibBuilder" % list(common_frameworks)[0].title() elif used_frameworks: clsname = "%sLibBuilder" % used_frameworks[0].title() - obj = getattr(sys.modules[__name__], clsname)(env, - path, - verbose=verbose) + obj = getattr(sys.modules[__name__], clsname)(env, path, verbose=verbose) assert isinstance(obj, LibBuilderBase) return obj @staticmethod def get_used_frameworks(env, path): if any( - isfile(join(path, fname)) - for fname in ("library.properties", "keywords.txt")): + isfile(join(path, fname)) + for fname in ("library.properties", "keywords.txt") + ): return ["arduino"] if isfile(join(path, "module.json")): return ["mbed"] - include_re = re.compile(r'^#include\s+(<|")(Arduino|mbed)\.h(<|")', - flags=re.MULTILINE) + include_re = re.compile( + r'^#include\s+(<|")(Arduino|mbed)\.h(<|")', flags=re.MULTILINE + ) # check source files for root, _, files in os.walk(path, followlinks=True): @@ -79,7 +89,8 @@ class LibBuilderFactory(object): return ["mbed"] for fname in files: if not fs.path_endswith_ext( - fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT): + fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT + ): continue content = get_file_contents(join(root, fname)) if not content: @@ -124,7 +135,7 @@ class LibBuilderBase(object): self._processed_files = list() # reset source filter, could be overridden with extra script - self.env['SRC_FILTER'] = "" + self.env["SRC_FILTER"] = "" # process extra options and append to build environment self.process_extra_options() @@ -153,7 +164,8 @@ class LibBuilderBase(object): @property def dependencies(self): return LibraryManager.normalize_dependencies( - self._manifest.get("dependencies", [])) + self._manifest.get("dependencies", []) + ) @property def src_filter(self): @@ -161,7 +173,7 @@ class LibBuilderBase(object): "-" % os.sep, "-" % os.sep, "-" % os.sep, - "-" % os.sep + "-" % os.sep, ] @property @@ -172,8 +184,7 @@ class LibBuilderBase(object): @property def src_dir(self): - return (join(self.path, "src") - if isdir(join(self.path, "src")) else self.path) + return join(self.path, "src") if isdir(join(self.path, "src")) else self.path def get_include_dirs(self): items = [] @@ -234,8 +245,7 @@ class LibBuilderBase(object): @property def lib_compat_mode(self): - return self.env.GetProjectOption("lib_compat_mode", - self.COMPAT_MODE_DEFAULT) + return self.env.GetProjectOption("lib_compat_mode", self.COMPAT_MODE_DEFAULT) @staticmethod def validate_compat_mode(mode): @@ -263,11 +273,10 @@ class LibBuilderBase(object): self.env.ProcessFlags(self.build_flags) if self.extra_script: self.env.SConscriptChdir(1) - self.env.SConscript(realpath(self.extra_script), - exports={ - "env": self.env, - "pio_lib_builder": self - }) + self.env.SConscript( + realpath(self.extra_script), + exports={"env": self.env, "pio_lib_builder": self}, + ) self.env.ProcessUnFlags(self.build_unflags) def process_dependencies(self): @@ -276,7 +285,7 @@ class LibBuilderBase(object): for item in self.dependencies: found = False for lb in self.env.GetLibBuilders(): - if item['name'] != lb.name: + if item["name"] != lb.name: continue found = True if lb not in self.depbuilders: @@ -284,37 +293,43 @@ class LibBuilderBase(object): break if not found and self.verbose: - sys.stderr.write("Warning: Ignored `%s` dependency for `%s` " - "library\n" % (item['name'], self.name)) + sys.stderr.write( + "Warning: Ignored `%s` dependency for `%s` " + "library\n" % (item["name"], self.name) + ) def get_search_files(self): items = [ - join(self.src_dir, item) for item in self.env.MatchSourceFiles( - self.src_dir, self.src_filter) + join(self.src_dir, item) + for item in self.env.MatchSourceFiles(self.src_dir, self.src_filter) ] include_dir = self.include_dir if include_dir: - items.extend([ - join(include_dir, item) - for item in self.env.MatchSourceFiles(include_dir) - ]) + items.extend( + [ + join(include_dir, item) + for item in self.env.MatchSourceFiles(include_dir) + ] + ) return items def _get_found_includes( # pylint: disable=too-many-branches - self, search_files=None): + self, search_files=None + ): # all include directories if not LibBuilderBase._INCLUDE_DIRS_CACHE: LibBuilderBase._INCLUDE_DIRS_CACHE = [] for lb in self.env.GetLibBuilders(): LibBuilderBase._INCLUDE_DIRS_CACHE.extend( - [self.env.Dir(d) for d in lb.get_include_dirs()]) + [self.env.Dir(d) for d in lb.get_include_dirs()] + ) # append self include directories include_dirs = [self.env.Dir(d) for d in self.get_include_dirs()] include_dirs.extend(LibBuilderBase._INCLUDE_DIRS_CACHE) result = [] - for path in (search_files or []): + for path in search_files or []: if path in self._processed_files: continue self._processed_files.append(path) @@ -325,19 +340,25 @@ class LibBuilderBase(object): self.env.File(path), self.env, tuple(include_dirs), - depth=self.CCONDITIONAL_SCANNER_DEPTH) + depth=self.CCONDITIONAL_SCANNER_DEPTH, + ) # mark candidates already processed via Conditional Scanner - self._processed_files.extend([ - c.get_abspath() for c in candidates - if c.get_abspath() not in self._processed_files - ]) + self._processed_files.extend( + [ + c.get_abspath() + for c in candidates + if c.get_abspath() not in self._processed_files + ] + ) except Exception as e: # pylint: disable=broad-except if self.verbose and "+" in self.lib_ldf_mode: sys.stderr.write( "Warning! Classic Pre Processor is used for `%s`, " - "advanced has failed with `%s`\n" % (path, e)) + "advanced has failed with `%s`\n" % (path, e) + ) candidates = LibBuilderBase.CLASSIC_SCANNER( - self.env.File(path), self.env, tuple(include_dirs)) + self.env.File(path), self.env, tuple(include_dirs) + ) # print(path, map(lambda n: n.get_abspath(), candidates)) for item in candidates: @@ -348,7 +369,7 @@ class LibBuilderBase(object): _h_path = item.get_abspath() if not fs.path_endswith_ext(_h_path, piotool.SRC_HEADER_EXT): continue - _f_part = _h_path[:_h_path.rindex(".")] + _f_part = _h_path[: _h_path.rindex(".")] for ext in piotool.SRC_C_EXT: if not isfile("%s.%s" % (_f_part, ext)): continue @@ -359,7 +380,6 @@ class LibBuilderBase(object): return result def depend_recursive(self, lb, search_files=None): - def _already_depends(_lb): if self in _lb.depbuilders: return True @@ -372,9 +392,10 @@ class LibBuilderBase(object): if self != lb: if _already_depends(lb): if self.verbose: - sys.stderr.write("Warning! Circular dependencies detected " - "between `%s` and `%s`\n" % - (self.path, lb.path)) + sys.stderr.write( + "Warning! Circular dependencies detected " + "between `%s` and `%s`\n" % (self.path, lb.path) + ) self._circular_deps.append(lb) elif lb not in self._depbuilders: self._depbuilders.append(lb) @@ -431,11 +452,10 @@ class LibBuilderBase(object): if self.lib_archive: libs.append( - self.env.BuildLibrary(self.build_dir, self.src_dir, - self.src_filter)) + self.env.BuildLibrary(self.build_dir, self.src_dir, self.src_filter) + ) else: - self.env.BuildSources(self.build_dir, self.src_dir, - self.src_filter) + self.env.BuildSources(self.build_dir, self.src_dir, self.src_filter) return libs @@ -444,7 +464,6 @@ class UnknownLibBuilder(LibBuilderBase): class ArduinoLibBuilder(LibBuilderBase): - def load_manifest(self): manifest = {} if not isfile(join(self.path, "library.properties")): @@ -508,7 +527,7 @@ class ArduinoLibBuilder(LibBuilderBase): "esp32": ["espressif32"], "arc32": ["intel_arc32"], "stm32": ["ststm32"], - "nrf5": ["nordicnrf51", "nordicnrf52"] + "nrf5": ["nordicnrf51", "nordicnrf52"], } items = [] for arch in self._manifest.get("architectures", "").split(","): @@ -524,7 +543,6 @@ class ArduinoLibBuilder(LibBuilderBase): class MbedLibBuilder(LibBuilderBase): - def load_manifest(self): if not isfile(join(self.path, "module.json")): return {} @@ -611,14 +629,15 @@ class MbedLibBuilder(LibBuilderBase): # default macros for macro in manifest.get("macros", []): macro = self._mbed_normalize_macro(macro) - macros[macro['name']] = macro + macros[macro["name"]] = macro # configuration items for key, options in manifest.get("config", {}).items(): if "value" not in options: continue - macros[key] = dict(name=options.get("macro_name"), - value=options.get("value")) + macros[key] = dict( + name=options.get("macro_name"), value=options.get("value") + ) # overrode items per target for target, options in manifest.get("target_overrides", {}).items(): @@ -626,25 +645,23 @@ class MbedLibBuilder(LibBuilderBase): continue for macro in options.get("target.macros_add", []): macro = self._mbed_normalize_macro(macro) - macros[macro['name']] = macro + macros[macro["name"]] = macro for key, value in options.items(): if not key.startswith("target.") and key in macros: - macros[key]['value'] = value + macros[key]["value"] = value # normalize macro names for key, macro in macros.items(): - if not macro['name']: - macro['name'] = key - if "." not in macro['name']: - macro['name'] = "%s.%s" % (manifest.get("name"), - macro['name']) - macro['name'] = re.sub(r"[^a-z\d]+", - "_", - macro['name'], - flags=re.I).upper() - macro['name'] = "MBED_CONF_" + macro['name'] - if isinstance(macro['value'], bool): - macro['value'] = 1 if macro['value'] else 0 + if not macro["name"]: + macro["name"] = key + if "." not in macro["name"]: + macro["name"] = "%s.%s" % (manifest.get("name"), macro["name"]) + macro["name"] = re.sub( + r"[^a-z\d]+", "_", macro["name"], flags=re.I + ).upper() + macro["name"] = "MBED_CONF_" + macro["name"] + if isinstance(macro["value"], bool): + macro["value"] = 1 if macro["value"] else 0 return {macro["name"]: macro["value"] for macro in macros.values()} @@ -654,13 +671,13 @@ class MbedLibBuilder(LibBuilderBase): for line in fp.readlines(): line = line.strip() if line == "#endif": - lines.append( - "// PlatformIO Library Dependency Finder (LDF)") - lines.extend([ - "#define %s %s" % - (name, value if value is not None else "") - for name, value in macros.items() - ]) + lines.append("// PlatformIO Library Dependency Finder (LDF)") + lines.extend( + [ + "#define %s %s" % (name, value if value is not None else "") + for name, value in macros.items() + ] + ) lines.append("") if not line.startswith("#define"): lines.append(line) @@ -674,7 +691,6 @@ class MbedLibBuilder(LibBuilderBase): class PlatformIOLibBuilder(LibBuilderBase): - def load_manifest(self): assert isfile(join(self.path, "library.json")) manifest = fs.load_json(join(self.path, "library.json")) @@ -682,9 +698,9 @@ class PlatformIOLibBuilder(LibBuilderBase): # replace "espressif" old name dev/platform with ESP8266 if "platforms" in manifest: - manifest['platforms'] = [ + manifest["platforms"] = [ "espressif8266" if p == "espressif" else p - for p in util.items_to_list(manifest['platforms']) + for p in util.items_to_list(manifest["platforms"]) ] return manifest @@ -710,8 +726,8 @@ class PlatformIOLibBuilder(LibBuilderBase): def src_filter(self): if "srcFilter" in self._manifest.get("build", {}): return self._manifest.get("build").get("srcFilter") - if self.env['SRC_FILTER']: - return self.env['SRC_FILTER'] + if self.env["SRC_FILTER"]: + return self.env["SRC_FILTER"] if self._is_arduino_manifest(): return ArduinoLibBuilder.src_filter.fget(self) return LibBuilderBase.src_filter.fget(self) @@ -740,7 +756,8 @@ class PlatformIOLibBuilder(LibBuilderBase): if global_value is not None: return global_value return self._manifest.get("build", {}).get( - "libArchive", LibBuilderBase.lib_archive.fget(self)) + "libArchive", LibBuilderBase.lib_archive.fget(self) + ) @property def lib_ldf_mode(self): @@ -748,7 +765,10 @@ class PlatformIOLibBuilder(LibBuilderBase): self.env.GetProjectOption( "lib_ldf_mode", self._manifest.get("build", {}).get( - "libLDFMode", LibBuilderBase.lib_ldf_mode.fget(self)))) + "libLDFMode", LibBuilderBase.lib_ldf_mode.fget(self) + ), + ) + ) @property def lib_compat_mode(self): @@ -756,8 +776,10 @@ class PlatformIOLibBuilder(LibBuilderBase): self.env.GetProjectOption( "lib_compat_mode", self._manifest.get("build", {}).get( - "libCompatMode", - LibBuilderBase.lib_compat_mode.fget(self)))) + "libCompatMode", LibBuilderBase.lib_compat_mode.fget(self) + ), + ) + ) def is_platforms_compatible(self, platforms): items = self._manifest.get("platforms") @@ -775,9 +797,12 @@ class PlatformIOLibBuilder(LibBuilderBase): include_dirs = LibBuilderBase.get_include_dirs(self) # backwards compatibility with PlatformIO 2.0 - if ("build" not in self._manifest and self._is_arduino_manifest() - and not isdir(join(self.path, "src")) - and isdir(join(self.path, "utility"))): + if ( + "build" not in self._manifest + and self._is_arduino_manifest() + and not isdir(join(self.path, "src")) + and isdir(join(self.path, "utility")) + ): include_dirs.append(join(self.path, "utility")) for path in self.env.get("CPPPATH", []): @@ -788,12 +813,11 @@ class PlatformIOLibBuilder(LibBuilderBase): class ProjectAsLibBuilder(LibBuilderBase): - def __init__(self, env, *args, **kwargs): # backup original value, will be reset in base.__init__ project_src_filter = env.get("SRC_FILTER") super(ProjectAsLibBuilder, self).__init__(env, *args, **kwargs) - self.env['SRC_FILTER'] = project_src_filter + self.env["SRC_FILTER"] = project_src_filter @property def include_dir(self): @@ -819,11 +843,14 @@ class ProjectAsLibBuilder(LibBuilderBase): items = LibBuilderBase.get_search_files(self) # test files if "__test" in COMMAND_LINE_TARGETS: - items.extend([ - join("$PROJECTTEST_DIR", - item) for item in self.env.MatchSourceFiles( - "$PROJECTTEST_DIR", "$PIOTEST_SRC_FILTER") - ]) + items.extend( + [ + join("$PROJECTTEST_DIR", item) + for item in self.env.MatchSourceFiles( + "$PROJECTTEST_DIR", "$PIOTEST_SRC_FILTER" + ) + ] + ) return items @property @@ -836,8 +863,7 @@ class ProjectAsLibBuilder(LibBuilderBase): @property def src_filter(self): - return (self.env.get("SRC_FILTER") - or LibBuilderBase.src_filter.fget(self)) + return self.env.get("SRC_FILTER") or LibBuilderBase.src_filter.fget(self) @property def dependencies(self): @@ -848,7 +874,6 @@ class ProjectAsLibBuilder(LibBuilderBase): pass def install_dependencies(self): - def _is_builtin(uri): for lb in self.env.GetLibBuilders(): if lb.name == uri: @@ -871,8 +896,7 @@ class ProjectAsLibBuilder(LibBuilderBase): not_found_uri.append(uri) did_install = False - lm = LibraryManager( - self.env.subst(join("$PROJECTLIBDEPS_DIR", "$PIOENV"))) + lm = LibraryManager(self.env.subst(join("$PROJECTLIBDEPS_DIR", "$PIOENV"))) for uri in not_found_uri: try: lm.install(uri) @@ -923,28 +947,27 @@ class ProjectAsLibBuilder(LibBuilderBase): def GetLibSourceDirs(env): items = env.GetProjectOption("lib_extra_dirs", []) - items.extend(env['LIBSOURCE_DIRS']) + items.extend(env["LIBSOURCE_DIRS"]) return [ - env.subst(expanduser(item) if item.startswith("~") else item) - for item in items + env.subst(expanduser(item) if item.startswith("~") else item) for item in items ] -def IsCompatibleLibBuilder(env, - lb, - verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))): +def IsCompatibleLibBuilder(env, lb, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))): compat_mode = lb.lib_compat_mode if lb.name in env.GetProjectOption("lib_ignore", []): if verbose: sys.stderr.write("Ignored library %s\n" % lb.path) return None - if compat_mode == "strict" and not lb.is_platforms_compatible( - env['PIOPLATFORM']): + if compat_mode == "strict" and not lb.is_platforms_compatible(env["PIOPLATFORM"]): if verbose: sys.stderr.write("Platform incompatible library %s\n" % lb.path) return False - if (compat_mode in ("soft", "strict") and "PIOFRAMEWORK" in env - and not lb.is_frameworks_compatible(env.get("PIOFRAMEWORK", []))): + if ( + compat_mode in ("soft", "strict") + and "PIOFRAMEWORK" in env + and not lb.is_frameworks_compatible(env.get("PIOFRAMEWORK", [])) + ): if verbose: sys.stderr.write("Framework incompatible library %s\n" % lb.path) return False @@ -953,8 +976,10 @@ def IsCompatibleLibBuilder(env, def GetLibBuilders(env): # pylint: disable=too-many-branches if DefaultEnvironment().get("__PIO_LIB_BUILDERS", None) is not None: - return sorted(DefaultEnvironment()['__PIO_LIB_BUILDERS'], - key=lambda lb: 0 if lb.dependent else 1) + return sorted( + DefaultEnvironment()["__PIO_LIB_BUILDERS"], + key=lambda lb: 0 if lb.dependent else 1, + ) DefaultEnvironment().Replace(__PIO_LIB_BUILDERS=[]) @@ -974,7 +999,8 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches except exception.InvalidJSONFile: if verbose: sys.stderr.write( - "Skip library with broken manifest: %s\n" % lib_dir) + "Skip library with broken manifest: %s\n" % lib_dir + ) continue if env.IsCompatibleLibBuilder(lb): DefaultEnvironment().Append(__PIO_LIB_BUILDERS=[lb]) @@ -989,15 +1015,15 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches if verbose and found_incompat: sys.stderr.write( - "More details about \"Library Compatibility Mode\": " + 'More details about "Library Compatibility Mode": ' "https://docs.platformio.org/page/librarymanager/ldf.html#" - "ldf-compat-mode\n") + "ldf-compat-mode\n" + ) - return DefaultEnvironment()['__PIO_LIB_BUILDERS'] + return DefaultEnvironment()["__PIO_LIB_BUILDERS"] def ConfigureProjectLibBuilder(env): - def _get_vcs_info(lb): path = LibraryManager.get_src_manifest_path(lb.path) return fs.load_json(path) if path else None @@ -1036,26 +1062,28 @@ def ConfigureProjectLibBuilder(env): project = ProjectAsLibBuilder(env, "$PROJECT_DIR") ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project) - print("LDF: Library Dependency Finder -> http://bit.ly/configure-pio-ldf") - print("LDF Modes: Finder ~ %s, Compatibility ~ %s" % - (ldf_mode, project.lib_compat_mode)) + print ("LDF: Library Dependency Finder -> http://bit.ly/configure-pio-ldf") + print ( + "LDF Modes: Finder ~ %s, Compatibility ~ %s" + % (ldf_mode, project.lib_compat_mode) + ) project.install_dependencies() lib_builders = env.GetLibBuilders() - print("Found %d compatible libraries" % len(lib_builders)) + print ("Found %d compatible libraries" % len(lib_builders)) - print("Scanning dependencies...") + print ("Scanning dependencies...") project.search_deps_recursive() if ldf_mode.startswith("chain") and project.depbuilders: _correct_found_libs(lib_builders) if project.depbuilders: - print("Dependency Graph") + print ("Dependency Graph") _print_deps_tree(project) else: - print("No dependencies") + print ("No dependencies") return project diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index d7e17b0b..78295ea3 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -39,7 +39,9 @@ class InoToCPPConverter(object): ([a-z_\d]+\s*) # name of prototype \([a-z_,\.\*\&\[\]\s\d]*\) # arguments )\s*(\{|;) # must end with `{` or `;` - """, re.X | re.M | re.I) + """, + re.X | re.M | re.I, + ) DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I) PROTOPTRS_TPLRE = r"\([^&\(]*&(%s)[^\)]*\)" @@ -61,9 +63,7 @@ class InoToCPPConverter(object): lines = [] for node in nodes: contents = get_file_contents(node.get_path()) - _lines = [ - '# 1 "%s"' % node.get_path().replace("\\", "/"), contents - ] + _lines = ['# 1 "%s"' % node.get_path().replace("\\", "/"), contents] if self.is_main_node(contents): lines = _lines + lines self._main_ino = node.get_path() @@ -91,8 +91,11 @@ class InoToCPPConverter(object): self.env.Execute( self.env.VerboseAction( '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format( - out_file, tmp_path), - "Converting " + basename(out_file[:-4]))) + out_file, tmp_path + ), + "Converting " + basename(out_file[:-4]), + ) + ) atexit.register(_delete_file, tmp_path) return isfile(out_file) @@ -120,8 +123,9 @@ class InoToCPPConverter(object): elif stropen and line.endswith(('",', '";')): newlines[len(newlines) - 1] += line stropen = False - newlines.append('#line %d "%s"' % - (linenum, self._main_ino.replace("\\", "/"))) + newlines.append( + '#line %d "%s"' % (linenum, self._main_ino.replace("\\", "/")) + ) continue newlines.append(line) @@ -141,8 +145,10 @@ class InoToCPPConverter(object): prototypes = [] reserved_keywords = set(["if", "else", "while"]) for match in self.PROTOTYPE_RE.finditer(contents): - if (set([match.group(2).strip(), - match.group(3).strip()]) & reserved_keywords): + if ( + set([match.group(2).strip(), match.group(3).strip()]) + & reserved_keywords + ): continue prototypes.append(match) return prototypes @@ -162,11 +168,8 @@ class InoToCPPConverter(object): prototypes = self._parse_prototypes(contents) or [] # skip already declared prototypes - declared = set( - m.group(1).strip() for m in prototypes if m.group(4) == ";") - prototypes = [ - m for m in prototypes if m.group(1).strip() not in declared - ] + declared = set(m.group(1).strip() for m in prototypes if m.group(4) == ";") + prototypes = [m for m in prototypes if m.group(1).strip() not in declared] if not prototypes: return contents @@ -175,23 +178,29 @@ class InoToCPPConverter(object): split_pos = prototypes[0].start() match_ptrs = re.search( self.PROTOPTRS_TPLRE % ("|".join(prototype_names)), - contents[:split_pos], re.M) + contents[:split_pos], + re.M, + ) if match_ptrs: split_pos = contents.rfind("\n", 0, match_ptrs.start()) + 1 result = [] result.append(contents[:split_pos].strip()) result.append("%s;" % ";\n".join([m.group(1) for m in prototypes])) - result.append('#line %d "%s"' % (self._get_total_lines( - contents[:split_pos]), self._main_ino.replace("\\", "/"))) + result.append( + '#line %d "%s"' + % ( + self._get_total_lines(contents[:split_pos]), + self._main_ino.replace("\\", "/"), + ) + ) result.append(contents[split_pos:].strip()) return "\n".join(result) def ConvertInoToCpp(env): src_dir = glob_escape(env.subst("$PROJECTSRC_DIR")) - ino_nodes = (env.Glob(join(src_dir, "*.ino")) + - env.Glob(join(src_dir, "*.pde"))) + ino_nodes = env.Glob(join(src_dir, "*.ino")) + env.Glob(join(src_dir, "*.pde")) if not ino_nodes: return c = InoToCPPConverter(env) @@ -214,13 +223,13 @@ def _get_compiler_type(env): return "gcc" try: sysenv = environ.copy() - sysenv['PATH'] = str(env['ENV']['PATH']) + sysenv["PATH"] = str(env["ENV"]["PATH"]) result = exec_command([env.subst("$CC"), "-v"], env=sysenv) except OSError: return None - if result['returncode'] != 0: + if result["returncode"] != 0: return None - output = "".join([result['out'], result['err']]).lower() + output = "".join([result["out"], result["err"]]).lower() if "clang" in output and "LLVM" in output: return "clang" if "gcc" in output: @@ -233,7 +242,6 @@ def GetCompilerType(env): def GetActualLDScript(env): - def _lookup_in_ldpath(script): for d in env.get("LIBPATH", []): path = join(env.subst(d), script) @@ -264,12 +272,13 @@ def GetActualLDScript(env): if script: sys.stderr.write( - "Error: Could not find '%s' LD script in LDPATH '%s'\n" % - (script, env.subst("$LIBPATH"))) + "Error: Could not find '%s' LD script in LDPATH '%s'\n" + % (script, env.subst("$LIBPATH")) + ) env.Exit(1) if not script and "LDSCRIPT_PATH" in env: - path = _lookup_in_ldpath(env['LDSCRIPT_PATH']) + path = _lookup_in_ldpath(env["LDSCRIPT_PATH"]) if path: return path @@ -285,16 +294,17 @@ def VerboseAction(_, act, actstr): def PioClean(env, clean_dir): if not isdir(clean_dir): - print("Build environment is clean") + print ("Build environment is clean") env.Exit(0) clean_rel_path = relpath(clean_dir) for root, _, files in walk(clean_dir): for f in files: dst = join(root, f) remove(dst) - print("Removed %s" % - (dst if clean_rel_path.startswith(".") else relpath(dst))) - print("Done cleaning") + print ( + "Removed %s" % (dst if clean_rel_path.startswith(".") else relpath(dst)) + ) + print ("Done cleaning") fs.rmtree(clean_dir) env.Exit(0) @@ -302,8 +312,9 @@ def PioClean(env, clean_dir): def ProcessDebug(env): if not env.subst("$PIODEBUGFLAGS"): env.Replace(PIODEBUGFLAGS=["-Og", "-g3", "-ggdb3"]) - env.Append(BUILD_FLAGS=list(env['PIODEBUGFLAGS']) + - ["-D__PLATFORMIO_BUILD_DEBUG__"]) + env.Append( + BUILD_FLAGS=list(env["PIODEBUGFLAGS"]) + ["-D__PLATFORMIO_BUILD_DEBUG__"] + ) unflags = ["-Os"] for level in [0, 1, 2]: for flag in ("O", "g", "ggdb"): @@ -312,15 +323,18 @@ def ProcessDebug(env): def ProcessTest(env): - env.Append(CPPDEFINES=["UNIT_TEST", "UNITY_INCLUDE_CONFIG_H"], - CPPPATH=[join("$BUILD_DIR", "UnityTestLib")]) - unitylib = env.BuildLibrary(join("$BUILD_DIR", "UnityTestLib"), - get_core_package_dir("tool-unity")) + env.Append( + CPPDEFINES=["UNIT_TEST", "UNITY_INCLUDE_CONFIG_H"], + CPPPATH=[join("$BUILD_DIR", "UnityTestLib")], + ) + unitylib = env.BuildLibrary( + join("$BUILD_DIR", "UnityTestLib"), get_core_package_dir("tool-unity") + ) env.Prepend(LIBS=[unitylib]) src_filter = ["+<*.cpp>", "+<*.c>"] if "PIOTEST_RUNNING_NAME" in env: - src_filter.append("+<%s%s>" % (env['PIOTEST_RUNNING_NAME'], sep)) + src_filter.append("+<%s%s>" % (env["PIOTEST_RUNNING_NAME"], sep)) env.Replace(PIOTEST_SRC_FILTER=src_filter) @@ -330,7 +344,7 @@ def GetExtraScripts(env, scope): if scope == "post" and ":" not in item: items.append(item) elif item.startswith("%s:" % scope): - items.append(item[len(scope) + 1:]) + items.append(item[len(scope) + 1 :]) if not items: return items with fs.cd(env.subst("$PROJECT_DIR")): diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 8179982e..9571ea75 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -33,8 +33,8 @@ def PioPlatform(env): variables = env.GetProjectOptions(as_dict=True) if "framework" in variables: # support PIO Core 3.0 dev/platforms - variables['pioframework'] = variables['framework'] - p = PlatformFactory.newPlatform(env['PLATFORM_MANIFEST']) + variables["pioframework"] = variables["framework"] + p = PlatformFactory.newPlatform(env["PLATFORM_MANIFEST"]) p.configure_default_packages(variables, COMMAND_LINE_TARGETS) return p @@ -54,7 +54,7 @@ def BoardConfig(env, board=None): def GetFrameworkScript(env, framework): p = env.PioPlatform() assert p.frameworks and framework in p.frameworks - script_path = env.subst(p.frameworks[framework]['script']) + script_path = env.subst(p.frameworks[framework]["script"]) if not isfile(script_path): script_path = join(p.get_dir(), script_path) return script_path @@ -65,7 +65,7 @@ def LoadPioPlatform(env): installed_packages = p.get_installed_packages() # Ensure real platform name - env['PIOPLATFORM'] = p.name + env["PIOPLATFORM"] = p.name # Add toolchains and uploaders to $PATH and $*_LIBRARY_PATH systype = util.get_systype() @@ -75,14 +75,13 @@ def LoadPioPlatform(env): continue pkg_dir = p.get_package_dir(name) env.PrependENVPath( - "PATH", - join(pkg_dir, "bin") if isdir(join(pkg_dir, "bin")) else pkg_dir) - if (not WINDOWS and isdir(join(pkg_dir, "lib")) - and type_ != "toolchain"): + "PATH", join(pkg_dir, "bin") if isdir(join(pkg_dir, "bin")) else pkg_dir + ) + if not WINDOWS and isdir(join(pkg_dir, "lib")) and type_ != "toolchain": env.PrependENVPath( - "DYLD_LIBRARY_PATH" - if "darwin" in systype else "LD_LIBRARY_PATH", - join(pkg_dir, "lib")) + "DYLD_LIBRARY_PATH" if "darwin" in systype else "LD_LIBRARY_PATH", + join(pkg_dir, "lib"), + ) # Platform specific LD Scripts if isdir(join(p.get_dir(), "ldscripts")): @@ -101,9 +100,11 @@ def LoadPioPlatform(env): for option_meta in ProjectOptions.values(): if not option_meta.buildenvvar or option_meta.buildenvvar in env: continue - data_path = (option_meta.name[6:] - if option_meta.name.startswith("board_") else - option_meta.name.replace("_", ".")) + data_path = ( + option_meta.name[6:] + if option_meta.name.startswith("board_") + else option_meta.name.replace("_", ".") + ) try: env[option_meta.buildenvvar] = board_config.get(data_path) except KeyError: @@ -118,22 +119,25 @@ def PrintConfiguration(env): # pylint: disable=too-many-statements board_config = env.BoardConfig() if "BOARD" in env else None def _get_configuration_data(): - return None if not board_config else [ - "CONFIGURATION:", - "https://docs.platformio.org/page/boards/%s/%s.html" % - (platform.name, board_config.id) - ] + return ( + None + if not board_config + else [ + "CONFIGURATION:", + "https://docs.platformio.org/page/boards/%s/%s.html" + % (platform.name, board_config.id), + ] + ) def _get_plaform_data(): data = ["PLATFORM: %s %s" % (platform.title, platform.version)] - src_manifest_path = platform.pm.get_src_manifest_path( - platform.get_dir()) + src_manifest_path = platform.pm.get_src_manifest_path(platform.get_dir()) if src_manifest_path: src_manifest = fs.load_json(src_manifest_path) if "version" in src_manifest: - data.append("#" + src_manifest['version']) + data.append("#" + src_manifest["version"]) if int(ARGUMENTS.get("PIOVERBOSE", 0)): - data.append("(%s)" % src_manifest['url']) + data.append("(%s)" % src_manifest["url"]) if board_config: data.extend([">", board_config.get("name")]) return data @@ -151,19 +155,22 @@ def PrintConfiguration(env): # pylint: disable=too-many-statements return data ram = board_config.get("upload", {}).get("maximum_ram_size") flash = board_config.get("upload", {}).get("maximum_size") - data.append("%s RAM, %s Flash" % - (fs.format_filesize(ram), fs.format_filesize(flash))) + data.append( + "%s RAM, %s Flash" % (fs.format_filesize(ram), fs.format_filesize(flash)) + ) return data def _get_debug_data(): - debug_tools = board_config.get( - "debug", {}).get("tools") if board_config else None + debug_tools = ( + board_config.get("debug", {}).get("tools") if board_config else None + ) if not debug_tools: return None data = [ - "DEBUG:", "Current", - "(%s)" % board_config.get_debug_tool_name( - env.GetProjectOption("debug_tool")) + "DEBUG:", + "Current", + "(%s)" + % board_config.get_debug_tool_name(env.GetProjectOption("debug_tool")), ] onboard = [] external = [] @@ -187,23 +194,27 @@ def PrintConfiguration(env): # pylint: disable=too-many-statements if not pkg_dir: continue manifest = platform.pm.load_manifest(pkg_dir) - original_version = util.get_original_version(manifest['version']) - info = "%s %s" % (manifest['name'], manifest['version']) + original_version = util.get_original_version(manifest["version"]) + info = "%s %s" % (manifest["name"], manifest["version"]) extra = [] if original_version: extra.append(original_version) if "__src_url" in manifest and int(ARGUMENTS.get("PIOVERBOSE", 0)): - extra.append(manifest['__src_url']) + extra.append(manifest["__src_url"]) if extra: info += " (%s)" % ", ".join(extra) data.append(info) return ["PACKAGES:", ", ".join(data)] - for data in (_get_configuration_data(), _get_plaform_data(), - _get_hardware_data(), _get_debug_data(), - _get_packages_data()): + for data in ( + _get_configuration_data(), + _get_plaform_data(), + _get_hardware_data(), + _get_debug_data(), + _get_packages_data(), + ): if data and len(data) > 1: - print(" ".join(data)) + print (" ".join(data)) def exists(_): diff --git a/platformio/builder/tools/pioproject.py b/platformio/builder/tools/pioproject.py index 614fb9bd..4a42b3d8 100644 --- a/platformio/builder/tools/pioproject.py +++ b/platformio/builder/tools/pioproject.py @@ -18,22 +18,25 @@ from platformio.project.config import ProjectConfig, ProjectOptions def GetProjectConfig(env): - return ProjectConfig.get_instance(env['PROJECT_CONFIG']) + return ProjectConfig.get_instance(env["PROJECT_CONFIG"]) def GetProjectOptions(env, as_dict=False): - return env.GetProjectConfig().items(env=env['PIOENV'], as_dict=as_dict) + return env.GetProjectConfig().items(env=env["PIOENV"], as_dict=as_dict) def GetProjectOption(env, option, default=None): - return env.GetProjectConfig().get("env:" + env['PIOENV'], option, default) + return env.GetProjectConfig().get("env:" + env["PIOENV"], option, default) def LoadProjectOptions(env): for option, value in env.GetProjectOptions(): option_meta = ProjectOptions.get("env." + option) - if (not option_meta or not option_meta.buildenvvar - or option_meta.buildenvvar in env): + if ( + not option_meta + or not option_meta.buildenvvar + or option_meta.buildenvvar in env + ): continue env[option_meta.buildenvvar] = value diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py index a7ea2f4e..a77c6726 100644 --- a/platformio/builder/tools/pioupload.py +++ b/platformio/builder/tools/pioupload.py @@ -45,7 +45,7 @@ def FlushSerialBuffer(env, port): def TouchSerialPort(env, port, baudrate): port = env.subst(port) - print("Forcing reset using %dbps open/close on port %s" % (baudrate, port)) + print ("Forcing reset using %dbps open/close on port %s" % (baudrate, port)) try: s = Serial(port=port, baudrate=baudrate) s.setDTR(False) @@ -56,13 +56,13 @@ def TouchSerialPort(env, port, baudrate): def WaitForNewSerialPort(env, before): - print("Waiting for the new upload port...") + print ("Waiting for the new upload port...") prev_port = env.subst("$UPLOAD_PORT") new_port = None elapsed = 0 - before = [p['port'] for p in before] + before = [p["port"] for p in before] while elapsed < 5 and new_port is None: - now = [p['port'] for p in util.get_serial_ports()] + now = [p["port"] for p in util.get_serial_ports()] for p in now: if p not in before: new_port = p @@ -84,10 +84,12 @@ def WaitForNewSerialPort(env, before): sleep(1) if not new_port: - sys.stderr.write("Error: Couldn't find a board on the selected port. " - "Check that you have the correct port selected. " - "If it is correct, try pressing the board's reset " - "button after initiating the upload.\n") + sys.stderr.write( + "Error: Couldn't find a board on the selected port. " + "Check that you have the correct port selected. " + "If it is correct, try pressing the board's reset " + "button after initiating the upload.\n" + ) env.Exit(1) return new_port @@ -99,8 +101,8 @@ def AutodetectUploadPort(*args, **kwargs): def _get_pattern(): if "UPLOAD_PORT" not in env: return None - if set(["*", "?", "[", "]"]) & set(env['UPLOAD_PORT']): - return env['UPLOAD_PORT'] + if set(["*", "?", "[", "]"]) & set(env["UPLOAD_PORT"]): + return env["UPLOAD_PORT"] return None def _is_match_pattern(port): @@ -112,17 +114,13 @@ def AutodetectUploadPort(*args, **kwargs): def _look_for_mbed_disk(): msdlabels = ("mbed", "nucleo", "frdm", "microbit") for item in util.get_logical_devices(): - if item['path'].startswith("/net") or not _is_match_pattern( - item['path']): + if item["path"].startswith("/net") or not _is_match_pattern(item["path"]): continue - mbed_pages = [ - join(item['path'], n) for n in ("mbed.htm", "mbed.html") - ] + mbed_pages = [join(item["path"], n) for n in ("mbed.htm", "mbed.html")] if any(isfile(p) for p in mbed_pages): - return item['path'] - if item['name'] \ - and any(l in item['name'].lower() for l in msdlabels): - return item['path'] + return item["path"] + if item["name"] and any(l in item["name"].lower() for l in msdlabels): + return item["path"] return None def _look_for_serial_port(): @@ -132,27 +130,27 @@ def AutodetectUploadPort(*args, **kwargs): if "BOARD" in env and "build.hwids" in env.BoardConfig(): board_hwids = env.BoardConfig().get("build.hwids") for item in util.get_serial_ports(filter_hwid=True): - if not _is_match_pattern(item['port']): + if not _is_match_pattern(item["port"]): continue - port = item['port'] + port = item["port"] if upload_protocol.startswith("blackmagic"): if WINDOWS and port.startswith("COM") and len(port) > 4: port = "\\\\.\\%s" % port - if "GDB" in item['description']: + if "GDB" in item["description"]: return port for hwid in board_hwids: hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "") - if hwid_str in item['hwid']: + if hwid_str in item["hwid"]: return port return port if "UPLOAD_PORT" in env and not _get_pattern(): - print(env.subst("Use manually specified: $UPLOAD_PORT")) + print (env.subst("Use manually specified: $UPLOAD_PORT")) return - if (env.subst("$UPLOAD_PROTOCOL") == "mbed" - or ("mbed" in env.subst("$PIOFRAMEWORK") - and not env.subst("$UPLOAD_PROTOCOL"))): + if env.subst("$UPLOAD_PROTOCOL") == "mbed" or ( + "mbed" in env.subst("$PIOFRAMEWORK") and not env.subst("$UPLOAD_PROTOCOL") + ): env.Replace(UPLOAD_PORT=_look_for_mbed_disk()) else: try: @@ -162,13 +160,14 @@ def AutodetectUploadPort(*args, **kwargs): env.Replace(UPLOAD_PORT=_look_for_serial_port()) if env.subst("$UPLOAD_PORT"): - print(env.subst("Auto-detected: $UPLOAD_PORT")) + print (env.subst("Auto-detected: $UPLOAD_PORT")) else: sys.stderr.write( "Error: Please specify `upload_port` for environment or use " "global `--upload-port` option.\n" "For some development platforms it can be a USB flash " - "drive (i.e. /media//)\n") + "drive (i.e. /media//)\n" + ) env.Exit(1) @@ -179,16 +178,17 @@ def UploadToDisk(_, target, source, env): fpath = join(env.subst("$BUILD_DIR"), "%s.%s" % (progname, ext)) if not isfile(fpath): continue - copyfile(fpath, - join(env.subst("$UPLOAD_PORT"), "%s.%s" % (progname, ext))) - print("Firmware has been successfully uploaded.\n" - "(Some boards may require manual hard reset)") + copyfile(fpath, join(env.subst("$UPLOAD_PORT"), "%s.%s" % (progname, ext))) + print ( + "Firmware has been successfully uploaded.\n" + "(Some boards may require manual hard reset)" + ) def CheckUploadSize(_, target, source, env): check_conditions = [ env.get("BOARD"), - env.get("SIZETOOL") or env.get("SIZECHECKCMD") + env.get("SIZETOOL") or env.get("SIZECHECKCMD"), ] if not all(check_conditions): return @@ -198,9 +198,11 @@ def CheckUploadSize(_, target, source, env): return def _configure_defaults(): - env.Replace(SIZECHECKCMD="$SIZETOOL -B -d $SOURCES", - SIZEPROGREGEXP=r"^(\d+)\s+(\d+)\s+\d+\s", - SIZEDATAREGEXP=r"^\d+\s+(\d+)\s+(\d+)\s+\d+") + env.Replace( + SIZECHECKCMD="$SIZETOOL -B -d $SOURCES", + SIZEPROGREGEXP=r"^(\d+)\s+(\d+)\s+\d+\s", + SIZEDATAREGEXP=r"^\d+\s+(\d+)\s+(\d+)\s+\d+", + ) def _get_size_output(): cmd = env.get("SIZECHECKCMD") @@ -210,11 +212,11 @@ def CheckUploadSize(_, target, source, env): cmd = cmd.split() cmd = [arg.replace("$SOURCES", str(source[0])) for arg in cmd if arg] sysenv = environ.copy() - sysenv['PATH'] = str(env['ENV']['PATH']) + sysenv["PATH"] = str(env["ENV"]["PATH"]) result = exec_command(env.subst(cmd), env=sysenv) - if result['returncode'] != 0: + if result["returncode"] != 0: return None - return result['out'].strip() + return result["out"].strip() def _calculate_size(output, pattern): if not output or not pattern: @@ -238,7 +240,8 @@ def CheckUploadSize(_, target, source, env): if used_blocks > blocks_per_progress: used_blocks = blocks_per_progress return "[{:{}}] {: 6.1%} (used {:d} bytes from {:d} bytes)".format( - "=" * used_blocks, blocks_per_progress, percent_raw, value, total) + "=" * used_blocks, blocks_per_progress, percent_raw, value, total + ) if not env.get("SIZECHECKCMD") and not env.get("SIZEPROGREGEXP"): _configure_defaults() @@ -246,14 +249,13 @@ def CheckUploadSize(_, target, source, env): program_size = _calculate_size(output, env.get("SIZEPROGREGEXP")) data_size = _calculate_size(output, env.get("SIZEDATAREGEXP")) - print("Memory Usage -> http://bit.ly/pio-memory-usage") + print ("Memory Usage -> http://bit.ly/pio-memory-usage") if data_max_size and data_size > -1: - print("DATA: %s" % _format_availale_bytes(data_size, data_max_size)) + print ("DATA: %s" % _format_availale_bytes(data_size, data_max_size)) if program_size > -1: - print("PROGRAM: %s" % - _format_availale_bytes(program_size, program_max_size)) + print ("PROGRAM: %s" % _format_availale_bytes(program_size, program_max_size)) if int(ARGUMENTS.get("PIOVERBOSE", 0)): - print(output) + print (output) # raise error # if data_max_size and data_size > data_max_size: @@ -262,9 +264,10 @@ def CheckUploadSize(_, target, source, env): # "than maximum allowed (%s bytes)\n" % (data_size, data_max_size)) # env.Exit(1) if program_size > program_max_size: - sys.stderr.write("Error: The program size (%d bytes) is greater " - "than maximum allowed (%s bytes)\n" % - (program_size, program_max_size)) + sys.stderr.write( + "Error: The program size (%d bytes) is greater " + "than maximum allowed (%s bytes)\n" % (program_size, program_max_size) + ) env.Exit(1) @@ -272,12 +275,11 @@ def PrintUploadInfo(env): configured = env.subst("$UPLOAD_PROTOCOL") available = [configured] if configured else [] if "BOARD" in env: - available.extend(env.BoardConfig().get("upload", - {}).get("protocols", [])) + available.extend(env.BoardConfig().get("upload", {}).get("protocols", [])) if available: - print("AVAILABLE: %s" % ", ".join(sorted(set(available)))) + print ("AVAILABLE: %s" % ", ".join(sorted(set(available)))) if configured: - print("CURRENT: upload_protocol = %s" % configured) + print ("CURRENT: upload_protocol = %s" % configured) def exists(_): diff --git a/platformio/builder/tools/piowinhooks.py b/platformio/builder/tools/piowinhooks.py index 3679897d..a3187291 100644 --- a/platformio/builder/tools/piowinhooks.py +++ b/platformio/builder/tools/piowinhooks.py @@ -61,8 +61,9 @@ def _file_long_data(env, data): build_dir = env.subst("$BUILD_DIR") if not isdir(build_dir): makedirs(build_dir) - tmp_file = join(build_dir, - "longcmd-%s" % md5(hashlib_encode_data(data)).hexdigest()) + tmp_file = join( + build_dir, "longcmd-%s" % md5(hashlib_encode_data(data)).hexdigest() + ) if isfile(tmp_file): return tmp_file with open(tmp_file, "w") as fp: @@ -83,10 +84,12 @@ def generate(env): coms = {} for key in ("ARCOM", "LINKCOM"): coms[key] = env.get(key, "").replace( - "$SOURCES", "${_long_sources_hook(__env__, SOURCES)}") + "$SOURCES", "${_long_sources_hook(__env__, SOURCES)}" + ) for key in ("_CCCOMCOM", "ASPPCOM"): coms[key] = env.get(key, "").replace( - "$_CPPINCFLAGS", "${_long_incflags_hook(__env__, _CPPINCFLAGS)}") + "$_CPPINCFLAGS", "${_long_incflags_hook(__env__, _CPPINCFLAGS)}" + ) env.Replace(**coms) return env diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 057d39aa..a33dafb3 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -54,7 +54,8 @@ def _build_project_deps(env): key: project_lib_builder.env.get(key) for key in ("LIBS", "LIBPATH", "LINKFLAGS") if project_lib_builder.env.get(key) - }) + } + ) projenv = env.Clone() @@ -65,27 +66,32 @@ def _build_project_deps(env): is_test = "__test" in COMMAND_LINE_TARGETS if is_test: - projenv.BuildSources("$BUILDTEST_DIR", "$PROJECTTEST_DIR", - "$PIOTEST_SRC_FILTER") + projenv.BuildSources( + "$BUILDTEST_DIR", "$PROJECTTEST_DIR", "$PIOTEST_SRC_FILTER" + ) if not is_test or env.GetProjectOption("test_build_project_src", False): - projenv.BuildSources("$BUILDSRC_DIR", "$PROJECTSRC_DIR", - env.get("SRC_FILTER")) + projenv.BuildSources("$BUILDSRC_DIR", "$PROJECTSRC_DIR", env.get("SRC_FILTER")) if not env.get("PIOBUILDFILES") and not COMMAND_LINE_TARGETS: sys.stderr.write( "Error: Nothing to build. Please put your source code files " - "to '%s' folder\n" % env.subst("$PROJECTSRC_DIR")) + "to '%s' folder\n" % env.subst("$PROJECTSRC_DIR") + ) env.Exit(1) Export("projenv") def BuildProgram(env): - def _append_pio_macros(): - env.AppendUnique(CPPDEFINES=[( - "PLATFORMIO", - int("{0:02d}{1:02d}{2:02d}".format(*pioversion_to_intstr())))]) + env.AppendUnique( + CPPDEFINES=[ + ( + "PLATFORMIO", + int("{0:02d}{1:02d}{2:02d}".format(*pioversion_to_intstr())), + ) + ] + ) _append_pio_macros() @@ -95,8 +101,7 @@ def BuildProgram(env): if not Util.case_sensitive_suffixes(".s", ".S"): env.Replace(AS="$CC", ASCOM="$ASPPCOM") - if ("debug" in COMMAND_LINE_TARGETS - or env.GetProjectOption("build_type") == "debug"): + if "debug" in COMMAND_LINE_TARGETS or env.GetProjectOption("build_type") == "debug": env.ProcessDebug() # process extra flags from board @@ -122,8 +127,7 @@ def BuildProgram(env): _build_project_deps(env) # append into the beginning a main LD script - if (env.get("LDSCRIPT_PATH") - and not any("-Wl,-T" in f for f in env['LINKFLAGS'])): + if env.get("LDSCRIPT_PATH") and not any("-Wl,-T" in f for f in env["LINKFLAGS"]): env.Prepend(LINKFLAGS=["-T", "$LDSCRIPT_PATH"]) # enable "cyclic reference" for linker @@ -131,15 +135,18 @@ def BuildProgram(env): env.Prepend(_LIBFLAGS="-Wl,--start-group ") env.Append(_LIBFLAGS=" -Wl,--end-group") - program = env.Program(join("$BUILD_DIR", env.subst("$PROGNAME")), - env['PIOBUILDFILES']) + program = env.Program( + join("$BUILD_DIR", env.subst("$PROGNAME")), env["PIOBUILDFILES"] + ) env.Replace(PIOMAINPROG=program) AlwaysBuild( env.Alias( - "checkprogsize", program, - env.VerboseAction(env.CheckUploadSize, - "Checking size $PIOMAINPROG"))) + "checkprogsize", + program, + env.VerboseAction(env.CheckUploadSize, "Checking size $PIOMAINPROG"), + ) + ) return program @@ -155,19 +162,19 @@ def ParseFlagsExtended(env, flags): # pylint: disable=too-many-branches result[key].extend(value) cppdefines = [] - for item in result['CPPDEFINES']: + for item in result["CPPDEFINES"]: if not Util.is_Sequence(item): cppdefines.append(item) continue name, value = item[:2] - if '\"' in value: - value = value.replace('\"', '\\\"') + if '"' in value: + value = value.replace('"', '\\"') elif value.isdigit(): value = int(value) elif value.replace(".", "", 1).isdigit(): value = float(value) cppdefines.append((name, value)) - result['CPPDEFINES'] = cppdefines + result["CPPDEFINES"] = cppdefines # fix relative CPPPATH & LIBPATH for k in ("CPPPATH", "LIBPATH"): @@ -178,7 +185,7 @@ def ParseFlagsExtended(env, flags): # pylint: disable=too-many-branches # fix relative path for "-include" for i, f in enumerate(result.get("CCFLAGS", [])): if isinstance(f, tuple) and f[0] == "-include": - result['CCFLAGS'][i] = (f[0], env.File(realpath(f[1].get_path()))) + result["CCFLAGS"][i] = (f[0], env.File(realpath(f[1].get_path()))) return result @@ -191,14 +198,15 @@ def ProcessFlags(env, flags): # pylint: disable=too-many-branches # Cancel any previous definition of name, either built in or # provided with a -U option // Issue #191 undefines = [ - u for u in env.get("CCFLAGS", []) + u + for u in env.get("CCFLAGS", []) if isinstance(u, string_types) and u.startswith("-U") ] if undefines: for undef in undefines: - env['CCFLAGS'].remove(undef) - if undef[2:] in env['CPPDEFINES']: - env['CPPDEFINES'].remove(undef[2:]) + env["CCFLAGS"].remove(undef) + if undef[2:] in env["CPPDEFINES"]: + env["CPPDEFINES"].remove(undef[2:]) env.Append(_CPPDEFFLAGS=" %s" % " ".join(undefines)) @@ -221,8 +229,7 @@ def ProcessUnFlags(env, flags): for current in env.get(key, []): conditions = [ unflag == current, - isinstance(current, (tuple, list)) - and unflag[0] == current[0] + isinstance(current, (tuple, list)) and unflag[0] == current[0], ] if any(conditions): env[key].remove(current) @@ -231,15 +238,12 @@ def ProcessUnFlags(env, flags): def MatchSourceFiles(env, src_dir, src_filter=None): src_filter = env.subst(src_filter) if src_filter else None src_filter = src_filter or SRC_FILTER_DEFAULT - return fs.match_src_files(env.subst(src_dir), src_filter, - SRC_BUILD_EXT + SRC_HEADER_EXT) + return fs.match_src_files( + env.subst(src_dir), src_filter, SRC_BUILD_EXT + SRC_HEADER_EXT + ) -def CollectBuildFiles(env, - variant_dir, - src_dir, - src_filter=None, - duplicate=False): +def CollectBuildFiles(env, variant_dir, src_dir, src_filter=None, duplicate=False): sources = [] variants = [] @@ -267,8 +271,10 @@ def BuildFrameworks(env, frameworks): return if "BOARD" not in env: - sys.stderr.write("Please specify `board` in `platformio.ini` to use " - "with '%s' framework\n" % ", ".join(frameworks)) + sys.stderr.write( + "Please specify `board` in `platformio.ini` to use " + "with '%s' framework\n" % ", ".join(frameworks) + ) env.Exit(1) board_frameworks = env.BoardConfig().get("frameworks", []) @@ -276,8 +282,7 @@ def BuildFrameworks(env, frameworks): if board_frameworks: frameworks.insert(0, board_frameworks[0]) else: - sys.stderr.write( - "Error: Please specify `board` in `platformio.ini`\n") + sys.stderr.write("Error: Please specify `board` in `platformio.ini`\n") env.Exit(1) for f in frameworks: @@ -290,22 +295,20 @@ def BuildFrameworks(env, frameworks): if f in board_frameworks: SConscript(env.GetFrameworkScript(f), exports="env") else: - sys.stderr.write( - "Error: This board doesn't support %s framework!\n" % f) + sys.stderr.write("Error: This board doesn't support %s framework!\n" % f) env.Exit(1) def BuildLibrary(env, variant_dir, src_dir, src_filter=None): env.ProcessUnFlags(env.get("BUILD_UNFLAGS")) return env.StaticLibrary( - env.subst(variant_dir), - env.CollectBuildFiles(variant_dir, src_dir, src_filter)) + env.subst(variant_dir), env.CollectBuildFiles(variant_dir, src_dir, src_filter) + ) def BuildSources(env, variant_dir, src_dir, src_filter=None): nodes = env.CollectBuildFiles(variant_dir, src_dir, src_filter) - DefaultEnvironment().Append( - PIOBUILDFILES=[env.Object(node) for node in nodes]) + DefaultEnvironment().Append(PIOBUILDFILES=[env.Object(node) for node in nodes]) def exists(_): diff --git a/platformio/commands/__init__.py b/platformio/commands/__init__.py index 5d53349e..5dd6d34b 100644 --- a/platformio/commands/__init__.py +++ b/platformio/commands/__init__.py @@ -25,10 +25,14 @@ class PlatformioCLI(click.MultiCommand): @staticmethod def in_silence(): args = PlatformioCLI.leftover_args - return args and any([ - args[0] == "debug" and "--interpreter" in " ".join(args), - args[0] == "upgrade", "--json-output" in args, "--version" in args - ]) + return args and any( + [ + args[0] == "debug" and "--interpreter" in " ".join(args), + args[0] == "upgrade", + "--json-output" in args, + "--version" in args, + ] + ) def invoke(self, ctx): PlatformioCLI.leftover_args = ctx.args @@ -52,8 +56,7 @@ class PlatformioCLI(click.MultiCommand): def get_command(self, ctx, cmd_name): mod = None try: - mod = __import__("platformio.commands." + cmd_name, None, None, - ["cli"]) + mod = __import__("platformio.commands." + cmd_name, None, None, ["cli"]) except ImportError: try: return self._handle_obsolate_command(cmd_name) @@ -65,8 +68,10 @@ class PlatformioCLI(click.MultiCommand): def _handle_obsolate_command(name): if name == "platforms": from platformio.commands import platform + return platform.cli if name == "serialports": from platformio.commands import device + return device.cli raise AttributeError() diff --git a/platformio/commands/boards.py b/platformio/commands/boards.py index bf3f68ef..335d1b34 100644 --- a/platformio/commands/boards.py +++ b/platformio/commands/boards.py @@ -34,9 +34,9 @@ def cli(query, installed, json_output): # pylint: disable=R0912 for board in _get_boards(installed): if query and query.lower() not in json.dumps(board).lower(): continue - if board['platform'] not in grpboards: - grpboards[board['platform']] = [] - grpboards[board['platform']].append(board) + if board["platform"] not in grpboards: + grpboards[board["platform"]] = [] + grpboards[board["platform"]].append(board) terminal_width, _ = click.get_terminal_size() for (platform, boards) in sorted(grpboards.items()): @@ -50,11 +50,21 @@ def cli(query, installed, json_output): # pylint: disable=R0912 def print_boards(boards): click.echo( - tabulate([(click.style(b['id'], fg="cyan"), b['mcu'], "%dMHz" % - (b['fcpu'] / 1000000), fs.format_filesize( - b['rom']), fs.format_filesize(b['ram']), b['name']) - for b in boards], - headers=["ID", "MCU", "Frequency", "Flash", "RAM", "Name"])) + tabulate( + [ + ( + click.style(b["id"], fg="cyan"), + b["mcu"], + "%dMHz" % (b["fcpu"] / 1000000), + fs.format_filesize(b["rom"]), + fs.format_filesize(b["ram"]), + b["name"], + ) + for b in boards + ], + headers=["ID", "MCU", "Frequency", "Flash", "RAM", "Name"], + ) + ) def _get_boards(installed=False): @@ -66,7 +76,7 @@ def _print_boards_json(query, installed=False): result = [] for board in _get_boards(installed): if query: - search_data = "%s %s" % (board['id'], json.dumps(board).lower()) + search_data = "%s %s" % (board["id"], json.dumps(board).lower()) if query.lower() not in search_data.lower(): continue result.append(board) diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py index 28c4431d..0bd0a1bd 100644 --- a/platformio/commands/check/command.py +++ b/platformio/commands/check/command.py @@ -28,39 +28,50 @@ from platformio.commands.check.defect import DefectItem from platformio.commands.check.tools import CheckToolFactory from platformio.compat import dump_json_to_unicode from platformio.project.config import ProjectConfig -from platformio.project.helpers import (find_project_dir_above, - get_project_dir, - get_project_include_dir, - get_project_src_dir) +from platformio.project.helpers import ( + find_project_dir_above, + get_project_dir, + get_project_include_dir, + get_project_src_dir, +) @click.command("check", short_help="Run a static analysis tool on code") @click.option("-e", "--environment", multiple=True) -@click.option("-d", - "--project-dir", - default=os.getcwd, - type=click.Path(exists=True, - file_okay=True, - dir_okay=True, - writable=True, - resolve_path=True)) -@click.option("-c", - "--project-conf", - type=click.Path(exists=True, - file_okay=True, - dir_okay=False, - readable=True, - resolve_path=True)) +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path( + exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True + ), +) +@click.option( + "-c", + "--project-conf", + type=click.Path( + exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True + ), +) @click.option("--filter", multiple=True, help="Pattern: + -") @click.option("--flags", multiple=True) -@click.option("--severity", - multiple=True, - type=click.Choice(DefectItem.SEVERITY_LABELS.values())) +@click.option( + "--severity", multiple=True, type=click.Choice(DefectItem.SEVERITY_LABELS.values()) +) @click.option("-s", "--silent", is_flag=True) @click.option("-v", "--verbose", is_flag=True) @click.option("--json-output", is_flag=True) -def cli(environment, project_dir, project_conf, filter, flags, severity, - silent, verbose, json_output): +def cli( + environment, + project_dir, + project_conf, + filter, + flags, + severity, + silent, + verbose, + json_output, +): # find project directory on upper level if isfile(project_dir): project_dir = find_project_dir_above(project_dir) @@ -68,15 +79,18 @@ def cli(environment, project_dir, project_conf, filter, flags, severity, results = [] with fs.cd(project_dir): config = ProjectConfig.get_instance( - project_conf or join(project_dir, "platformio.ini")) + project_conf or join(project_dir, "platformio.ini") + ) config.validate(environment) default_envs = config.default_envs() for envname in config.envs(): - skipenv = any([ - environment and envname not in environment, not environment - and default_envs and envname not in default_envs - ]) + skipenv = any( + [ + environment and envname not in environment, + not environment and default_envs and envname not in default_envs, + ] + ) env_options = config.items(env=envname, as_dict=True) env_dump = [] @@ -84,7 +98,8 @@ def cli(environment, project_dir, project_conf, filter, flags, severity, if k not in ("platform", "framework", "board"): continue env_dump.append( - "%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v)) + "%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v) + ) default_filter = [ "+<%s/>" % basename(d) @@ -94,13 +109,12 @@ def cli(environment, project_dir, project_conf, filter, flags, severity, tool_options = dict( verbose=verbose, silent=silent, - filter=filter - or env_options.get("check_filter", default_filter), + filter=filter or env_options.get("check_filter", default_filter), flags=flags or env_options.get("check_flags"), - severity=[ - DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_HIGH] - ] if silent else - (severity or env_options.get("check_severity"))) + severity=[DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_HIGH]] + if silent + else (severity or env_options.get("check_severity")), + ) for tool in env_options.get("check_tool", ["cppcheck"]): if skipenv: @@ -109,26 +123,29 @@ def cli(environment, project_dir, project_conf, filter, flags, severity, if not silent and not json_output: print_processing_header(tool, envname, env_dump) - ct = CheckToolFactory.new(tool, project_dir, config, envname, - tool_options) + ct = CheckToolFactory.new( + tool, project_dir, config, envname, tool_options + ) result = {"env": envname, "tool": tool, "duration": time()} - rc = ct.check(on_defect_callback=None if ( - json_output or verbose - ) else lambda defect: click.echo(repr(defect))) + rc = ct.check( + on_defect_callback=None + if (json_output or verbose) + else lambda defect: click.echo(repr(defect)) + ) - result['defects'] = ct.get_defects() - result['duration'] = time() - result['duration'] - result['succeeded'] = ( - rc == 0 and not any(d.severity == DefectItem.SEVERITY_HIGH - for d in result['defects'])) + result["defects"] = ct.get_defects() + result["duration"] = time() - result["duration"] + result["succeeded"] = rc == 0 and not any( + d.severity == DefectItem.SEVERITY_HIGH for d in result["defects"] + ) results.append(result) if verbose: - click.echo("\n".join(repr(d) for d in result['defects'])) + click.echo("\n".join(repr(d) for d in result["defects"])) if not json_output and not silent: - if not result['defects']: + if not result["defects"]: click.echo("No defects found") print_processing_footer(result) @@ -145,11 +162,13 @@ def cli(environment, project_dir, project_conf, filter, flags, severity, def results_to_json(raw): results = [] for item in raw: - item.update({ - "ignored": item.get("succeeded") is None, - "succeeded": bool(item.get("succeeded")), - "defects": [d.to_json() for d in item.get("defects", [])] - }) + item.update( + { + "ignored": item.get("succeeded") is None, + "succeeded": bool(item.get("succeeded")), + "defects": [d.to_json() for d in item.get("defects", [])], + } + ) results.append(item) return results @@ -157,8 +176,9 @@ def results_to_json(raw): def print_processing_header(tool, envname, envdump): click.echo( - "Checking %s > %s (%s)" % - (click.style(envname, fg="cyan", bold=True), tool, "; ".join(envdump))) + "Checking %s > %s (%s)" + % (click.style(envname, fg="cyan", bold=True), tool, "; ".join(envdump)) + ) terminal_width, _ = click.get_terminal_size() click.secho("-" * terminal_width, bold=True) @@ -166,10 +186,17 @@ def print_processing_header(tool, envname, envdump): def print_processing_footer(result): is_failed = not result.get("succeeded") util.print_labeled_bar( - "[%s] Took %.2f seconds" % - ((click.style("FAILED", fg="red", bold=True) if is_failed else - click.style("PASSED", fg="green", bold=True)), result['duration']), - is_error=is_failed) + "[%s] Took %.2f seconds" + % ( + ( + click.style("FAILED", fg="red", bold=True) + if is_failed + else click.style("PASSED", fg="green", bold=True) + ), + result["duration"], + ), + is_error=is_failed, + ) def print_defects_stats(results): @@ -178,8 +205,7 @@ def print_defects_stats(results): def _append_defect(component, defect): if not components.get(component): components[component] = Counter() - components[component].update( - {DefectItem.SEVERITY_LABELS[defect.severity]: 1}) + components[component].update({DefectItem.SEVERITY_LABELS[defect.severity]: 1}) for result in results: for defect in result.get("defects", []): @@ -235,20 +261,32 @@ def print_check_summary(results): status_str = click.style("PASSED", fg="green") tabular_data.append( - (click.style(result['env'], fg="cyan"), result['tool'], status_str, - util.humanize_duration_time(result.get("duration")))) + ( + click.style(result["env"], fg="cyan"), + result["tool"], + status_str, + util.humanize_duration_time(result.get("duration")), + ) + ) - click.echo(tabulate(tabular_data, - headers=[ - click.style(s, bold=True) - for s in ("Environment", "Tool", "Status", - "Duration") - ]), - err=failed_nums) + click.echo( + tabulate( + tabular_data, + headers=[ + click.style(s, bold=True) + for s in ("Environment", "Tool", "Status", "Duration") + ], + ), + err=failed_nums, + ) util.print_labeled_bar( - "%s%d succeeded in %s" % - ("%d failed, " % failed_nums if failed_nums else "", succeeded_nums, - util.humanize_duration_time(duration)), + "%s%d succeeded in %s" + % ( + "%d failed, " % failed_nums if failed_nums else "", + succeeded_nums, + util.humanize_duration_time(duration), + ), is_error=failed_nums, - fg="red" if failed_nums else "green") + fg="red" if failed_nums else "green", + ) diff --git a/platformio/commands/check/defect.py b/platformio/commands/check/defect.py index 7c2fbcff..0b25084c 100644 --- a/platformio/commands/check/defect.py +++ b/platformio/commands/check/defect.py @@ -29,18 +29,19 @@ class DefectItem(object): SEVERITY_LOW = 4 SEVERITY_LABELS = {4: "low", 2: "medium", 1: "high"} - def __init__(self, - severity, - category, - message, - file="unknown", - line=0, - column=0, - id=None, - callstack=None, - cwe=None): - assert severity in (self.SEVERITY_HIGH, self.SEVERITY_MEDIUM, - self.SEVERITY_LOW) + def __init__( + self, + severity, + category, + message, + file="unknown", + line=0, + column=0, + id=None, + callstack=None, + cwe=None, + ): + assert severity in (self.SEVERITY_HIGH, self.SEVERITY_MEDIUM, self.SEVERITY_LOW) self.severity = severity self.category = category self.message = message @@ -61,14 +62,14 @@ class DefectItem(object): defect_color = "yellow" format_str = "{file}:{line}: [{severity}:{category}] {message} {id}" - return format_str.format(severity=click.style( - self.SEVERITY_LABELS[self.severity], fg=defect_color), - category=click.style(self.category.lower(), - fg=defect_color), - file=click.style(self.file, bold=True), - message=self.message, - line=self.line, - id="%s" % "[%s]" % self.id if self.id else "") + return format_str.format( + severity=click.style(self.SEVERITY_LABELS[self.severity], fg=defect_color), + category=click.style(self.category.lower(), fg=defect_color), + file=click.style(self.file, bold=True), + message=self.message, + line=self.line, + id="%s" % "[%s]" % self.id if self.id else "", + ) def __or__(self, defect): return self.severity | defect.severity @@ -90,5 +91,5 @@ class DefectItem(object): "column": self.column, "callstack": self.callstack, "id": self.id, - "cwe": self.cwe + "cwe": self.cwe, } diff --git a/platformio/commands/check/tools/__init__.py b/platformio/commands/check/tools/__init__.py index e76cf48b..e3c4b190 100644 --- a/platformio/commands/check/tools/__init__.py +++ b/platformio/commands/check/tools/__init__.py @@ -18,7 +18,6 @@ from platformio.commands.check.tools.cppcheck import CppcheckCheckTool class CheckToolFactory(object): - @staticmethod def new(tool, project_dir, config, envname, options): cls = None @@ -27,6 +26,5 @@ class CheckToolFactory(object): elif tool == "clangtidy": cls = ClangtidyCheckTool else: - raise exception.PlatformioException("Unknown check tool `%s`" % - tool) + raise exception.PlatformioException("Unknown check tool `%s`" % tool) return cls(project_dir, config, envname, options) diff --git a/platformio/commands/check/tools/base.py b/platformio/commands/check/tools/base.py index 7d6224b2..d9454dd0 100644 --- a/platformio/commands/check/tools/base.py +++ b/platformio/commands/check/tools/base.py @@ -20,7 +20,6 @@ from platformio.project.helpers import get_project_dir, load_project_ide_data class CheckToolBase(object): # pylint: disable=too-many-instance-attributes - def __init__(self, project_dir, config, envname, options): self.config = config self.envname = envname @@ -35,14 +34,15 @@ class CheckToolBase(object): # pylint: disable=too-many-instance-attributes # detect all defects by default if not self.options.get("severity"): - self.options['severity'] = [ - DefectItem.SEVERITY_LOW, DefectItem.SEVERITY_MEDIUM, - DefectItem.SEVERITY_HIGH + self.options["severity"] = [ + DefectItem.SEVERITY_LOW, + DefectItem.SEVERITY_MEDIUM, + DefectItem.SEVERITY_HIGH, ] # cast to severity by ids - self.options['severity'] = [ + self.options["severity"] = [ s if isinstance(s, int) else DefectItem.severity_to_int(s) - for s in self.options['severity'] + for s in self.options["severity"] ] def _load_cpp_data(self, project_dir, envname): @@ -51,8 +51,7 @@ class CheckToolBase(object): # pylint: disable=too-many-instance-attributes return self.cpp_includes = data.get("includes", []) self.cpp_defines = data.get("defines", []) - self.cpp_defines.extend( - self._get_toolchain_defines(data.get("cc_path"))) + self.cpp_defines.extend(self._get_toolchain_defines(data.get("cc_path"))) def get_flags(self, tool): result = [] @@ -61,18 +60,16 @@ class CheckToolBase(object): # pylint: disable=too-many-instance-attributes if ":" not in flag: result.extend([f for f in flag.split(" ") if f]) elif flag.startswith("%s:" % tool): - result.extend( - [f for f in flag.split(":", 1)[1].split(" ") if f]) + result.extend([f for f in flag.split(":", 1)[1].split(" ") if f]) return result @staticmethod def _get_toolchain_defines(cc_path): defines = [] - result = proc.exec_command("echo | %s -dM -E -x c++ -" % cc_path, - shell=True) + result = proc.exec_command("echo | %s -dM -E -x c++ -" % cc_path, shell=True) - for line in result['out'].split("\n"): + for line in result["out"].split("\n"): tokens = line.strip().split(" ", 2) if not tokens or tokens[0] != "#define": continue @@ -105,7 +102,7 @@ class CheckToolBase(object): # pylint: disable=too-many-instance-attributes click.echo(line) return - if defect.severity not in self.options['severity']: + if defect.severity not in self.options["severity"]: return self._defects.append(defect) @@ -125,8 +122,9 @@ class CheckToolBase(object): # pylint: disable=too-many-instance-attributes def get_project_src_files(self): file_extensions = ["h", "hpp", "c", "cc", "cpp", "ino"] - return fs.match_src_files(get_project_dir(), - self.options.get("filter"), file_extensions) + return fs.match_src_files( + get_project_dir(), self.options.get("filter"), file_extensions + ) def check(self, on_defect_callback=None): self._on_defect_callback = on_defect_callback @@ -137,7 +135,8 @@ class CheckToolBase(object): # pylint: disable=too-many-instance-attributes proc.exec_command( cmd, stdout=proc.LineBufferedAsyncPipe(self.on_tool_output), - stderr=proc.LineBufferedAsyncPipe(self.on_tool_output)) + stderr=proc.LineBufferedAsyncPipe(self.on_tool_output), + ) self.clean_up() diff --git a/platformio/commands/check/tools/clangtidy.py b/platformio/commands/check/tools/clangtidy.py index 843d02dc..c4154c52 100644 --- a/platformio/commands/check/tools/clangtidy.py +++ b/platformio/commands/check/tools/clangtidy.py @@ -21,10 +21,8 @@ from platformio.managers.core import get_core_package_dir class ClangtidyCheckTool(CheckToolBase): - def tool_output_filter(self, line): - if not self.options.get( - "verbose") and "[clang-diagnostic-error]" in line: + if not self.options.get("verbose") and "[clang-diagnostic-error]" in line: return "" if "[CommonOptionsParser]" in line: @@ -37,8 +35,7 @@ class ClangtidyCheckTool(CheckToolBase): return "" def parse_defect(self, raw_line): - match = re.match(r"^(.*):(\d+):(\d+):\s+([^:]+):\s(.+)\[([^]]+)\]$", - raw_line) + match = re.match(r"^(.*):(\d+):(\d+):\s+([^:]+):\s(.+)\[([^]]+)\]$", raw_line) if not match: return raw_line @@ -50,8 +47,7 @@ class ClangtidyCheckTool(CheckToolBase): elif category == "warning": severity = DefectItem.SEVERITY_MEDIUM - return DefectItem(severity, category, message, file_, line, column, - defect_id) + return DefectItem(severity, category, message, file_, line, column, defect_id) def configure_command(self): tool_path = join(get_core_package_dir("tool-clangtidy"), "clang-tidy") diff --git a/platformio/commands/check/tools/cppcheck.py b/platformio/commands/check/tools/cppcheck.py index e74031bd..3279a704 100644 --- a/platformio/commands/check/tools/cppcheck.py +++ b/platformio/commands/check/tools/cppcheck.py @@ -23,29 +23,42 @@ from platformio.project.helpers import get_project_core_dir class CppcheckCheckTool(CheckToolBase): - def __init__(self, *args, **kwargs): self._tmp_files = [] self.defect_fields = [ - "severity", "message", "file", "line", "column", "callstack", - "cwe", "id" + "severity", + "message", + "file", + "line", + "column", + "callstack", + "cwe", + "id", ] super(CppcheckCheckTool, self).__init__(*args, **kwargs) def tool_output_filter(self, line): - if not self.options.get( - "verbose") and "--suppress=unmatchedSuppression:" in line: + if ( + not self.options.get("verbose") + and "--suppress=unmatchedSuppression:" in line + ): return "" - if any(msg in line for msg in ("No C or C++ source files found", - "unrecognized command line option")): + if any( + msg in line + for msg in ( + "No C or C++ source files found", + "unrecognized command line option", + ) + ): self._bad_input = True return line def parse_defect(self, raw_line): - if "<&PIO&>" not in raw_line or any(f not in raw_line - for f in self.defect_fields): + if "<&PIO&>" not in raw_line or any( + f not in raw_line for f in self.defect_fields + ): return None args = dict() @@ -54,13 +67,13 @@ class CppcheckCheckTool(CheckToolBase): name, value = field.split("=", 1) args[name] = value - args['category'] = args['severity'] - if args['severity'] == "error": - args['severity'] = DefectItem.SEVERITY_HIGH - elif args['severity'] == "warning": - args['severity'] = DefectItem.SEVERITY_MEDIUM + args["category"] = args["severity"] + if args["severity"] == "error": + args["severity"] = DefectItem.SEVERITY_HIGH + elif args["severity"] == "warning": + args["severity"] = DefectItem.SEVERITY_MEDIUM else: - args['severity'] = DefectItem.SEVERITY_LOW + args["severity"] = DefectItem.SEVERITY_LOW return DefectItem(**args) @@ -68,20 +81,26 @@ class CppcheckCheckTool(CheckToolBase): tool_path = join(get_core_package_dir("tool-cppcheck"), "cppcheck") cmd = [ - tool_path, "--error-exitcode=1", - "--verbose" if self.options.get("verbose") else "--quiet" + tool_path, + "--error-exitcode=1", + "--verbose" if self.options.get("verbose") else "--quiet", ] - cmd.append('--template="%s"' % "<&PIO&>".join( - ["{0}={{{0}}}".format(f) for f in self.defect_fields])) + cmd.append( + '--template="%s"' + % "<&PIO&>".join(["{0}={{{0}}}".format(f) for f in self.defect_fields]) + ) flags = self.get_flags("cppcheck") if not self.is_flag_set("--platform", flags): cmd.append("--platform=unspecified") if not self.is_flag_set("--enable", flags): enabled_checks = [ - "warning", "style", "performance", "portability", - "unusedFunction" + "warning", + "style", + "performance", + "portability", + "unusedFunction", ] cmd.append("--enable=%s" % ",".join(enabled_checks)) diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index 8ad68c2b..6b98aacd 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -48,37 +48,37 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument @click.command("ci", short_help="Continuous Integration") @click.argument("src", nargs=-1, callback=validate_path) -@click.option("-l", - "--lib", - multiple=True, - callback=validate_path, - metavar="DIRECTORY") +@click.option("-l", "--lib", multiple=True, callback=validate_path, metavar="DIRECTORY") @click.option("--exclude", multiple=True) -@click.option("-b", - "--board", - multiple=True, - metavar="ID", - callback=validate_boards) -@click.option("--build-dir", - default=mkdtemp, - type=click.Path(file_okay=False, - dir_okay=True, - writable=True, - resolve_path=True)) +@click.option("-b", "--board", multiple=True, metavar="ID", callback=validate_boards) +@click.option( + "--build-dir", + default=mkdtemp, + type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True), +) @click.option("--keep-build-dir", is_flag=True) -@click.option("-c", - "--project-conf", - type=click.Path(exists=True, - file_okay=True, - dir_okay=False, - readable=True, - resolve_path=True)) +@click.option( + "-c", + "--project-conf", + type=click.Path( + exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True + ), +) @click.option("-O", "--project-option", multiple=True) @click.option("-v", "--verbose", is_flag=True) @click.pass_context def cli( # pylint: disable=too-many-arguments, too-many-branches - ctx, src, lib, exclude, board, build_dir, keep_build_dir, project_conf, - project_option, verbose): + ctx, + src, + lib, + exclude, + board, + build_dir, + keep_build_dir, + project_conf, + project_option, + verbose, +): if not src and getenv("PLATFORMIO_CI_SRC"): src = validate_path(ctx, None, getenv("PLATFORMIO_CI_SRC").split(":")) @@ -110,10 +110,9 @@ def cli( # pylint: disable=too-many-arguments, too-many-branches _exclude_contents(build_dir, exclude) # initialise project - ctx.invoke(cmd_init, - project_dir=build_dir, - board=board, - project_option=project_option) + ctx.invoke( + cmd_init, project_dir=build_dir, board=board, project_option=project_option + ) # process project ctx.invoke(cmd_run, project_dir=build_dir, verbose=verbose) @@ -127,27 +126,27 @@ def _copy_contents(dst_dir, contents): for path in contents: if isdir(path): - items['dirs'].add(path) + items["dirs"].add(path) elif isfile(path): - items['files'].add(path) + items["files"].add(path) dst_dir_name = basename(dst_dir) - if dst_dir_name == "src" and len(items['dirs']) == 1: - copytree(list(items['dirs']).pop(), dst_dir, symlinks=True) + if dst_dir_name == "src" and len(items["dirs"]) == 1: + copytree(list(items["dirs"]).pop(), dst_dir, symlinks=True) else: if not isdir(dst_dir): makedirs(dst_dir) - for d in items['dirs']: + for d in items["dirs"]: copytree(d, join(dst_dir, basename(d)), symlinks=True) - if not items['files']: + if not items["files"]: return if dst_dir_name == "lib": dst_dir = join(dst_dir, mkdtemp(dir=dst_dir)) - for f in items['files']: + for f in items["files"]: dst_file = join(dst_dir, basename(f)) if f == dst_file: continue diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py index 4a00cb9e..2334fbf9 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/client.py @@ -53,8 +53,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes if not isdir(get_project_cache_dir()): os.makedirs(get_project_cache_dir()) - self._gdbsrc_dir = mkdtemp(dir=get_project_cache_dir(), - prefix=".piodebug-") + self._gdbsrc_dir = mkdtemp(dir=get_project_cache_dir(), prefix=".piodebug-") self._target_is_run = False self._last_server_activity = 0 @@ -70,25 +69,28 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes "PROG_PATH": prog_path, "PROG_DIR": dirname(prog_path), "PROG_NAME": basename(splitext(prog_path)[0]), - "DEBUG_PORT": self.debug_options['port'], - "UPLOAD_PROTOCOL": self.debug_options['upload_protocol'], - "INIT_BREAK": self.debug_options['init_break'] or "", - "LOAD_CMDS": "\n".join(self.debug_options['load_cmds'] or []), + "DEBUG_PORT": self.debug_options["port"], + "UPLOAD_PROTOCOL": self.debug_options["upload_protocol"], + "INIT_BREAK": self.debug_options["init_break"] or "", + "LOAD_CMDS": "\n".join(self.debug_options["load_cmds"] or []), } self._debug_server.spawn(patterns) - if not patterns['DEBUG_PORT']: - patterns['DEBUG_PORT'] = self._debug_server.get_debug_port() + if not patterns["DEBUG_PORT"]: + patterns["DEBUG_PORT"] = self._debug_server.get_debug_port() self.generate_pioinit(self._gdbsrc_dir, patterns) # start GDB client args = [ "piogdb", "-q", - "--directory", self._gdbsrc_dir, - "--directory", self.project_dir, - "-l", "10" + "--directory", + self._gdbsrc_dir, + "--directory", + self.project_dir, + "-l", + "10", ] # yapf: disable args.extend(self.args) if not gdb_path: @@ -96,13 +98,11 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes gdb_data_dir = self._get_data_dir(gdb_path) if gdb_data_dir: args.extend(["--data-directory", gdb_data_dir]) - args.append(patterns['PROG_PATH']) + args.append(patterns["PROG_PATH"]) - return reactor.spawnProcess(self, - gdb_path, - args, - path=self.project_dir, - env=os.environ) + return reactor.spawnProcess( + self, gdb_path, args, path=self.project_dir, env=os.environ + ) @staticmethod def _get_data_dir(gdb_path): @@ -112,8 +112,9 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes return gdb_data_dir if isdir(gdb_data_dir) else None def generate_pioinit(self, dst_dir, patterns): - server_exe = (self.debug_options.get("server") - or {}).get("executable", "").lower() + server_exe = ( + (self.debug_options.get("server") or {}).get("executable", "").lower() + ) if "jlink" in server_exe: cfg = initcfgs.GDB_JLINK_INIT_CONFIG elif "st-util" in server_exe: @@ -122,43 +123,43 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes cfg = initcfgs.GDB_MSPDEBUG_INIT_CONFIG elif "qemu" in server_exe: cfg = initcfgs.GDB_QEMU_INIT_CONFIG - elif self.debug_options['require_debug_port']: + elif self.debug_options["require_debug_port"]: cfg = initcfgs.GDB_BLACKMAGIC_INIT_CONFIG else: cfg = initcfgs.GDB_DEFAULT_INIT_CONFIG commands = cfg.split("\n") - if self.debug_options['init_cmds']: - commands = self.debug_options['init_cmds'] - commands.extend(self.debug_options['extra_cmds']) + if self.debug_options["init_cmds"]: + commands = self.debug_options["init_cmds"] + commands.extend(self.debug_options["extra_cmds"]) if not any("define pio_reset_target" in cmd for cmd in commands): commands = [ "define pio_reset_target", " echo Warning! Undefined pio_reset_target command\\n", " mon reset", - "end" + "end", ] + commands # yapf: disable if not any("define pio_reset_halt_target" in cmd for cmd in commands): commands = [ "define pio_reset_halt_target", " echo Warning! Undefined pio_reset_halt_target command\\n", " mon reset halt", - "end" + "end", ] + commands # yapf: disable if not any("define pio_restart_target" in cmd for cmd in commands): commands += [ "define pio_restart_target", " pio_reset_halt_target", " $INIT_BREAK", - " %s" % ("continue" if patterns['INIT_BREAK'] else "next"), - "end" + " %s" % ("continue" if patterns["INIT_BREAK"] else "next"), + "end", ] # yapf: disable banner = [ "echo PlatformIO Unified Debugger -> http://bit.ly/pio-debug\\n", - "echo PlatformIO: debug_tool = %s\\n" % self.debug_options['tool'], - "echo PlatformIO: Initializing remote target...\\n" + "echo PlatformIO: debug_tool = %s\\n" % self.debug_options["tool"], + "echo PlatformIO: Initializing remote target...\\n", ] footer = ["echo %s\\n" % self.INIT_COMPLETED_BANNER] commands = banner + commands + footer @@ -214,8 +215,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes self._handle_error(data) # go to init break automatically if self.INIT_COMPLETED_BANNER.encode() in data: - self._auto_continue_timer = task.LoopingCall( - self._auto_exec_continue) + self._auto_continue_timer = task.LoopingCall(self._auto_exec_continue) self._auto_continue_timer.start(0.1) def errReceived(self, data): @@ -236,29 +236,34 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes self._auto_continue_timer.stop() self._auto_continue_timer = None - if not self.debug_options['init_break'] or self._target_is_run: + if not self.debug_options["init_break"] or self._target_is_run: return self.console_log( - "PlatformIO: Resume the execution to `debug_init_break = %s`" % - self.debug_options['init_break']) - self.console_log("PlatformIO: More configuration options -> " - "http://bit.ly/pio-debug") - self.transport.write(b"0-exec-continue\n" if helpers. - is_mi_mode(self.args) else b"continue\n") + "PlatformIO: Resume the execution to `debug_init_break = %s`" + % self.debug_options["init_break"] + ) + self.console_log( + "PlatformIO: More configuration options -> " "http://bit.ly/pio-debug" + ) + self.transport.write( + b"0-exec-continue\n" if helpers.is_mi_mode(self.args) else b"continue\n" + ) self._target_is_run = True def _handle_error(self, data): - if (self.PIO_SRC_NAME.encode() not in data - or b"Error in sourced" not in data): + if self.PIO_SRC_NAME.encode() not in data or b"Error in sourced" not in data: return configuration = {"debug": self.debug_options, "env": self.env_options} exd = re.sub(r'\\(?!")', "/", json.dumps(configuration)) - exd = re.sub(r'"(?:[a-z]\:)?((/[^"/]+)+)"', - lambda m: '"%s"' % join(*m.group(1).split("/")[-2:]), exd, - re.I | re.M) + exd = re.sub( + r'"(?:[a-z]\:)?((/[^"/]+)+)"', + lambda m: '"%s"' % join(*m.group(1).split("/")[-2:]), + exd, + re.I | re.M, + ) mp = MeasurementProtocol() - mp['exd'] = "DebugGDBPioInitError: %s" % exd - mp['exf'] = 1 + mp["exd"] = "DebugGDBPioInitError: %s" % exd + mp["exf"] = 1 mp.send("exception") self.transport.loseConnection() diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index 8e78e728..64e62765 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -25,35 +25,35 @@ from platformio import exception, fs, proc, util from platformio.commands.debug import helpers from platformio.managers.core import inject_contrib_pysite from platformio.project.config import ProjectConfig -from platformio.project.helpers import (is_platformio_project, - load_project_ide_data) +from platformio.project.helpers import is_platformio_project, load_project_ide_data -@click.command("debug", - context_settings=dict(ignore_unknown_options=True), - short_help="PIO Unified Debugger") -@click.option("-d", - "--project-dir", - default=os.getcwd, - type=click.Path(exists=True, - file_okay=False, - dir_okay=True, - writable=True, - resolve_path=True)) -@click.option("-c", - "--project-conf", - type=click.Path(exists=True, - file_okay=True, - dir_okay=False, - readable=True, - resolve_path=True)) +@click.command( + "debug", + context_settings=dict(ignore_unknown_options=True), + short_help="PIO Unified Debugger", +) +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path( + exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True + ), +) +@click.option( + "-c", + "--project-conf", + type=click.Path( + exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True + ), +) @click.option("--environment", "-e", metavar="") @click.option("--verbose", "-v", is_flag=True) @click.option("--interface", type=click.Choice(["gdb"])) @click.argument("__unprocessed", nargs=-1, type=click.UNPROCESSED) @click.pass_context -def cli(ctx, project_dir, project_conf, environment, verbose, interface, - __unprocessed): +def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unprocessed): # use env variables from Eclipse or CLion for sysenv in ("CWD", "PWD", "PLATFORMIO_PROJECT_DIR"): if is_platformio_project(project_dir): @@ -63,7 +63,8 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, with fs.cd(project_dir): config = ProjectConfig.get_instance( - project_conf or join(project_dir, "platformio.ini")) + project_conf or join(project_dir, "platformio.ini") + ) config.validate(envs=[environment] if environment else None) env_name = environment or helpers.get_default_debug_env(config) @@ -74,68 +75,64 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, assert debug_options if not interface: - return helpers.predebug_project(ctx, project_dir, env_name, False, - verbose) + return helpers.predebug_project(ctx, project_dir, env_name, False, verbose) configuration = load_project_ide_data(project_dir, env_name) if not configuration: - raise exception.DebugInvalidOptions( - "Could not load debug configuration") + raise exception.DebugInvalidOptions("Could not load debug configuration") if "--version" in __unprocessed: - result = proc.exec_command([configuration['gdb_path'], "--version"]) - if result['returncode'] == 0: - return click.echo(result['out']) - raise exception.PlatformioException("\n".join( - [result['out'], result['err']])) + result = proc.exec_command([configuration["gdb_path"], "--version"]) + if result["returncode"] == 0: + return click.echo(result["out"]) + raise exception.PlatformioException("\n".join([result["out"], result["err"]])) try: fs.ensure_udev_rules() except exception.InvalidUdevRules as e: for line in str(e).split("\n") + [""]: click.echo( - ('~"%s\\n"' if helpers.is_mi_mode(__unprocessed) else "%s") % - line) + ('~"%s\\n"' if helpers.is_mi_mode(__unprocessed) else "%s") % line + ) - debug_options['load_cmds'] = helpers.configure_esp32_load_cmds( - debug_options, configuration) + debug_options["load_cmds"] = helpers.configure_esp32_load_cmds( + debug_options, configuration + ) rebuild_prog = False - preload = debug_options['load_cmds'] == ["preload"] - load_mode = debug_options['load_mode'] + preload = debug_options["load_cmds"] == ["preload"] + load_mode = debug_options["load_mode"] if load_mode == "always": - rebuild_prog = ( - preload - or not helpers.has_debug_symbols(configuration['prog_path'])) + rebuild_prog = preload or not helpers.has_debug_symbols( + configuration["prog_path"] + ) elif load_mode == "modified": - rebuild_prog = ( - helpers.is_prog_obsolete(configuration['prog_path']) - or not helpers.has_debug_symbols(configuration['prog_path'])) + rebuild_prog = helpers.is_prog_obsolete( + configuration["prog_path"] + ) or not helpers.has_debug_symbols(configuration["prog_path"]) else: - rebuild_prog = not isfile(configuration['prog_path']) + rebuild_prog = not isfile(configuration["prog_path"]) if preload or (not rebuild_prog and load_mode != "always"): # don't load firmware through debug server - debug_options['load_cmds'] = [] + debug_options["load_cmds"] = [] if rebuild_prog: if helpers.is_mi_mode(__unprocessed): click.echo('~"Preparing firmware for debugging...\\n"') output = helpers.GDBBytesIO() with util.capture_std_streams(output): - helpers.predebug_project(ctx, project_dir, env_name, preload, - verbose) + helpers.predebug_project(ctx, project_dir, env_name, preload, verbose) output.close() else: click.echo("Preparing firmware for debugging...") - helpers.predebug_project(ctx, project_dir, env_name, preload, - verbose) + helpers.predebug_project(ctx, project_dir, env_name, preload, verbose) # save SHA sum of newly created prog if load_mode == "modified": - helpers.is_prog_obsolete(configuration['prog_path']) + helpers.is_prog_obsolete(configuration["prog_path"]) - if not isfile(configuration['prog_path']): + if not isfile(configuration["prog_path"]): raise exception.DebugInvalidOptions("Program/firmware is missed") # run debugging client @@ -143,7 +140,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, from platformio.commands.debug.client import GDBClient, reactor client = GDBClient(project_dir, __unprocessed, debug_options, env_options) - client.spawn(configuration['gdb_path'], configuration['prog_path']) + client.spawn(configuration["gdb_path"], configuration["prog_path"]) signal.signal(signal.SIGINT, lambda *args, **kwargs: None) reactor.run() diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index 33e1e855..d88307e8 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -20,8 +20,7 @@ from io import BytesIO from os.path import isfile from platformio import exception, fs, util -from platformio.commands.platform import \ - platform_install as cmd_platform_install +from platformio.commands.platform import platform_install as cmd_platform_install from platformio.commands.run import cli as cmd_run from platformio.managers.platform import PlatformFactory from platformio.project.config import ProjectConfig @@ -57,41 +56,41 @@ def get_default_debug_env(config): def predebug_project(ctx, project_dir, env_name, preload, verbose): - ctx.invoke(cmd_run, - project_dir=project_dir, - environment=[env_name], - target=["debug"] + (["upload"] if preload else []), - verbose=verbose) + ctx.invoke( + cmd_run, + project_dir=project_dir, + environment=[env_name], + target=["debug"] + (["upload"] if preload else []), + verbose=verbose, + ) if preload: time.sleep(5) def validate_debug_options(cmd_ctx, env_options): - def _cleanup_cmds(items): items = ProjectConfig.parse_multi_values(items) - return [ - "$LOAD_CMDS" if item == "$LOAD_CMD" else item for item in items - ] + return ["$LOAD_CMDS" if item == "$LOAD_CMD" else item for item in items] try: - platform = PlatformFactory.newPlatform(env_options['platform']) + platform = PlatformFactory.newPlatform(env_options["platform"]) except exception.UnknownPlatform: - cmd_ctx.invoke(cmd_platform_install, - platforms=[env_options['platform']], - skip_default_package=True) - platform = PlatformFactory.newPlatform(env_options['platform']) + cmd_ctx.invoke( + cmd_platform_install, + platforms=[env_options["platform"]], + skip_default_package=True, + ) + platform = PlatformFactory.newPlatform(env_options["platform"]) - board_config = platform.board_config(env_options['board']) + board_config = platform.board_config(env_options["board"]) tool_name = board_config.get_debug_tool_name(env_options.get("debug_tool")) - tool_settings = board_config.get("debug", {}).get("tools", - {}).get(tool_name, {}) + tool_settings = board_config.get("debug", {}).get("tools", {}).get(tool_name, {}) server_options = None # specific server per a system if isinstance(tool_settings.get("server", {}), list): - for item in tool_settings['server'][:]: - tool_settings['server'] = item + for item in tool_settings["server"][:]: + tool_settings["server"] = item if util.get_systype() in item.get("system", []): break @@ -100,76 +99,87 @@ def validate_debug_options(cmd_ctx, env_options): server_options = { "cwd": None, "executable": None, - "arguments": env_options.get("debug_server") + "arguments": env_options.get("debug_server"), } - server_options['executable'] = server_options['arguments'][0] - server_options['arguments'] = server_options['arguments'][1:] + server_options["executable"] = server_options["arguments"][0] + server_options["arguments"] = server_options["arguments"][1:] elif "server" in tool_settings: - server_package = tool_settings['server'].get("package") - server_package_dir = platform.get_package_dir( - server_package) if server_package else None + server_package = tool_settings["server"].get("package") + server_package_dir = ( + platform.get_package_dir(server_package) if server_package else None + ) if server_package and not server_package_dir: - platform.install_packages(with_packages=[server_package], - skip_default_package=True, - silent=True) + platform.install_packages( + with_packages=[server_package], skip_default_package=True, silent=True + ) server_package_dir = platform.get_package_dir(server_package) server_options = dict( cwd=server_package_dir if server_package else None, - executable=tool_settings['server'].get("executable"), + executable=tool_settings["server"].get("executable"), arguments=[ a.replace("$PACKAGE_DIR", server_package_dir) - if server_package_dir else a - for a in tool_settings['server'].get("arguments", []) - ]) + if server_package_dir + else a + for a in tool_settings["server"].get("arguments", []) + ], + ) extra_cmds = _cleanup_cmds(env_options.get("debug_extra_cmds")) extra_cmds.extend(_cleanup_cmds(tool_settings.get("extra_cmds"))) result = dict( tool=tool_name, upload_protocol=env_options.get( - "upload_protocol", - board_config.get("upload", {}).get("protocol")), + "upload_protocol", board_config.get("upload", {}).get("protocol") + ), load_cmds=_cleanup_cmds( env_options.get( "debug_load_cmds", - tool_settings.get("load_cmds", - tool_settings.get("load_cmd", "load")))), - load_mode=env_options.get("debug_load_mode", - tool_settings.get("load_mode", "always")), + tool_settings.get("load_cmds", tool_settings.get("load_cmd", "load")), + ) + ), + load_mode=env_options.get( + "debug_load_mode", tool_settings.get("load_mode", "always") + ), init_break=env_options.get( - "debug_init_break", tool_settings.get("init_break", - "tbreak main")), + "debug_init_break", tool_settings.get("init_break", "tbreak main") + ), init_cmds=_cleanup_cmds( - env_options.get("debug_init_cmds", - tool_settings.get("init_cmds"))), + env_options.get("debug_init_cmds", tool_settings.get("init_cmds")) + ), extra_cmds=extra_cmds, require_debug_port=tool_settings.get("require_debug_port", False), port=reveal_debug_port( env_options.get("debug_port", tool_settings.get("port")), - tool_name, tool_settings), - server=server_options) + tool_name, + tool_settings, + ), + server=server_options, + ) return result def configure_esp32_load_cmds(debug_options, configuration): ignore_conds = [ - debug_options['load_cmds'] != ["load"], + debug_options["load_cmds"] != ["load"], "xtensa-esp32" not in configuration.get("cc_path", ""), - not configuration.get("flash_extra_images"), not all([ - isfile(item['path']) - for item in configuration.get("flash_extra_images") - ]) + not configuration.get("flash_extra_images"), + not all( + [isfile(item["path"]) for item in configuration.get("flash_extra_images")] + ), ] if any(ignore_conds): - return debug_options['load_cmds'] + return debug_options["load_cmds"] mon_cmds = [ 'monitor program_esp32 "{{{path}}}" {offset} verify'.format( - path=fs.to_unix_path(item['path']), offset=item['offset']) + path=fs.to_unix_path(item["path"]), offset=item["offset"] + ) for item in configuration.get("flash_extra_images") ] - mon_cmds.append('monitor program_esp32 "{%s.bin}" 0x10000 verify' % - fs.to_unix_path(configuration['prog_path'][:-4])) + mon_cmds.append( + 'monitor program_esp32 "{%s.bin}" 0x10000 verify' + % fs.to_unix_path(configuration["prog_path"][:-4]) + ) return mon_cmds @@ -181,7 +191,7 @@ def has_debug_symbols(prog_path): b".debug_abbrev": False, b" -Og": False, b" -g": False, - b"__PLATFORMIO_BUILD_DEBUG__": False + b"__PLATFORMIO_BUILD_DEBUG__": False, } with open(prog_path, "rb") as fp: last_data = b"" @@ -222,7 +232,6 @@ def is_prog_obsolete(prog_path): def reveal_debug_port(env_debug_port, tool_name, tool_settings): - def _get_pattern(): if not env_debug_port: return None @@ -238,18 +247,21 @@ def reveal_debug_port(env_debug_port, tool_name, tool_settings): def _look_for_serial_port(hwids): for item in util.get_serialports(filter_hwid=True): - if not _is_match_pattern(item['port']): + if not _is_match_pattern(item["port"]): continue - port = item['port'] + port = item["port"] if tool_name.startswith("blackmagic"): - if "windows" in util.get_systype() and \ - port.startswith("COM") and len(port) > 4: + if ( + "windows" in util.get_systype() + and port.startswith("COM") + and len(port) > 4 + ): port = "\\\\.\\%s" % port - if "GDB" in item['description']: + if "GDB" in item["description"]: return port for hwid in hwids: hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "") - if hwid_str in item['hwid']: + if hwid_str in item["hwid"]: return port return None @@ -261,5 +273,6 @@ def reveal_debug_port(env_debug_port, tool_name, tool_settings): debug_port = _look_for_serial_port(tool_settings.get("hwids", [])) if not debug_port: raise exception.DebugInvalidOptions( - "Please specify `debug_port` for environment") + "Please specify `debug_port` for environment" + ) return debug_port diff --git a/platformio/commands/debug/process.py b/platformio/commands/debug/process.py index e45b6c7a..d23f26b3 100644 --- a/platformio/commands/debug/process.py +++ b/platformio/commands/debug/process.py @@ -32,7 +32,7 @@ class BaseProcess(protocol.ProcessProtocol, object): COMMON_PATTERNS = { "PLATFORMIO_HOME_DIR": get_project_core_dir(), "PLATFORMIO_CORE_DIR": get_project_core_dir(), - "PYTHONEXE": get_pythonexe_path() + "PYTHONEXE": get_pythonexe_path(), } def apply_patterns(self, source, patterns=None): @@ -52,8 +52,7 @@ class BaseProcess(protocol.ProcessProtocol, object): if isinstance(source, string_types): source = _replace(source) elif isinstance(source, (list, dict)): - items = enumerate(source) if isinstance(source, - list) else source.items() + items = enumerate(source) if isinstance(source, list) else source.items() for key, value in items: if isinstance(value, string_types): source[key] = _replace(value) @@ -67,9 +66,9 @@ class BaseProcess(protocol.ProcessProtocol, object): with open(LOG_FILE, "ab") as fp: fp.write(data) while data: - chunk = data[:self.STDOUT_CHUNK_SIZE] + chunk = data[: self.STDOUT_CHUNK_SIZE] click.echo(chunk, nl=False) - data = data[self.STDOUT_CHUNK_SIZE:] + data = data[self.STDOUT_CHUNK_SIZE :] @staticmethod def errReceived(data): diff --git a/platformio/commands/debug/server.py b/platformio/commands/debug/server.py index e5bc6e45..213c413e 100644 --- a/platformio/commands/debug/server.py +++ b/platformio/commands/debug/server.py @@ -24,7 +24,6 @@ from platformio.proc import where_is_program class DebugServer(BaseProcess): - def __init__(self, debug_options, env_options): self.debug_options = debug_options self.env_options = env_options @@ -39,13 +38,16 @@ class DebugServer(BaseProcess): if not server: return None server = self.apply_patterns(server, patterns) - server_executable = server['executable'] + server_executable = server["executable"] if not server_executable: return None - if server['cwd']: - server_executable = join(server['cwd'], server_executable) - if ("windows" in systype and not server_executable.endswith(".exe") - and isfile(server_executable + ".exe")): + if server["cwd"]: + server_executable = join(server["cwd"], server_executable) + if ( + "windows" in systype + and not server_executable.endswith(".exe") + and isfile(server_executable + ".exe") + ): server_executable = server_executable + ".exe" if not isfile(server_executable): @@ -55,48 +57,56 @@ class DebugServer(BaseProcess): "\nCould not launch Debug Server '%s'. Please check that it " "is installed and is included in a system PATH\n\n" "See documentation or contact contact@platformio.org:\n" - "http://docs.platformio.org/page/plus/debugging.html\n" % - server_executable) + "http://docs.platformio.org/page/plus/debugging.html\n" + % server_executable + ) self._debug_port = ":3333" - openocd_pipe_allowed = all([ - not self.debug_options['port'], - "openocd" in server_executable - ]) # yapf: disable + openocd_pipe_allowed = all( + [not self.debug_options["port"], "openocd" in server_executable] + ) # yapf: disable if openocd_pipe_allowed: args = [] - if server['cwd']: - args.extend(["-s", server['cwd']]) - args.extend([ - "-c", "gdb_port pipe; tcl_port disabled; telnet_port disabled" - ]) - args.extend(server['arguments']) + if server["cwd"]: + args.extend(["-s", server["cwd"]]) + args.extend( + ["-c", "gdb_port pipe; tcl_port disabled; telnet_port disabled"] + ) + args.extend(server["arguments"]) str_args = " ".join( - [arg if arg.startswith("-") else '"%s"' % arg for arg in args]) + [arg if arg.startswith("-") else '"%s"' % arg for arg in args] + ) self._debug_port = '| "%s" %s' % (server_executable, str_args) self._debug_port = fs.to_unix_path(self._debug_port) else: env = os.environ.copy() # prepend server "lib" folder to LD path - if ("windows" not in systype and server['cwd'] - and isdir(join(server['cwd'], "lib"))): - ld_key = ("DYLD_LIBRARY_PATH" - if "darwin" in systype else "LD_LIBRARY_PATH") - env[ld_key] = join(server['cwd'], "lib") + if ( + "windows" not in systype + and server["cwd"] + and isdir(join(server["cwd"], "lib")) + ): + ld_key = ( + "DYLD_LIBRARY_PATH" if "darwin" in systype else "LD_LIBRARY_PATH" + ) + env[ld_key] = join(server["cwd"], "lib") if os.environ.get(ld_key): - env[ld_key] = "%s:%s" % (env[ld_key], - os.environ.get(ld_key)) + env[ld_key] = "%s:%s" % (env[ld_key], os.environ.get(ld_key)) # prepend BIN to PATH - if server['cwd'] and isdir(join(server['cwd'], "bin")): - env['PATH'] = "%s%s%s" % ( - join(server['cwd'], "bin"), os.pathsep, - os.environ.get("PATH", os.environ.get("Path", ""))) + if server["cwd"] and isdir(join(server["cwd"], "bin")): + env["PATH"] = "%s%s%s" % ( + join(server["cwd"], "bin"), + os.pathsep, + os.environ.get("PATH", os.environ.get("Path", "")), + ) self._transport = reactor.spawnProcess( self, - server_executable, [server_executable] + server['arguments'], - path=server['cwd'], - env=env) + server_executable, + [server_executable] + server["arguments"], + path=server["cwd"], + env=env, + ) if "mspdebug" in server_executable.lower(): self._debug_port = ":2000" elif "jlink" in server_executable.lower(): diff --git a/platformio/commands/device.py b/platformio/commands/device.py index d5ffe030..351ba641 100644 --- a/platformio/commands/device.py +++ b/platformio/commands/device.py @@ -36,27 +36,29 @@ def cli(): @click.option("--mdns", is_flag=True, help="List multicast DNS services") @click.option("--json-output", is_flag=True) def device_list( # pylint: disable=too-many-branches - serial, logical, mdns, json_output): + serial, logical, mdns, json_output +): if not logical and not mdns: serial = True data = {} if serial: - data['serial'] = util.get_serial_ports() + data["serial"] = util.get_serial_ports() if logical: - data['logical'] = util.get_logical_devices() + data["logical"] = util.get_logical_devices() if mdns: - data['mdns'] = util.get_mdns_services() + data["mdns"] = util.get_mdns_services() single_key = list(data)[0] if len(list(data)) == 1 else None if json_output: return click.echo( - dump_json_to_unicode(data[single_key] if single_key else data)) + dump_json_to_unicode(data[single_key] if single_key else data) + ) titles = { "serial": "Serial Ports", "logical": "Logical Devices", - "mdns": "Multicast DNS Services" + "mdns": "Multicast DNS Services", } for key, value in data.items(): @@ -66,31 +68,38 @@ def device_list( # pylint: disable=too-many-branches if key == "serial": for item in value: - click.secho(item['port'], fg="cyan") - click.echo("-" * len(item['port'])) - click.echo("Hardware ID: %s" % item['hwid']) - click.echo("Description: %s" % item['description']) + click.secho(item["port"], fg="cyan") + click.echo("-" * len(item["port"])) + click.echo("Hardware ID: %s" % item["hwid"]) + click.echo("Description: %s" % item["description"]) click.echo("") if key == "logical": for item in value: - click.secho(item['path'], fg="cyan") - click.echo("-" * len(item['path'])) - click.echo("Name: %s" % item['name']) + click.secho(item["path"], fg="cyan") + click.echo("-" * len(item["path"])) + click.echo("Name: %s" % item["name"]) click.echo("") if key == "mdns": for item in value: - click.secho(item['name'], fg="cyan") - click.echo("-" * len(item['name'])) - click.echo("Type: %s" % item['type']) - click.echo("IP: %s" % item['ip']) - click.echo("Port: %s" % item['port']) - if item['properties']: - click.echo("Properties: %s" % ("; ".join([ - "%s=%s" % (k, v) - for k, v in item['properties'].items() - ]))) + click.secho(item["name"], fg="cyan") + click.echo("-" * len(item["name"])) + click.echo("Type: %s" % item["type"]) + click.echo("IP: %s" % item["ip"]) + click.echo("Port: %s" % item["port"]) + if item["properties"]: + click.echo( + "Properties: %s" + % ( + "; ".join( + [ + "%s=%s" % (k, v) + for k, v in item["properties"].items() + ] + ) + ) + ) click.echo("") if single_key: @@ -102,66 +111,71 @@ def device_list( # pylint: disable=too-many-branches @cli.command("monitor", short_help="Monitor device (Serial)") @click.option("--port", "-p", help="Port, a number or a device name") @click.option("--baud", "-b", type=int, help="Set baud rate, default=9600") -@click.option("--parity", - default="N", - type=click.Choice(["N", "E", "O", "S", "M"]), - help="Set parity, default=N") -@click.option("--rtscts", - is_flag=True, - help="Enable RTS/CTS flow control, default=Off") -@click.option("--xonxoff", - is_flag=True, - help="Enable software flow control, default=Off") -@click.option("--rts", - default=None, - type=click.IntRange(0, 1), - help="Set initial RTS line state") -@click.option("--dtr", - default=None, - type=click.IntRange(0, 1), - help="Set initial DTR line state") +@click.option( + "--parity", + default="N", + type=click.Choice(["N", "E", "O", "S", "M"]), + help="Set parity, default=N", +) +@click.option("--rtscts", is_flag=True, help="Enable RTS/CTS flow control, default=Off") +@click.option( + "--xonxoff", is_flag=True, help="Enable software flow control, default=Off" +) +@click.option( + "--rts", default=None, type=click.IntRange(0, 1), help="Set initial RTS line state" +) +@click.option( + "--dtr", default=None, type=click.IntRange(0, 1), help="Set initial DTR line state" +) @click.option("--echo", is_flag=True, help="Enable local echo, default=Off") -@click.option("--encoding", - default="UTF-8", - help="Set the encoding for the serial port (e.g. hexlify, " - "Latin1, UTF-8), default: UTF-8") +@click.option( + "--encoding", + default="UTF-8", + help="Set the encoding for the serial port (e.g. hexlify, " + "Latin1, UTF-8), default: UTF-8", +) @click.option("--filter", "-f", multiple=True, help="Add text transformation") -@click.option("--eol", - default="CRLF", - type=click.Choice(["CR", "LF", "CRLF"]), - help="End of line mode, default=CRLF") -@click.option("--raw", - is_flag=True, - help="Do not apply any encodings/transformations") -@click.option("--exit-char", - type=int, - default=3, - help="ASCII code of special character that is used to exit " - "the application, default=3 (Ctrl+C)") -@click.option("--menu-char", - type=int, - default=20, - help="ASCII code of special character that is used to " - "control miniterm (menu), default=20 (DEC)") -@click.option("--quiet", - is_flag=True, - help="Diagnostics: suppress non-error messages, default=Off") -@click.option("-d", - "--project-dir", - default=getcwd, - type=click.Path(exists=True, - file_okay=False, - dir_okay=True, - resolve_path=True)) +@click.option( + "--eol", + default="CRLF", + type=click.Choice(["CR", "LF", "CRLF"]), + help="End of line mode, default=CRLF", +) +@click.option("--raw", is_flag=True, help="Do not apply any encodings/transformations") +@click.option( + "--exit-char", + type=int, + default=3, + help="ASCII code of special character that is used to exit " + "the application, default=3 (Ctrl+C)", +) +@click.option( + "--menu-char", + type=int, + default=20, + help="ASCII code of special character that is used to " + "control miniterm (menu), default=20 (DEC)", +) +@click.option( + "--quiet", + is_flag=True, + help="Diagnostics: suppress non-error messages, default=Off", +) +@click.option( + "-d", + "--project-dir", + default=getcwd, + type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), +) @click.option( "-e", "--environment", - help="Load configuration from `platformio.ini` and specified environment") + help="Load configuration from `platformio.ini` and specified environment", +) def device_monitor(**kwargs): # pylint: disable=too-many-branches env_options = {} try: - env_options = get_project_options(kwargs['project_dir'], - kwargs['environment']) + env_options = get_project_options(kwargs["project_dir"], kwargs["environment"]) for k in ("port", "speed", "rts", "dtr"): k2 = "monitor_%s" % k if k == "speed": @@ -173,10 +187,10 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches except exception.NotPlatformIOProject: pass - if not kwargs['port']: + if not kwargs["port"]: ports = util.get_serial_ports(filter_hwid=True) if len(ports) == 1: - kwargs['port'] = ports[0]['port'] + kwargs["port"] = ports[0]["port"] sys.argv = ["monitor"] + env_options.get("monitor_flags", []) for k, v in kwargs.items(): @@ -194,17 +208,19 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches else: sys.argv.extend([k, str(v)]) - if kwargs['port'] and (set(["*", "?", "[", "]"]) & set(kwargs['port'])): + if kwargs["port"] and (set(["*", "?", "[", "]"]) & set(kwargs["port"])): for item in util.get_serial_ports(): - if fnmatch(item['port'], kwargs['port']): - kwargs['port'] = item['port'] + if fnmatch(item["port"], kwargs["port"]): + kwargs["port"] = item["port"] break try: - miniterm.main(default_port=kwargs['port'], - default_baudrate=kwargs['baud'] or 9600, - default_rts=kwargs['rts'], - default_dtr=kwargs['dtr']) + miniterm.main( + default_port=kwargs["port"], + default_baudrate=kwargs["baud"] or 9600, + default_rts=kwargs["rts"], + default_dtr=kwargs["dtr"], + ) except Exception as e: raise exception.MinitermException(e) diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py index d1b7d607..f3c748a9 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home/command.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=too-many-locals + import mimetypes import socket from os.path import isdir @@ -19,8 +21,7 @@ from os.path import isdir import click from platformio import exception -from platformio.managers.core import (get_core_package_dir, - inject_contrib_pysite) +from platformio.managers.core import get_core_package_dir, inject_contrib_pysite @click.command("home", short_help="PIO Home") @@ -28,9 +29,12 @@ from platformio.managers.core import (get_core_package_dir, @click.option( "--host", default="127.0.0.1", - help="HTTP host, default=127.0.0.1. " - "You can open PIO Home for inbound connections with --host=0.0.0.0") -@click.option("--no-open", is_flag=True) # pylint: disable=too-many-locals + help=( + "HTTP host, default=127.0.0.1. You can open PIO Home for inbound " + "connections with --host=0.0.0.0" + ), +) +@click.option("--no-open", is_flag=True) def cli(port, host, no_open): # import contrib modules inject_contrib_pysite() @@ -38,6 +42,7 @@ def cli(port, host, no_open): from autobahn.twisted.resource import WebSocketResource from twisted.internet import reactor from twisted.web import server + # pylint: enable=import-error from platformio.commands.home.rpc.handlers.app import AppRPC from platformio.commands.home.rpc.handlers.ide import IDERPC @@ -89,14 +94,18 @@ def cli(port, host, no_open): else: reactor.callLater(1, lambda: click.launch(home_url)) - click.echo("\n".join([ - "", - " ___I_", - " /\\-_--\\ PlatformIO Home", - "/ \\_-__\\", - "|[]| [] | %s" % home_url, - "|__|____|______________%s" % ("_" * len(host)), - ])) + click.echo( + "\n".join( + [ + "", + " ___I_", + " /\\-_--\\ PlatformIO Home", + "/ \\_-__\\", + "|[]| [] | %s" % home_url, + "|__|____|______________%s" % ("_" * len(host)), + ] + ) + ) click.echo("") click.echo("Open PIO Home in your browser by this URL => %s" % home_url) diff --git a/platformio/commands/home/helpers.py b/platformio/commands/home/helpers.py index 46c1750c..9497e49c 100644 --- a/platformio/commands/home/helpers.py +++ b/platformio/commands/home/helpers.py @@ -27,7 +27,6 @@ from platformio.proc import where_is_program class AsyncSession(requests.Session): - def __init__(self, n=None, *args, **kwargs): if n: pool = reactor.getThreadPool() @@ -51,7 +50,8 @@ def requests_session(): @util.memoized(expire="60s") def get_core_fullpath(): return where_is_program( - "platformio" + (".exe" if "windows" in util.get_systype() else "")) + "platformio" + (".exe" if "windows" in util.get_systype() else "") + ) @util.memoized(expire="10s") @@ -60,9 +60,7 @@ def is_twitter_blocked(): timeout = 2 try: if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")): - requests.get("http://%s" % ip, - allow_redirects=False, - timeout=timeout) + requests.get("http://%s" % ip, allow_redirects=False, timeout=timeout) else: socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((ip, 80)) return False diff --git a/platformio/commands/home/rpc/handlers/app.py b/platformio/commands/home/rpc/handlers/app.py index 9e19edf1..cc3be81f 100644 --- a/platformio/commands/home/rpc/handlers/app.py +++ b/platformio/commands/home/rpc/handlers/app.py @@ -17,8 +17,7 @@ from __future__ import absolute_import from os.path import expanduser, join from platformio import __version__, app, util -from platformio.project.helpers import (get_project_core_dir, - is_platformio_project) +from platformio.project.helpers import get_project_core_dir, is_platformio_project class AppRPC(object): @@ -26,8 +25,13 @@ class AppRPC(object): APPSTATE_PATH = join(get_project_core_dir(), "homestate.json") IGNORE_STORAGE_KEYS = [ - "cid", "coreVersion", "coreSystype", "coreCaller", "coreSettings", - "homeDir", "projectsDir" + "cid", + "coreVersion", + "coreSystype", + "coreCaller", + "coreSettings", + "homeDir", + "projectsDir", ] @staticmethod @@ -37,31 +41,28 @@ class AppRPC(object): # base data caller_id = app.get_session_var("caller_id") - storage['cid'] = app.get_cid() - storage['coreVersion'] = __version__ - storage['coreSystype'] = util.get_systype() - storage['coreCaller'] = (str(caller_id).lower() - if caller_id else None) - storage['coreSettings'] = { + storage["cid"] = app.get_cid() + storage["coreVersion"] = __version__ + storage["coreSystype"] = util.get_systype() + storage["coreCaller"] = str(caller_id).lower() if caller_id else None + storage["coreSettings"] = { name: { - "description": data['description'], - "default_value": data['value'], - "value": app.get_setting(name) + "description": data["description"], + "default_value": data["value"], + "value": app.get_setting(name), } for name, data in app.DEFAULT_SETTINGS.items() } - storage['homeDir'] = expanduser("~") - storage['projectsDir'] = storage['coreSettings']['projects_dir'][ - 'value'] + storage["homeDir"] = expanduser("~") + storage["projectsDir"] = storage["coreSettings"]["projects_dir"]["value"] # skip non-existing recent projects - storage['recentProjects'] = [ - p for p in storage.get("recentProjects", []) - if is_platformio_project(p) + storage["recentProjects"] = [ + p for p in storage.get("recentProjects", []) if is_platformio_project(p) ] - state['storage'] = storage + state["storage"] = storage state.modified = False # skip saving extra fields return state.as_dict() diff --git a/platformio/commands/home/rpc/handlers/ide.py b/platformio/commands/home/rpc/handlers/ide.py index 85bf98fa..af5b474e 100644 --- a/platformio/commands/home/rpc/handlers/ide.py +++ b/platformio/commands/home/rpc/handlers/ide.py @@ -19,20 +19,18 @@ from twisted.internet import defer # pylint: disable=import-error class IDERPC(object): - def __init__(self): self._queue = {} def send_command(self, command, params, sid=0): if not self._queue.get(sid): raise jsonrpc.exceptions.JSONRPCDispatchException( - code=4005, message="PIO Home IDE agent is not started") + code=4005, message="PIO Home IDE agent is not started" + ) while self._queue[sid]: - self._queue[sid].pop().callback({ - "id": time.time(), - "method": command, - "params": params - }) + self._queue[sid].pop().callback( + {"id": time.time(), "method": command, "params": params} + ) def listen_commands(self, sid=0): if sid not in self._queue: diff --git a/platformio/commands/home/rpc/handlers/misc.py b/platformio/commands/home/rpc/handlers/misc.py index 2422da78..004c14a3 100644 --- a/platformio/commands/home/rpc/handlers/misc.py +++ b/platformio/commands/home/rpc/handlers/misc.py @@ -22,7 +22,6 @@ from platformio.commands.home.rpc.handlers.os import OSRPC class MiscRPC(object): - def load_latest_tweets(self, username): cache_key = "piohome_latest_tweets_" + str(username) cache_valid = "7d" @@ -31,10 +30,11 @@ class MiscRPC(object): if cache_data: cache_data = json.loads(cache_data) # automatically update cache in background every 12 hours - if cache_data['time'] < (time.time() - (3600 * 12)): - reactor.callLater(5, self._preload_latest_tweets, username, - cache_key, cache_valid) - return cache_data['result'] + if cache_data["time"] < (time.time() - (3600 * 12)): + reactor.callLater( + 5, self._preload_latest_tweets, username, cache_key, cache_valid + ) + return cache_data["result"] result = self._preload_latest_tweets(username, cache_key, cache_valid) return result @@ -43,12 +43,13 @@ class MiscRPC(object): @defer.inlineCallbacks def _preload_latest_tweets(username, cache_key, cache_valid): result = yield OSRPC.fetch_content( - "https://api.platformio.org/tweets/" + username) + "https://api.platformio.org/tweets/" + username + ) result = json.loads(result) with app.ContentCache() as cc: - cc.set(cache_key, - json.dumps({ - "time": int(time.time()), - "result": result - }), cache_valid) + cc.set( + cache_key, + json.dumps({"time": int(time.time()), "result": result}), + cache_valid, + ) defer.returnValue(result) diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index c84f486e..960f04e6 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -30,19 +30,18 @@ from platformio.compat import PY2, get_filesystem_encoding class OSRPC(object): - @staticmethod @defer.inlineCallbacks def fetch_content(uri, data=None, headers=None, cache_valid=None): if not headers: headers = { - "User-Agent": - ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) " - "AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 " - "Safari/603.3.8") + "User-Agent": ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) " + "AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 " + "Safari/603.3.8" + ) } - cache_key = (app.ContentCache.key_from_args(uri, data) - if cache_valid else None) + cache_key = app.ContentCache.key_from_args(uri, data) if cache_valid else None with app.ContentCache() as cc: if cache_key: result = cc.get(cache_key) @@ -66,7 +65,7 @@ class OSRPC(object): defer.returnValue(result) def request_content(self, uri, data=None, headers=None, cache_valid=None): - if uri.startswith('http'): + if uri.startswith("http"): return self.fetch_content(uri, data, headers, cache_valid) if not isfile(uri): return None @@ -80,8 +79,8 @@ class OSRPC(object): @staticmethod def reveal_file(path): return click.launch( - path.encode(get_filesystem_encoding()) if PY2 else path, - locate=True) + path.encode(get_filesystem_encoding()) if PY2 else path, locate=True + ) @staticmethod def is_file(path): @@ -109,13 +108,11 @@ class OSRPC(object): pathnames = [pathnames] result = set() for pathname in pathnames: - result |= set( - glob.glob(join(root, pathname) if root else pathname)) + result |= set(glob.glob(join(root, pathname) if root else pathname)) return list(result) @staticmethod def list_dir(path): - def _cmp(x, y): if x[1] and not y[1]: return -1 @@ -146,7 +143,7 @@ class OSRPC(object): def get_logical_devices(): items = [] for item in util.get_logical_devices(): - if item['name']: - item['name'] = item['name'] + if item["name"]: + item["name"] = item["name"] items.append(item) return items diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index 4fa13b1a..9ef39a03 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -27,8 +27,7 @@ from twisted.internet import utils # pylint: disable=import-error from platformio import __main__, __version__, fs from platformio.commands.home import helpers -from platformio.compat import (PY2, get_filesystem_encoding, is_bytes, - string_types) +from platformio.compat import PY2, get_filesystem_encoding, is_bytes, string_types try: from thread import get_ident as thread_get_ident @@ -37,7 +36,6 @@ except ImportError: class MultiThreadingStdStream(object): - def __init__(self, parent_stream): self._buffers = {thread_get_ident(): parent_stream} @@ -54,7 +52,8 @@ class MultiThreadingStdStream(object): thread_id = thread_get_ident() self._ensure_thread_buffer(thread_id) return self._buffers[thread_id].write( - value.decode() if is_bytes(value) else value) + value.decode() if is_bytes(value) else value + ) def get_value_and_reset(self): result = "" @@ -68,7 +67,6 @@ class MultiThreadingStdStream(object): class PIOCoreRPC(object): - @staticmethod def version(): return __version__ @@ -104,16 +102,15 @@ class PIOCoreRPC(object): else: result = yield PIOCoreRPC._call_inline(args, options) try: - defer.returnValue( - PIOCoreRPC._process_result(result, to_json)) + defer.returnValue(PIOCoreRPC._process_result(result, to_json)) except ValueError: # fall-back to subprocess method result = yield PIOCoreRPC._call_subprocess(args, options) - defer.returnValue( - PIOCoreRPC._process_result(result, to_json)) + defer.returnValue(PIOCoreRPC._process_result(result, to_json)) except Exception as e: # pylint: disable=bare-except raise jsonrpc.exceptions.JSONRPCDispatchException( - code=4003, message="PIO Core Call Error", data=str(e)) + code=4003, message="PIO Core Call Error", data=str(e) + ) @staticmethod def _call_inline(args, options): @@ -123,8 +120,11 @@ class PIOCoreRPC(object): def _thread_task(): with fs.cd(cwd): exit_code = __main__.main(["-c"] + args) - return (PIOCoreRPC.thread_stdout.get_value_and_reset(), - PIOCoreRPC.thread_stderr.get_value_and_reset(), exit_code) + return ( + PIOCoreRPC.thread_stdout.get_value_and_reset(), + PIOCoreRPC.thread_stderr.get_value_and_reset(), + exit_code, + ) return threads.deferToThread(_thread_task) @@ -135,8 +135,8 @@ class PIOCoreRPC(object): helpers.get_core_fullpath(), args, path=cwd, - env={k: v - for k, v in os.environ.items() if "%" not in k}) + env={k: v for k, v in os.environ.items() if "%" not in k}, + ) @staticmethod def _process_result(result, to_json=False): diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 795bde77..c6f6a89f 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -17,8 +17,7 @@ from __future__ import absolute_import import os import shutil import time -from os.path import (basename, expanduser, getmtime, isdir, isfile, join, - realpath, sep) +from os.path import basename, expanduser, getmtime, isdir, isfile, join, realpath, sep import jsonrpc # pylint: disable=import-error @@ -29,38 +28,37 @@ from platformio.compat import PY2, get_filesystem_encoding from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager from platformio.project.config import ProjectConfig -from platformio.project.helpers import (get_project_libdeps_dir, - get_project_src_dir, - is_platformio_project) +from platformio.project.helpers import ( + get_project_libdeps_dir, + get_project_src_dir, + is_platformio_project, +) class ProjectRPC(object): - @staticmethod def _get_projects(project_dirs=None): - def _get_project_data(project_dir): data = {"boards": [], "envLibdepsDirs": [], "libExtraDirs": []} config = ProjectConfig(join(project_dir, "platformio.ini")) libdeps_dir = get_project_libdeps_dir() - data['libExtraDirs'].extend( - config.get("platformio", "lib_extra_dirs", [])) + data["libExtraDirs"].extend(config.get("platformio", "lib_extra_dirs", [])) for section in config.sections(): if not section.startswith("env:"): continue - data['envLibdepsDirs'].append(join(libdeps_dir, section[4:])) + data["envLibdepsDirs"].append(join(libdeps_dir, section[4:])) if config.has_option(section, "board"): - data['boards'].append(config.get(section, "board")) - data['libExtraDirs'].extend( - config.get(section, "lib_extra_dirs", [])) + data["boards"].append(config.get(section, "board")) + data["libExtraDirs"].extend(config.get(section, "lib_extra_dirs", [])) # skip non existing folders and resolve full path for key in ("envLibdepsDirs", "libExtraDirs"): data[key] = [ expanduser(d) if d.startswith("~") else realpath(d) - for d in data[key] if isdir(d) + for d in data[key] + if isdir(d) ] return data @@ -69,7 +67,7 @@ class ProjectRPC(object): return (sep).join(path.split(sep)[-2:]) if not project_dirs: - project_dirs = AppRPC.load_state()['storage']['recentProjects'] + project_dirs = AppRPC.load_state()["storage"]["recentProjects"] result = [] pm = PlatformManager() @@ -85,29 +83,27 @@ class ProjectRPC(object): for board_id in data.get("boards", []): name = board_id try: - name = pm.board_config(board_id)['name'] + name = pm.board_config(board_id)["name"] except exception.PlatformioException: pass boards.append({"id": board_id, "name": name}) - result.append({ - "path": - project_dir, - "name": - _path_to_name(project_dir), - "modified": - int(getmtime(project_dir)), - "boards": - boards, - "envLibStorages": [{ - "name": basename(d), - "path": d - } for d in data.get("envLibdepsDirs", [])], - "extraLibStorages": [{ - "name": _path_to_name(d), - "path": d - } for d in data.get("libExtraDirs", [])] - }) + result.append( + { + "path": project_dir, + "name": _path_to_name(project_dir), + "modified": int(getmtime(project_dir)), + "boards": boards, + "envLibStorages": [ + {"name": basename(d), "path": d} + for d in data.get("envLibdepsDirs", []) + ], + "extraLibStorages": [ + {"name": _path_to_name(d), "path": d} + for d in data.get("libExtraDirs", []) + ], + } + ) return result def get_projects(self, project_dirs=None): @@ -117,7 +113,7 @@ class ProjectRPC(object): def get_project_examples(): result = [] for manifest in PlatformManager().get_installed(): - examples_dir = join(manifest['__pkg_dir'], "examples") + examples_dir = join(manifest["__pkg_dir"], "examples") if not isdir(examples_dir): continue items = [] @@ -126,28 +122,30 @@ class ProjectRPC(object): try: config = ProjectConfig(join(project_dir, "platformio.ini")) config.validate(silent=True) - project_description = config.get("platformio", - "description") + project_description = config.get("platformio", "description") except exception.PlatformIOProjectException: continue path_tokens = project_dir.split(sep) - items.append({ - "name": - "/".join(path_tokens[path_tokens.index("examples") + 1:]), - "path": - project_dir, - "description": - project_description - }) - result.append({ - "platform": { - "title": manifest['title'], - "version": manifest['version'] - }, - "items": sorted(items, key=lambda item: item['name']) - }) - return sorted(result, key=lambda data: data['platform']['title']) + items.append( + { + "name": "/".join( + path_tokens[path_tokens.index("examples") + 1 :] + ), + "path": project_dir, + "description": project_description, + } + ) + result.append( + { + "platform": { + "title": manifest["title"], + "version": manifest["version"], + }, + "items": sorted(items, key=lambda item: item["name"]), + } + ) + return sorted(result, key=lambda data: data["platform"]["title"]) def init(self, board, framework, project_dir): assert project_dir @@ -157,9 +155,11 @@ class ProjectRPC(object): args = ["init", "--board", board] if framework: args.extend(["--project-option", "framework = %s" % framework]) - if (state['storage']['coreCaller'] and state['storage']['coreCaller'] - in ProjectGenerator.get_supported_ides()): - args.extend(["--ide", state['storage']['coreCaller']]) + if ( + state["storage"]["coreCaller"] + and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides() + ): + args.extend(["--ide", state["storage"]["coreCaller"]]) d = PIOCoreRPC.call(args, options={"cwd": project_dir}) d.addCallback(self._generate_project_main, project_dir, framework) return d @@ -168,32 +168,35 @@ class ProjectRPC(object): def _generate_project_main(_, project_dir, framework): main_content = None if framework == "arduino": - main_content = "\n".join([ - "#include ", - "", - "void setup() {", - " // put your setup code here, to run once:", - "}", - "", - "void loop() {", - " // put your main code here, to run repeatedly:", - "}" - "" - ]) # yapf: disable + main_content = "\n".join( + [ + "#include ", + "", + "void setup() {", + " // put your setup code here, to run once:", + "}", + "", + "void loop() {", + " // put your main code here, to run repeatedly:", + "}" "", + ] + ) # yapf: disable elif framework == "mbed": - main_content = "\n".join([ - "#include ", - "", - "int main() {", - "", - " // put your setup code here, to run once:", - "", - " while(1) {", - " // put your main code here, to run repeatedly:", - " }", - "}", - "" - ]) # yapf: disable + main_content = "\n".join( + [ + "#include ", + "", + "int main() {", + "", + " // put your setup code here, to run once:", + "", + " while(1) {", + " // put your main code here, to run repeatedly:", + " }", + "}", + "", + ] + ) # yapf: disable if not main_content: return project_dir with fs.cd(project_dir): @@ -210,41 +213,46 @@ class ProjectRPC(object): def import_arduino(self, board, use_arduino_libs, arduino_project_dir): board = str(board) if arduino_project_dir and PY2: - arduino_project_dir = arduino_project_dir.encode( - get_filesystem_encoding()) + arduino_project_dir = arduino_project_dir.encode(get_filesystem_encoding()) # don't import PIO Project if is_platformio_project(arduino_project_dir): return arduino_project_dir - is_arduino_project = any([ - isfile( - join(arduino_project_dir, - "%s.%s" % (basename(arduino_project_dir), ext))) - for ext in ("ino", "pde") - ]) + is_arduino_project = any( + [ + isfile( + join( + arduino_project_dir, + "%s.%s" % (basename(arduino_project_dir), ext), + ) + ) + for ext in ("ino", "pde") + ] + ) if not is_arduino_project: raise jsonrpc.exceptions.JSONRPCDispatchException( - code=4000, - message="Not an Arduino project: %s" % arduino_project_dir) + code=4000, message="Not an Arduino project: %s" % arduino_project_dir + ) state = AppRPC.load_state() - project_dir = join(state['storage']['projectsDir'], - time.strftime("%y%m%d-%H%M%S-") + board) + project_dir = join( + state["storage"]["projectsDir"], time.strftime("%y%m%d-%H%M%S-") + board + ) if not isdir(project_dir): os.makedirs(project_dir) args = ["init", "--board", board] args.extend(["--project-option", "framework = arduino"]) if use_arduino_libs: - args.extend([ - "--project-option", - "lib_extra_dirs = ~/Documents/Arduino/libraries" - ]) - if (state['storage']['coreCaller'] and state['storage']['coreCaller'] - in ProjectGenerator.get_supported_ides()): - args.extend(["--ide", state['storage']['coreCaller']]) + args.extend( + ["--project-option", "lib_extra_dirs = ~/Documents/Arduino/libraries"] + ) + if ( + state["storage"]["coreCaller"] + and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides() + ): + args.extend(["--ide", state["storage"]["coreCaller"]]) d = PIOCoreRPC.call(args, options={"cwd": project_dir}) - d.addCallback(self._finalize_arduino_import, project_dir, - arduino_project_dir) + d.addCallback(self._finalize_arduino_import, project_dir, arduino_project_dir) return d @staticmethod @@ -260,18 +268,21 @@ class ProjectRPC(object): def import_pio(project_dir): if not project_dir or not is_platformio_project(project_dir): raise jsonrpc.exceptions.JSONRPCDispatchException( - code=4001, - message="Not an PlatformIO project: %s" % project_dir) + code=4001, message="Not an PlatformIO project: %s" % project_dir + ) new_project_dir = join( - AppRPC.load_state()['storage']['projectsDir'], - time.strftime("%y%m%d-%H%M%S-") + basename(project_dir)) + AppRPC.load_state()["storage"]["projectsDir"], + time.strftime("%y%m%d-%H%M%S-") + basename(project_dir), + ) shutil.copytree(project_dir, new_project_dir) state = AppRPC.load_state() args = ["init"] - if (state['storage']['coreCaller'] and state['storage']['coreCaller'] - in ProjectGenerator.get_supported_ides()): - args.extend(["--ide", state['storage']['coreCaller']]) + if ( + state["storage"]["coreCaller"] + and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides() + ): + args.extend(["--ide", state["storage"]["coreCaller"]]) d = PIOCoreRPC.call(args, options={"cwd": new_project_dir}) d.addCallback(lambda _: new_project_dir) return d diff --git a/platformio/commands/home/rpc/server.py b/platformio/commands/home/rpc/server.py index 36aa1dff..d7b73bb4 100644 --- a/platformio/commands/home/rpc/server.py +++ b/platformio/commands/home/rpc/server.py @@ -16,8 +16,7 @@ import click import jsonrpc -from autobahn.twisted.websocket import (WebSocketServerFactory, - WebSocketServerProtocol) +from autobahn.twisted.websocket import WebSocketServerFactory, WebSocketServerProtocol from jsonrpc.exceptions import JSONRPCDispatchException from twisted.internet import defer @@ -25,40 +24,39 @@ from platformio.compat import PY2, dump_json_to_unicode, is_bytes class JSONRPCServerProtocol(WebSocketServerProtocol): - def onMessage(self, payload, isBinary): # pylint: disable=unused-argument # click.echo("> %s" % payload) response = jsonrpc.JSONRPCResponseManager.handle( - payload, self.factory.dispatcher).data + payload, self.factory.dispatcher + ).data # if error if "result" not in response: self.sendJSONResponse(response) return None - d = defer.maybeDeferred(lambda: response['result']) + d = defer.maybeDeferred(lambda: response["result"]) d.addCallback(self._callback, response) d.addErrback(self._errback, response) return None def _callback(self, result, response): - response['result'] = result + response["result"] = result self.sendJSONResponse(response) def _errback(self, failure, response): if isinstance(failure.value, JSONRPCDispatchException): e = failure.value else: - e = JSONRPCDispatchException(code=4999, - message=failure.getErrorMessage()) + e = JSONRPCDispatchException(code=4999, message=failure.getErrorMessage()) del response["result"] - response['error'] = e.error._data # pylint: disable=protected-access + response["error"] = e.error._data # pylint: disable=protected-access self.sendJSONResponse(response) def sendJSONResponse(self, response): # click.echo("< %s" % response) if "error" in response: - click.secho("Error: %s" % response['error'], fg="red", err=True) + click.secho("Error: %s" % response["error"], fg="red", err=True) response = dump_json_to_unicode(response) if not PY2 and not is_bytes(response): response = response.encode("utf-8") diff --git a/platformio/commands/home/web.py b/platformio/commands/home/web.py index df48b1fa..45b6c37d 100644 --- a/platformio/commands/home/web.py +++ b/platformio/commands/home/web.py @@ -17,14 +17,12 @@ from twisted.web import static # pylint: disable=import-error class WebRoot(static.File): - def render_GET(self, request): if request.args.get("__shutdown__", False): reactor.stop() return "Server has been stopped" - request.setHeader("cache-control", - "no-cache, no-store, must-revalidate") + request.setHeader("cache-control", "no-cache, no-store, must-revalidate") request.setHeader("pragma", "no-cache") request.setHeader("expires", "0") return static.File.render_GET(self, request) diff --git a/platformio/commands/init.py b/platformio/commands/init.py index e23d24e7..03a08fab 100644 --- a/platformio/commands/init.py +++ b/platformio/commands/init.py @@ -20,16 +20,17 @@ from os.path import isdir, isfile, join import click from platformio import exception, fs -from platformio.commands.platform import \ - platform_install as cli_platform_install +from platformio.commands.platform import platform_install as cli_platform_install from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager from platformio.project.config import ProjectConfig -from platformio.project.helpers import (get_project_include_dir, - get_project_lib_dir, - get_project_src_dir, - get_project_test_dir, - is_platformio_project) +from platformio.project.helpers import ( + get_project_include_dir, + get_project_lib_dir, + get_project_src_dir, + get_project_test_dir, + is_platformio_project, +) def validate_boards(ctx, param, value): # pylint: disable=W0613 @@ -40,66 +41,66 @@ def validate_boards(ctx, param, value): # pylint: disable=W0613 except exception.UnknownBoard: raise click.BadParameter( "`%s`. Please search for board ID using `platformio boards` " - "command" % id_) + "command" % id_ + ) return value -@click.command("init", - short_help="Initialize PlatformIO project or update existing") -@click.option("--project-dir", - "-d", - default=getcwd, - type=click.Path(exists=True, - file_okay=False, - dir_okay=True, - writable=True, - resolve_path=True)) -@click.option("-b", - "--board", - multiple=True, - metavar="ID", - callback=validate_boards) -@click.option("--ide", - type=click.Choice(ProjectGenerator.get_supported_ides())) +@click.command("init", short_help="Initialize PlatformIO project or update existing") +@click.option( + "--project-dir", + "-d", + default=getcwd, + type=click.Path( + exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True + ), +) +@click.option("-b", "--board", multiple=True, metavar="ID", callback=validate_boards) +@click.option("--ide", type=click.Choice(ProjectGenerator.get_supported_ides())) @click.option("-O", "--project-option", multiple=True) @click.option("--env-prefix", default="") @click.option("-s", "--silent", is_flag=True) @click.pass_context def cli( - ctx, # pylint: disable=R0913 - project_dir, - board, - ide, - project_option, - env_prefix, - silent): + ctx, # pylint: disable=R0913 + project_dir, + board, + ide, + project_option, + env_prefix, + silent, +): if not silent: if project_dir == getcwd(): - click.secho("\nThe current working directory", - fg="yellow", - nl=False) + click.secho("\nThe current working directory", fg="yellow", nl=False) click.secho(" %s " % project_dir, fg="cyan", nl=False) click.secho("will be used for the project.", fg="yellow") click.echo("") - click.echo("The next files/directories have been created in %s" % - click.style(project_dir, fg="cyan")) - click.echo("%s - Put project header files here" % - click.style("include", fg="cyan")) - click.echo("%s - Put here project specific (private) libraries" % - click.style("lib", fg="cyan")) - click.echo("%s - Put project source files here" % - click.style("src", fg="cyan")) - click.echo("%s - Project Configuration File" % - click.style("platformio.ini", fg="cyan")) + click.echo( + "The next files/directories have been created in %s" + % click.style(project_dir, fg="cyan") + ) + click.echo( + "%s - Put project header files here" % click.style("include", fg="cyan") + ) + click.echo( + "%s - Put here project specific (private) libraries" + % click.style("lib", fg="cyan") + ) + click.echo("%s - Put project source files here" % click.style("src", fg="cyan")) + click.echo( + "%s - Project Configuration File" % click.style("platformio.ini", fg="cyan") + ) is_new_project = not is_platformio_project(project_dir) if is_new_project: init_base_project(project_dir) if board: - fill_project_envs(ctx, project_dir, board, project_option, env_prefix, - ide is not None) + fill_project_envs( + ctx, project_dir, board, project_option, env_prefix, ide is not None + ) if ide: pg = ProjectGenerator(project_dir, ide, board) @@ -115,9 +116,9 @@ def cli( if ide: click.secho( "\nProject has been successfully %s including configuration files " - "for `%s` IDE." % - ("initialized" if is_new_project else "updated", ide), - fg="green") + "for `%s` IDE." % ("initialized" if is_new_project else "updated", ide), + fg="green", + ) else: click.secho( "\nProject has been successfully %s! Useful commands:\n" @@ -125,9 +126,10 @@ def cli( "`pio run --target upload` or `pio run -t upload` " "- upload firmware to a target\n" "`pio run --target clean` - clean project (remove compiled files)" - "\n`pio run --help` - additional information" % - ("initialized" if is_new_project else "updated"), - fg="green") + "\n`pio run --help` - additional information" + % ("initialized" if is_new_project else "updated"), + fg="green", + ) def init_base_project(project_dir): @@ -149,7 +151,8 @@ def init_base_project(project_dir): def init_include_readme(include_dir): with open(join(include_dir, "README"), "w") as f: - f.write(""" + f.write( + """ This directory is intended for project header files. A header file is a file containing C declarations and macro definitions @@ -188,12 +191,15 @@ Read more about using header files in official GCC documentation: * Computed Includes https://gcc.gnu.org/onlinedocs/cpp/Header-Files.html -""") +""" + ) def init_lib_readme(lib_dir): with open(join(lib_dir, "README"), "w") as f: - f.write(""" + # pylint: disable=line-too-long + f.write( + """ This directory is intended for project specific (private) libraries. PlatformIO will compile them to static libraries and link into executable file. @@ -239,12 +245,14 @@ libraries scanning project source files. More information about PlatformIO Library Dependency Finder - https://docs.platformio.org/page/librarymanager/ldf.html -""") +""" + ) def init_test_readme(test_dir): with open(join(test_dir, "README"), "w") as f: - f.write(""" + f.write( + """ This directory is intended for PIO Unit Testing and project tests. Unit Testing is a software testing method by which individual units of @@ -255,7 +263,8 @@ in the development cycle. More information about PIO Unit Testing: - https://docs.platformio.org/page/plus/unit-testing.html -""") +""" + ) def init_ci_conf(project_dir): @@ -263,7 +272,8 @@ def init_ci_conf(project_dir): if isfile(conf_path): return with open(conf_path, "w") as f: - f.write("""# Continuous Integration (CI) is the practice, in software + f.write( + """# Continuous Integration (CI) is the practice, in software # engineering, of merging all developer working copies with a shared mainline # several times a day < https://docs.platformio.org/page/ci/index.html > # @@ -330,7 +340,8 @@ def init_ci_conf(project_dir): # # script: # - platformio ci --lib="." --board=ID_1 --board=ID_2 --board=ID_N -""") +""" + ) def init_cvs_ignore(project_dir): @@ -341,16 +352,13 @@ def init_cvs_ignore(project_dir): fp.write(".pio\n") -def fill_project_envs(ctx, project_dir, board_ids, project_option, env_prefix, - force_download): - config = ProjectConfig(join(project_dir, "platformio.ini"), - parse_extra=False) +def fill_project_envs( + ctx, project_dir, board_ids, project_option, env_prefix, force_download +): + config = ProjectConfig(join(project_dir, "platformio.ini"), parse_extra=False) used_boards = [] for section in config.sections(): - cond = [ - section.startswith("env:"), - config.has_option(section, "board") - ] + cond = [section.startswith("env:"), config.has_option(section, "board")] if all(cond): used_boards.append(config.get(section, "board")) @@ -359,17 +367,17 @@ def fill_project_envs(ctx, project_dir, board_ids, project_option, env_prefix, modified = False for id_ in board_ids: board_config = pm.board_config(id_) - used_platforms.append(board_config['platform']) + used_platforms.append(board_config["platform"]) if id_ in used_boards: continue used_boards.append(id_) modified = True - envopts = {"platform": board_config['platform'], "board": id_} + envopts = {"platform": board_config["platform"], "board": id_} # find default framework for board frameworks = board_config.get("frameworks") if frameworks: - envopts['framework'] = frameworks[0] + envopts["framework"] = frameworks[0] for item in project_option: if "=" not in item: @@ -391,10 +399,9 @@ def fill_project_envs(ctx, project_dir, board_ids, project_option, env_prefix, def _install_dependent_platforms(ctx, platforms): - installed_platforms = [ - p['name'] for p in PlatformManager().get_installed() - ] + installed_platforms = [p["name"] for p in PlatformManager().get_installed()] if set(platforms) <= set(installed_platforms): return - ctx.invoke(cli_platform_install, - platforms=list(set(platforms) - set(installed_platforms))) + ctx.invoke( + cli_platform_install, platforms=list(set(platforms) - set(installed_platforms)) + ) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 1de0f960..6244da99 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -24,14 +24,15 @@ from tabulate import tabulate from platformio import exception, fs, util from platformio.commands import PlatformioCLI from platformio.compat import dump_json_to_unicode -from platformio.managers.lib import (LibraryManager, get_builtin_libs, - is_builtin_lib) +from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib from platformio.proc import is_ci from platformio.project.config import ProjectConfig -from platformio.project.helpers import (get_project_dir, - get_project_global_lib_dir, - get_project_libdeps_dir, - is_platformio_project) +from platformio.project.helpers import ( + get_project_dir, + get_project_global_lib_dir, + get_project_libdeps_dir, + is_platformio_project, +) try: from urllib.parse import quote @@ -45,35 +46,38 @@ CTX_META_STORAGE_LIBDEPS_KEY = __name__ + ".storage_lib_deps" @click.group(short_help="Library Manager") -@click.option("-d", - "--storage-dir", - multiple=True, - default=None, - type=click.Path(exists=True, - file_okay=False, - dir_okay=True, - writable=True, - resolve_path=True), - help="Manage custom library storage") -@click.option("-g", - "--global", - is_flag=True, - help="Manage global PlatformIO library storage") +@click.option( + "-d", + "--storage-dir", + multiple=True, + default=None, + type=click.Path( + exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True + ), + help="Manage custom library storage", +) +@click.option( + "-g", "--global", is_flag=True, help="Manage global PlatformIO library storage" +) @click.option( "-e", "--environment", multiple=True, - help=("Manage libraries for the specific project build environments " - "declared in `platformio.ini`")) + help=( + "Manage libraries for the specific project build environments " + "declared in `platformio.ini`" + ), +) @click.pass_context def cli(ctx, **options): storage_cmds = ("install", "uninstall", "update", "list") # skip commands that don't need storage folder - if ctx.invoked_subcommand not in storage_cmds or \ - (len(ctx.args) == 2 and ctx.args[1] in ("-h", "--help")): + if ctx.invoked_subcommand not in storage_cmds or ( + len(ctx.args) == 2 and ctx.args[1] in ("-h", "--help") + ): return - storage_dirs = list(options['storage_dir']) - if options['global']: + storage_dirs = list(options["storage_dir"]) + if options["global"]: storage_dirs.append(get_project_global_lib_dir()) if not storage_dirs: if is_platformio_project(): @@ -84,15 +88,16 @@ def cli(ctx, **options): "Warning! Global library storage is used automatically. " "Please use `platformio lib --global %s` command to remove " "this warning." % ctx.invoked_subcommand, - fg="yellow") + fg="yellow", + ) if not storage_dirs: - raise exception.NotGlobalLibDir(get_project_dir(), - get_project_global_lib_dir(), - ctx.invoked_subcommand) + raise exception.NotGlobalLibDir( + get_project_dir(), get_project_global_lib_dir(), ctx.invoked_subcommand + ) in_silence = PlatformioCLI.in_silence() - ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] = options['environment'] + ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] = options["environment"] ctx.meta[CTX_META_INPUT_DIRS_KEY] = storage_dirs ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [] ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY] = {} @@ -102,16 +107,16 @@ def cli(ctx, **options): continue with fs.cd(storage_dir): libdeps_dir = get_project_libdeps_dir() - config = ProjectConfig.get_instance(join(storage_dir, - "platformio.ini")) - config.validate(options['environment'], silent=in_silence) + config = ProjectConfig.get_instance(join(storage_dir, "platformio.ini")) + config.validate(options["environment"], silent=in_silence) for env in config.envs(): - if options['environment'] and env not in options['environment']: + if options["environment"] and env not in options["environment"]: continue storage_dir = join(libdeps_dir, env) ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY][storage_dir] = config.get( - "env:" + env, "lib_deps", []) + "env:" + env, "lib_deps", [] + ) @cli.command("install", short_help="Install library") @@ -119,21 +124,19 @@ def cli(ctx, **options): @click.option( "--save", is_flag=True, - help="Save installed libraries into the `platformio.ini` dependency list") -@click.option("-s", - "--silent", - is_flag=True, - help="Suppress progress reporting") -@click.option("--interactive", - is_flag=True, - help="Allow to make a choice for all prompts") -@click.option("-f", - "--force", - is_flag=True, - help="Reinstall/redownload library if exists") + help="Save installed libraries into the `platformio.ini` dependency list", +) +@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") +@click.option( + "--interactive", is_flag=True, help="Allow to make a choice for all prompts" +) +@click.option( + "-f", "--force", is_flag=True, help="Reinstall/redownload library if exists" +) @click.pass_context def lib_install( # pylint: disable=too-many-arguments - ctx, libraries, save, silent, interactive, force): + ctx, libraries, save, silent, interactive, force +): storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] storage_libdeps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, []) @@ -144,25 +147,22 @@ def lib_install( # pylint: disable=too-many-arguments lm = LibraryManager(storage_dir) if libraries: for library in libraries: - pkg_dir = lm.install(library, - silent=silent, - interactive=interactive, - force=force) + pkg_dir = lm.install( + library, silent=silent, interactive=interactive, force=force + ) installed_manifests[library] = lm.load_manifest(pkg_dir) elif storage_dir in storage_libdeps: builtin_lib_storages = None for library in storage_libdeps[storage_dir]: try: - pkg_dir = lm.install(library, - silent=silent, - interactive=interactive, - force=force) + pkg_dir = lm.install( + library, silent=silent, interactive=interactive, force=force + ) installed_manifests[library] = lm.load_manifest(pkg_dir) except exception.LibNotFound as e: if builtin_lib_storages is None: builtin_lib_storages = get_builtin_libs() - if not silent or not is_builtin_lib( - builtin_lib_storages, library): + if not silent or not is_builtin_lib(builtin_lib_storages, library): click.secho("Warning! %s" % e, fg="yellow") if not save or not libraries: @@ -183,8 +183,8 @@ def lib_install( # pylint: disable=too-many-arguments continue manifest = installed_manifests[library] try: - assert library.lower() == manifest['name'].lower() - assert semantic_version.Version(manifest['version']) + assert library.lower() == manifest["name"].lower() + assert semantic_version.Version(manifest["version"]) lib_deps.append("{name}@^{version}".format(**manifest)) except (AssertionError, ValueError): lib_deps.append(library) @@ -206,13 +206,15 @@ def lib_uninstall(ctx, libraries): @cli.command("update", short_help="Update installed libraries") @click.argument("libraries", required=False, nargs=-1, metavar="[LIBRARY...]") -@click.option("-c", - "--only-check", - is_flag=True, - help="DEPRECATED. Please use `--dry-run` instead") -@click.option("--dry-run", - is_flag=True, - help="Do not update, only check for the new versions") +@click.option( + "-c", + "--only-check", + is_flag=True, + help="DEPRECATED. Please use `--dry-run` instead", +) +@click.option( + "--dry-run", is_flag=True, help="Do not update, only check for the new versions" +) @click.option("--json-output", is_flag=True) @click.pass_context def lib_update(ctx, libraries, only_check, dry_run, json_output): @@ -226,9 +228,7 @@ def lib_update(ctx, libraries, only_check, dry_run, json_output): _libraries = libraries if not _libraries: - _libraries = [ - manifest['__pkg_dir'] for manifest in lm.get_installed() - ] + _libraries = [manifest["__pkg_dir"] for manifest in lm.get_installed()] if only_check and json_output: result = [] @@ -245,7 +245,7 @@ def lib_update(ctx, libraries, only_check, dry_run, json_output): if not latest: continue manifest = lm.load_manifest(pkg_dir) - manifest['versionLatest'] = latest + manifest["versionLatest"] = latest result.append(manifest) json_result[storage_dir] = result else: @@ -254,8 +254,10 @@ def lib_update(ctx, libraries, only_check, dry_run, json_output): if json_output: return click.echo( - dump_json_to_unicode(json_result[storage_dirs[0]] - if len(storage_dirs) == 1 else json_result)) + dump_json_to_unicode( + json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result + ) + ) return True @@ -274,15 +276,17 @@ def lib_list(ctx, json_output): if json_output: json_result[storage_dir] = items elif items: - for item in sorted(items, key=lambda i: i['name']): + for item in sorted(items, key=lambda i: i["name"]): print_lib_item(item) else: click.echo("No items found") if json_output: return click.echo( - dump_json_to_unicode(json_result[storage_dirs[0]] - if len(storage_dirs) == 1 else json_result)) + dump_json_to_unicode( + json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result + ) + ) return True @@ -298,9 +302,11 @@ def lib_list(ctx, json_output): @click.option("-f", "--framework", multiple=True) @click.option("-p", "--platform", multiple=True) @click.option("-i", "--header", multiple=True) -@click.option("--noninteractive", - is_flag=True, - help="Do not prompt, automatically paginate with delay") +@click.option( + "--noninteractive", + is_flag=True, + help="Do not prompt, automatically paginate with delay", +) def lib_search(query, json_output, page, noninteractive, **filters): if not query: query = [] @@ -311,55 +317,61 @@ def lib_search(query, json_output, page, noninteractive, **filters): for value in values: query.append('%s:"%s"' % (key, value)) - result = util.get_api_result("/v2/lib/search", - dict(query=" ".join(query), page=page), - cache_valid="1d") + result = util.get_api_result( + "/v2/lib/search", dict(query=" ".join(query), page=page), cache_valid="1d" + ) if json_output: click.echo(dump_json_to_unicode(result)) return - if result['total'] == 0: + if result["total"] == 0: click.secho( "Nothing has been found by your request\n" "Try a less-specific search or use truncation (or wildcard) " "operator", fg="yellow", - nl=False) + nl=False, + ) click.secho(" *", fg="green") click.secho("For example: DS*, PCA*, DHT* and etc.\n", fg="yellow") - click.echo("For more examples and advanced search syntax, " - "please use documentation:") + click.echo( + "For more examples and advanced search syntax, " "please use documentation:" + ) click.secho( "https://docs.platformio.org/page/userguide/lib/cmd_search.html\n", - fg="cyan") + fg="cyan", + ) return - click.secho("Found %d libraries:\n" % result['total'], - fg="green" if result['total'] else "yellow") + click.secho( + "Found %d libraries:\n" % result["total"], + fg="green" if result["total"] else "yellow", + ) while True: - for item in result['items']: + for item in result["items"]: print_lib_item(item) - if (int(result['page']) * int(result['perpage']) >= int( - result['total'])): + if int(result["page"]) * int(result["perpage"]) >= int(result["total"]): break if noninteractive: click.echo() - click.secho("Loading next %d libraries... Press Ctrl+C to stop!" % - result['perpage'], - fg="yellow") + click.secho( + "Loading next %d libraries... Press Ctrl+C to stop!" + % result["perpage"], + fg="yellow", + ) click.echo() time.sleep(5) elif not click.confirm("Show next libraries?"): break - result = util.get_api_result("/v2/lib/search", { - "query": " ".join(query), - "page": int(result['page']) + 1 - }, - cache_valid="1d") + result = util.get_api_result( + "/v2/lib/search", + {"query": " ".join(query), "page": int(result["page"]) + 1}, + cache_valid="1d", + ) @cli.command("builtin", short_help="List built-in libraries") @@ -371,13 +383,13 @@ def lib_builtin(storage, json_output): return click.echo(dump_json_to_unicode(items)) for storage_ in items: - if not storage_['items']: + if not storage_["items"]: continue - click.secho(storage_['name'], fg="green") - click.echo("*" * len(storage_['name'])) + click.secho(storage_["name"], fg="green") + click.echo("*" * len(storage_["name"])) click.echo() - for item in sorted(storage_['items'], key=lambda i: i['name']): + for item in sorted(storage_["items"], key=lambda i: i["name"]): print_lib_item(item) return True @@ -389,27 +401,29 @@ def lib_builtin(storage, json_output): def lib_show(library, json_output): lm = LibraryManager() name, requirements, _ = lm.parse_pkg_uri(library) - lib_id = lm.search_lib_id({ - "name": name, - "requirements": requirements - }, - silent=json_output, - interactive=not json_output) + lib_id = lm.search_lib_id( + {"name": name, "requirements": requirements}, + silent=json_output, + interactive=not json_output, + ) lib = util.get_api_result("/lib/info/%d" % lib_id, cache_valid="1d") if json_output: return click.echo(dump_json_to_unicode(lib)) - click.secho(lib['name'], fg="cyan") - click.echo("=" * len(lib['name'])) - click.secho("#ID: %d" % lib['id'], bold=True) - click.echo(lib['description']) + click.secho(lib["name"], fg="cyan") + click.echo("=" * len(lib["name"])) + click.secho("#ID: %d" % lib["id"], bold=True) + click.echo(lib["description"]) click.echo() click.echo( - "Version: %s, released %s" % - (lib['version']['name'], - time.strftime("%c", util.parse_date(lib['version']['released'])))) - click.echo("Manifest: %s" % lib['confurl']) + "Version: %s, released %s" + % ( + lib["version"]["name"], + time.strftime("%c", util.parse_date(lib["version"]["released"])), + ) + ) + click.echo("Manifest: %s" % lib["confurl"]) for key in ("homepage", "repository", "license"): if key not in lib or not lib[key]: continue @@ -436,23 +450,33 @@ def lib_show(library, json_output): if _authors: blocks.append(("Authors", _authors)) - blocks.append(("Keywords", lib['keywords'])) + blocks.append(("Keywords", lib["keywords"])) for key in ("frameworks", "platforms"): if key not in lib or not lib[key]: continue - blocks.append(("Compatible %s" % key, [i['title'] for i in lib[key]])) - blocks.append(("Headers", lib['headers'])) - blocks.append(("Examples", lib['examples'])) - blocks.append(("Versions", [ - "%s, released %s" % - (v['name'], time.strftime("%c", util.parse_date(v['released']))) - for v in lib['versions'] - ])) - blocks.append(("Unique Downloads", [ - "Today: %s" % lib['dlstats']['day'], - "Week: %s" % lib['dlstats']['week'], - "Month: %s" % lib['dlstats']['month'] - ])) + blocks.append(("Compatible %s" % key, [i["title"] for i in lib[key]])) + blocks.append(("Headers", lib["headers"])) + blocks.append(("Examples", lib["examples"])) + blocks.append( + ( + "Versions", + [ + "%s, released %s" + % (v["name"], time.strftime("%c", util.parse_date(v["released"]))) + for v in lib["versions"] + ], + ) + ) + blocks.append( + ( + "Unique Downloads", + [ + "Today: %s" % lib["dlstats"]["day"], + "Week: %s" % lib["dlstats"]["week"], + "Month: %s" % lib["dlstats"]["month"], + ], + ) + ) for (title, rows) in blocks: click.echo() @@ -467,16 +491,15 @@ def lib_show(library, json_output): @cli.command("register", short_help="Register a new library") @click.argument("config_url") def lib_register(config_url): - if (not config_url.startswith("http://") - and not config_url.startswith("https://")): + if not config_url.startswith("http://") and not config_url.startswith("https://"): raise exception.InvalidLibConfURL(config_url) - result = util.get_api_result("/lib/register", - data=dict(config_url=config_url)) - if "message" in result and result['message']: - click.secho(result['message'], - fg="green" if "successed" in result and result['successed'] - else "red") + result = util.get_api_result("/lib/register", data=dict(config_url=config_url)) + if "message" in result and result["message"]: + click.secho( + result["message"], + fg="green" if "successed" in result and result["successed"] else "red", + ) @cli.command("stats", short_help="Library Registry Statistics") @@ -488,46 +511,56 @@ def lib_stats(json_output): return click.echo(dump_json_to_unicode(result)) for key in ("updated", "added"): - tabular_data = [(click.style(item['name'], fg="cyan"), - time.strftime("%c", util.parse_date(item['date'])), - "https://platformio.org/lib/show/%s/%s" % - (item['id'], quote(item['name']))) - for item in result.get(key, [])] - table = tabulate(tabular_data, - headers=[ - click.style("RECENTLY " + key.upper(), bold=True), - "Date", "URL" - ]) + tabular_data = [ + ( + click.style(item["name"], fg="cyan"), + time.strftime("%c", util.parse_date(item["date"])), + "https://platformio.org/lib/show/%s/%s" + % (item["id"], quote(item["name"])), + ) + for item in result.get(key, []) + ] + table = tabulate( + tabular_data, + headers=[click.style("RECENTLY " + key.upper(), bold=True), "Date", "URL"], + ) click.echo(table) click.echo() for key in ("lastkeywords", "topkeywords"): - tabular_data = [(click.style(name, fg="cyan"), - "https://platformio.org/lib/search?query=" + - quote("keyword:%s" % name)) - for name in result.get(key, [])] + tabular_data = [ + ( + click.style(name, fg="cyan"), + "https://platformio.org/lib/search?query=" + quote("keyword:%s" % name), + ) + for name in result.get(key, []) + ] table = tabulate( tabular_data, headers=[ click.style( - ("RECENT" if key == "lastkeywords" else "POPULAR") + - " KEYWORDS", - bold=True), "URL" - ]) + ("RECENT" if key == "lastkeywords" else "POPULAR") + " KEYWORDS", + bold=True, + ), + "URL", + ], + ) click.echo(table) click.echo() - for key, title in (("dlday", "Today"), ("dlweek", "Week"), ("dlmonth", - "Month")): - tabular_data = [(click.style(item['name'], fg="cyan"), - "https://platformio.org/lib/show/%s/%s" % - (item['id'], quote(item['name']))) - for item in result.get(key, [])] - table = tabulate(tabular_data, - headers=[ - click.style("FEATURED: " + title.upper(), - bold=True), "URL" - ]) + for key, title in (("dlday", "Today"), ("dlweek", "Week"), ("dlmonth", "Month")): + tabular_data = [ + ( + click.style(item["name"], fg="cyan"), + "https://platformio.org/lib/show/%s/%s" + % (item["id"], quote(item["name"])), + ) + for item in result.get(key, []) + ] + table = tabulate( + tabular_data, + headers=[click.style("FEATURED: " + title.upper(), bold=True), "URL"], + ) click.echo(table) click.echo() @@ -538,15 +571,16 @@ def print_storage_header(storage_dirs, storage_dir): if storage_dirs and storage_dirs[0] != storage_dir: click.echo("") click.echo( - click.style("Library Storage: ", bold=True) + - click.style(storage_dir, fg="blue")) + click.style("Library Storage: ", bold=True) + + click.style(storage_dir, fg="blue") + ) def print_lib_item(item): - click.secho(item['name'], fg="cyan") - click.echo("=" * len(item['name'])) + click.secho(item["name"], fg="cyan") + click.echo("=" * len(item["name"])) if "id" in item: - click.secho("#ID: %d" % item['id'], bold=True) + click.secho("#ID: %d" % item["id"], bold=True) if "description" in item or "url" in item: click.echo(item.get("description", item.get("url", ""))) click.echo() @@ -562,14 +596,26 @@ def print_lib_item(item): for key in ("frameworks", "platforms"): if key not in item: continue - click.echo("Compatible %s: %s" % (key, ", ".join( - [i['title'] if isinstance(i, dict) else i for i in item[key]]))) + click.echo( + "Compatible %s: %s" + % ( + key, + ", ".join( + [i["title"] if isinstance(i, dict) else i for i in item[key]] + ), + ) + ) if "authors" in item or "authornames" in item: - click.echo("Authors: %s" % ", ".join( - item.get("authornames", - [a.get("name", "") for a in item.get("authors", [])]))) + click.echo( + "Authors: %s" + % ", ".join( + item.get( + "authornames", [a.get("name", "") for a in item.get("authors", [])] + ) + ) + ) if "__src_url" in item: - click.secho("Source: %s" % item['__src_url']) + click.secho("Source: %s" % item["__src_url"]) click.echo() diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 5b35c4df..30afd2c9 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -29,24 +29,27 @@ def cli(): def _print_platforms(platforms): for platform in platforms: - click.echo("{name} ~ {title}".format(name=click.style(platform['name'], - fg="cyan"), - title=platform['title'])) - click.echo("=" * (3 + len(platform['name'] + platform['title']))) - click.echo(platform['description']) + click.echo( + "{name} ~ {title}".format( + name=click.style(platform["name"], fg="cyan"), title=platform["title"] + ) + ) + click.echo("=" * (3 + len(platform["name"] + platform["title"]))) + click.echo(platform["description"]) click.echo() if "homepage" in platform: - click.echo("Home: %s" % platform['homepage']) - if "frameworks" in platform and platform['frameworks']: - click.echo("Frameworks: %s" % ", ".join(platform['frameworks'])) + click.echo("Home: %s" % platform["homepage"]) + if "frameworks" in platform and platform["frameworks"]: + click.echo("Frameworks: %s" % ", ".join(platform["frameworks"])) if "packages" in platform: - click.echo("Packages: %s" % ", ".join(platform['packages'])) + click.echo("Packages: %s" % ", ".join(platform["packages"])) if "version" in platform: if "__src_url" in platform: - click.echo("Version: #%s (%s)" % - (platform['version'], platform['__src_url'])) + click.echo( + "Version: #%s (%s)" % (platform["version"], platform["__src_url"]) + ) else: - click.echo("Version: " + platform['version']) + click.echo("Version: " + platform["version"]) click.echo() @@ -54,7 +57,7 @@ def _get_registry_platforms(): platforms = util.get_api_result("/platforms", cache_valid="7d") pm = PlatformManager() for platform in platforms or []: - platform['versions'] = pm.get_all_repo_versions(platform['name']) + platform["versions"] = pm.get_all_repo_versions(platform["name"]) return platforms @@ -65,22 +68,22 @@ def _get_platform_data(*args, **kwargs): return _get_registry_platform_data(*args, **kwargs) -def _get_installed_platform_data(platform, - with_boards=True, - expose_packages=True): +def _get_installed_platform_data(platform, with_boards=True, expose_packages=True): p = PlatformFactory.newPlatform(platform) - data = dict(name=p.name, - title=p.title, - description=p.description, - version=p.version, - homepage=p.homepage, - repository=p.repository_url, - url=p.vendor_url, - docs=p.docs_url, - license=p.license, - forDesktop=not p.is_embedded(), - frameworks=sorted(list(p.frameworks) if p.frameworks else []), - packages=list(p.packages) if p.packages else []) + data = dict( + name=p.name, + title=p.title, + description=p.description, + version=p.version, + homepage=p.homepage, + repository=p.repository_url, + url=p.vendor_url, + docs=p.docs_url, + license=p.license, + forDesktop=not p.is_embedded(), + frameworks=sorted(list(p.frameworks) if p.frameworks else []), + packages=list(p.packages) if p.packages else [], + ) # if dump to API # del data['version'] @@ -94,18 +97,20 @@ def _get_installed_platform_data(platform, data[key] = manifest[key] if with_boards: - data['boards'] = [c.get_brief_data() for c in p.get_boards().values()] + data["boards"] = [c.get_brief_data() for c in p.get_boards().values()] - if not data['packages'] or not expose_packages: + if not data["packages"] or not expose_packages: return data - data['packages'] = [] + data["packages"] = [] installed_pkgs = p.get_installed_packages() for name, opts in p.packages.items(): - item = dict(name=name, - type=p.get_package_type(name), - requirements=opts.get("version"), - optional=opts.get("optional") is True) + item = dict( + name=name, + type=p.get_package_type(name), + requirements=opts.get("version"), + optional=opts.get("optional") is True, + ) if name in installed_pkgs: for key, value in installed_pkgs[name].items(): if key not in ("url", "version", "description"): @@ -113,40 +118,42 @@ def _get_installed_platform_data(platform, item[key] = value if key == "version": item["originalVersion"] = util.get_original_version(value) - data['packages'].append(item) + data["packages"].append(item) return data def _get_registry_platform_data( # pylint: disable=unused-argument - platform, - with_boards=True, - expose_packages=True): + platform, with_boards=True, expose_packages=True +): _data = None for p in _get_registry_platforms(): - if p['name'] == platform: + if p["name"] == platform: _data = p break if not _data: return None - data = dict(name=_data['name'], - title=_data['title'], - description=_data['description'], - homepage=_data['homepage'], - repository=_data['repository'], - url=_data['url'], - license=_data['license'], - forDesktop=_data['forDesktop'], - frameworks=_data['frameworks'], - packages=_data['packages'], - versions=_data['versions']) + data = dict( + name=_data["name"], + title=_data["title"], + description=_data["description"], + homepage=_data["homepage"], + repository=_data["repository"], + url=_data["url"], + license=_data["license"], + forDesktop=_data["forDesktop"], + frameworks=_data["frameworks"], + packages=_data["packages"], + versions=_data["versions"], + ) if with_boards: - data['boards'] = [ - board for board in PlatformManager().get_registered_boards() - if board['platform'] == _data['name'] + data["boards"] = [ + board + for board in PlatformManager().get_registered_boards() + if board["platform"] == _data["name"] ] return data @@ -164,9 +171,10 @@ def platform_search(query, json_output): if query and query.lower() not in search_data.lower(): continue platforms.append( - _get_registry_platform_data(platform['name'], - with_boards=False, - expose_packages=False)) + _get_registry_platform_data( + platform["name"], with_boards=False, expose_packages=False + ) + ) if json_output: click.echo(dump_json_to_unicode(platforms)) @@ -185,15 +193,15 @@ def platform_frameworks(query, json_output): search_data = dump_json_to_unicode(framework) if query and query.lower() not in search_data.lower(): continue - framework['homepage'] = ("https://platformio.org/frameworks/" + - framework['name']) - framework['platforms'] = [ - platform['name'] for platform in _get_registry_platforms() - if framework['name'] in platform['frameworks'] + framework["homepage"] = "https://platformio.org/frameworks/" + framework["name"] + framework["platforms"] = [ + platform["name"] + for platform in _get_registry_platforms() + if framework["name"] in platform["frameworks"] ] frameworks.append(framework) - frameworks = sorted(frameworks, key=lambda manifest: manifest['name']) + frameworks = sorted(frameworks, key=lambda manifest: manifest["name"]) if json_output: click.echo(dump_json_to_unicode(frameworks)) else: @@ -207,11 +215,12 @@ def platform_list(json_output): pm = PlatformManager() for manifest in pm.get_installed(): platforms.append( - _get_installed_platform_data(manifest['__pkg_dir'], - with_boards=False, - expose_packages=False)) + _get_installed_platform_data( + manifest["__pkg_dir"], with_boards=False, expose_packages=False + ) + ) - platforms = sorted(platforms, key=lambda manifest: manifest['name']) + platforms = sorted(platforms, key=lambda manifest: manifest["name"]) if json_output: click.echo(dump_json_to_unicode(platforms)) else: @@ -228,55 +237,58 @@ def platform_show(platform, json_output): # pylint: disable=too-many-branches if json_output: return click.echo(dump_json_to_unicode(data)) - click.echo("{name} ~ {title}".format(name=click.style(data['name'], - fg="cyan"), - title=data['title'])) - click.echo("=" * (3 + len(data['name'] + data['title']))) - click.echo(data['description']) + click.echo( + "{name} ~ {title}".format( + name=click.style(data["name"], fg="cyan"), title=data["title"] + ) + ) + click.echo("=" * (3 + len(data["name"] + data["title"]))) + click.echo(data["description"]) click.echo() if "version" in data: - click.echo("Version: %s" % data['version']) - if data['homepage']: - click.echo("Home: %s" % data['homepage']) - if data['repository']: - click.echo("Repository: %s" % data['repository']) - if data['url']: - click.echo("Vendor: %s" % data['url']) - if data['license']: - click.echo("License: %s" % data['license']) - if data['frameworks']: - click.echo("Frameworks: %s" % ", ".join(data['frameworks'])) + click.echo("Version: %s" % data["version"]) + if data["homepage"]: + click.echo("Home: %s" % data["homepage"]) + if data["repository"]: + click.echo("Repository: %s" % data["repository"]) + if data["url"]: + click.echo("Vendor: %s" % data["url"]) + if data["license"]: + click.echo("License: %s" % data["license"]) + if data["frameworks"]: + click.echo("Frameworks: %s" % ", ".join(data["frameworks"])) - if not data['packages']: + if not data["packages"]: return None - if not isinstance(data['packages'][0], dict): - click.echo("Packages: %s" % ", ".join(data['packages'])) + if not isinstance(data["packages"][0], dict): + click.echo("Packages: %s" % ", ".join(data["packages"])) else: click.echo() click.secho("Packages", bold=True) click.echo("--------") - for item in data['packages']: + for item in data["packages"]: click.echo() - click.echo("Package %s" % click.style(item['name'], fg="yellow")) - click.echo("-" * (8 + len(item['name']))) - if item['type']: - click.echo("Type: %s" % item['type']) - click.echo("Requirements: %s" % item['requirements']) - click.echo("Installed: %s" % - ("Yes" if item.get("version") else "No (optional)")) + click.echo("Package %s" % click.style(item["name"], fg="yellow")) + click.echo("-" * (8 + len(item["name"]))) + if item["type"]: + click.echo("Type: %s" % item["type"]) + click.echo("Requirements: %s" % item["requirements"]) + click.echo( + "Installed: %s" % ("Yes" if item.get("version") else "No (optional)") + ) if "version" in item: - click.echo("Version: %s" % item['version']) + click.echo("Version: %s" % item["version"]) if "originalVersion" in item: - click.echo("Original version: %s" % item['originalVersion']) + click.echo("Original version: %s" % item["originalVersion"]) if "description" in item: - click.echo("Description: %s" % item['description']) + click.echo("Description: %s" % item["description"]) - if data['boards']: + if data["boards"]: click.echo() click.secho("Boards", bold=True) click.echo("------") - print_boards(data['boards']) + print_boards(data["boards"]) return True @@ -290,20 +302,26 @@ def platform_show(platform, json_output): # pylint: disable=too-many-branches "-f", "--force", is_flag=True, - help="Reinstall/redownload dev/platform and its packages if exist") -def platform_install(platforms, with_package, without_package, - skip_default_package, force): + help="Reinstall/redownload dev/platform and its packages if exist", +) +def platform_install( + platforms, with_package, without_package, skip_default_package, force +): pm = PlatformManager() for platform in platforms: - if pm.install(name=platform, - with_packages=with_package, - without_packages=without_package, - skip_default_package=skip_default_package, - force=force): - click.secho("The platform '%s' has been successfully installed!\n" - "The rest of packages will be installed automatically " - "depending on your build environment." % platform, - fg="green") + if pm.install( + name=platform, + with_packages=with_package, + without_packages=without_package, + skip_default_package=skip_default_package, + force=force, + ): + click.secho( + "The platform '%s' has been successfully installed!\n" + "The rest of packages will be installed automatically " + "depending on your build environment." % platform, + fg="green", + ) @cli.command("uninstall", short_help="Uninstall development platform") @@ -312,35 +330,39 @@ def platform_uninstall(platforms): pm = PlatformManager() for platform in platforms: if pm.uninstall(platform): - click.secho("The platform '%s' has been successfully " - "uninstalled!" % platform, - fg="green") + click.secho( + "The platform '%s' has been successfully " "uninstalled!" % platform, + fg="green", + ) @cli.command("update", short_help="Update installed development platforms") @click.argument("platforms", nargs=-1, required=False, metavar="[PLATFORM...]") -@click.option("-p", - "--only-packages", - is_flag=True, - help="Update only the platform packages") -@click.option("-c", - "--only-check", - is_flag=True, - help="DEPRECATED. Please use `--dry-run` instead") -@click.option("--dry-run", - is_flag=True, - help="Do not update, only check for the new versions") +@click.option( + "-p", "--only-packages", is_flag=True, help="Update only the platform packages" +) +@click.option( + "-c", + "--only-check", + is_flag=True, + help="DEPRECATED. Please use `--dry-run` instead", +) +@click.option( + "--dry-run", is_flag=True, help="Do not update, only check for the new versions" +) @click.option("--json-output", is_flag=True) def platform_update( # pylint: disable=too-many-locals - platforms, only_packages, only_check, dry_run, json_output): + platforms, only_packages, only_check, dry_run, json_output +): pm = PlatformManager() pkg_dir_to_name = {} if not platforms: platforms = [] for manifest in pm.get_installed(): - platforms.append(manifest['__pkg_dir']) - pkg_dir_to_name[manifest['__pkg_dir']] = manifest.get( - "title", manifest['name']) + platforms.append(manifest["__pkg_dir"]) + pkg_dir_to_name[manifest["__pkg_dir"]] = manifest.get( + "title", manifest["name"] + ) only_check = dry_run or only_check @@ -356,14 +378,16 @@ def platform_update( # pylint: disable=too-many-locals if not pkg_dir: continue latest = pm.outdated(pkg_dir, requirements) - if (not latest and not PlatformFactory.newPlatform( - pkg_dir).are_outdated_packages()): + if ( + not latest + and not PlatformFactory.newPlatform(pkg_dir).are_outdated_packages() + ): continue - data = _get_installed_platform_data(pkg_dir, - with_boards=False, - expose_packages=False) + data = _get_installed_platform_data( + pkg_dir, with_boards=False, expose_packages=False + ) if latest: - data['versionLatest'] = latest + data["versionLatest"] = latest result.append(data) return click.echo(dump_json_to_unicode(result)) @@ -371,8 +395,9 @@ def platform_update( # pylint: disable=too-many-locals app.clean_cache() for platform in platforms: click.echo( - "Platform %s" % - click.style(pkg_dir_to_name.get(platform, platform), fg="cyan")) + "Platform %s" + % click.style(pkg_dir_to_name.get(platform, platform), fg="cyan") + ) click.echo("--------") pm.update(platform, only_packages=only_packages, only_check=only_check) click.echo() diff --git a/platformio/commands/remote.py b/platformio/commands/remote.py index b5649979..e15bbf7a 100644 --- a/platformio/commands/remote.py +++ b/platformio/commands/remote.py @@ -43,13 +43,12 @@ def remote_agent(): @remote_agent.command("start", short_help="Start agent") @click.option("-n", "--name") @click.option("-s", "--share", multiple=True, metavar="E-MAIL") -@click.option("-d", - "--working-dir", - envvar="PLATFORMIO_REMOTE_AGENT_DIR", - type=click.Path(file_okay=False, - dir_okay=True, - writable=True, - resolve_path=True)) +@click.option( + "-d", + "--working-dir", + envvar="PLATFORMIO_REMOTE_AGENT_DIR", + type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True), +) def remote_agent_start(**kwargs): pioplus_call(sys.argv[1:]) @@ -64,15 +63,16 @@ def remote_agent_list(): pioplus_call(sys.argv[1:]) -@cli.command("update", - short_help="Update installed Platforms, Packages and Libraries") -@click.option("-c", - "--only-check", - is_flag=True, - help="DEPRECATED. Please use `--dry-run` instead") -@click.option("--dry-run", - is_flag=True, - help="Do not update, only check for the new versions") +@cli.command("update", short_help="Update installed Platforms, Packages and Libraries") +@click.option( + "-c", + "--only-check", + is_flag=True, + help="DEPRECATED. Please use `--dry-run` instead", +) +@click.option( + "--dry-run", is_flag=True, help="Do not update, only check for the new versions" +) def remote_update(only_check, dry_run): pioplus_call(sys.argv[1:]) @@ -81,14 +81,14 @@ def remote_update(only_check, dry_run): @click.option("-e", "--environment", multiple=True) @click.option("-t", "--target", multiple=True) @click.option("--upload-port") -@click.option("-d", - "--project-dir", - default=getcwd, - type=click.Path(exists=True, - file_okay=True, - dir_okay=True, - writable=True, - resolve_path=True)) +@click.option( + "-d", + "--project-dir", + default=getcwd, + type=click.Path( + exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True + ), +) @click.option("--disable-auto-clean", is_flag=True) @click.option("-r", "--force-remote", is_flag=True) @click.option("-s", "--silent", is_flag=True) @@ -102,14 +102,14 @@ def remote_run(**kwargs): @click.option("--ignore", "-i", multiple=True, metavar="") @click.option("--upload-port") @click.option("--test-port") -@click.option("-d", - "--project-dir", - default=getcwd, - type=click.Path(exists=True, - file_okay=False, - dir_okay=True, - writable=True, - resolve_path=True)) +@click.option( + "-d", + "--project-dir", + default=getcwd, + type=click.Path( + exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True + ), +) @click.option("-r", "--force-remote", is_flag=True) @click.option("--without-building", is_flag=True) @click.option("--without-uploading", is_flag=True) @@ -131,58 +131,61 @@ def device_list(json_output): @remote_device.command("monitor", short_help="Monitor remote device") @click.option("--port", "-p", help="Port, a number or a device name") -@click.option("--baud", - "-b", - type=int, - default=9600, - help="Set baud rate, default=9600") -@click.option("--parity", - default="N", - type=click.Choice(["N", "E", "O", "S", "M"]), - help="Set parity, default=N") -@click.option("--rtscts", - is_flag=True, - help="Enable RTS/CTS flow control, default=Off") -@click.option("--xonxoff", - is_flag=True, - help="Enable software flow control, default=Off") -@click.option("--rts", - default=None, - type=click.IntRange(0, 1), - help="Set initial RTS line state") -@click.option("--dtr", - default=None, - type=click.IntRange(0, 1), - help="Set initial DTR line state") +@click.option( + "--baud", "-b", type=int, default=9600, help="Set baud rate, default=9600" +) +@click.option( + "--parity", + default="N", + type=click.Choice(["N", "E", "O", "S", "M"]), + help="Set parity, default=N", +) +@click.option("--rtscts", is_flag=True, help="Enable RTS/CTS flow control, default=Off") +@click.option( + "--xonxoff", is_flag=True, help="Enable software flow control, default=Off" +) +@click.option( + "--rts", default=None, type=click.IntRange(0, 1), help="Set initial RTS line state" +) +@click.option( + "--dtr", default=None, type=click.IntRange(0, 1), help="Set initial DTR line state" +) @click.option("--echo", is_flag=True, help="Enable local echo, default=Off") -@click.option("--encoding", - default="UTF-8", - help="Set the encoding for the serial port (e.g. hexlify, " - "Latin1, UTF-8), default: UTF-8") +@click.option( + "--encoding", + default="UTF-8", + help="Set the encoding for the serial port (e.g. hexlify, " + "Latin1, UTF-8), default: UTF-8", +) @click.option("--filter", "-f", multiple=True, help="Add text transformation") -@click.option("--eol", - default="CRLF", - type=click.Choice(["CR", "LF", "CRLF"]), - help="End of line mode, default=CRLF") -@click.option("--raw", - is_flag=True, - help="Do not apply any encodings/transformations") -@click.option("--exit-char", - type=int, - default=3, - help="ASCII code of special character that is used to exit " - "the application, default=3 (Ctrl+C)") -@click.option("--menu-char", - type=int, - default=20, - help="ASCII code of special character that is used to " - "control miniterm (menu), default=20 (DEC)") -@click.option("--quiet", - is_flag=True, - help="Diagnostics: suppress non-error messages, default=Off") +@click.option( + "--eol", + default="CRLF", + type=click.Choice(["CR", "LF", "CRLF"]), + help="End of line mode, default=CRLF", +) +@click.option("--raw", is_flag=True, help="Do not apply any encodings/transformations") +@click.option( + "--exit-char", + type=int, + default=3, + help="ASCII code of special character that is used to exit " + "the application, default=3 (Ctrl+C)", +) +@click.option( + "--menu-char", + type=int, + default=20, + help="ASCII code of special character that is used to " + "control miniterm (menu), default=20 (DEC)", +) +@click.option( + "--quiet", + is_flag=True, + help="Diagnostics: suppress non-error messages, default=Off", +) @click.pass_context def device_monitor(ctx, **kwargs): - def _tx_target(sock_dir): try: pioplus_call(sys.argv[1:] + ["--sock", sock_dir]) @@ -192,13 +195,13 @@ def device_monitor(ctx, **kwargs): sock_dir = mkdtemp(suffix="pioplus") sock_file = join(sock_dir, "sock") try: - t = threading.Thread(target=_tx_target, args=(sock_dir, )) + t = threading.Thread(target=_tx_target, args=(sock_dir,)) t.start() while t.is_alive() and not isfile(sock_file): sleep(0.1) if not t.is_alive(): return - kwargs['port'] = get_file_contents(sock_file) + kwargs["port"] = get_file_contents(sock_file) ctx.invoke(cmd_device_monitor, **kwargs) t.join(2) finally: diff --git a/platformio/commands/run/command.py b/platformio/commands/run/command.py index 7b5ca50c..fd53e5fc 100644 --- a/platformio/commands/run/command.py +++ b/platformio/commands/run/command.py @@ -22,13 +22,11 @@ from tabulate import tabulate from platformio import exception, fs, util from platformio.commands.device import device_monitor as cmd_device_monitor -from platformio.commands.run.helpers import (clean_build_dir, - handle_legacy_libdeps) +from platformio.commands.run.helpers import clean_build_dir, handle_legacy_libdeps from platformio.commands.run.processor import EnvironmentProcessor from platformio.commands.test.processor import CTX_META_TEST_IS_RUNNING from platformio.project.config import ProjectConfig -from platformio.project.helpers import (find_project_dir_above, - get_project_build_dir) +from platformio.project.helpers import find_project_dir_above, get_project_build_dir # pylint: disable=too-many-arguments,too-many-locals,too-many-branches @@ -42,34 +40,47 @@ except NotImplementedError: @click.option("-e", "--environment", multiple=True) @click.option("-t", "--target", multiple=True) @click.option("--upload-port") -@click.option("-d", - "--project-dir", - default=getcwd, - type=click.Path(exists=True, - file_okay=True, - dir_okay=True, - writable=True, - resolve_path=True)) -@click.option("-c", - "--project-conf", - type=click.Path(exists=True, - file_okay=True, - dir_okay=False, - readable=True, - resolve_path=True)) -@click.option("-j", - "--jobs", - type=int, - default=DEFAULT_JOB_NUMS, - help=("Allow N jobs at once. " - "Default is a number of CPUs in a system (N=%d)" % - DEFAULT_JOB_NUMS)) +@click.option( + "-d", + "--project-dir", + default=getcwd, + type=click.Path( + exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True + ), +) +@click.option( + "-c", + "--project-conf", + type=click.Path( + exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True + ), +) +@click.option( + "-j", + "--jobs", + type=int, + default=DEFAULT_JOB_NUMS, + help=( + "Allow N jobs at once. " + "Default is a number of CPUs in a system (N=%d)" % DEFAULT_JOB_NUMS + ), +) @click.option("-s", "--silent", is_flag=True) @click.option("-v", "--verbose", is_flag=True) @click.option("--disable-auto-clean", is_flag=True) @click.pass_context -def cli(ctx, environment, target, upload_port, project_dir, project_conf, jobs, - silent, verbose, disable_auto_clean): +def cli( + ctx, + environment, + target, + upload_port, + project_dir, + project_conf, + jobs, + silent, + verbose, + disable_auto_clean, +): # find project directory on upper level if isfile(project_dir): project_dir = find_project_dir_above(project_dir) @@ -78,7 +89,8 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, jobs, with fs.cd(project_dir): config = ProjectConfig.get_instance( - project_conf or join(project_dir, "platformio.ini")) + project_conf or join(project_dir, "platformio.ini") + ) config.validate(environment) # clean obsolete build dir @@ -88,36 +100,48 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, jobs, except: # pylint: disable=bare-except click.secho( "Can not remove temporary directory `%s`. Please remove " - "it manually to avoid build issues" % - get_project_build_dir(force=True), - fg="yellow") + "it manually to avoid build issues" + % get_project_build_dir(force=True), + fg="yellow", + ) handle_legacy_libdeps(project_dir, config) default_envs = config.default_envs() results = [] for env in config.envs(): - skipenv = any([ - environment and env not in environment, not environment - and default_envs and env not in default_envs - ]) + skipenv = any( + [ + environment and env not in environment, + not environment and default_envs and env not in default_envs, + ] + ) if skipenv: results.append({"env": env}) continue # print empty line between multi environment project - if not silent and any( - r.get("succeeded") is not None for r in results): + if not silent and any(r.get("succeeded") is not None for r in results): click.echo() results.append( - process_env(ctx, env, config, environment, target, upload_port, - silent, verbose, jobs, is_test_running)) + process_env( + ctx, + env, + config, + environment, + target, + upload_port, + silent, + verbose, + jobs, + is_test_running, + ) + ) command_failed = any(r.get("succeeded") is False for r in results) - if (not is_test_running and (command_failed or not silent) - and len(results) > 1): + if not is_test_running and (command_failed or not silent) and len(results) > 1: print_processing_summary(results) if command_failed: @@ -125,24 +149,39 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, jobs, return True -def process_env(ctx, name, config, environments, targets, upload_port, silent, - verbose, jobs, is_test_running): +def process_env( + ctx, + name, + config, + environments, + targets, + upload_port, + silent, + verbose, + jobs, + is_test_running, +): if not is_test_running and not silent: print_processing_header(name, config, verbose) - ep = EnvironmentProcessor(ctx, name, config, targets, upload_port, silent, - verbose, jobs) + ep = EnvironmentProcessor( + ctx, name, config, targets, upload_port, silent, verbose, jobs + ) result = {"env": name, "duration": time(), "succeeded": ep.process()} - result['duration'] = time() - result['duration'] + result["duration"] = time() - result["duration"] # print footer on error or when is not unit testing - if not is_test_running and (not silent or not result['succeeded']): + if not is_test_running and (not silent or not result["succeeded"]): print_processing_footer(result) - if (result['succeeded'] and "monitor" in ep.get_build_targets() - and "nobuild" not in ep.get_build_targets()): - ctx.invoke(cmd_device_monitor, - environment=environments[0] if environments else None) + if ( + result["succeeded"] + and "monitor" in ep.get_build_targets() + and "nobuild" not in ep.get_build_targets() + ): + ctx.invoke( + cmd_device_monitor, environment=environments[0] if environments else None + ) return result @@ -151,10 +190,11 @@ def print_processing_header(env, config, verbose=False): env_dump = [] for k, v in config.items(env=env): if verbose or k in ("platform", "framework", "board"): - env_dump.append("%s: %s" % - (k, ", ".join(v) if isinstance(v, list) else v)) - click.echo("Processing %s (%s)" % - (click.style(env, fg="cyan", bold=True), "; ".join(env_dump))) + env_dump.append("%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v)) + click.echo( + "Processing %s (%s)" + % (click.style(env, fg="cyan", bold=True), "; ".join(env_dump)) + ) terminal_width, _ = click.get_terminal_size() click.secho("-" * terminal_width, bold=True) @@ -162,10 +202,17 @@ def print_processing_header(env, config, verbose=False): def print_processing_footer(result): is_failed = not result.get("succeeded") util.print_labeled_bar( - "[%s] Took %.2f seconds" % - ((click.style("FAILED", fg="red", bold=True) if is_failed else - click.style("SUCCESS", fg="green", bold=True)), result['duration']), - is_error=is_failed) + "[%s] Took %.2f seconds" + % ( + ( + click.style("FAILED", fg="red", bold=True) + if is_failed + else click.style("SUCCESS", fg="green", bold=True) + ), + result["duration"], + ), + is_error=is_failed, + ) def print_processing_summary(results): @@ -186,20 +233,31 @@ def print_processing_summary(results): status_str = click.style("SUCCESS", fg="green") tabular_data.append( - (click.style(result['env'], fg="cyan"), status_str, - util.humanize_duration_time(result.get("duration")))) + ( + click.style(result["env"], fg="cyan"), + status_str, + util.humanize_duration_time(result.get("duration")), + ) + ) click.echo() - click.echo(tabulate(tabular_data, - headers=[ - click.style(s, bold=True) - for s in ("Environment", "Status", "Duration") - ]), - err=failed_nums) + click.echo( + tabulate( + tabular_data, + headers=[ + click.style(s, bold=True) for s in ("Environment", "Status", "Duration") + ], + ), + err=failed_nums, + ) util.print_labeled_bar( - "%s%d succeeded in %s" % - ("%d failed, " % failed_nums if failed_nums else "", succeeded_nums, - util.humanize_duration_time(duration)), + "%s%d succeeded in %s" + % ( + "%d failed, " % failed_nums if failed_nums else "", + succeeded_nums, + util.humanize_duration_time(duration), + ), is_error=failed_nums, - fg="red" if failed_nums else "green") + fg="red" if failed_nums else "green", + ) diff --git a/platformio/commands/run/helpers.py b/platformio/commands/run/helpers.py index 3e6497f5..d9481433 100644 --- a/platformio/commands/run/helpers.py +++ b/platformio/commands/run/helpers.py @@ -18,15 +18,16 @@ from os.path import isdir, isfile, join import click from platformio import fs -from platformio.project.helpers import (compute_project_checksum, - get_project_dir, - get_project_libdeps_dir) +from platformio.project.helpers import ( + compute_project_checksum, + get_project_dir, + get_project_libdeps_dir, +) def handle_legacy_libdeps(project_dir, config): legacy_libdeps_dir = join(project_dir, ".piolibdeps") - if (not isdir(legacy_libdeps_dir) - or legacy_libdeps_dir == get_project_libdeps_dir()): + if not isdir(legacy_libdeps_dir) or legacy_libdeps_dir == get_project_libdeps_dir(): return if not config.has_section("env"): config.add_section("env") @@ -39,7 +40,8 @@ def handle_legacy_libdeps(project_dir, config): " file using `lib_deps` option and remove `{0}` folder." "\nMore details -> http://docs.platformio.org/page/projectconf/" "section_env_library.html#lib-deps".format(legacy_libdeps_dir), - fg="yellow") + fg="yellow", + ) def clean_build_dir(build_dir, config): diff --git a/platformio/commands/run/processor.py b/platformio/commands/run/processor.py index b061c00a..032286e5 100644 --- a/platformio/commands/run/processor.py +++ b/platformio/commands/run/processor.py @@ -13,8 +13,7 @@ # limitations under the License. from platformio import exception, telemetry -from platformio.commands.platform import \ - platform_install as cmd_platform_install +from platformio.commands.platform import platform_install as cmd_platform_install from platformio.commands.test.processor import CTX_META_TEST_RUNNING_NAME from platformio.managers.platform import PlatformFactory @@ -22,10 +21,9 @@ from platformio.managers.platform import PlatformFactory class EnvironmentProcessor(object): - def __init__( # pylint: disable=too-many-arguments - self, cmd_ctx, name, config, targets, upload_port, silent, verbose, - jobs): + self, cmd_ctx, name, config, targets, upload_port, silent, verbose, jobs + ): self.cmd_ctx = cmd_ctx self.name = name self.config = config @@ -40,12 +38,13 @@ class EnvironmentProcessor(object): variables = {"pioenv": self.name, "project_config": self.config.path} if CTX_META_TEST_RUNNING_NAME in self.cmd_ctx.meta: - variables['piotest_running_name'] = self.cmd_ctx.meta[ - CTX_META_TEST_RUNNING_NAME] + variables["piotest_running_name"] = self.cmd_ctx.meta[ + CTX_META_TEST_RUNNING_NAME + ] if self.upload_port: # override upload port with a custom from CLI - variables['upload_port'] = self.upload_port + variables["upload_port"] = self.upload_port return variables def get_build_targets(self): @@ -67,13 +66,14 @@ class EnvironmentProcessor(object): build_targets.remove("monitor") try: - p = PlatformFactory.newPlatform(self.options['platform']) + p = PlatformFactory.newPlatform(self.options["platform"]) except exception.UnknownPlatform: - self.cmd_ctx.invoke(cmd_platform_install, - platforms=[self.options['platform']], - skip_default_package=True) - p = PlatformFactory.newPlatform(self.options['platform']) + self.cmd_ctx.invoke( + cmd_platform_install, + platforms=[self.options["platform"]], + skip_default_package=True, + ) + p = PlatformFactory.newPlatform(self.options["platform"]) - result = p.run(build_vars, build_targets, self.silent, self.verbose, - self.jobs) - return result['returncode'] == 0 + result = p.run(build_vars, build_targets, self.silent, self.verbose, self.jobs) + return result["returncode"] == 0 diff --git a/platformio/commands/settings.py b/platformio/commands/settings.py index c626e4ca..7f03f81b 100644 --- a/platformio/commands/settings.py +++ b/platformio/commands/settings.py @@ -42,20 +42,24 @@ def settings_get(name): raw_value = app.get_setting(key) formatted_value = format_value(raw_value) - if raw_value != options['value']: - default_formatted_value = format_value(options['value']) + if raw_value != options["value"]: + default_formatted_value = format_value(options["value"]) formatted_value += "%s" % ( - "\n" if len(default_formatted_value) > 10 else " ") - formatted_value += "[%s]" % click.style(default_formatted_value, - fg="yellow") + "\n" if len(default_formatted_value) > 10 else " " + ) + formatted_value += "[%s]" % click.style( + default_formatted_value, fg="yellow" + ) tabular_data.append( - (click.style(key, - fg="cyan"), formatted_value, options['description'])) + (click.style(key, fg="cyan"), formatted_value, options["description"]) + ) click.echo( - tabulate(tabular_data, - headers=["Name", "Current value [Default]", "Description"])) + tabulate( + tabular_data, headers=["Name", "Current value [Default]", "Description"] + ) + ) @cli.command("set", short_help="Set new value for the setting") diff --git a/platformio/commands/test/command.py b/platformio/commands/test/command.py index ce50b5d9..17aec86f 100644 --- a/platformio/commands/test/command.py +++ b/platformio/commands/test/command.py @@ -31,51 +31,72 @@ from platformio.project.helpers import get_project_test_dir @click.command("test", short_help="Unit Testing") @click.option("--environment", "-e", multiple=True, metavar="") -@click.option("--filter", - "-f", - multiple=True, - metavar="", - help="Filter tests by a pattern") -@click.option("--ignore", - "-i", - multiple=True, - metavar="", - help="Ignore tests by a pattern") +@click.option( + "--filter", + "-f", + multiple=True, + metavar="", + help="Filter tests by a pattern", +) +@click.option( + "--ignore", + "-i", + multiple=True, + metavar="", + help="Ignore tests by a pattern", +) @click.option("--upload-port") @click.option("--test-port") -@click.option("-d", - "--project-dir", - default=getcwd, - type=click.Path(exists=True, - file_okay=False, - dir_okay=True, - writable=True, - resolve_path=True)) -@click.option("-c", - "--project-conf", - type=click.Path(exists=True, - file_okay=True, - dir_okay=False, - readable=True, - resolve_path=True)) +@click.option( + "-d", + "--project-dir", + default=getcwd, + type=click.Path( + exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True + ), +) +@click.option( + "-c", + "--project-conf", + type=click.Path( + exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True + ), +) @click.option("--without-building", is_flag=True) @click.option("--without-uploading", is_flag=True) @click.option("--without-testing", is_flag=True) @click.option("--no-reset", is_flag=True) -@click.option("--monitor-rts", - default=None, - type=click.IntRange(0, 1), - help="Set initial RTS line state for Serial Monitor") -@click.option("--monitor-dtr", - default=None, - type=click.IntRange(0, 1), - help="Set initial DTR line state for Serial Monitor") +@click.option( + "--monitor-rts", + default=None, + type=click.IntRange(0, 1), + help="Set initial RTS line state for Serial Monitor", +) +@click.option( + "--monitor-dtr", + default=None, + type=click.IntRange(0, 1), + help="Set initial DTR line state for Serial Monitor", +) @click.option("--verbose", "-v", is_flag=True) @click.pass_context def cli( # pylint: disable=redefined-builtin - ctx, environment, ignore, filter, upload_port, test_port, project_dir, - project_conf, without_building, without_uploading, without_testing, - no_reset, monitor_rts, monitor_dtr, verbose): + ctx, + environment, + ignore, + filter, + upload_port, + test_port, + project_dir, + project_conf, + without_building, + without_uploading, + without_testing, + no_reset, + monitor_rts, + monitor_dtr, + verbose, +): with fs.cd(project_dir): test_dir = get_project_test_dir() if not isdir(test_dir): @@ -83,7 +104,8 @@ def cli( # pylint: disable=redefined-builtin test_names = get_test_names(test_dir) config = ProjectConfig.get_instance( - project_conf or join(project_dir, "platformio.ini")) + project_conf or join(project_dir, "platformio.ini") + ) config.validate(envs=environment) click.echo("Verbose mode can be enabled via `-v, --verbose` option") @@ -99,19 +121,16 @@ def cli( # pylint: disable=redefined-builtin # filter and ignore patterns patterns = dict(filter=list(filter), ignore=list(ignore)) for key in patterns: - patterns[key].extend( - config.get(section, "test_%s" % key, [])) + patterns[key].extend(config.get(section, "test_%s" % key, [])) skip_conditions = [ environment and envname not in environment, - not environment and default_envs - and envname not in default_envs, - testname != "*" and patterns['filter'] and - not any([fnmatch(testname, p) - for p in patterns['filter']]), + not environment and default_envs and envname not in default_envs, testname != "*" - and any([fnmatch(testname, p) - for p in patterns['ignore']]), + and patterns["filter"] + and not any([fnmatch(testname, p) for p in patterns["filter"]]), + testname != "*" + and any([fnmatch(testname, p) for p in patterns["ignore"]]), ] if any(skip_conditions): results.append({"env": envname, "test": testname}) @@ -120,29 +139,36 @@ def cli( # pylint: disable=redefined-builtin click.echo() print_processing_header(testname, envname) - cls = (NativeTestProcessor - if config.get(section, "platform") == "native" else - EmbeddedTestProcessor) + cls = ( + NativeTestProcessor + if config.get(section, "platform") == "native" + else EmbeddedTestProcessor + ) tp = cls( - ctx, testname, envname, - dict(project_config=config, - project_dir=project_dir, - upload_port=upload_port, - test_port=test_port, - without_building=without_building, - without_uploading=without_uploading, - without_testing=without_testing, - no_reset=no_reset, - monitor_rts=monitor_rts, - monitor_dtr=monitor_dtr, - verbose=verbose)) + ctx, + testname, + envname, + dict( + project_config=config, + project_dir=project_dir, + upload_port=upload_port, + test_port=test_port, + without_building=without_building, + without_uploading=without_uploading, + without_testing=without_testing, + no_reset=no_reset, + monitor_rts=monitor_rts, + monitor_dtr=monitor_dtr, + verbose=verbose, + ), + ) result = { "env": envname, "test": testname, "duration": time(), - "succeeded": tp.process() + "succeeded": tp.process(), } - result['duration'] = time() - result['duration'] + result["duration"] = time() - result["duration"] results.append(result) print_processing_footer(result) @@ -168,8 +194,13 @@ def get_test_names(test_dir): def print_processing_header(test, env): - click.echo("Processing %s in %s environment" % (click.style( - test, fg="yellow", bold=True), click.style(env, fg="cyan", bold=True))) + click.echo( + "Processing %s in %s environment" + % ( + click.style(test, fg="yellow", bold=True), + click.style(env, fg="cyan", bold=True), + ) + ) terminal_width, _ = click.get_terminal_size() click.secho("-" * terminal_width, bold=True) @@ -177,10 +208,17 @@ def print_processing_header(test, env): def print_processing_footer(result): is_failed = not result.get("succeeded") util.print_labeled_bar( - "[%s] Took %.2f seconds" % - ((click.style("FAILED", fg="red", bold=True) if is_failed else - click.style("PASSED", fg="green", bold=True)), result['duration']), - is_error=is_failed) + "[%s] Took %.2f seconds" + % ( + ( + click.style("FAILED", fg="red", bold=True) + if is_failed + else click.style("PASSED", fg="green", bold=True) + ), + result["duration"], + ), + is_error=is_failed, + ) def print_testing_summary(results): @@ -203,20 +241,32 @@ def print_testing_summary(results): status_str = click.style("PASSED", fg="green") tabular_data.append( - (result['test'], click.style(result['env'], fg="cyan"), status_str, - util.humanize_duration_time(result.get("duration")))) + ( + result["test"], + click.style(result["env"], fg="cyan"), + status_str, + util.humanize_duration_time(result.get("duration")), + ) + ) - click.echo(tabulate(tabular_data, - headers=[ - click.style(s, bold=True) - for s in ("Test", "Environment", "Status", - "Duration") - ]), - err=failed_nums) + click.echo( + tabulate( + tabular_data, + headers=[ + click.style(s, bold=True) + for s in ("Test", "Environment", "Status", "Duration") + ], + ), + err=failed_nums, + ) util.print_labeled_bar( - "%s%d succeeded in %s" % - ("%d failed, " % failed_nums if failed_nums else "", succeeded_nums, - util.humanize_duration_time(duration)), + "%s%d succeeded in %s" + % ( + "%d failed, " % failed_nums if failed_nums else "", + succeeded_nums, + util.humanize_duration_time(duration), + ), is_error=failed_nums, - fg="red" if failed_nums else "green") + fg="red" if failed_nums else "green", + ) diff --git a/platformio/commands/test/embedded.py b/platformio/commands/test/embedded.py index 6870dcd5..dc0d9ef0 100644 --- a/platformio/commands/test/embedded.py +++ b/platformio/commands/test/embedded.py @@ -27,47 +27,50 @@ class EmbeddedTestProcessor(TestProcessorBase): SERIAL_TIMEOUT = 600 def process(self): - if not self.options['without_building']: + if not self.options["without_building"]: self.print_progress("Building...") target = ["__test"] - if self.options['without_uploading']: + if self.options["without_uploading"]: target.append("checkprogsize") if not self.build_or_upload(target): return False - if not self.options['without_uploading']: + if not self.options["without_uploading"]: self.print_progress("Uploading...") target = ["upload"] - if self.options['without_building']: + if self.options["without_building"]: target.append("nobuild") else: target.append("__test") if not self.build_or_upload(target): return False - if self.options['without_testing']: + if self.options["without_testing"]: return None self.print_progress("Testing...") return self.run() def run(self): - click.echo("If you don't see any output for the first 10 secs, " - "please reset board (press reset button)") + click.echo( + "If you don't see any output for the first 10 secs, " + "please reset board (press reset button)" + ) click.echo() try: - ser = serial.Serial(baudrate=self.get_baudrate(), - timeout=self.SERIAL_TIMEOUT) + ser = serial.Serial( + baudrate=self.get_baudrate(), timeout=self.SERIAL_TIMEOUT + ) ser.port = self.get_test_port() - ser.rts = self.options['monitor_rts'] - ser.dtr = self.options['monitor_dtr'] + ser.rts = self.options["monitor_rts"] + ser.dtr = self.options["monitor_dtr"] ser.open() except serial.SerialException as e: click.secho(str(e), fg="red", err=True) return False - if not self.options['no_reset']: + if not self.options["no_reset"]: ser.flushInput() ser.setDTR(False) ser.setRTS(False) @@ -105,17 +108,16 @@ class EmbeddedTestProcessor(TestProcessorBase): return self.env_options.get("test_port") assert set(["platform", "board"]) & set(self.env_options.keys()) - p = PlatformFactory.newPlatform(self.env_options['platform']) - board_hwids = p.board_config(self.env_options['board']).get( - "build.hwids", []) + p = PlatformFactory.newPlatform(self.env_options["platform"]) + board_hwids = p.board_config(self.env_options["board"]).get("build.hwids", []) port = None elapsed = 0 while elapsed < 5 and not port: for item in util.get_serialports(): - port = item['port'] + port = item["port"] for hwid in board_hwids: hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "") - if hwid_str in item['hwid']: + if hwid_str in item["hwid"]: return port # check if port is already configured @@ -131,5 +133,6 @@ class EmbeddedTestProcessor(TestProcessorBase): if not port: raise exception.PlatformioException( "Please specify `test_port` for environment or use " - "global `--test-port` option.") + "global `--test-port` option." + ) return port diff --git a/platformio/commands/test/native.py b/platformio/commands/test/native.py index e5ba8f3d..75d4fcf8 100644 --- a/platformio/commands/test/native.py +++ b/platformio/commands/test/native.py @@ -21,23 +21,23 @@ from platformio.project.helpers import get_project_build_dir class NativeTestProcessor(TestProcessorBase): - def process(self): - if not self.options['without_building']: + if not self.options["without_building"]: self.print_progress("Building...") if not self.build_or_upload(["__test"]): return False - if self.options['without_testing']: + if self.options["without_testing"]: return None self.print_progress("Testing...") return self.run() def run(self): - with fs.cd(self.options['project_dir']): + with fs.cd(self.options["project_dir"]): build_dir = get_project_build_dir() result = proc.exec_command( [join(build_dir, self.env_name, "program")], stdout=LineBufferedAsyncPipe(self.on_run_out), - stderr=LineBufferedAsyncPipe(self.on_run_out)) + stderr=LineBufferedAsyncPipe(self.on_run_out), + ) assert "returncode" in result - return result['returncode'] == 0 and not self._run_failed + return result["returncode"] == 0 and not self._run_failed diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index 085ffb2c..b04a2789 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -29,7 +29,7 @@ TRANSPORT_OPTIONS = { "putchar": "Serial.write(c)", "flush": "Serial.flush()", "begin": "Serial.begin($baudrate)", - "end": "Serial.end()" + "end": "Serial.end()", }, "mbed": { "include": "#include ", @@ -37,7 +37,7 @@ TRANSPORT_OPTIONS = { "putchar": "pc.putc(c)", "flush": "", "begin": "pc.baud($baudrate)", - "end": "" + "end": "", }, "espidf": { "include": "#include ", @@ -45,7 +45,7 @@ TRANSPORT_OPTIONS = { "putchar": "putchar(c)", "flush": "fflush(stdout)", "begin": "", - "end": "" + "end": "", }, "native": { "include": "#include ", @@ -53,7 +53,7 @@ TRANSPORT_OPTIONS = { "putchar": "putchar(c)", "flush": "fflush(stdout)", "begin": "", - "end": "" + "end": "", }, "custom": { "include": '#include "unittest_transport.h"', @@ -61,8 +61,8 @@ TRANSPORT_OPTIONS = { "putchar": "unittest_uart_putchar(c)", "flush": "unittest_uart_flush()", "begin": "unittest_uart_begin()", - "end": "unittest_uart_end()" - } + "end": "unittest_uart_end()", + }, } CTX_META_TEST_IS_RUNNING = __name__ + ".test_running" @@ -79,8 +79,7 @@ class TestProcessorBase(object): self.test_name = testname self.options = options self.env_name = envname - self.env_options = options['project_config'].items(env=envname, - as_dict=True) + self.env_options = options["project_config"].items(env=envname, as_dict=True) self._run_failed = False self._outputcpp_generated = False @@ -90,10 +89,11 @@ class TestProcessorBase(object): elif "framework" in self.env_options: transport = self.env_options.get("framework")[0] if "test_transport" in self.env_options: - transport = self.env_options['test_transport'] + transport = self.env_options["test_transport"] if transport not in TRANSPORT_OPTIONS: raise exception.PlatformioException( - "Unknown Unit Test transport `%s`" % transport) + "Unknown Unit Test transport `%s`" % transport + ) return transport.lower() def get_baudrate(self): @@ -112,13 +112,16 @@ class TestProcessorBase(object): try: from platformio.commands.run import cli as cmd_run - return self.cmd_ctx.invoke(cmd_run, - project_dir=self.options['project_dir'], - upload_port=self.options['upload_port'], - silent=not self.options['verbose'], - environment=[self.env_name], - disable_auto_clean="nobuild" in target, - target=target) + + return self.cmd_ctx.invoke( + cmd_run, + project_dir=self.options["project_dir"], + upload_port=self.options["upload_port"], + silent=not self.options["verbose"], + environment=[self.env_name], + disable_auto_clean="nobuild" in target, + target=target, + ) except exception.ReturnErrorCode: return False @@ -131,8 +134,7 @@ class TestProcessorBase(object): def on_run_out(self, line): line = line.strip() if line.endswith(":PASS"): - click.echo("%s\t[%s]" % - (line[:-5], click.style("PASSED", fg="green"))) + click.echo("%s\t[%s]" % (line[:-5], click.style("PASSED", fg="green"))) elif ":FAIL" in line: self._run_failed = True click.echo("%s\t[%s]" % (line, click.style("FAILED", fg="red"))) @@ -142,36 +144,38 @@ class TestProcessorBase(object): def generate_outputcpp(self, test_dir): assert isdir(test_dir) - cpp_tpl = "\n".join([ - "$include", - "#include ", - "", - "$object", - "", - "#ifdef __GNUC__", - "void output_start(unsigned int baudrate __attribute__((unused)))", - "#else", - "void output_start(unsigned int baudrate)", - "#endif", - "{", - " $begin;", - "}", - "", - "void output_char(int c)", - "{", - " $putchar;", - "}", - "", - "void output_flush(void)", - "{", - " $flush;", - "}", - "", - "void output_complete(void)", - "{", - " $end;", - "}" - ]) # yapf: disable + cpp_tpl = "\n".join( + [ + "$include", + "#include ", + "", + "$object", + "", + "#ifdef __GNUC__", + "void output_start(unsigned int baudrate __attribute__((unused)))", + "#else", + "void output_start(unsigned int baudrate)", + "#endif", + "{", + " $begin;", + "}", + "", + "void output_char(int c)", + "{", + " $putchar;", + "}", + "", + "void output_flush(void)", + "{", + " $flush;", + "}", + "", + "void output_complete(void)", + "{", + " $end;", + "}", + ] + ) # yapf: disable def delete_tmptest_file(file_): try: @@ -181,10 +185,10 @@ class TestProcessorBase(object): click.secho( "Warning: Could not remove temporary file '%s'. " "Please remove it manually." % file_, - fg="yellow") + fg="yellow", + ) - tpl = Template(cpp_tpl).substitute( - TRANSPORT_OPTIONS[self.get_transport()]) + tpl = Template(cpp_tpl).substitute(TRANSPORT_OPTIONS[self.get_transport()]) data = Template(tpl).substitute(baudrate=self.get_baudrate()) tmp_file = join(test_dir, "output_export.cpp") diff --git a/platformio/commands/update.py b/platformio/commands/update.py index dc6ed1c6..1bac4f77 100644 --- a/platformio/commands/update.py +++ b/platformio/commands/update.py @@ -22,18 +22,19 @@ from platformio.managers.core import update_core_packages from platformio.managers.lib import LibraryManager -@click.command("update", - short_help="Update installed platforms, packages and libraries") -@click.option("--core-packages", - is_flag=True, - help="Update only the core packages") -@click.option("-c", - "--only-check", - is_flag=True, - help="DEPRECATED. Please use `--dry-run` instead") -@click.option("--dry-run", - is_flag=True, - help="Do not update, only check for the new versions") +@click.command( + "update", short_help="Update installed platforms, packages and libraries" +) +@click.option("--core-packages", is_flag=True, help="Update only the core packages") +@click.option( + "-c", + "--only-check", + is_flag=True, + help="DEPRECATED. Please use `--dry-run` instead", +) +@click.option( + "--dry-run", is_flag=True, help="Do not update, only check for the new versions" +) @click.pass_context def cli(ctx, core_packages, only_check, dry_run): # cleanup lib search results, cached board and platform lists diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index a70e7780..8c0d3496 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -25,21 +25,23 @@ from platformio.proc import exec_command, get_pythonexe_path from platformio.project.helpers import get_project_cache_dir -@click.command("upgrade", - short_help="Upgrade PlatformIO to the latest version") +@click.command("upgrade", short_help="Upgrade PlatformIO to the latest version") @click.option("--dev", is_flag=True, help="Use development branch") def cli(dev): if not dev and __version__ == get_latest_version(): return click.secho( "You're up-to-date!\nPlatformIO %s is currently the " "newest version available." % __version__, - fg="green") + fg="green", + ) click.secho("Please wait while upgrading PlatformIO ...", fg="yellow") to_develop = dev or not all(c.isdigit() for c in __version__ if c != ".") - cmds = (["pip", "install", "--upgrade", - get_pip_package(to_develop)], ["platformio", "--version"]) + cmds = ( + ["pip", "install", "--upgrade", get_pip_package(to_develop)], + ["platformio", "--version"], + ) cmd = None r = {} @@ -49,26 +51,26 @@ def cli(dev): r = exec_command(cmd) # try pip with disabled cache - if r['returncode'] != 0 and cmd[2] == "pip": + if r["returncode"] != 0 and cmd[2] == "pip": cmd.insert(3, "--no-cache-dir") r = exec_command(cmd) - assert r['returncode'] == 0 - assert "version" in r['out'] - actual_version = r['out'].strip().split("version", 1)[1].strip() - click.secho("PlatformIO has been successfully upgraded to %s" % - actual_version, - fg="green") + assert r["returncode"] == 0 + assert "version" in r["out"] + actual_version = r["out"].strip().split("version", 1)[1].strip() + click.secho( + "PlatformIO has been successfully upgraded to %s" % actual_version, + fg="green", + ) click.echo("Release notes: ", nl=False) - click.secho("https://docs.platformio.org/en/latest/history.html", - fg="cyan") + click.secho("https://docs.platformio.org/en/latest/history.html", fg="cyan") except Exception as e: # pylint: disable=broad-except if not r: raise exception.UpgradeError("\n".join([str(cmd), str(e)])) permission_errors = ("permission denied", "not permitted") - if (any(m in r['err'].lower() for m in permission_errors) - and not WINDOWS): - click.secho(""" + if any(m in r["err"].lower() for m in permission_errors) and not WINDOWS: + click.secho( + """ ----------------- Permission denied ----------------- @@ -78,10 +80,11 @@ You need the `sudo` permission to install Python packages. Try WARNING! Don't use `sudo` for the rest PlatformIO commands. """, - fg="yellow", - err=True) + fg="yellow", + err=True, + ) raise exception.ReturnErrorCode(1) - raise exception.UpgradeError("\n".join([str(cmd), r['out'], r['err']])) + raise exception.UpgradeError("\n".join([str(cmd), r["out"], r["err"]])) return True @@ -89,18 +92,17 @@ WARNING! Don't use `sudo` for the rest PlatformIO commands. def get_pip_package(to_develop): if not to_develop: return "platformio" - dl_url = ("https://github.com/platformio/" - "platformio-core/archive/develop.zip") + dl_url = "https://github.com/platformio/" "platformio-core/archive/develop.zip" cache_dir = get_project_cache_dir() if not os.path.isdir(cache_dir): os.makedirs(cache_dir) pkg_name = os.path.join(cache_dir, "piocoredevelop.zip") try: with open(pkg_name, "w") as fp: - r = exec_command(["curl", "-fsSL", dl_url], - stdout=fp, - universal_newlines=True) - assert r['returncode'] == 0 + r = exec_command( + ["curl", "-fsSL", dl_url], stdout=fp, universal_newlines=True + ) + assert r["returncode"] == 0 # check ZIP structure with ZipFile(pkg_name) as zp: assert zp.testzip() is None @@ -127,7 +129,8 @@ def get_develop_latest_version(): r = requests.get( "https://raw.githubusercontent.com/platformio/platformio" "/develop/platformio/__init__.py", - headers=util.get_request_defheaders()) + headers=util.get_request_defheaders(), + ) r.raise_for_status() for line in r.text.split("\n"): line = line.strip() @@ -145,7 +148,8 @@ def get_develop_latest_version(): def get_pypi_latest_version(): - r = requests.get("https://pypi.org/pypi/platformio/json", - headers=util.get_request_defheaders()) + r = requests.get( + "https://pypi.org/pypi/platformio/json", headers=util.get_request_defheaders() + ) r.raise_for_status() - return r.json()['info']['version'] + return r.json()["info"]["version"] diff --git a/platformio/compat.py b/platformio/compat.py index 8b082b4b..712f65bc 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -20,8 +20,8 @@ import re import sys PY2 = sys.version_info[0] == 2 -CYGWIN = sys.platform.startswith('cygwin') -WINDOWS = sys.platform.startswith('win') +CYGWIN = sys.platform.startswith("cygwin") +WINDOWS = sys.platform.startswith("win") def get_filesystem_encoding(): @@ -56,13 +56,12 @@ if PY2: def dump_json_to_unicode(obj): if isinstance(obj, unicode): return obj - return json.dumps(obj, - encoding=get_filesystem_encoding(), - ensure_ascii=False, - sort_keys=True).encode("utf8") + return json.dumps( + obj, encoding=get_filesystem_encoding(), ensure_ascii=False, sort_keys=True + ).encode("utf8") - _magic_check = re.compile('([*?[])') - _magic_check_bytes = re.compile(b'([*?[])') + _magic_check = re.compile("([*?[])") + _magic_check_bytes = re.compile(b"([*?[])") def glob_escape(pathname): """Escape all special characters.""" @@ -72,14 +71,16 @@ if PY2: # escaped. drive, pathname = os.path.splitdrive(pathname) if isinstance(pathname, bytes): - pathname = _magic_check_bytes.sub(br'[\1]', pathname) + pathname = _magic_check_bytes.sub(br"[\1]", pathname) else: - pathname = _magic_check.sub(r'[\1]', pathname) + pathname = _magic_check.sub(r"[\1]", pathname) return drive + pathname + + else: from glob import escape as glob_escape # pylint: disable=no-name-in-module - string_types = (str, ) + string_types = (str,) def is_bytes(x): return isinstance(x, (bytes, memoryview, bytearray)) diff --git a/platformio/downloader.py b/platformio/downloader.py index 60f0e159..57c712ed 100644 --- a/platformio/downloader.py +++ b/platformio/downloader.py @@ -22,8 +22,11 @@ import click import requests from platformio import util -from platformio.exception import (FDSHASumMismatch, FDSizeMismatch, - FDUnrecognizedStatusCode) +from platformio.exception import ( + FDSHASumMismatch, + FDSizeMismatch, + FDUnrecognizedStatusCode, +) from platformio.proc import exec_command @@ -34,17 +37,22 @@ class FileDownloader(object): def __init__(self, url, dest_dir=None): self._request = None # make connection - self._request = requests.get(url, - stream=True, - headers=util.get_request_defheaders(), - verify=version_info >= (2, 7, 9)) + self._request = requests.get( + url, + stream=True, + headers=util.get_request_defheaders(), + verify=version_info >= (2, 7, 9), + ) if self._request.status_code != 200: raise FDUnrecognizedStatusCode(self._request.status_code, url) disposition = self._request.headers.get("content-disposition") if disposition and "filename=" in disposition: - self._fname = disposition[disposition.index("filename=") + - 9:].replace('"', "").replace("'", "") + self._fname = ( + disposition[disposition.index("filename=") + 9 :] + .replace('"', "") + .replace("'", "") + ) else: self._fname = [p for p in url.split("/") if p][-1] self._fname = str(self._fname) @@ -64,7 +72,7 @@ class FileDownloader(object): def get_size(self): if "content-length" not in self._request.headers: return -1 - return int(self._request.headers['content-length']) + return int(self._request.headers["content-length"]) def start(self, with_progress=True): label = "Downloading" @@ -101,11 +109,11 @@ class FileDownloader(object): dlsha1 = None try: result = exec_command(["sha1sum", self._destination]) - dlsha1 = result['out'] + dlsha1 = result["out"] except (OSError, ValueError): try: result = exec_command(["shasum", "-a", "1", self._destination]) - dlsha1 = result['out'] + dlsha1 = result["out"] except (OSError, ValueError): pass if not dlsha1: diff --git a/platformio/exception.py b/platformio/exception.py index 823f68af..c9f3f509 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -64,8 +64,10 @@ class IncompatiblePlatform(PlatformioException): class PlatformNotInstalledYet(PlatformioException): - MESSAGE = ("The platform '{0}' has not been installed yet. " - "Use `platformio platform install {0}` command") + MESSAGE = ( + "The platform '{0}' has not been installed yet. " + "Use `platformio platform install {0}` command" + ) class UnknownBoard(PlatformioException): @@ -102,22 +104,27 @@ class MissingPackageManifest(PlatformIOPackageException): class UndefinedPackageVersion(PlatformIOPackageException): - MESSAGE = ("Could not find a version that satisfies the requirement '{0}'" - " for your system '{1}'") + MESSAGE = ( + "Could not find a version that satisfies the requirement '{0}'" + " for your system '{1}'" + ) class PackageInstallError(PlatformIOPackageException): - MESSAGE = ("Could not install '{0}' with version requirements '{1}' " - "for your system '{2}'.\n\n" - "Please try this solution -> http://bit.ly/faq-package-manager") + MESSAGE = ( + "Could not install '{0}' with version requirements '{1}' " + "for your system '{2}'.\n\n" + "Please try this solution -> http://bit.ly/faq-package-manager" + ) class ExtractArchiveItemError(PlatformIOPackageException): MESSAGE = ( "Could not extract `{0}` to `{1}`. Try to disable antivirus " - "tool or check this solution -> http://bit.ly/faq-package-manager") + "tool or check this solution -> http://bit.ly/faq-package-manager" + ) class UnsupportedArchiveType(PlatformIOPackageException): @@ -132,14 +139,17 @@ class FDUnrecognizedStatusCode(PlatformIOPackageException): class FDSizeMismatch(PlatformIOPackageException): - MESSAGE = ("The size ({0:d} bytes) of downloaded file '{1}' " - "is not equal to remote size ({2:d} bytes)") + MESSAGE = ( + "The size ({0:d} bytes) of downloaded file '{1}' " + "is not equal to remote size ({2:d} bytes)" + ) class FDSHASumMismatch(PlatformIOPackageException): - MESSAGE = ("The 'sha1' sum '{0}' of downloaded file '{1}' " - "is not equal to remote '{2}'") + MESSAGE = ( + "The 'sha1' sum '{0}' of downloaded file '{1}' " "is not equal to remote '{2}'" + ) # @@ -156,12 +166,13 @@ class NotPlatformIOProject(PlatformIOProjectException): MESSAGE = ( "Not a PlatformIO project. `platformio.ini` file has not been " "found in current working directory ({0}). To initialize new project " - "please use `platformio init` command") + "please use `platformio init` command" + ) class InvalidProjectConf(PlatformIOProjectException): - MESSAGE = ("Invalid '{0}' (project configuration file): '{1}'") + MESSAGE = "Invalid '{0}' (project configuration file): '{1}'" class UndefinedEnvPlatform(PlatformIOProjectException): @@ -191,9 +202,11 @@ class ProjectOptionValueError(PlatformIOProjectException): class LibNotFound(PlatformioException): - MESSAGE = ("Library `{0}` has not been found in PlatformIO Registry.\n" - "You can ignore this message, if `{0}` is a built-in library " - "(included in framework, SDK). E.g., SPI, Wire, etc.") + MESSAGE = ( + "Library `{0}` has not been found in PlatformIO Registry.\n" + "You can ignore this message, if `{0}` is a built-in library " + "(included in framework, SDK). E.g., SPI, Wire, etc." + ) class NotGlobalLibDir(UserSideException): @@ -203,7 +216,8 @@ class NotGlobalLibDir(UserSideException): "To manage libraries in global storage `{1}`,\n" "please use `platformio lib --global {2}` or specify custom storage " "`platformio lib --storage-dir /path/to/storage/ {2}`.\n" - "Check `platformio lib --help` for details.") + "Check `platformio lib --help` for details." + ) class InvalidLibConfURL(PlatformioException): @@ -224,7 +238,8 @@ class MissedUdevRules(InvalidUdevRules): MESSAGE = ( "Warning! Please install `99-platformio-udev.rules`. \nMode details: " - "https://docs.platformio.org/en/latest/faq.html#platformio-udev-rules") + "https://docs.platformio.org/en/latest/faq.html#platformio-udev-rules" + ) class OutdatedUdevRules(InvalidUdevRules): @@ -232,7 +247,8 @@ class OutdatedUdevRules(InvalidUdevRules): MESSAGE = ( "Warning! Your `{0}` are outdated. Please update or reinstall them." "\n Mode details: https://docs.platformio.org" - "/en/latest/faq.html#platformio-udev-rules") + "/en/latest/faq.html#platformio-udev-rules" + ) # @@ -260,7 +276,8 @@ class InternetIsOffline(UserSideException): MESSAGE = ( "You are not connected to the Internet.\n" "If you build a project first time, we need Internet connection " - "to install all dependencies and toolchains.") + "to install all dependencies and toolchains." + ) class BuildScriptNotFound(PlatformioException): @@ -285,9 +302,11 @@ class InvalidJSONFile(PlatformioException): class CIBuildEnvsEmpty(PlatformioException): - MESSAGE = ("Can't find PlatformIO build environments.\n" - "Please specify `--board` or path to `platformio.ini` with " - "predefined environments using `--project-conf` option") + MESSAGE = ( + "Can't find PlatformIO build environments.\n" + "Please specify `--board` or path to `platformio.ini` with " + "predefined environments using `--project-conf` option" + ) class UpgradeError(PlatformioException): @@ -307,13 +326,16 @@ class HomeDirPermissionsError(PlatformioException): "current user and PlatformIO can not store configuration data.\n" "Please check the permissions and owner of that directory.\n" "Otherwise, please remove manually `{0}` directory and PlatformIO " - "will create new from the current user.") + "will create new from the current user." + ) class CygwinEnvDetected(PlatformioException): - MESSAGE = ("PlatformIO does not work within Cygwin environment. " - "Use native Terminal instead.") + MESSAGE = ( + "PlatformIO does not work within Cygwin environment. " + "Use native Terminal instead." + ) class DebugSupportError(PlatformioException): @@ -322,7 +344,8 @@ class DebugSupportError(PlatformioException): "Currently, PlatformIO does not support debugging for `{0}`.\n" "Please request support at https://github.com/platformio/" "platformio-core/issues \nor visit -> https://docs.platformio.org" - "/page/plus/debugging.html") + "/page/plus/debugging.html" + ) class DebugInvalidOptions(PlatformioException): @@ -331,8 +354,10 @@ class DebugInvalidOptions(PlatformioException): class TestDirNotExists(PlatformioException): - MESSAGE = "A test folder '{0}' does not exist.\nPlease create 'test' "\ - "directory in project's root and put a test set.\n"\ - "More details about Unit "\ - "Testing: http://docs.platformio.org/page/plus/"\ - "unit-testing.html" + MESSAGE = ( + "A test folder '{0}' does not exist.\nPlease create 'test' " + "directory in project's root and put a test set.\n" + "More details about Unit " + "Testing: http://docs.platformio.org/page/plus/" + "unit-testing.html" + ) diff --git a/platformio/fs.py b/platformio/fs.py index 1f330abe..3e708484 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -27,7 +27,6 @@ from platformio.compat import WINDOWS, get_file_contents, glob_escape class cd(object): - def __init__(self, new_path): self.new_path = new_path self.prev_path = os.getcwd() @@ -65,10 +64,10 @@ def format_filesize(filesize): if filesize < base: return "%d%s" % (filesize, suffix) for i, suffix in enumerate("KMGTPEZY"): - unit = base**(i + 2) + unit = base ** (i + 2) if filesize >= unit: continue - if filesize % (base**(i + 1)): + if filesize % (base ** (i + 1)): return "%.2f%sB" % ((base * filesize / unit), suffix) break return "%d%sB" % ((base * filesize / unit), suffix) @@ -78,21 +77,24 @@ def ensure_udev_rules(): from platformio.util import get_systype def _rules_to_set(rules_path): - return set(l.strip() for l in get_file_contents(rules_path).split("\n") - if l.strip() and not l.startswith("#")) + return set( + l.strip() + for l in get_file_contents(rules_path).split("\n") + if l.strip() and not l.startswith("#") + ) if "linux" not in get_systype(): return None installed_rules = [ "/etc/udev/rules.d/99-platformio-udev.rules", - "/lib/udev/rules.d/99-platformio-udev.rules" + "/lib/udev/rules.d/99-platformio-udev.rules", ] if not any(os.path.isfile(p) for p in installed_rules): raise exception.MissedUdevRules origin_path = os.path.abspath( - os.path.join(get_source_dir(), "..", "scripts", - "99-platformio-udev.rules")) + os.path.join(get_source_dir(), "..", "scripts", "99-platformio-udev.rules") + ) if not os.path.isfile(origin_path): return None @@ -117,7 +119,6 @@ def path_endswith_ext(path, extensions): def match_src_files(src_dir, src_filter=None, src_exts=None): - def _append_build_item(items, item, src_dir): if not src_exts or path_endswith_ext(item, src_exts): items.add(item.replace(src_dir + os.sep, "")) @@ -135,8 +136,7 @@ def match_src_files(src_dir, src_filter=None, src_exts=None): if os.path.isdir(item): for root, _, files in os.walk(item, followlinks=True): for f in files: - _append_build_item(items, os.path.join(root, f), - src_dir) + _append_build_item(items, os.path.join(root, f), src_dir) else: _append_build_item(items, item, src_dir) if action == "+": @@ -153,7 +153,6 @@ def to_unix_path(path): def rmtree(path): - def _onerror(func, path, __): try: st_mode = os.stat(path).st_mode @@ -161,9 +160,10 @@ def rmtree(path): os.chmod(path, st_mode | stat.S_IWRITE) func(path) except Exception as e: # pylint: disable=broad-except - click.secho("%s \nPlease manually remove the file `%s`" % - (str(e), path), - fg="red", - err=True) + click.secho( + "%s \nPlease manually remove the file `%s`" % (str(e), path), + fg="red", + err=True, + ) return shutil.rmtree(path, onerror=_onerror) diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index fb276af6..75252866 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -23,17 +23,17 @@ from platformio import fs, util from platformio.compat import get_file_contents from platformio.proc import where_is_program from platformio.project.config import ProjectConfig -from platformio.project.helpers import (get_project_lib_dir, - get_project_libdeps_dir, - get_project_src_dir, - load_project_ide_data) +from platformio.project.helpers import ( + get_project_lib_dir, + get_project_libdeps_dir, + get_project_src_dir, + load_project_ide_data, +) class ProjectGenerator(object): - def __init__(self, project_dir, ide, boards): - self.config = ProjectConfig.get_instance( - join(project_dir, "platformio.ini")) + self.config = ProjectConfig.get_instance(join(project_dir, "platformio.ini")) self.config.validate() self.project_dir = project_dir self.ide = str(ide) @@ -42,8 +42,7 @@ class ProjectGenerator(object): @staticmethod def get_supported_ides(): tpls_dir = join(fs.get_source_dir(), "ide", "tpls") - return sorted( - [d for d in os.listdir(tpls_dir) if isdir(join(tpls_dir, d))]) + return sorted([d for d in os.listdir(tpls_dir) if isdir(join(tpls_dir, d))]) def get_best_envname(self, boards=None): envname = None @@ -72,28 +71,29 @@ class ProjectGenerator(object): "project_dir": self.project_dir, "env_name": self.env_name, "user_home_dir": abspath(expanduser("~")), - "platformio_path": - sys.argv[0] if isfile(sys.argv[0]) - else where_is_program("platformio"), + "platformio_path": sys.argv[0] + if isfile(sys.argv[0]) + else where_is_program("platformio"), "env_path": os.getenv("PATH"), - "env_pathsep": os.pathsep - } # yapf: disable + "env_pathsep": os.pathsep, + } # yapf: disable # default env configuration tpl_vars.update(self.config.items(env=self.env_name, as_dict=True)) # build data - tpl_vars.update( - load_project_ide_data(self.project_dir, self.env_name) or {}) + tpl_vars.update(load_project_ide_data(self.project_dir, self.env_name) or {}) with fs.cd(self.project_dir): - tpl_vars.update({ - "src_files": self.get_src_files(), - "project_src_dir": get_project_src_dir(), - "project_lib_dir": get_project_lib_dir(), - "project_libdeps_dir": join( - get_project_libdeps_dir(), self.env_name) - - }) # yapf: disable + tpl_vars.update( + { + "src_files": self.get_src_files(), + "project_src_dir": get_project_src_dir(), + "project_lib_dir": get_project_lib_dir(), + "project_libdeps_dir": join( + get_project_libdeps_dir(), self.env_name + ), + } + ) # yapf: disable for key, value in tpl_vars.items(): if key.endswith(("_path", "_dir")): @@ -103,7 +103,7 @@ class ProjectGenerator(object): continue tpl_vars[key] = [fs.to_unix_path(inc) for inc in tpl_vars[key]] - tpl_vars['to_unix_path'] = fs.to_unix_path + tpl_vars["to_unix_path"] = fs.to_unix_path return tpl_vars def get_src_files(self): diff --git a/platformio/lockfile.py b/platformio/lockfile.py index f00c16e5..8c7a4658 100644 --- a/platformio/lockfile.py +++ b/platformio/lockfile.py @@ -26,10 +26,12 @@ LOCKFILE_INTERFACE_MSVCRT = 2 try: import fcntl + LOCKFILE_CURRENT_INTERFACE = LOCKFILE_INTERFACE_FCNTL except ImportError: try: import msvcrt + LOCKFILE_CURRENT_INTERFACE = LOCKFILE_INTERFACE_MSVCRT except ImportError: LOCKFILE_CURRENT_INTERFACE = None @@ -40,7 +42,6 @@ class LockFileExists(Exception): class LockFile(object): - def __init__(self, path, timeout=LOCKFILE_TIMEOUT, delay=LOCKFILE_DELAY): self.timeout = timeout self.delay = delay diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 012644d1..046555a2 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -49,12 +49,16 @@ def on_platformio_end(ctx, result): # pylint: disable=unused-argument check_platformio_upgrade() check_internal_updates(ctx, "platforms") check_internal_updates(ctx, "libraries") - except (exception.InternetIsOffline, exception.GetLatestVersionError, - exception.APIRequestError): + except ( + exception.InternetIsOffline, + exception.GetLatestVersionError, + exception.APIRequestError, + ): click.secho( "Failed to check for PlatformIO upgrades. " "Please check your Internet connection.", - fg="red") + fg="red", + ) def on_platformio_exception(e): @@ -78,15 +82,17 @@ def set_caller(caller=None): class Upgrader(object): - def __init__(self, from_version, to_version): self.from_version = semantic_version.Version.coerce( - util.pepver_to_semver(from_version)) + util.pepver_to_semver(from_version) + ) self.to_version = semantic_version.Version.coerce( - util.pepver_to_semver(to_version)) + util.pepver_to_semver(to_version) + ) - self._upgraders = [(semantic_version.Version("3.5.0-a.2"), - self._update_dev_platforms)] + self._upgraders = [ + (semantic_version.Version("3.5.0-a.2"), self._update_dev_platforms) + ] def run(self, ctx): if self.from_version > self.to_version: @@ -114,19 +120,21 @@ def after_upgrade(ctx): if last_version == "0.0.0": app.set_state_item("last_version", __version__) - elif semantic_version.Version.coerce(util.pepver_to_semver( - last_version)) > semantic_version.Version.coerce( - util.pepver_to_semver(__version__)): + elif semantic_version.Version.coerce( + util.pepver_to_semver(last_version) + ) > semantic_version.Version.coerce(util.pepver_to_semver(__version__)): click.secho("*" * terminal_width, fg="yellow") - click.secho("Obsolete PIO Core v%s is used (previous was %s)" % - (__version__, last_version), - fg="yellow") - click.secho("Please remove multiple PIO Cores from a system:", - fg="yellow") + click.secho( + "Obsolete PIO Core v%s is used (previous was %s)" + % (__version__, last_version), + fg="yellow", + ) + click.secho("Please remove multiple PIO Cores from a system:", fg="yellow") click.secho( "https://docs.platformio.org/page/faq.html" "#multiple-pio-cores-in-a-system", - fg="cyan") + fg="cyan", + ) click.secho("*" * terminal_width, fg="yellow") return else: @@ -139,37 +147,53 @@ def after_upgrade(ctx): u = Upgrader(last_version, __version__) if u.run(ctx): app.set_state_item("last_version", __version__) - click.secho("PlatformIO has been successfully upgraded to %s!\n" % - __version__, - fg="green") - telemetry.on_event(category="Auto", - action="Upgrade", - label="%s > %s" % (last_version, __version__)) + click.secho( + "PlatformIO has been successfully upgraded to %s!\n" % __version__, + fg="green", + ) + telemetry.on_event( + category="Auto", + action="Upgrade", + label="%s > %s" % (last_version, __version__), + ) else: raise exception.UpgradeError("Auto upgrading...") click.echo("") # PlatformIO banner click.echo("*" * terminal_width) - click.echo("If you like %s, please:" % - (click.style("PlatformIO", fg="cyan"))) - click.echo("- %s us on Twitter to stay up-to-date " - "on the latest project news > %s" % - (click.style("follow", fg="cyan"), - click.style("https://twitter.com/PlatformIO_Org", fg="cyan"))) + click.echo("If you like %s, please:" % (click.style("PlatformIO", fg="cyan"))) click.echo( - "- %s it on GitHub > %s" % - (click.style("star", fg="cyan"), - click.style("https://github.com/platformio/platformio", fg="cyan"))) + "- %s us on Twitter to stay up-to-date " + "on the latest project news > %s" + % ( + click.style("follow", fg="cyan"), + click.style("https://twitter.com/PlatformIO_Org", fg="cyan"), + ) + ) + click.echo( + "- %s it on GitHub > %s" + % ( + click.style("star", fg="cyan"), + click.style("https://github.com/platformio/platformio", fg="cyan"), + ) + ) if not getenv("PLATFORMIO_IDE"): click.echo( - "- %s PlatformIO IDE for IoT development > %s" % - (click.style("try", fg="cyan"), - click.style("https://platformio.org/platformio-ide", fg="cyan"))) + "- %s PlatformIO IDE for IoT development > %s" + % ( + click.style("try", fg="cyan"), + click.style("https://platformio.org/platformio-ide", fg="cyan"), + ) + ) if not is_ci(): - click.echo("- %s us with PlatformIO Plus > %s" % - (click.style("support", fg="cyan"), - click.style("https://pioplus.com", fg="cyan"))) + click.echo( + "- %s us with PlatformIO Plus > %s" + % ( + click.style("support", fg="cyan"), + click.style("https://pioplus.com", fg="cyan"), + ) + ) click.echo("*" * terminal_width) click.echo("") @@ -181,7 +205,7 @@ def check_platformio_upgrade(): if (time() - interval) < last_check.get("platformio_upgrade", 0): return - last_check['platformio_upgrade'] = int(time()) + last_check["platformio_upgrade"] = int(time()) app.set_state_item("last_check", last_check) util.internet_on(raise_exception=True) @@ -190,23 +214,23 @@ def check_platformio_upgrade(): update_core_packages(silent=True) latest_version = get_latest_version() - if semantic_version.Version.coerce(util.pepver_to_semver( - latest_version)) <= semantic_version.Version.coerce( - util.pepver_to_semver(__version__)): + if semantic_version.Version.coerce( + util.pepver_to_semver(latest_version) + ) <= semantic_version.Version.coerce(util.pepver_to_semver(__version__)): return terminal_width, _ = click.get_terminal_size() click.echo("") click.echo("*" * terminal_width) - click.secho("There is a new version %s of PlatformIO available.\n" - "Please upgrade it via `" % latest_version, - fg="yellow", - nl=False) + click.secho( + "There is a new version %s of PlatformIO available.\n" + "Please upgrade it via `" % latest_version, + fg="yellow", + nl=False, + ) if getenv("PLATFORMIO_IDE"): - click.secho("PlatformIO IDE Menu: Upgrade PlatformIO", - fg="cyan", - nl=False) + click.secho("PlatformIO IDE Menu: Upgrade PlatformIO", fg="cyan", nl=False) click.secho("`.", fg="yellow") elif join("Cellar", "platformio") in fs.get_source_dir(): click.secho("brew update && brew upgrade", fg="cyan", nl=False) @@ -217,8 +241,7 @@ def check_platformio_upgrade(): click.secho("pip install -U platformio", fg="cyan", nl=False) click.secho("` command.", fg="yellow") click.secho("Changes: ", fg="yellow", nl=False) - click.secho("https://docs.platformio.org/en/latest/history.html", - fg="cyan") + click.secho("https://docs.platformio.org/en/latest/history.html", fg="cyan") click.echo("*" * terminal_width) click.echo("") @@ -229,7 +252,7 @@ def check_internal_updates(ctx, what): if (time() - interval) < last_check.get(what + "_update", 0): return - last_check[what + '_update'] = int(time()) + last_check[what + "_update"] = int(time()) app.set_state_item("last_check", last_check) util.internet_on(raise_exception=True) @@ -237,15 +260,17 @@ def check_internal_updates(ctx, what): pm = PlatformManager() if what == "platforms" else LibraryManager() outdated_items = [] for manifest in pm.get_installed(): - if manifest['name'] in outdated_items: + if manifest["name"] in outdated_items: continue conds = [ - pm.outdated(manifest['__pkg_dir']), what == "platforms" + pm.outdated(manifest["__pkg_dir"]), + what == "platforms" and PlatformFactory.newPlatform( - manifest['__pkg_dir']).are_outdated_packages() + manifest["__pkg_dir"] + ).are_outdated_packages(), ] if any(conds): - outdated_items.append(manifest['name']) + outdated_items.append(manifest["name"]) if not outdated_items: return @@ -254,26 +279,32 @@ def check_internal_updates(ctx, what): click.echo("") click.echo("*" * terminal_width) - click.secho("There are the new updates for %s (%s)" % - (what, ", ".join(outdated_items)), - fg="yellow") + click.secho( + "There are the new updates for %s (%s)" % (what, ", ".join(outdated_items)), + fg="yellow", + ) if not app.get_setting("auto_update_" + what): click.secho("Please update them via ", fg="yellow", nl=False) - click.secho("`platformio %s update`" % - ("lib --global" if what == "libraries" else "platform"), - fg="cyan", - nl=False) + click.secho( + "`platformio %s update`" + % ("lib --global" if what == "libraries" else "platform"), + fg="cyan", + nl=False, + ) click.secho(" command.\n", fg="yellow") click.secho( "If you want to manually check for the new versions " "without updating, please use ", fg="yellow", - nl=False) - click.secho("`platformio %s update --dry-run`" % - ("lib --global" if what == "libraries" else "platform"), - fg="cyan", - nl=False) + nl=False, + ) + click.secho( + "`platformio %s update --dry-run`" + % ("lib --global" if what == "libraries" else "platform"), + fg="cyan", + nl=False, + ) click.secho(" command.", fg="yellow") else: click.secho("Please wait while updating %s ..." % what, fg="yellow") @@ -284,9 +315,7 @@ def check_internal_updates(ctx, what): ctx.invoke(cmd_lib_update, libraries=outdated_items) click.echo() - telemetry.on_event(category="Auto", - action="Update", - label=what.title()) + telemetry.on_event(category="Auto", action="Update", label=what.title()) click.echo("*" * terminal_width) click.echo("") diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 0d386326..9d1487d6 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -25,13 +25,12 @@ from platformio.project.helpers import get_project_packages_dir CORE_PACKAGES = { "contrib-piohome": "^2.3.2", - "contrib-pysite": - "~2.%d%d.190418" % (sys.version_info[0], sys.version_info[1]), + "contrib-pysite": "~2.%d%d.190418" % (sys.version_info[0], sys.version_info[1]), "tool-pioplus": "^2.5.2", "tool-unity": "~1.20403.0", "tool-scons": "~2.20501.7" if PY2 else "~3.30101.0", "tool-cppcheck": "~1.189.0", - "tool-clangtidy": "^1.80000.0" + "tool-clangtidy": "^1.80000.0", } PIOPLUS_AUTO_UPDATES_MAX = 100 @@ -40,20 +39,19 @@ PIOPLUS_AUTO_UPDATES_MAX = 100 class CorePackageManager(PackageManager): - def __init__(self): - super(CorePackageManager, self).__init__(get_project_packages_dir(), [ - "https://dl.bintray.com/platformio/dl-packages/manifest.json", - "http%s://dl.platformio.org/packages/manifest.json" % - ("" if sys.version_info < (2, 7, 9) else "s") - ]) + super(CorePackageManager, self).__init__( + get_project_packages_dir(), + [ + "https://dl.bintray.com/platformio/dl-packages/manifest.json", + "http%s://dl.platformio.org/packages/manifest.json" + % ("" if sys.version_info < (2, 7, 9) else "s"), + ], + ) def install( # pylint: disable=keyword-arg-before-vararg - self, - name, - requirements=None, - *args, - **kwargs): + self, name, requirements=None, *args, **kwargs + ): PackageManager.install(self, name, requirements, *args, **kwargs) self.cleanup_packages() return self.get_package_dir(name, requirements) @@ -70,12 +68,12 @@ class CorePackageManager(PackageManager): pkg_dir = self.get_package_dir(name, requirements) if not pkg_dir: continue - best_pkg_versions[name] = self.load_manifest(pkg_dir)['version'] + best_pkg_versions[name] = self.load_manifest(pkg_dir)["version"] for manifest in self.get_installed(): - if manifest['name'] not in best_pkg_versions: + if manifest["name"] not in best_pkg_versions: continue - if manifest['version'] != best_pkg_versions[manifest['name']]: - self.uninstall(manifest['__pkg_dir'], after_update=True) + if manifest["version"] != best_pkg_versions[manifest["name"]]: + self.uninstall(manifest["__pkg_dir"], after_update=True) self.cache_reset() return True @@ -104,6 +102,7 @@ def update_core_packages(only_check=False, silent=False): def inject_contrib_pysite(): from site import addsitedir + contrib_pysite_dir = get_core_package_dir("contrib-pysite") if contrib_pysite_dir in sys.path: return @@ -116,16 +115,18 @@ def pioplus_call(args, **kwargs): raise exception.PlatformioException( "PlatformIO Core Plus v%s does not run under Python version %s.\n" "Minimum supported version is 2.7.6, please upgrade Python.\n" - "Python 3 is not yet supported.\n" % (__version__, sys.version)) + "Python 3 is not yet supported.\n" % (__version__, sys.version) + ) pioplus_path = join(get_core_package_dir("tool-pioplus"), "pioplus") pythonexe_path = get_pythonexe_path() - os.environ['PYTHONEXEPATH'] = pythonexe_path - os.environ['PYTHONPYSITEDIR'] = get_core_package_dir("contrib-pysite") - os.environ['PIOCOREPYSITEDIR'] = dirname(fs.get_source_dir() or "") - if dirname(pythonexe_path) not in os.environ['PATH'].split(os.pathsep): - os.environ['PATH'] = (os.pathsep).join( - [dirname(pythonexe_path), os.environ['PATH']]) + os.environ["PYTHONEXEPATH"] = pythonexe_path + os.environ["PYTHONPYSITEDIR"] = get_core_package_dir("contrib-pysite") + os.environ["PIOCOREPYSITEDIR"] = dirname(fs.get_source_dir() or "") + if dirname(pythonexe_path) not in os.environ["PATH"].split(os.pathsep): + os.environ["PATH"] = (os.pathsep).join( + [dirname(pythonexe_path), os.environ["PATH"]] + ) copy_pythonpath_to_osenv() code = subprocess.call([pioplus_path] + args, **kwargs) diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index 3f46be61..943126d7 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -41,10 +41,7 @@ class LibraryManager(BasePkgManager): @property def manifest_names(self): - return [ - ".library.json", "library.json", "library.properties", - "module.json" - ] + return [".library.json", "library.json", "library.properties", "module.json"] def get_manifest_path(self, pkg_dir): path = BasePkgManager.get_manifest_path(self, pkg_dir) @@ -71,36 +68,37 @@ class LibraryManager(BasePkgManager): # if Arduino library.properties if "sentence" in manifest: - manifest['frameworks'] = ["arduino"] - manifest['description'] = manifest['sentence'] - del manifest['sentence'] + manifest["frameworks"] = ["arduino"] + manifest["description"] = manifest["sentence"] + del manifest["sentence"] if "author" in manifest: - if isinstance(manifest['author'], dict): - manifest['authors'] = [manifest['author']] + if isinstance(manifest["author"], dict): + manifest["authors"] = [manifest["author"]] else: - manifest['authors'] = [{"name": manifest['author']}] - del manifest['author'] + manifest["authors"] = [{"name": manifest["author"]}] + del manifest["author"] - if "authors" in manifest and not isinstance(manifest['authors'], list): - manifest['authors'] = [manifest['authors']] + if "authors" in manifest and not isinstance(manifest["authors"], list): + manifest["authors"] = [manifest["authors"]] if "keywords" not in manifest: keywords = [] - for keyword in re.split(r"[\s/]+", - manifest.get("category", "Uncategorized")): + for keyword in re.split( + r"[\s/]+", manifest.get("category", "Uncategorized") + ): keyword = keyword.strip() if not keyword: continue keywords.append(keyword.lower()) - manifest['keywords'] = keywords + manifest["keywords"] = keywords if "category" in manifest: - del manifest['category'] + del manifest["category"] # don't replace VCS URL if "url" in manifest and "description" in manifest: - manifest['homepage'] = manifest['url'] - del manifest['url'] + manifest["homepage"] = manifest["url"] + del manifest["url"] if "architectures" in manifest: platforms = [] @@ -110,26 +108,23 @@ class LibraryManager(BasePkgManager): "samd": "atmelsam", "esp8266": "espressif8266", "esp32": "espressif32", - "arc32": "intel_arc32" + "arc32": "intel_arc32", } - for arch in manifest['architectures'].split(","): + for arch in manifest["architectures"].split(","): arch = arch.strip() if arch == "*": platforms = "*" break if arch in platforms_map: platforms.append(platforms_map[arch]) - manifest['platforms'] = platforms - del manifest['architectures'] + manifest["platforms"] = platforms + del manifest["architectures"] # convert listed items via comma to array for key in ("keywords", "frameworks", "platforms"): - if key not in manifest or \ - not isinstance(manifest[key], string_types): + if key not in manifest or not isinstance(manifest[key], string_types): continue - manifest[key] = [ - i.strip() for i in manifest[key].split(",") if i.strip() - ] + manifest[key] = [i.strip() for i in manifest[key].split(",") if i.strip()] return manifest @@ -153,13 +148,10 @@ class LibraryManager(BasePkgManager): if item[k] == "*": del item[k] elif isinstance(item[k], string_types): - item[k] = [ - i.strip() for i in item[k].split(",") if i.strip() - ] + item[k] = [i.strip() for i in item[k].split(",") if i.strip()] return items def max_satisfying_repo_version(self, versions, requirements=None): - def _cmp_dates(datestr1, datestr2): date1 = util.parse_date(datestr1) date2 = util.parse_date(datestr2) @@ -169,61 +161,66 @@ class LibraryManager(BasePkgManager): semver_spec = None try: - semver_spec = semantic_version.SimpleSpec( - requirements) if requirements else None + semver_spec = ( + semantic_version.SimpleSpec(requirements) if requirements else None + ) except ValueError: pass item = {} for v in versions: - semver_new = self.parse_semver_version(v['name']) + semver_new = self.parse_semver_version(v["name"]) if semver_spec: if not semver_new or semver_new not in semver_spec: continue - if not item or self.parse_semver_version( - item['name']) < semver_new: + if not item or self.parse_semver_version(item["name"]) < semver_new: item = v elif requirements: - if requirements == v['name']: + if requirements == v["name"]: return v else: - if not item or _cmp_dates(item['released'], - v['released']) == -1: + if not item or _cmp_dates(item["released"], v["released"]) == -1: item = v return item def get_latest_repo_version(self, name, requirements, silent=False): item = self.max_satisfying_repo_version( - util.get_api_result("/lib/info/%d" % self.search_lib_id( - { - "name": name, - "requirements": requirements - }, silent=silent), - cache_valid="1h")['versions'], requirements) - return item['name'] if item else None + util.get_api_result( + "/lib/info/%d" + % self.search_lib_id( + {"name": name, "requirements": requirements}, silent=silent + ), + cache_valid="1h", + )["versions"], + requirements, + ) + return item["name"] if item else None def _install_from_piorepo(self, name, requirements): assert name.startswith("id="), name version = self.get_latest_repo_version(name, requirements) if not version: - raise exception.UndefinedPackageVersion(requirements or "latest", - util.get_systype()) - dl_data = util.get_api_result("/lib/download/" + str(name[3:]), - dict(version=version), - cache_valid="30d") + raise exception.UndefinedPackageVersion( + requirements or "latest", util.get_systype() + ) + dl_data = util.get_api_result( + "/lib/download/" + str(name[3:]), dict(version=version), cache_valid="30d" + ) assert dl_data return self._install_from_url( - name, dl_data['url'].replace("http://", "https://") - if app.get_setting("strict_ssl") else dl_data['url'], requirements) + name, + dl_data["url"].replace("http://", "https://") + if app.get_setting("strict_ssl") + else dl_data["url"], + requirements, + ) def search_lib_id( # pylint: disable=too-many-branches - self, - filters, - silent=False, - interactive=False): + self, filters, silent=False, interactive=False + ): assert isinstance(filters, dict) assert "name" in filters @@ -234,8 +231,10 @@ class LibraryManager(BasePkgManager): # looking in PIO Library Registry if not silent: - click.echo("Looking for %s library in registry" % - click.style(filters['name'], fg="cyan")) + click.echo( + "Looking for %s library in registry" + % click.style(filters["name"], fg="cyan") + ) query = [] for key in filters: if key not in ("name", "authors", "frameworks", "platforms"): @@ -244,25 +243,29 @@ class LibraryManager(BasePkgManager): if not isinstance(values, list): values = [v.strip() for v in values.split(",") if v] for value in values: - query.append('%s:"%s"' % - (key[:-1] if key.endswith("s") else key, value)) + query.append( + '%s:"%s"' % (key[:-1] if key.endswith("s") else key, value) + ) lib_info = None - result = util.get_api_result("/v2/lib/search", - dict(query=" ".join(query)), - cache_valid="1h") - if result['total'] == 1: - lib_info = result['items'][0] - elif result['total'] > 1: + result = util.get_api_result( + "/v2/lib/search", dict(query=" ".join(query)), cache_valid="1h" + ) + if result["total"] == 1: + lib_info = result["items"][0] + elif result["total"] > 1: if silent and not interactive: - lib_info = result['items'][0] + lib_info = result["items"][0] else: - click.secho("Conflict: More than one library has been found " - "by request %s:" % json.dumps(filters), - fg="yellow", - err=True) + click.secho( + "Conflict: More than one library has been found " + "by request %s:" % json.dumps(filters), + fg="yellow", + err=True, + ) from platformio.commands.lib import print_lib_item - for item in result['items']: + + for item in result["items"]: print_lib_item(item) if not interactive: @@ -270,36 +273,39 @@ class LibraryManager(BasePkgManager): "Automatically chose the first available library " "(use `--interactive` option to make a choice)", fg="yellow", - err=True) - lib_info = result['items'][0] + err=True, + ) + lib_info = result["items"][0] else: - deplib_id = click.prompt("Please choose library ID", - type=click.Choice([ - str(i['id']) - for i in result['items'] - ])) - for item in result['items']: - if item['id'] == int(deplib_id): + deplib_id = click.prompt( + "Please choose library ID", + type=click.Choice([str(i["id"]) for i in result["items"]]), + ) + for item in result["items"]: + if item["id"] == int(deplib_id): lib_info = item break if not lib_info: if list(filters) == ["name"]: - raise exception.LibNotFound(filters['name']) + raise exception.LibNotFound(filters["name"]) raise exception.LibNotFound(str(filters)) if not silent: - click.echo("Found: %s" % click.style( - "https://platformio.org/lib/show/{id}/{name}".format( - **lib_info), - fg="blue")) - return int(lib_info['id']) + click.echo( + "Found: %s" + % click.style( + "https://platformio.org/lib/show/{id}/{name}".format(**lib_info), + fg="blue", + ) + ) + return int(lib_info["id"]) def _get_lib_id_from_installed(self, filters): - if filters['name'].startswith("id="): - return int(filters['name'][3:]) + if filters["name"].startswith("id="): + return int(filters["name"][3:]) package_dir = self.get_package_dir( - filters['name'], filters.get("requirements", - filters.get("version"))) + filters["name"], filters.get("requirements", filters.get("version")) + ) if not package_dir: return None manifest = self.load_manifest(package_dir) @@ -311,52 +317,55 @@ class LibraryManager(BasePkgManager): continue if key not in manifest: return None - if not util.items_in_list(util.items_to_list(filters[key]), - util.items_to_list(manifest[key])): + if not util.items_in_list( + util.items_to_list(filters[key]), util.items_to_list(manifest[key]) + ): return None if "authors" in filters: if "authors" not in manifest: return None - manifest_authors = manifest['authors'] + manifest_authors = manifest["authors"] if not isinstance(manifest_authors, list): manifest_authors = [manifest_authors] manifest_authors = [ - a['name'] for a in manifest_authors + a["name"] + for a in manifest_authors if isinstance(a, dict) and "name" in a ] - filter_authors = filters['authors'] + filter_authors = filters["authors"] if not isinstance(filter_authors, list): filter_authors = [filter_authors] if not set(filter_authors) <= set(manifest_authors): return None - return int(manifest['id']) + return int(manifest["id"]) def install( # pylint: disable=arguments-differ - self, - name, - requirements=None, - silent=False, - after_update=False, - interactive=False, - force=False): + self, + name, + requirements=None, + silent=False, + after_update=False, + interactive=False, + force=False, + ): _name, _requirements, _url = self.parse_pkg_uri(name, requirements) if not _url: name = "id=%d" % self.search_lib_id( - { - "name": _name, - "requirements": _requirements - }, + {"name": _name, "requirements": _requirements}, silent=silent, - interactive=interactive) + interactive=interactive, + ) requirements = _requirements - pkg_dir = BasePkgManager.install(self, - name, - requirements, - silent=silent, - after_update=after_update, - force=force) + pkg_dir = BasePkgManager.install( + self, + name, + requirements, + silent=silent, + after_update=after_update, + force=force, + ) if not pkg_dir: return None @@ -369,7 +378,7 @@ class LibraryManager(BasePkgManager): click.secho("Installing dependencies", fg="yellow") builtin_lib_storages = None - for filters in self.normalize_dependencies(manifest['dependencies']): + for filters in self.normalize_dependencies(manifest["dependencies"]): assert "name" in filters # avoid circle dependencies @@ -381,35 +390,42 @@ class LibraryManager(BasePkgManager): self.INSTALL_HISTORY.append(history_key) if any(s in filters.get("version", "") for s in ("\\", "/")): - self.install("{name}={version}".format(**filters), - silent=silent, - after_update=after_update, - interactive=interactive, - force=force) + self.install( + "{name}={version}".format(**filters), + silent=silent, + after_update=after_update, + interactive=interactive, + force=force, + ) else: try: lib_id = self.search_lib_id(filters, silent, interactive) except exception.LibNotFound as e: if builtin_lib_storages is None: builtin_lib_storages = get_builtin_libs() - if not silent or is_builtin_lib(builtin_lib_storages, - filters['name']): + if not silent or is_builtin_lib( + builtin_lib_storages, filters["name"] + ): click.secho("Warning! %s" % e, fg="yellow") continue if filters.get("version"): - self.install(lib_id, - filters.get("version"), - silent=silent, - after_update=after_update, - interactive=interactive, - force=force) + self.install( + lib_id, + filters.get("version"), + silent=silent, + after_update=after_update, + interactive=interactive, + force=force, + ) else: - self.install(lib_id, - silent=silent, - after_update=after_update, - interactive=interactive, - force=force) + self.install( + lib_id, + silent=silent, + after_update=after_update, + interactive=interactive, + force=force, + ) return pkg_dir @@ -418,21 +434,23 @@ def get_builtin_libs(storage_names=None): storage_names = storage_names or [] pm = PlatformManager() for manifest in pm.get_installed(): - p = PlatformFactory.newPlatform(manifest['__pkg_dir']) + p = PlatformFactory.newPlatform(manifest["__pkg_dir"]) for storage in p.get_lib_storages(): - if storage_names and storage['name'] not in storage_names: + if storage_names and storage["name"] not in storage_names: continue - lm = LibraryManager(storage['path']) - items.append({ - "name": storage['name'], - "path": storage['path'], - "items": lm.get_installed() - }) + lm = LibraryManager(storage["path"]) + items.append( + { + "name": storage["name"], + "path": storage["path"], + "items": lm.get_installed(), + } + ) return items def is_builtin_lib(storages, name): for storage in storages or []: - if any(l.get("name") == name for l in storage['items']): + if any(l.get("name") == name for l in storage["items"]): return True return False diff --git a/platformio/managers/package.py b/platformio/managers/package.py index f84ba472..01cf5254 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -36,7 +36,6 @@ from platformio.vcsclient import VCSClientFactory class PackageRepoIterator(object): - def __init__(self, package, repositories): assert isinstance(repositories, list) self.package = package @@ -87,8 +86,9 @@ class PkgRepoMixin(object): item = None reqspec = None try: - reqspec = semantic_version.SimpleSpec( - requirements) if requirements else None + reqspec = ( + semantic_version.SimpleSpec(requirements) if requirements else None + ) except ValueError: pass @@ -99,33 +99,32 @@ class PkgRepoMixin(object): # if PkgRepoMixin.PIO_VERSION not in requirements.SimpleSpec( # v['engines']['platformio']): # continue - specver = semantic_version.Version(v['version']) + specver = semantic_version.Version(v["version"]) if reqspec and specver not in reqspec: continue - if not item or semantic_version.Version(item['version']) < specver: + if not item or semantic_version.Version(item["version"]) < specver: item = v return item def get_latest_repo_version( # pylint: disable=unused-argument - self, - name, - requirements, - silent=False): + self, name, requirements, silent=False + ): version = None for versions in PackageRepoIterator(name, self.repositories): pkgdata = self.max_satisfying_repo_version(versions, requirements) if not pkgdata: continue - if not version or semantic_version.compare(pkgdata['version'], - version) == 1: - version = pkgdata['version'] + if ( + not version + or semantic_version.compare(pkgdata["version"], version) == 1 + ): + version = pkgdata["version"] return version def get_all_repo_versions(self, name): result = [] for versions in PackageRepoIterator(name, self.repositories): - result.extend( - [semantic_version.Version(v['version']) for v in versions]) + result.extend([semantic_version.Version(v["version"]) for v in versions]) return [str(v) for v in sorted(set(result))] @@ -154,7 +153,8 @@ class PkgInstallerMixin(object): if result: return result result = [ - join(src_dir, name) for name in sorted(os.listdir(src_dir)) + join(src_dir, name) + for name in sorted(os.listdir(src_dir)) if isdir(join(src_dir, name)) ] self.cache_set(cache_key, result) @@ -189,14 +189,17 @@ class PkgInstallerMixin(object): click.secho( "Error: Please read http://bit.ly/package-manager-ioerror", fg="red", - err=True) + err=True, + ) raise e if sha1: fd.verify(sha1) dst_path = fd.get_filepath() - if not self.FILE_CACHE_VALID or getsize( - dst_path) > PkgInstallerMixin.FILE_CACHE_MAX_SIZE: + if ( + not self.FILE_CACHE_VALID + or getsize(dst_path) > PkgInstallerMixin.FILE_CACHE_MAX_SIZE + ): return dst_path with app.ContentCache() as cc: @@ -232,15 +235,15 @@ class PkgInstallerMixin(object): return None @staticmethod - def parse_pkg_uri( # pylint: disable=too-many-branches - text, requirements=None): + def parse_pkg_uri(text, requirements=None): # pylint: disable=too-many-branches text = str(text) name, url = None, None # Parse requirements req_conditions = [ - "@" in text, not requirements, ":" not in text - or text.rfind("/") < text.rfind("@") + "@" in text, + not requirements, + ":" not in text or text.rfind("/") < text.rfind("@"), ] if all(req_conditions): text, requirements = text.rsplit("@", 1) @@ -259,17 +262,16 @@ class PkgInstallerMixin(object): elif "/" in text or "\\" in text: git_conditions = [ # Handle GitHub URL (https://github.com/user/package) - text.startswith("https://github.com/") and not text.endswith( - (".zip", ".tar.gz")), - (text.split("#", 1)[0] - if "#" in text else text).endswith(".git") + text.startswith("https://github.com/") + and not text.endswith((".zip", ".tar.gz")), + (text.split("#", 1)[0] if "#" in text else text).endswith(".git"), ] hg_conditions = [ # Handle Developer Mbed URL # (https://developer.mbed.org/users/user/code/package/) # (https://os.mbed.com/users/user/code/package/) text.startswith("https://developer.mbed.org"), - text.startswith("https://os.mbed.com") + text.startswith("https://os.mbed.com"), ] if any(git_conditions): url = "git+" + text @@ -296,9 +298,9 @@ class PkgInstallerMixin(object): @staticmethod def get_install_dirname(manifest): - name = re.sub(r"[^\da-z\_\-\. ]", "_", manifest['name'], flags=re.I) + name = re.sub(r"[^\da-z\_\-\. ]", "_", manifest["name"], flags=re.I) if "id" in manifest: - name += "_ID%d" % manifest['id'] + name += "_ID%d" % manifest["id"] return str(name) @classmethod @@ -322,8 +324,7 @@ class PkgInstallerMixin(object): return None def manifest_exists(self, pkg_dir): - return self.get_manifest_path(pkg_dir) or \ - self.get_src_manifest_path(pkg_dir) + return self.get_manifest_path(pkg_dir) or self.get_src_manifest_path(pkg_dir) def load_manifest(self, pkg_dir): cache_key = "load_manifest-%s" % pkg_dir @@ -353,19 +354,19 @@ class PkgInstallerMixin(object): if src_manifest: if "version" in src_manifest: - manifest['version'] = src_manifest['version'] - manifest['__src_url'] = src_manifest['url'] + manifest["version"] = src_manifest["version"] + manifest["__src_url"] = src_manifest["url"] # handle a custom package name - autogen_name = self.parse_pkg_uri(manifest['__src_url'])[0] - if "name" not in manifest or autogen_name != src_manifest['name']: - manifest['name'] = src_manifest['name'] + autogen_name = self.parse_pkg_uri(manifest["__src_url"])[0] + if "name" not in manifest or autogen_name != src_manifest["name"]: + manifest["name"] = src_manifest["name"] if "name" not in manifest: - manifest['name'] = basename(pkg_dir) + manifest["name"] = basename(pkg_dir) if "version" not in manifest: - manifest['version'] = "0.0.0" + manifest["version"] = "0.0.0" - manifest['__pkg_dir'] = pkg_dir + manifest["__pkg_dir"] = pkg_dir self.cache_set(cache_key, manifest) return manifest @@ -390,25 +391,24 @@ class PkgInstallerMixin(object): continue elif pkg_id and manifest.get("id") != pkg_id: continue - elif not pkg_id and manifest['name'] != name: + elif not pkg_id and manifest["name"] != name: continue elif not PkgRepoMixin.is_system_compatible(manifest.get("system")): continue # strict version or VCS HASH - if requirements and requirements == manifest['version']: + if requirements and requirements == manifest["version"]: return manifest try: - if requirements and not semantic_version.SimpleSpec( - requirements).match( - self.parse_semver_version(manifest['version'], - raise_exception=True)): + if requirements and not semantic_version.SimpleSpec(requirements).match( + self.parse_semver_version(manifest["version"], raise_exception=True) + ): continue - elif not best or (self.parse_semver_version( - manifest['version'], raise_exception=True) > - self.parse_semver_version( - best['version'], raise_exception=True)): + elif not best or ( + self.parse_semver_version(manifest["version"], raise_exception=True) + > self.parse_semver_version(best["version"], raise_exception=True) + ): best = manifest except ValueError: pass @@ -417,12 +417,15 @@ class PkgInstallerMixin(object): def get_package_dir(self, name, requirements=None, url=None): manifest = self.get_package(name, requirements, url) - return manifest.get("__pkg_dir") if manifest and isdir( - manifest.get("__pkg_dir")) else None + return ( + manifest.get("__pkg_dir") + if manifest and isdir(manifest.get("__pkg_dir")) + else None + ) def get_package_by_dir(self, pkg_dir): for manifest in self.get_installed(): - if manifest['__pkg_dir'] == abspath(pkg_dir): + if manifest["__pkg_dir"] == abspath(pkg_dir): return manifest return None @@ -443,9 +446,9 @@ class PkgInstallerMixin(object): if not pkgdata: continue try: - pkg_dir = self._install_from_url(name, pkgdata['url'], - requirements, - pkgdata.get("sha1")) + pkg_dir = self._install_from_url( + name, pkgdata["url"], requirements, pkgdata.get("sha1") + ) break except Exception as e: # pylint: disable=broad-except click.secho("Warning! Package Mirror: %s" % e, fg="yellow") @@ -455,16 +458,12 @@ class PkgInstallerMixin(object): util.internet_on(raise_exception=True) raise exception.UnknownPackage(name) if not pkgdata: - raise exception.UndefinedPackageVersion(requirements or "latest", - util.get_systype()) + raise exception.UndefinedPackageVersion( + requirements or "latest", util.get_systype() + ) return pkg_dir - def _install_from_url(self, - name, - url, - requirements=None, - sha1=None, - track=False): + def _install_from_url(self, name, url, requirements=None, sha1=None, track=False): tmp_dir = mkdtemp("-package", self.TMP_FOLDER_PREFIX, self.package_dir) src_manifest_dir = None src_manifest = {"name": name, "url": url, "requirements": requirements} @@ -486,7 +485,7 @@ class PkgInstallerMixin(object): vcs = VCSClientFactory.newClient(tmp_dir, url) assert vcs.export() src_manifest_dir = vcs.storage_dir - src_manifest['version'] = vcs.get_current_revision() + src_manifest["version"] = vcs.get_current_revision() _tmp_dir = tmp_dir if not src_manifest_dir: @@ -515,7 +514,8 @@ class PkgInstallerMixin(object): json.dump(_data, fp) def _install_from_tmp_dir( # pylint: disable=too-many-branches - self, tmp_dir, requirements=None): + self, tmp_dir, requirements=None + ): tmp_manifest = self.load_manifest(tmp_dir) assert set(["name", "version"]) <= set(tmp_manifest) @@ -523,28 +523,30 @@ class PkgInstallerMixin(object): pkg_dir = join(self.package_dir, pkg_dirname) cur_manifest = self.load_manifest(pkg_dir) - tmp_semver = self.parse_semver_version(tmp_manifest['version']) + tmp_semver = self.parse_semver_version(tmp_manifest["version"]) cur_semver = None if cur_manifest: - cur_semver = self.parse_semver_version(cur_manifest['version']) + cur_semver = self.parse_semver_version(cur_manifest["version"]) # package should satisfy requirements if requirements: - mismatch_error = ( - "Package version %s doesn't satisfy requirements %s" % - (tmp_manifest['version'], requirements)) + mismatch_error = "Package version %s doesn't satisfy requirements %s" % ( + tmp_manifest["version"], + requirements, + ) try: assert tmp_semver and tmp_semver in semantic_version.SimpleSpec( - requirements), mismatch_error + requirements + ), mismatch_error except (AssertionError, ValueError): - assert tmp_manifest['version'] == requirements, mismatch_error + assert tmp_manifest["version"] == requirements, mismatch_error # check if package already exists if cur_manifest: # 0-overwrite, 1-rename, 2-fix to a version action = 0 if "__src_url" in cur_manifest: - if cur_manifest['__src_url'] != tmp_manifest.get("__src_url"): + if cur_manifest["__src_url"] != tmp_manifest.get("__src_url"): action = 1 elif "__src_url" in tmp_manifest: action = 2 @@ -556,25 +558,25 @@ class PkgInstallerMixin(object): # rename if action == 1: - target_dirname = "%s@%s" % (pkg_dirname, - cur_manifest['version']) + target_dirname = "%s@%s" % (pkg_dirname, cur_manifest["version"]) if "__src_url" in cur_manifest: target_dirname = "%s@src-%s" % ( pkg_dirname, hashlib.md5( - hashlib_encode_data( - cur_manifest['__src_url'])).hexdigest()) + hashlib_encode_data(cur_manifest["__src_url"]) + ).hexdigest(), + ) shutil.move(pkg_dir, join(self.package_dir, target_dirname)) # fix to a version elif action == 2: - target_dirname = "%s@%s" % (pkg_dirname, - tmp_manifest['version']) + target_dirname = "%s@%s" % (pkg_dirname, tmp_manifest["version"]) if "__src_url" in tmp_manifest: target_dirname = "%s@src-%s" % ( pkg_dirname, hashlib.md5( - hashlib_encode_data( - tmp_manifest['__src_url'])).hexdigest()) + hashlib_encode_data(tmp_manifest["__src_url"]) + ).hexdigest(), + ) pkg_dir = join(self.package_dir, target_dirname) # remove previous/not-satisfied package @@ -622,9 +624,9 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): if "__src_url" in manifest: try: - vcs = VCSClientFactory.newClient(pkg_dir, - manifest['__src_url'], - silent=True) + vcs = VCSClientFactory.newClient( + pkg_dir, manifest["__src_url"], silent=True + ) except (AttributeError, exception.PlatformioException): return None if not vcs.can_be_updated: @@ -633,10 +635,10 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): else: try: latest = self.get_latest_repo_version( - "id=%d" % - manifest['id'] if "id" in manifest else manifest['name'], + "id=%d" % manifest["id"] if "id" in manifest else manifest["name"], requirements, - silent=True) + silent=True, + ) except (exception.PlatformioException, ValueError): return None @@ -646,21 +648,17 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): up_to_date = False try: assert "__src_url" not in manifest - up_to_date = (self.parse_semver_version(manifest['version'], - raise_exception=True) >= - self.parse_semver_version(latest, - raise_exception=True)) + up_to_date = self.parse_semver_version( + manifest["version"], raise_exception=True + ) >= self.parse_semver_version(latest, raise_exception=True) except (AssertionError, ValueError): - up_to_date = latest == manifest['version'] + up_to_date = latest == manifest["version"] return False if up_to_date else latest - def install(self, - name, - requirements=None, - silent=False, - after_update=False, - force=False): + def install( + self, name, requirements=None, silent=False, after_update=False, force=False + ): pkg_dir = None # interprocess lock with LockFile(self.package_dir): @@ -690,34 +688,38 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): if not silent: click.secho( "{name} @ {version} is already installed".format( - **self.load_manifest(package_dir)), - fg="yellow") + **self.load_manifest(package_dir) + ), + fg="yellow", + ) return package_dir if url: - pkg_dir = self._install_from_url(name, - url, - requirements, - track=True) + pkg_dir = self._install_from_url(name, url, requirements, track=True) else: pkg_dir = self._install_from_piorepo(name, requirements) if not pkg_dir or not self.manifest_exists(pkg_dir): - raise exception.PackageInstallError(name, requirements or "*", - util.get_systype()) + raise exception.PackageInstallError( + name, requirements or "*", util.get_systype() + ) manifest = self.load_manifest(pkg_dir) assert manifest if not after_update: - telemetry.on_event(category=self.__class__.__name__, - action="Install", - label=manifest['name']) + telemetry.on_event( + category=self.__class__.__name__, + action="Install", + label=manifest["name"], + ) click.secho( "{name} @ {version} has been successfully installed!".format( - **manifest), - fg="green") + **manifest + ), + fg="green", + ) return pkg_dir @@ -729,18 +731,20 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): if isdir(package) and self.get_package_by_dir(package): pkg_dir = package else: - name, requirements, url = self.parse_pkg_uri( - package, requirements) + name, requirements, url = self.parse_pkg_uri(package, requirements) pkg_dir = self.get_package_dir(name, requirements, url) if not pkg_dir: - raise exception.UnknownPackage("%s @ %s" % - (package, requirements or "*")) + raise exception.UnknownPackage( + "%s @ %s" % (package, requirements or "*") + ) manifest = self.load_manifest(pkg_dir) - click.echo("Uninstalling %s @ %s: \t" % (click.style( - manifest['name'], fg="cyan"), manifest['version']), - nl=False) + click.echo( + "Uninstalling %s @ %s: \t" + % (click.style(manifest["name"], fg="cyan"), manifest["version"]), + nl=False, + ) if islink(pkg_dir): os.unlink(pkg_dir) @@ -749,19 +753,21 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): self.cache_reset() # unfix package with the same name - pkg_dir = self.get_package_dir(manifest['name']) + pkg_dir = self.get_package_dir(manifest["name"]) if pkg_dir and "@" in pkg_dir: shutil.move( - pkg_dir, - join(self.package_dir, self.get_install_dirname(manifest))) + pkg_dir, join(self.package_dir, self.get_install_dirname(manifest)) + ) self.cache_reset() click.echo("[%s]" % click.style("OK", fg="green")) if not after_update: - telemetry.on_event(category=self.__class__.__name__, - action="Uninstall", - label=manifest['name']) + telemetry.on_event( + category=self.__class__.__name__, + action="Uninstall", + label=manifest["name"], + ) return True @@ -773,16 +779,19 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): pkg_dir = self.get_package_dir(*self.parse_pkg_uri(package)) if not pkg_dir: - raise exception.UnknownPackage("%s @ %s" % - (package, requirements or "*")) + raise exception.UnknownPackage("%s @ %s" % (package, requirements or "*")) manifest = self.load_manifest(pkg_dir) - name = manifest['name'] + name = manifest["name"] - click.echo("{} {:<40} @ {:<15}".format( - "Checking" if only_check else "Updating", - click.style(manifest['name'], fg="cyan"), manifest['version']), - nl=False) + click.echo( + "{} {:<40} @ {:<15}".format( + "Checking" if only_check else "Updating", + click.style(manifest["name"], fg="cyan"), + manifest["version"], + ), + nl=False, + ) if not util.internet_on(): click.echo("[%s]" % (click.style("Off-line", fg="yellow"))) return None @@ -799,22 +808,22 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): return True if "__src_url" in manifest: - vcs = VCSClientFactory.newClient(pkg_dir, manifest['__src_url']) + vcs = VCSClientFactory.newClient(pkg_dir, manifest["__src_url"]) assert vcs.update() - self._update_src_manifest(dict(version=vcs.get_current_revision()), - vcs.storage_dir) + self._update_src_manifest( + dict(version=vcs.get_current_revision()), vcs.storage_dir + ) else: self.uninstall(pkg_dir, after_update=True) self.install(name, latest, after_update=True) - telemetry.on_event(category=self.__class__.__name__, - action="Update", - label=manifest['name']) + telemetry.on_event( + category=self.__class__.__name__, action="Update", label=manifest["name"] + ) return True class PackageManager(BasePkgManager): - @property def manifest_names(self): return ["package.json"] diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index dafdb855..4ab20b8d 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -26,13 +26,19 @@ from platformio import __version__, app, exception, fs, util from platformio.compat import PY2, hashlib_encode_data, is_bytes from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager -from platformio.proc import (BuildAsyncPipe, copy_pythonpath_to_osenv, - exec_command, get_pythonexe_path) +from platformio.proc import ( + BuildAsyncPipe, + copy_pythonpath_to_osenv, + exec_command, + get_pythonexe_path, +) from platformio.project.config import ProjectConfig -from platformio.project.helpers import (get_project_boards_dir, - get_project_core_dir, - get_project_packages_dir, - get_project_platforms_dir) +from platformio.project.helpers import ( + get_project_boards_dir, + get_project_core_dir, + get_project_packages_dir, + get_project_platforms_dir, +) try: from urllib.parse import quote @@ -41,16 +47,17 @@ except ImportError: class PlatformManager(BasePkgManager): - def __init__(self, package_dir=None, repositories=None): if not repositories: repositories = [ "https://dl.bintray.com/platformio/dl-platforms/manifest.json", "{0}://dl.platformio.org/platforms/manifest.json".format( - "https" if app.get_setting("strict_ssl") else "http") + "https" if app.get_setting("strict_ssl") else "http" + ), ] - BasePkgManager.__init__(self, package_dir - or get_project_platforms_dir(), repositories) + BasePkgManager.__init__( + self, package_dir or get_project_platforms_dir(), repositories + ) @property def manifest_names(self): @@ -65,21 +72,21 @@ class PlatformManager(BasePkgManager): return manifest_path return None - def install(self, - name, - requirements=None, - with_packages=None, - without_packages=None, - skip_default_package=False, - after_update=False, - silent=False, - force=False, - **_): # pylint: disable=too-many-arguments, arguments-differ - platform_dir = BasePkgManager.install(self, - name, - requirements, - silent=silent, - force=force) + def install( + self, + name, + requirements=None, + with_packages=None, + without_packages=None, + skip_default_package=False, + after_update=False, + silent=False, + force=False, + **_ + ): # pylint: disable=too-many-arguments, arguments-differ + platform_dir = BasePkgManager.install( + self, name, requirements, silent=silent, force=force + ) p = PlatformFactory.newPlatform(platform_dir) # don't cleanup packages or install them after update @@ -87,11 +94,13 @@ class PlatformManager(BasePkgManager): if after_update: return True - p.install_packages(with_packages, - without_packages, - skip_default_package, - silent=silent, - force=force) + p.install_packages( + with_packages, + without_packages, + skip_default_package, + silent=silent, + force=force, + ) return self.cleanup_packages(list(p.packages)) def uninstall(self, package, requirements=None, after_update=False): @@ -115,11 +124,8 @@ class PlatformManager(BasePkgManager): return self.cleanup_packages(list(p.packages)) def update( # pylint: disable=arguments-differ - self, - package, - requirements=None, - only_check=False, - only_packages=False): + self, package, requirements=None, only_check=False, only_packages=False + ): if isdir(package): pkg_dir = package else: @@ -143,8 +149,9 @@ class PlatformManager(BasePkgManager): self.cleanup_packages(list(p.packages)) if missed_pkgs: - p.install_packages(with_packages=list(missed_pkgs), - skip_default_package=True) + p.install_packages( + with_packages=list(missed_pkgs), skip_default_package=True + ) return True @@ -152,20 +159,22 @@ class PlatformManager(BasePkgManager): self.cache_reset() deppkgs = {} for manifest in PlatformManager().get_installed(): - p = PlatformFactory.newPlatform(manifest['__pkg_dir']) + p = PlatformFactory.newPlatform(manifest["__pkg_dir"]) for pkgname, pkgmanifest in p.get_installed_packages().items(): if pkgname not in deppkgs: deppkgs[pkgname] = set() - deppkgs[pkgname].add(pkgmanifest['version']) + deppkgs[pkgname].add(pkgmanifest["version"]) pm = PackageManager(get_project_packages_dir()) for manifest in pm.get_installed(): - if manifest['name'] not in names: + if manifest["name"] not in names: continue - if (manifest['name'] not in deppkgs - or manifest['version'] not in deppkgs[manifest['name']]): + if ( + manifest["name"] not in deppkgs + or manifest["version"] not in deppkgs[manifest["name"]] + ): try: - pm.uninstall(manifest['__pkg_dir'], after_update=True) + pm.uninstall(manifest["__pkg_dir"], after_update=True) except exception.UnknownPackage: pass @@ -176,7 +185,7 @@ class PlatformManager(BasePkgManager): def get_installed_boards(self): boards = [] for manifest in self.get_installed(): - p = PlatformFactory.newPlatform(manifest['__pkg_dir']) + p = PlatformFactory.newPlatform(manifest["__pkg_dir"]) for config in p.get_boards().values(): board = config.get_brief_data() if board not in boards: @@ -189,30 +198,31 @@ class PlatformManager(BasePkgManager): def get_all_boards(self): boards = self.get_installed_boards() - know_boards = ["%s:%s" % (b['platform'], b['id']) for b in boards] + know_boards = ["%s:%s" % (b["platform"], b["id"]) for b in boards] try: for board in self.get_registered_boards(): - key = "%s:%s" % (board['platform'], board['id']) + key = "%s:%s" % (board["platform"], board["id"]) if key not in know_boards: boards.append(board) except (exception.APIRequestError, exception.InternetIsOffline): pass - return sorted(boards, key=lambda b: b['name']) + return sorted(boards, key=lambda b: b["name"]) def board_config(self, id_, platform=None): for manifest in self.get_installed_boards(): - if manifest['id'] == id_ and (not platform - or manifest['platform'] == platform): + if manifest["id"] == id_ and ( + not platform or manifest["platform"] == platform + ): return manifest for manifest in self.get_registered_boards(): - if manifest['id'] == id_ and (not platform - or manifest['platform'] == platform): + if manifest["id"] == id_ and ( + not platform or manifest["platform"] == platform + ): return manifest raise exception.UnknownBoard(id_) class PlatformFactory(object): - @staticmethod def get_clsname(name): name = re.sub(r"[^\da-z\_]+", "", name, flags=re.I) @@ -222,8 +232,7 @@ class PlatformFactory(object): def load_module(name, path): module = None try: - module = load_source("platformio.managers.platform.%s" % name, - path) + module = load_source("platformio.managers.platform.%s" % name, path) except ImportError: raise exception.UnknownPlatform(name) return module @@ -234,28 +243,29 @@ class PlatformFactory(object): platform_dir = None if isdir(name): platform_dir = name - name = pm.load_manifest(platform_dir)['name'] + name = pm.load_manifest(platform_dir)["name"] elif name.endswith("platform.json") and isfile(name): platform_dir = dirname(name) - name = fs.load_json(name)['name'] + name = fs.load_json(name)["name"] else: name, requirements, url = pm.parse_pkg_uri(name, requirements) platform_dir = pm.get_package_dir(name, requirements, url) if platform_dir: - name = pm.load_manifest(platform_dir)['name'] + name = pm.load_manifest(platform_dir)["name"] if not platform_dir: raise exception.UnknownPlatform( - name if not requirements else "%s@%s" % (name, requirements)) + name if not requirements else "%s@%s" % (name, requirements) + ) platform_cls = None if isfile(join(platform_dir, "platform.py")): platform_cls = getattr( cls.load_module(name, join(platform_dir, "platform.py")), - cls.get_clsname(name)) + cls.get_clsname(name), + ) else: - platform_cls = type(str(cls.get_clsname(name)), (PlatformBase, ), - {}) + platform_cls = type(str(cls.get_clsname(name)), (PlatformBase,), {}) _instance = platform_cls(join(platform_dir, "platform.json")) assert isinstance(_instance, PlatformBase) @@ -263,14 +273,14 @@ class PlatformFactory(object): class PlatformPackagesMixin(object): - def install_packages( # pylint: disable=too-many-arguments - self, - with_packages=None, - without_packages=None, - skip_default_package=False, - silent=False, - force=False): + self, + with_packages=None, + without_packages=None, + skip_default_package=False, + silent=False, + force=False, + ): with_packages = set(self.find_pkg_names(with_packages or [])) without_packages = set(self.find_pkg_names(without_packages or [])) @@ -283,12 +293,13 @@ class PlatformPackagesMixin(object): version = opts.get("version", "") if name in without_packages: continue - elif (name in with_packages or - not (skip_default_package or opts.get("optional", False))): + elif name in with_packages or not ( + skip_default_package or opts.get("optional", False) + ): if ":" in version: - self.pm.install("%s=%s" % (name, version), - silent=silent, - force=force) + self.pm.install( + "%s=%s" % (name, version), silent=silent, force=force + ) else: self.pm.install(name, version, silent=silent, force=force) @@ -305,9 +316,12 @@ class PlatformPackagesMixin(object): result.append(_name) found = True - if (self.frameworks and candidate.startswith("framework-") - and candidate[10:] in self.frameworks): - result.append(self.frameworks[candidate[10:]]['package']) + if ( + self.frameworks + and candidate.startswith("framework-") + and candidate[10:] in self.frameworks + ): + result.append(self.frameworks[candidate[10:]]["package"]) found = True if not found: @@ -320,7 +334,7 @@ class PlatformPackagesMixin(object): requirements = self.packages[name].get("version", "") if ":" in requirements: _, requirements, __ = self.pm.parse_pkg_uri(requirements) - self.pm.update(manifest['__pkg_dir'], requirements, only_check) + self.pm.update(manifest["__pkg_dir"], requirements, only_check) def get_installed_packages(self): items = {} @@ -335,7 +349,7 @@ class PlatformPackagesMixin(object): requirements = self.packages[name].get("version", "") if ":" in requirements: _, requirements, __ = self.pm.parse_pkg_uri(requirements) - if self.pm.outdated(manifest['__pkg_dir'], requirements): + if self.pm.outdated(manifest["__pkg_dir"], requirements): return True return False @@ -343,7 +357,8 @@ class PlatformPackagesMixin(object): version = self.packages[name].get("version", "") if ":" in version: return self.pm.get_package_dir( - *self.pm.parse_pkg_uri("%s=%s" % (name, version))) + *self.pm.parse_pkg_uri("%s=%s" % (name, version)) + ) return self.pm.get_package_dir(name, version) def get_package_version(self, name): @@ -368,15 +383,16 @@ class PlatformRunMixin(object): return value.decode() if is_bytes(value) else value def run( # pylint: disable=too-many-arguments - self, variables, targets, silent, verbose, jobs): + self, variables, targets, silent, verbose, jobs + ): assert isinstance(variables, dict) assert isinstance(targets, list) - config = ProjectConfig.get_instance(variables['project_config']) - options = config.items(env=variables['pioenv'], as_dict=True) + config = ProjectConfig.get_instance(variables["project_config"]) + options = config.items(env=variables["pioenv"], as_dict=True) if "framework" in options: # support PIO Core 3.0 dev/platforms - options['pioframework'] = options['framework'] + options["pioframework"] = options["framework"] self.configure_default_packages(options, targets) self.install_packages(silent=True) @@ -386,12 +402,12 @@ class PlatformRunMixin(object): if "clean" in targets: targets = ["-c", "."] - variables['platform_manifest'] = self.manifest_path + variables["platform_manifest"] = self.manifest_path if "build_script" not in variables: - variables['build_script'] = self.get_build_script() - if not isfile(variables['build_script']): - raise exception.BuildScriptNotFound(variables['build_script']) + variables["build_script"] = self.get_build_script() + if not isfile(variables["build_script"]): + raise exception.BuildScriptNotFound(variables["build_script"]) result = self._run_scons(variables, targets, jobs) assert "returncode" in result @@ -402,14 +418,16 @@ class PlatformRunMixin(object): args = [ get_pythonexe_path(), join(get_core_package_dir("tool-scons"), "script", "scons"), - "-Q", "--warn=no-no-parallel-support", - "--jobs", str(jobs), - "--sconstruct", join(fs.get_source_dir(), "builder", "main.py") + "-Q", + "--warn=no-no-parallel-support", + "--jobs", + str(jobs), + "--sconstruct", + join(fs.get_source_dir(), "builder", "main.py"), ] # yapf: disable args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0)) # pylint: disable=protected-access - args.append("ISATTY=%d" % - (1 if click._compat.isatty(sys.stdout) else 0)) + args.append("ISATTY=%d" % (1 if click._compat.isatty(sys.stdout) else 0)) args += targets # encode and append variables @@ -428,10 +446,13 @@ class PlatformRunMixin(object): args, stdout=BuildAsyncPipe( line_callback=self._on_stdout_line, - data_callback=lambda data: _write_and_flush(sys.stdout, data)), + data_callback=lambda data: _write_and_flush(sys.stdout, data), + ), stderr=BuildAsyncPipe( line_callback=self._on_stderr_line, - data_callback=lambda data: _write_and_flush(sys.stderr, data))) + data_callback=lambda data: _write_and_flush(sys.stderr, data), + ), + ) return result def _on_stdout_line(self, line): @@ -447,7 +468,7 @@ class PlatformRunMixin(object): b_pos = line.rfind(": No such file or directory") if a_pos == -1 or b_pos == -1: return - self._echo_missed_dependency(line[a_pos + 12:b_pos].strip()) + self._echo_missed_dependency(line[a_pos + 12 : b_pos].strip()) def _echo_line(self, line, level): if line.startswith("scons: "): @@ -472,18 +493,22 @@ class PlatformRunMixin(object): * Web > {link} * {dots} -""".format(filename=filename, - filename_styled=click.style(filename, fg="cyan"), - link=click.style( - "https://platformio.org/lib/search?query=header:%s" % - quote(filename, safe=""), - fg="blue"), - dots="*" * (56 + len(filename))) +""".format( + filename=filename, + filename_styled=click.style(filename, fg="cyan"), + link=click.style( + "https://platformio.org/lib/search?query=header:%s" + % quote(filename, safe=""), + fg="blue", + ), + dots="*" * (56 + len(filename)), + ) click.echo(banner, err=True) class PlatformBase( # pylint: disable=too-many-public-methods - PlatformPackagesMixin, PlatformRunMixin): + PlatformPackagesMixin, PlatformRunMixin +): PIO_VERSION = semantic_version.Version(util.pepver_to_semver(__version__)) _BOARDS_CACHE = {} @@ -497,8 +522,7 @@ class PlatformBase( # pylint: disable=too-many-public-methods self._manifest = fs.load_json(manifest_path) self._custom_packages = None - self.pm = PackageManager(get_project_packages_dir(), - self.package_repositories) + self.pm = PackageManager(get_project_packages_dir(), self.package_repositories) # if self.engines and "platformio" in self.engines: # if self.PIO_VERSION not in semantic_version.SimpleSpec( @@ -508,19 +532,19 @@ class PlatformBase( # pylint: disable=too-many-public-methods @property def name(self): - return self._manifest['name'] + return self._manifest["name"] @property def title(self): - return self._manifest['title'] + return self._manifest["title"] @property def description(self): - return self._manifest['description'] + return self._manifest["description"] @property def version(self): - return self._manifest['version'] + return self._manifest["version"] @property def homepage(self): @@ -561,7 +585,7 @@ class PlatformBase( # pylint: disable=too-many-public-methods @property def packages(self): packages = self._manifest.get("packages", {}) - for item in (self._custom_packages or []): + for item in self._custom_packages or []: name = item version = "*" if "@" in item: @@ -569,10 +593,7 @@ class PlatformBase( # pylint: disable=too-many-public-methods name = name.strip() if name not in packages: packages[name] = {} - packages[name].update({ - "version": version.strip(), - "optional": False - }) + packages[name].update({"version": version.strip(), "optional": False}) return packages def get_dir(self): @@ -591,15 +612,13 @@ class PlatformBase( # pylint: disable=too-many-public-methods return False def get_boards(self, id_=None): - def _append_board(board_id, manifest_path): config = PlatformBoardConfig(manifest_path) if "platform" in config and config.get("platform") != self.name: return - if "platforms" in config \ - and self.name not in config.get("platforms"): + if "platforms" in config and self.name not in config.get("platforms"): return - config.manifest['platform'] = self.name + config.manifest["platform"] = self.name self._BOARDS_CACHE[board_id] = config bdirs = [ @@ -649,28 +668,28 @@ class PlatformBase( # pylint: disable=too-many-public-methods continue _pkg_name = self.frameworks[framework].get("package") if _pkg_name: - self.packages[_pkg_name]['optional'] = False + self.packages[_pkg_name]["optional"] = False # enable upload tools for upload targets if any(["upload" in t for t in targets] + ["program" in targets]): for name, opts in self.packages.items(): if opts.get("type") == "uploader": - self.packages[name]['optional'] = False + self.packages[name]["optional"] = False # skip all packages in "nobuild" mode # allow only upload tools and frameworks elif "nobuild" in targets and opts.get("type") != "framework": - self.packages[name]['optional'] = True + self.packages[name]["optional"] = True def get_lib_storages(self): storages = [] for opts in (self.frameworks or {}).values(): if "package" not in opts: continue - pkg_dir = self.get_package_dir(opts['package']) + pkg_dir = self.get_package_dir(opts["package"]) if not pkg_dir or not isdir(join(pkg_dir, "libraries")): continue libs_dir = join(pkg_dir, "libraries") - storages.append({"name": opts['package'], "path": libs_dir}) + storages.append({"name": opts["package"], "path": libs_dir}) libcores_dir = join(libs_dir, "__cores__") if not isdir(libcores_dir): continue @@ -678,16 +697,17 @@ class PlatformBase( # pylint: disable=too-many-public-methods libcore_dir = join(libcores_dir, item) if not isdir(libcore_dir): continue - storages.append({ - "name": "%s-core-%s" % (opts['package'], item), - "path": libcore_dir - }) + storages.append( + { + "name": "%s-core-%s" % (opts["package"], item), + "path": libcore_dir, + } + ) return storages class PlatformBoardConfig(object): - def __init__(self, manifest_path): self._id = basename(manifest_path)[:-5] assert isfile(manifest_path) @@ -698,8 +718,8 @@ class PlatformBoardConfig(object): raise exception.InvalidBoardManifest(manifest_path) if not set(["name", "url", "vendor"]) <= set(self._manifest): raise exception.PlatformioException( - "Please specify name, url and vendor fields for " + - manifest_path) + "Please specify name, url and vendor fields for " + manifest_path + ) def get(self, path, default=None): try: @@ -751,41 +771,33 @@ class PlatformBoardConfig(object): def get_brief_data(self): return { - "id": - self.id, - "name": - self._manifest['name'], - "platform": - self._manifest.get("platform"), - "mcu": - self._manifest.get("build", {}).get("mcu", "").upper(), - "fcpu": - int("".join([ - c for c in str( - self._manifest.get("build", {}).get("f_cpu", "0L")) - if c.isdigit() - ])), - "ram": - self._manifest.get("upload", {}).get("maximum_ram_size", 0), - "rom": - self._manifest.get("upload", {}).get("maximum_size", 0), - "connectivity": - self._manifest.get("connectivity"), - "frameworks": - self._manifest.get("frameworks"), - "debug": - self.get_debug_data(), - "vendor": - self._manifest['vendor'], - "url": - self._manifest['url'] + "id": self.id, + "name": self._manifest["name"], + "platform": self._manifest.get("platform"), + "mcu": self._manifest.get("build", {}).get("mcu", "").upper(), + "fcpu": int( + "".join( + [ + c + for c in str(self._manifest.get("build", {}).get("f_cpu", "0L")) + if c.isdigit() + ] + ) + ), + "ram": self._manifest.get("upload", {}).get("maximum_ram_size", 0), + "rom": self._manifest.get("upload", {}).get("maximum_size", 0), + "connectivity": self._manifest.get("connectivity"), + "frameworks": self._manifest.get("frameworks"), + "debug": self.get_debug_data(), + "vendor": self._manifest["vendor"], + "url": self._manifest["url"], } def get_debug_data(self): if not self._manifest.get("debug", {}).get("tools"): return None tools = {} - for name, options in self._manifest['debug']['tools'].items(): + for name, options in self._manifest["debug"]["tools"].items(): tools[name] = {} for key, value in options.items(): if key in ("default", "onboard"): @@ -798,22 +810,23 @@ class PlatformBoardConfig(object): if tool_name == "custom": return tool_name if not debug_tools: - raise exception.DebugSupportError(self._manifest['name']) + raise exception.DebugSupportError(self._manifest["name"]) if tool_name: if tool_name in debug_tools: return tool_name raise exception.DebugInvalidOptions( - "Unknown debug tool `%s`. Please use one of `%s` or `custom`" % - (tool_name, ", ".join(sorted(list(debug_tools))))) + "Unknown debug tool `%s`. Please use one of `%s` or `custom`" + % (tool_name, ", ".join(sorted(list(debug_tools)))) + ) # automatically select best tool data = {"default": [], "onboard": [], "external": []} for key, value in debug_tools.items(): if value.get("default"): - data['default'].append(key) + data["default"].append(key) elif value.get("onboard"): - data['onboard'].append(key) - data['external'].append(key) + data["onboard"].append(key) + data["external"].append(key) for key, value in data.items(): if not value: diff --git a/platformio/proc.py b/platformio/proc.py index 066813ed..b17170dc 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -23,7 +23,6 @@ from platformio.compat import WINDOWS, string_types class AsyncPipeBase(object): - def __init__(self): self._fd_read, self._fd_write = os.pipe() self._pipe_reader = os.fdopen(self._fd_read) @@ -53,7 +52,6 @@ class AsyncPipeBase(object): class BuildAsyncPipe(AsyncPipeBase): - def __init__(self, line_callback, data_callback): self.line_callback = line_callback self.data_callback = data_callback @@ -88,7 +86,6 @@ class BuildAsyncPipe(AsyncPipeBase): class LineBufferedAsyncPipe(AsyncPipeBase): - def __init__(self, line_callback): self.line_callback = line_callback super(LineBufferedAsyncPipe, self).__init__() @@ -109,8 +106,8 @@ def exec_command(*args, **kwargs): p = subprocess.Popen(*args, **kwargs) try: - result['out'], result['err'] = p.communicate() - result['returncode'] = p.returncode + result["out"], result["err"] = p.communicate() + result["returncode"] = p.returncode except KeyboardInterrupt: raise exception.AbortedByUser() finally: @@ -160,24 +157,22 @@ def copy_pythonpath_to_osenv(): for p in os.sys.path: conditions = [p not in _PYTHONPATH] if not WINDOWS: - conditions.append( - isdir(join(p, "click")) or isdir(join(p, "platformio"))) + conditions.append(isdir(join(p, "click")) or isdir(join(p, "platformio"))) if all(conditions): _PYTHONPATH.append(p) - os.environ['PYTHONPATH'] = os.pathsep.join(_PYTHONPATH) + os.environ["PYTHONPATH"] = os.pathsep.join(_PYTHONPATH) def where_is_program(program, envpath=None): env = os.environ if envpath: - env['PATH'] = envpath + env["PATH"] = envpath # try OS's built-in commands try: - result = exec_command(["where" if WINDOWS else "which", program], - env=env) - if result['returncode'] == 0 and isfile(result['out'].strip()): - return result['out'].strip() + result = exec_command(["where" if WINDOWS else "which", program], env=env) + if result["returncode"] == 0 and isfile(result["out"].strip()): + return result["out"].strip() except OSError: pass diff --git a/platformio/project/config.py b/platformio/project/config.py index 63ac5eec..28e4c955 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -119,23 +119,26 @@ class ProjectConfig(object): def _maintain_renaimed_options(self): # legacy `lib_extra_dirs` in [platformio] - if (self._parser.has_section("platformio") - and self._parser.has_option("platformio", "lib_extra_dirs")): + if self._parser.has_section("platformio") and self._parser.has_option( + "platformio", "lib_extra_dirs" + ): if not self._parser.has_section("env"): self._parser.add_section("env") - self._parser.set("env", "lib_extra_dirs", - self._parser.get("platformio", "lib_extra_dirs")) + self._parser.set( + "env", + "lib_extra_dirs", + self._parser.get("platformio", "lib_extra_dirs"), + ) self._parser.remove_option("platformio", "lib_extra_dirs") self.warnings.append( "`lib_extra_dirs` configuration option is deprecated in " - "section [platformio]! Please move it to global `env` section") + "section [platformio]! Please move it to global `env` section" + ) renamed_options = {} for option in ProjectOptions.values(): if option.oldnames: - renamed_options.update( - {name: option.name - for name in option.oldnames}) + renamed_options.update({name: option.name for name in option.oldnames}) for section in self._parser.sections(): scope = section.split(":", 1)[0] @@ -146,29 +149,34 @@ class ProjectConfig(object): self.warnings.append( "`%s` configuration option in section [%s] is " "deprecated and will be removed in the next release! " - "Please use `%s` instead" % - (option, section, renamed_options[option])) + "Please use `%s` instead" + % (option, section, renamed_options[option]) + ) # rename on-the-fly - self._parser.set(section, renamed_options[option], - self._parser.get(section, option)) + self._parser.set( + section, + renamed_options[option], + self._parser.get(section, option), + ) self._parser.remove_option(section, option) continue # unknown unknown_conditions = [ ("%s.%s" % (scope, option)) not in ProjectOptions, - scope != "env" or - not option.startswith(("custom_", "board_")) + scope != "env" or not option.startswith(("custom_", "board_")), ] # yapf: disable if all(unknown_conditions): self.warnings.append( "Ignore unknown configuration option `%s` " - "in section [%s]" % (option, section)) + "in section [%s]" % (option, section) + ) return True def walk_options(self, root_section): - extends_queue = (["env", root_section] if - root_section.startswith("env:") else [root_section]) + extends_queue = ( + ["env", root_section] if root_section.startswith("env:") else [root_section] + ) extends_done = [] while extends_queue: section = extends_queue.pop() @@ -179,8 +187,8 @@ class ProjectConfig(object): yield (section, option) if self._parser.has_option(section, "extends"): extends_queue.extend( - self.parse_multi_values( - self._parser.get(section, "extends"))[::-1]) + self.parse_multi_values(self._parser.get(section, "extends"))[::-1] + ) def options(self, section=None, env=None): result = [] @@ -213,11 +221,9 @@ class ProjectConfig(object): section = "env:" + env if as_dict: return { - option: self.get(section, option) - for option in self.options(section) + option: self.get(section, option) for option in self.options(section) } - return [(option, self.get(section, option)) - for option in self.options(section)] + return [(option, self.get(section, option)) for option in self.options(section)] def set(self, section, option, value): if isinstance(value, (list, tuple)): @@ -260,8 +266,7 @@ class ProjectConfig(object): except ConfigParser.Error as e: raise exception.InvalidProjectConf(self.path, str(e)) - option_meta = ProjectOptions.get("%s.%s" % - (section.split(":", 1)[0], option)) + option_meta = ProjectOptions.get("%s.%s" % (section.split(":", 1)[0], option)) if not option_meta: return value or default @@ -288,8 +293,7 @@ class ProjectConfig(object): try: return self._cast_to(value, option_meta.type) except click.BadParameter as e: - raise exception.ProjectOptionValueError(e.format_message(), option, - section) + raise exception.ProjectOptionValueError(e.format_message(), option, section) @staticmethod def _cast_to(value, to_type): @@ -317,8 +321,7 @@ class ProjectConfig(object): raise exception.ProjectEnvsNotAvailable() unknown = set(list(envs or []) + self.default_envs()) - known if unknown: - raise exception.UnknownEnvNames(", ".join(unknown), - ", ".join(known)) + raise exception.UnknownEnvNames(", ".join(unknown), ", ".join(known)) if not silent: for warning in self.warnings: click.secho("Warning! %s" % warning, fg="yellow") diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 9de1dbe2..f1e6702e 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -16,8 +16,16 @@ import json import os from hashlib import sha1 from os import walk -from os.path import (basename, dirname, expanduser, isdir, isfile, join, - realpath, splitdrive) +from os.path import ( + basename, + dirname, + expanduser, + isdir, + isfile, + join, + realpath, + splitdrive, +) from click.testing import CliRunner @@ -56,9 +64,13 @@ def get_project_optional_dir(name, default=None): if "$PROJECT_HASH" in optional_dir: optional_dir = optional_dir.replace( - "$PROJECT_HASH", "%s-%s" % - (basename(project_dir), sha1( - hashlib_encode_data(project_dir)).hexdigest()[:10])) + "$PROJECT_HASH", + "%s-%s" + % ( + basename(project_dir), + sha1(hashlib_encode_data(project_dir)).hexdigest()[:10], + ), + ) if optional_dir.startswith("~"): optional_dir = expanduser(optional_dir) @@ -69,7 +81,8 @@ def get_project_optional_dir(name, default=None): def get_project_core_dir(): default = join(expanduser("~"), ".platformio") core_dir = get_project_optional_dir( - "core_dir", get_project_optional_dir("home_dir", default)) + "core_dir", get_project_optional_dir("home_dir", default) + ) win_core_dir = None if WINDOWS and core_dir == default: win_core_dir = splitdrive(core_dir)[0] + "\\.platformio" @@ -91,33 +104,35 @@ def get_project_core_dir(): def get_project_global_lib_dir(): - return get_project_optional_dir("globallib_dir", - join(get_project_core_dir(), "lib")) + return get_project_optional_dir( + "globallib_dir", join(get_project_core_dir(), "lib") + ) def get_project_platforms_dir(): - return get_project_optional_dir("platforms_dir", - join(get_project_core_dir(), "platforms")) + return get_project_optional_dir( + "platforms_dir", join(get_project_core_dir(), "platforms") + ) def get_project_packages_dir(): - return get_project_optional_dir("packages_dir", - join(get_project_core_dir(), "packages")) + return get_project_optional_dir( + "packages_dir", join(get_project_core_dir(), "packages") + ) def get_project_cache_dir(): - return get_project_optional_dir("cache_dir", - join(get_project_core_dir(), ".cache")) + return get_project_optional_dir("cache_dir", join(get_project_core_dir(), ".cache")) def get_project_workspace_dir(): - return get_project_optional_dir("workspace_dir", - join(get_project_dir(), ".pio")) + return get_project_optional_dir("workspace_dir", join(get_project_dir(), ".pio")) def get_project_build_dir(force=False): - path = get_project_optional_dir("build_dir", - join(get_project_workspace_dir(), "build")) + path = get_project_optional_dir( + "build_dir", join(get_project_workspace_dir(), "build") + ) try: if not isdir(path): os.makedirs(path) @@ -129,7 +144,8 @@ def get_project_build_dir(force=False): def get_project_libdeps_dir(): return get_project_optional_dir( - "libdeps_dir", join(get_project_workspace_dir(), "libdeps")) + "libdeps_dir", join(get_project_workspace_dir(), "libdeps") + ) def get_project_lib_dir(): @@ -137,8 +153,7 @@ def get_project_lib_dir(): def get_project_include_dir(): - return get_project_optional_dir("include_dir", - join(get_project_dir(), "include")) + return get_project_optional_dir("include_dir", join(get_project_dir(), "include")) def get_project_src_dir(): @@ -146,23 +161,19 @@ def get_project_src_dir(): def get_project_test_dir(): - return get_project_optional_dir("test_dir", join(get_project_dir(), - "test")) + return get_project_optional_dir("test_dir", join(get_project_dir(), "test")) def get_project_boards_dir(): - return get_project_optional_dir("boards_dir", - join(get_project_dir(), "boards")) + return get_project_optional_dir("boards_dir", join(get_project_dir(), "boards")) def get_project_data_dir(): - return get_project_optional_dir("data_dir", join(get_project_dir(), - "data")) + return get_project_optional_dir("data_dir", join(get_project_dir(), "data")) def get_project_shared_dir(): - return get_project_optional_dir("shared_dir", - join(get_project_dir(), "shared")) + return get_project_optional_dir("shared_dir", join(get_project_dir(), "shared")) def compute_project_checksum(config): @@ -174,8 +185,7 @@ def compute_project_checksum(config): # project file structure check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S") - for d in (get_project_include_dir(), get_project_src_dir(), - get_project_lib_dir()): + for d in (get_project_include_dir(), get_project_src_dir(), get_project_lib_dir()): if not isdir(d): continue chunks = [] @@ -196,6 +206,7 @@ def compute_project_checksum(config): def load_project_ide_data(project_dir, env_or_envs): from platformio.commands.run import cli as cmd_run + assert env_or_envs envs = env_or_envs if not isinstance(envs, list): @@ -204,8 +215,9 @@ def load_project_ide_data(project_dir, env_or_envs): for env in envs: args.extend(["-e", env]) result = CliRunner().invoke(cmd_run, args) - if result.exit_code != 0 and not isinstance(result.exception, - exception.ReturnErrorCode): + if result.exit_code != 0 and not isinstance( + result.exception, exception.ReturnErrorCode + ): raise result.exception if '"includes":' not in result.output: raise exception.PlatformioException(result.output) @@ -213,11 +225,10 @@ def load_project_ide_data(project_dir, env_or_envs): data = {} for line in result.output.split("\n"): line = line.strip() - if (line.startswith('{"') and line.endswith("}") - and "env_name" in line): + if line.startswith('{"') and line.endswith("}") and "env_name" in line: _data = json.loads(line) if "env_name" in _data: - data[_data['env_name']] = _data + data[_data["env_name"]] = _data if not isinstance(env_or_envs, list) and env_or_envs in data: return data[env_or_envs] return data or None diff --git a/platformio/project/options.py b/platformio/project/options.py index 59a11081..0dc66343 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -18,20 +18,24 @@ from collections import OrderedDict, namedtuple import click -ConfigOptionClass = namedtuple("ConfigOption", [ - "scope", "name", "type", "multiple", "sysenvvar", "buildenvvar", "oldnames" -]) +ConfigOptionClass = namedtuple( + "ConfigOption", + ["scope", "name", "type", "multiple", "sysenvvar", "buildenvvar", "oldnames"], +) -def ConfigOption(scope, - name, - type=str, - multiple=False, - sysenvvar=None, - buildenvvar=None, - oldnames=None): - return ConfigOptionClass(scope, name, type, multiple, sysenvvar, - buildenvvar, oldnames) +def ConfigOption( + scope, + name, + type=str, + multiple=False, + sysenvvar=None, + buildenvvar=None, + oldnames=None, +): + return ConfigOptionClass( + scope, name, type, multiple, sysenvvar, buildenvvar, oldnames + ) def ConfigPlatformioOption(*args, **kwargs): @@ -42,171 +46,202 @@ def ConfigEnvOption(*args, **kwargs): return ConfigOption("env", *args, **kwargs) -ProjectOptions = OrderedDict([ - ("%s.%s" % (option.scope, option.name), option) for option in [ - # - # [platformio] - # - ConfigPlatformioOption(name="description"), - ConfigPlatformioOption(name="default_envs", - oldnames=["env_default"], - multiple=True, - sysenvvar="PLATFORMIO_DEFAULT_ENVS"), - ConfigPlatformioOption(name="extra_configs", multiple=True), - - # Dirs - ConfigPlatformioOption(name="core_dir", - oldnames=["home_dir"], - sysenvvar="PLATFORMIO_CORE_DIR"), - ConfigPlatformioOption(name="globallib_dir", - sysenvvar="PLATFORMIO_GLOBALLIB_DIR"), - ConfigPlatformioOption(name="platforms_dir", - sysenvvar="PLATFORMIO_PLATFORMS_DIR"), - ConfigPlatformioOption(name="packages_dir", - sysenvvar="PLATFORMIO_PACKAGES_DIR"), - ConfigPlatformioOption(name="cache_dir", - sysenvvar="PLATFORMIO_CACHE_DIR"), - ConfigPlatformioOption(name="build_cache_dir", - sysenvvar="PLATFORMIO_BUILD_CACHE_DIR"), - ConfigPlatformioOption(name="workspace_dir", - sysenvvar="PLATFORMIO_WORKSPACE_DIR"), - ConfigPlatformioOption(name="build_dir", - sysenvvar="PLATFORMIO_BUILD_DIR"), - ConfigPlatformioOption(name="libdeps_dir", - sysenvvar="PLATFORMIO_LIBDEPS_DIR"), - ConfigPlatformioOption(name="lib_dir", sysenvvar="PLATFORMIO_LIB_DIR"), - ConfigPlatformioOption(name="include_dir", - sysenvvar="PLATFORMIO_INCLUDE_DIR"), - ConfigPlatformioOption(name="src_dir", sysenvvar="PLATFORMIO_SRC_DIR"), - ConfigPlatformioOption(name="test_dir", - sysenvvar="PLATFORMIO_TEST_DIR"), - ConfigPlatformioOption(name="boards_dir", - sysenvvar="PLATFORMIO_BOARDS_DIR"), - ConfigPlatformioOption(name="data_dir", - sysenvvar="PLATFORMIO_DATA_DIR"), - ConfigPlatformioOption(name="shared_dir", - sysenvvar="PLATFORMIO_SHARED_DIR"), - - # - # [env] - # - ConfigEnvOption(name="extends", multiple=True), - - # Generic - ConfigEnvOption(name="platform", buildenvvar="PIOPLATFORM"), - ConfigEnvOption(name="platform_packages", multiple=True), - ConfigEnvOption( - name="framework", multiple=True, buildenvvar="PIOFRAMEWORK"), - - # Board - ConfigEnvOption(name="board", buildenvvar="BOARD"), - ConfigEnvOption(name="board_build.mcu", - oldnames=["board_mcu"], - buildenvvar="BOARD_MCU"), - ConfigEnvOption(name="board_build.f_cpu", - oldnames=["board_f_cpu"], - buildenvvar="BOARD_F_CPU"), - ConfigEnvOption(name="board_build.f_flash", - oldnames=["board_f_flash"], - buildenvvar="BOARD_F_FLASH"), - ConfigEnvOption(name="board_build.flash_mode", - oldnames=["board_flash_mode"], - buildenvvar="BOARD_FLASH_MODE"), - - # Build - ConfigEnvOption(name="build_type", - type=click.Choice(["release", "debug"])), - ConfigEnvOption(name="build_flags", - multiple=True, - sysenvvar="PLATFORMIO_BUILD_FLAGS", - buildenvvar="BUILD_FLAGS"), - ConfigEnvOption(name="src_build_flags", - multiple=True, - sysenvvar="PLATFORMIO_SRC_BUILD_FLAGS", - buildenvvar="SRC_BUILD_FLAGS"), - ConfigEnvOption(name="build_unflags", - multiple=True, - sysenvvar="PLATFORMIO_BUILD_UNFLAGS", - buildenvvar="BUILD_UNFLAGS"), - ConfigEnvOption(name="src_filter", - multiple=True, - sysenvvar="PLATFORMIO_SRC_FILTER", - buildenvvar="SRC_FILTER"), - ConfigEnvOption(name="targets", multiple=True), - - # Upload - ConfigEnvOption(name="upload_port", - sysenvvar="PLATFORMIO_UPLOAD_PORT", - buildenvvar="UPLOAD_PORT"), - ConfigEnvOption(name="upload_protocol", buildenvvar="UPLOAD_PROTOCOL"), - ConfigEnvOption( - name="upload_speed", type=click.INT, buildenvvar="UPLOAD_SPEED"), - ConfigEnvOption(name="upload_flags", - multiple=True, - sysenvvar="PLATFORMIO_UPLOAD_FLAGS", - buildenvvar="UPLOAD_FLAGS"), - ConfigEnvOption(name="upload_resetmethod", - buildenvvar="UPLOAD_RESETMETHOD"), - ConfigEnvOption(name="upload_command", buildenvvar="UPLOADCMD"), - - # Monitor - ConfigEnvOption(name="monitor_port"), - ConfigEnvOption(name="monitor_speed", oldnames=["monitor_baud"]), - ConfigEnvOption(name="monitor_rts", type=click.IntRange(0, 1)), - ConfigEnvOption(name="monitor_dtr", type=click.IntRange(0, 1)), - ConfigEnvOption(name="monitor_flags", multiple=True), - - # Library - ConfigEnvOption(name="lib_deps", - oldnames=["lib_use", "lib_force", "lib_install"], - multiple=True), - ConfigEnvOption(name="lib_ignore", multiple=True), - ConfigEnvOption(name="lib_extra_dirs", - multiple=True, - sysenvvar="PLATFORMIO_LIB_EXTRA_DIRS"), - ConfigEnvOption(name="lib_ldf_mode", - type=click.Choice( - ["off", "chain", "deep", "chain+", "deep+"])), - ConfigEnvOption(name="lib_compat_mode", - type=click.Choice(["off", "soft", "strict"])), - ConfigEnvOption(name="lib_archive", type=click.BOOL), - - # Test - ConfigEnvOption(name="test_filter", multiple=True), - ConfigEnvOption(name="test_ignore", multiple=True), - ConfigEnvOption(name="test_port"), - ConfigEnvOption(name="test_speed", type=click.INT), - ConfigEnvOption(name="test_transport"), - ConfigEnvOption(name="test_build_project_src", type=click.BOOL), - - # Debug - ConfigEnvOption(name="debug_tool"), - ConfigEnvOption(name="debug_init_break"), - ConfigEnvOption(name="debug_init_cmds", multiple=True), - ConfigEnvOption(name="debug_extra_cmds", multiple=True), - ConfigEnvOption(name="debug_load_cmds", - oldnames=["debug_load_cmd"], - multiple=True), - ConfigEnvOption(name="debug_load_mode", - type=click.Choice(["always", "modified", "manual"])), - ConfigEnvOption(name="debug_server", multiple=True), - ConfigEnvOption(name="debug_port"), - ConfigEnvOption(name="debug_svd_path", - type=click.Path( - exists=True, file_okay=True, dir_okay=False)), - - # Check - ConfigEnvOption(name="check_tool", multiple=True), - ConfigEnvOption(name="check_filter", multiple=True), - ConfigEnvOption(name="check_flags", multiple=True), - ConfigEnvOption(name="check_severity", - multiple=True, - type=click.Choice(["low", "medium", "high"])), - - # Other - ConfigEnvOption(name="extra_scripts", - oldnames=["extra_script"], - multiple=True, - sysenvvar="PLATFORMIO_EXTRA_SCRIPTS") +ProjectOptions = OrderedDict( + [ + ("%s.%s" % (option.scope, option.name), option) + for option in [ + # + # [platformio] + # + ConfigPlatformioOption(name="description"), + ConfigPlatformioOption( + name="default_envs", + oldnames=["env_default"], + multiple=True, + sysenvvar="PLATFORMIO_DEFAULT_ENVS", + ), + ConfigPlatformioOption(name="extra_configs", multiple=True), + # Dirs + ConfigPlatformioOption( + name="core_dir", oldnames=["home_dir"], sysenvvar="PLATFORMIO_CORE_DIR" + ), + ConfigPlatformioOption( + name="globallib_dir", sysenvvar="PLATFORMIO_GLOBALLIB_DIR" + ), + ConfigPlatformioOption( + name="platforms_dir", sysenvvar="PLATFORMIO_PLATFORMS_DIR" + ), + ConfigPlatformioOption( + name="packages_dir", sysenvvar="PLATFORMIO_PACKAGES_DIR" + ), + ConfigPlatformioOption(name="cache_dir", sysenvvar="PLATFORMIO_CACHE_DIR"), + ConfigPlatformioOption( + name="build_cache_dir", sysenvvar="PLATFORMIO_BUILD_CACHE_DIR" + ), + ConfigPlatformioOption( + name="workspace_dir", sysenvvar="PLATFORMIO_WORKSPACE_DIR" + ), + ConfigPlatformioOption(name="build_dir", sysenvvar="PLATFORMIO_BUILD_DIR"), + ConfigPlatformioOption( + name="libdeps_dir", sysenvvar="PLATFORMIO_LIBDEPS_DIR" + ), + ConfigPlatformioOption(name="lib_dir", sysenvvar="PLATFORMIO_LIB_DIR"), + ConfigPlatformioOption( + name="include_dir", sysenvvar="PLATFORMIO_INCLUDE_DIR" + ), + ConfigPlatformioOption(name="src_dir", sysenvvar="PLATFORMIO_SRC_DIR"), + ConfigPlatformioOption(name="test_dir", sysenvvar="PLATFORMIO_TEST_DIR"), + ConfigPlatformioOption( + name="boards_dir", sysenvvar="PLATFORMIO_BOARDS_DIR" + ), + ConfigPlatformioOption(name="data_dir", sysenvvar="PLATFORMIO_DATA_DIR"), + ConfigPlatformioOption( + name="shared_dir", sysenvvar="PLATFORMIO_SHARED_DIR" + ), + # + # [env] + # + ConfigEnvOption(name="extends", multiple=True), + # Generic + ConfigEnvOption(name="platform", buildenvvar="PIOPLATFORM"), + ConfigEnvOption(name="platform_packages", multiple=True), + ConfigEnvOption( + name="framework", multiple=True, buildenvvar="PIOFRAMEWORK" + ), + # Board + ConfigEnvOption(name="board", buildenvvar="BOARD"), + ConfigEnvOption( + name="board_build.mcu", oldnames=["board_mcu"], buildenvvar="BOARD_MCU" + ), + ConfigEnvOption( + name="board_build.f_cpu", + oldnames=["board_f_cpu"], + buildenvvar="BOARD_F_CPU", + ), + ConfigEnvOption( + name="board_build.f_flash", + oldnames=["board_f_flash"], + buildenvvar="BOARD_F_FLASH", + ), + ConfigEnvOption( + name="board_build.flash_mode", + oldnames=["board_flash_mode"], + buildenvvar="BOARD_FLASH_MODE", + ), + # Build + ConfigEnvOption(name="build_type", type=click.Choice(["release", "debug"])), + ConfigEnvOption( + name="build_flags", + multiple=True, + sysenvvar="PLATFORMIO_BUILD_FLAGS", + buildenvvar="BUILD_FLAGS", + ), + ConfigEnvOption( + name="src_build_flags", + multiple=True, + sysenvvar="PLATFORMIO_SRC_BUILD_FLAGS", + buildenvvar="SRC_BUILD_FLAGS", + ), + ConfigEnvOption( + name="build_unflags", + multiple=True, + sysenvvar="PLATFORMIO_BUILD_UNFLAGS", + buildenvvar="BUILD_UNFLAGS", + ), + ConfigEnvOption( + name="src_filter", + multiple=True, + sysenvvar="PLATFORMIO_SRC_FILTER", + buildenvvar="SRC_FILTER", + ), + ConfigEnvOption(name="targets", multiple=True), + # Upload + ConfigEnvOption( + name="upload_port", + sysenvvar="PLATFORMIO_UPLOAD_PORT", + buildenvvar="UPLOAD_PORT", + ), + ConfigEnvOption(name="upload_protocol", buildenvvar="UPLOAD_PROTOCOL"), + ConfigEnvOption( + name="upload_speed", type=click.INT, buildenvvar="UPLOAD_SPEED" + ), + ConfigEnvOption( + name="upload_flags", + multiple=True, + sysenvvar="PLATFORMIO_UPLOAD_FLAGS", + buildenvvar="UPLOAD_FLAGS", + ), + ConfigEnvOption( + name="upload_resetmethod", buildenvvar="UPLOAD_RESETMETHOD" + ), + ConfigEnvOption(name="upload_command", buildenvvar="UPLOADCMD"), + # Monitor + ConfigEnvOption(name="monitor_port"), + ConfigEnvOption(name="monitor_speed", oldnames=["monitor_baud"]), + ConfigEnvOption(name="monitor_rts", type=click.IntRange(0, 1)), + ConfigEnvOption(name="monitor_dtr", type=click.IntRange(0, 1)), + ConfigEnvOption(name="monitor_flags", multiple=True), + # Library + ConfigEnvOption( + name="lib_deps", + oldnames=["lib_use", "lib_force", "lib_install"], + multiple=True, + ), + ConfigEnvOption(name="lib_ignore", multiple=True), + ConfigEnvOption( + name="lib_extra_dirs", + multiple=True, + sysenvvar="PLATFORMIO_LIB_EXTRA_DIRS", + ), + ConfigEnvOption( + name="lib_ldf_mode", + type=click.Choice(["off", "chain", "deep", "chain+", "deep+"]), + ), + ConfigEnvOption( + name="lib_compat_mode", type=click.Choice(["off", "soft", "strict"]) + ), + ConfigEnvOption(name="lib_archive", type=click.BOOL), + # Test + ConfigEnvOption(name="test_filter", multiple=True), + ConfigEnvOption(name="test_ignore", multiple=True), + ConfigEnvOption(name="test_port"), + ConfigEnvOption(name="test_speed", type=click.INT), + ConfigEnvOption(name="test_transport"), + ConfigEnvOption(name="test_build_project_src", type=click.BOOL), + # Debug + ConfigEnvOption(name="debug_tool"), + ConfigEnvOption(name="debug_init_break"), + ConfigEnvOption(name="debug_init_cmds", multiple=True), + ConfigEnvOption(name="debug_extra_cmds", multiple=True), + ConfigEnvOption( + name="debug_load_cmds", oldnames=["debug_load_cmd"], multiple=True + ), + ConfigEnvOption( + name="debug_load_mode", + type=click.Choice(["always", "modified", "manual"]), + ), + ConfigEnvOption(name="debug_server", multiple=True), + ConfigEnvOption(name="debug_port"), + ConfigEnvOption( + name="debug_svd_path", + type=click.Path(exists=True, file_okay=True, dir_okay=False), + ), + # Check + ConfigEnvOption(name="check_tool", multiple=True), + ConfigEnvOption(name="check_filter", multiple=True), + ConfigEnvOption(name="check_flags", multiple=True), + ConfigEnvOption( + name="check_severity", + multiple=True, + type=click.Choice(["low", "medium", "high"]), + ), + # Other + ConfigEnvOption( + name="extra_scripts", + oldnames=["extra_script"], + multiple=True, + sysenvvar="PLATFORMIO_EXTRA_SCRIPTS", + ), + ] ] -]) +) diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 4325dd15..4a55f968 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -38,7 +38,6 @@ except ImportError: class TelemetryBase(object): - def __init__(self): self._params = {} @@ -64,17 +63,17 @@ class MeasurementProtocol(TelemetryBase): "event_category": "ec", "event_action": "ea", "event_label": "el", - "event_value": "ev" + "event_value": "ev", } def __init__(self): super(MeasurementProtocol, self).__init__() - self['v'] = 1 - self['tid'] = self.TID - self['cid'] = app.get_cid() + self["v"] = 1 + self["tid"] = self.TID + self["cid"] = app.get_cid() try: - self['sr'] = "%dx%d" % click.get_terminal_size() + self["sr"] = "%dx%d" % click.get_terminal_size() except ValueError: pass @@ -93,7 +92,7 @@ class MeasurementProtocol(TelemetryBase): super(MeasurementProtocol, self).__setitem__(name, value) def _prefill_appinfo(self): - self['av'] = __version__ + self["av"] = __version__ # gather dependent packages dpdata = [] @@ -102,10 +101,9 @@ class MeasurementProtocol(TelemetryBase): dpdata.append("Caller/%s" % app.get_session_var("caller_id")) if getenv("PLATFORMIO_IDE"): dpdata.append("IDE/%s" % getenv("PLATFORMIO_IDE")) - self['an'] = " ".join(dpdata) + self["an"] = " ".join(dpdata) def _prefill_custom_data(self): - def _filter_args(items): result = [] stop = False @@ -119,17 +117,16 @@ class MeasurementProtocol(TelemetryBase): return result caller_id = str(app.get_session_var("caller_id")) - self['cd1'] = util.get_systype() - self['cd2'] = "Python/%s %s" % (platform.python_version(), - platform.platform()) + self["cd1"] = util.get_systype() + self["cd2"] = "Python/%s %s" % (platform.python_version(), platform.platform()) # self['cd3'] = " ".join(_filter_args(sys.argv[1:])) - self['cd4'] = 1 if (not util.is_ci() and - (caller_id or not is_container())) else 0 + self["cd4"] = ( + 1 if (not util.is_ci() and (caller_id or not is_container())) else 0 + ) if caller_id: - self['cd5'] = caller_id.lower() + self["cd5"] = caller_id.lower() def _prefill_screen_name(self): - def _first_arg_from_list(args_, list_): for _arg in args_: if _arg in list_: @@ -146,12 +143,27 @@ class MeasurementProtocol(TelemetryBase): return cmd_path = args[:1] - if args[0] in ("platform", "platforms", "serialports", "device", - "settings", "account"): + if args[0] in ( + "platform", + "platforms", + "serialports", + "device", + "settings", + "account", + ): cmd_path = args[:2] if args[0] == "lib" and len(args) > 1: - lib_subcmds = ("builtin", "install", "list", "register", "search", - "show", "stats", "uninstall", "update") + lib_subcmds = ( + "builtin", + "install", + "list", + "register", + "search", + "show", + "stats", + "uninstall", + "update", + ) sub_cmd = _first_arg_from_list(args[1:], lib_subcmds) if sub_cmd: cmd_path.append(sub_cmd) @@ -165,24 +177,25 @@ class MeasurementProtocol(TelemetryBase): sub_cmd = _first_arg_from_list(args[2:], remote2_subcmds) if sub_cmd: cmd_path.append(sub_cmd) - self['screen_name'] = " ".join([p.title() for p in cmd_path]) + self["screen_name"] = " ".join([p.title() for p in cmd_path]) @staticmethod def _ignore_hit(): if not app.get_setting("enable_telemetry"): return True - if app.get_session_var("caller_id") and \ - all(c in sys.argv for c in ("run", "idedata")): + if app.get_session_var("caller_id") and all( + c in sys.argv for c in ("run", "idedata") + ): return True return False def send(self, hittype): if self._ignore_hit(): return - self['t'] = hittype + self["t"] = hittype # correct queue time - if "qt" in self._params and isinstance(self['qt'], float): - self['qt'] = int((time() - self['qt']) * 1000) + if "qt" in self._params and isinstance(self["qt"], float): + self["qt"] = int((time() - self["qt"]) * 1000) MPDataPusher().push(self._params) @@ -202,7 +215,7 @@ class MPDataPusher(object): # if network is off-line if self._http_offline: if "qt" not in item: - item['qt'] = time() + item["qt"] = time() self._failedque.append(item) return @@ -243,7 +256,7 @@ class MPDataPusher(object): item = self._queue.get() _item = item.copy() if "qt" not in _item: - _item['qt'] = time() + _item["qt"] = time() self._failedque.append(_item) if self._send_data(item): self._failedque.remove(_item) @@ -259,7 +272,8 @@ class MPDataPusher(object): "https://ssl.google-analytics.com/collect", data=data, headers=util.get_request_defheaders(), - timeout=1) + timeout=1, + ) r.raise_for_status() return True except requests.exceptions.HTTPError as e: @@ -284,11 +298,10 @@ def on_command(): def measure_ci(): event = {"category": "CI", "action": "NoName", "label": None} - known_cis = ("TRAVIS", "APPVEYOR", "GITLAB_CI", "CIRCLECI", "SHIPPABLE", - "DRONE") + known_cis = ("TRAVIS", "APPVEYOR", "GITLAB_CI", "CIRCLECI", "SHIPPABLE", "DRONE") for name in known_cis: if getenv(name, "false").lower() == "true": - event['action'] = name + event["action"] = name break on_event(**event) @@ -307,32 +320,37 @@ def on_run_environment(options, targets): def on_event(category, action, label=None, value=None, screen_name=None): mp = MeasurementProtocol() - mp['event_category'] = category[:150] - mp['event_action'] = action[:500] + mp["event_category"] = category[:150] + mp["event_action"] = action[:500] if label: - mp['event_label'] = label[:500] + mp["event_label"] = label[:500] if value: - mp['event_value'] = int(value) + mp["event_value"] = int(value) if screen_name: - mp['screen_name'] = screen_name[:2048] + mp["screen_name"] = screen_name[:2048] mp.send("event") def on_exception(e): - def _cleanup_description(text): text = text.replace("Traceback (most recent call last):", "") - text = re.sub(r'File "([^"]+)"', - lambda m: join(*m.group(1).split(sep)[-2:]), - text, - flags=re.M) + text = re.sub( + r'File "([^"]+)"', + lambda m: join(*m.group(1).split(sep)[-2:]), + text, + flags=re.M, + ) text = re.sub(r"\s+", " ", text, flags=re.M) return text.strip() skip_conditions = [ - isinstance(e, cls) for cls in (IOError, exception.ReturnErrorCode, - exception.UserSideException, - exception.PlatformIOProjectException) + isinstance(e, cls) + for cls in ( + IOError, + exception.ReturnErrorCode, + exception.UserSideException, + exception.PlatformIOProjectException, + ) ] try: skip_conditions.append("[API] Account: " in str(e)) @@ -340,14 +358,16 @@ def on_exception(e): e = ue if any(skip_conditions): return - is_crash = any([ - not isinstance(e, exception.PlatformioException), - "Error" in e.__class__.__name__ - ]) + is_crash = any( + [ + not isinstance(e, exception.PlatformioException), + "Error" in e.__class__.__name__, + ] + ) mp = MeasurementProtocol() description = _cleanup_description(format_exc() if is_crash else str(e)) - mp['exd'] = ("%s: %s" % (type(e).__name__, description))[:2048] - mp['exf'] = 1 if is_crash else 0 + mp["exd"] = ("%s: %s" % (type(e).__name__, description))[:2048] + mp["exf"] = 1 if is_crash else 0 mp.send("exception") @@ -373,7 +393,7 @@ def backup_reports(items): KEEP_MAX_REPORTS = 100 tm = app.get_state_item("telemetry", {}) if "backup" not in tm: - tm['backup'] = [] + tm["backup"] = [] for params in items: # skip static options @@ -383,28 +403,28 @@ def backup_reports(items): # store time in UNIX format if "qt" not in params: - params['qt'] = time() - elif not isinstance(params['qt'], float): - params['qt'] = time() - (params['qt'] / 1000) + params["qt"] = time() + elif not isinstance(params["qt"], float): + params["qt"] = time() - (params["qt"] / 1000) - tm['backup'].append(params) + tm["backup"].append(params) - tm['backup'] = tm['backup'][KEEP_MAX_REPORTS * -1:] + tm["backup"] = tm["backup"][KEEP_MAX_REPORTS * -1 :] app.set_state_item("telemetry", tm) def resend_backuped_reports(): tm = app.get_state_item("telemetry", {}) - if "backup" not in tm or not tm['backup']: + if "backup" not in tm or not tm["backup"]: return False - for report in tm['backup']: + for report in tm["backup"]: mp = MeasurementProtocol() for key, value in report.items(): mp[key] = value - mp.send(report['t']) + mp.send(report["t"]) # clean - tm['backup'] = [] + tm["backup"] = [] app.set_state_item("telemetry", tm) return True diff --git a/platformio/unpacker.py b/platformio/unpacker.py index 4d018072..41b455a2 100644 --- a/platformio/unpacker.py +++ b/platformio/unpacker.py @@ -24,7 +24,6 @@ from platformio import exception, util class ArchiveBase(object): - def __init__(self, arhfileobj): self._afo = arhfileobj @@ -46,7 +45,6 @@ class ArchiveBase(object): class TARArchive(ArchiveBase): - def __init__(self, archpath): super(TARArchive, self).__init__(tarfile_open(archpath)) @@ -62,7 +60,6 @@ class TARArchive(ArchiveBase): class ZIPArchive(ArchiveBase): - def __init__(self, archpath): super(ZIPArchive, self).__init__(ZipFile(archpath)) @@ -74,8 +71,10 @@ class ZIPArchive(ArchiveBase): @staticmethod def preserve_mtime(item, dest_dir): - util.change_filemtime(join(dest_dir, item.filename), - mktime(tuple(item.date_time) + tuple([0, 0, 0]))) + util.change_filemtime( + join(dest_dir, item.filename), + mktime(tuple(item.date_time) + tuple([0, 0, 0])), + ) def get_items(self): return self._afo.infolist() @@ -92,7 +91,6 @@ class ZIPArchive(ArchiveBase): class FileUnpacker(object): - def __init__(self, archpath): self.archpath = archpath self._unpacker = None diff --git a/platformio/util.py b/platformio/util.py index 1903f04c..bfd4abfb 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -40,7 +40,6 @@ from platformio.proc import is_ci # pylint: disable=unused-import class memoized(object): - def __init__(self, expire=0): expire = str(expire) if expire.isdigit(): @@ -51,13 +50,12 @@ class memoized(object): self.cache = {} def __call__(self, func): - @wraps(func) def wrapper(*args, **kwargs): key = str(args) + str(kwargs) - if (key not in self.cache - or (self.expire > 0 - and self.cache[key][0] < time.time() - self.expire)): + if key not in self.cache or ( + self.expire > 0 and self.cache[key][0] < time.time() - self.expire + ): self.cache[key] = (time.time(), func(*args, **kwargs)) return self.cache[key][1] @@ -69,13 +67,11 @@ class memoized(object): class throttle(object): - def __init__(self, threshhold): self.threshhold = threshhold # milliseconds self.last = 0 def __call__(self, func): - @wraps(func) def wrapper(*args, **kwargs): diff = int(round((time.time() - self.last) * 1000)) @@ -166,17 +162,14 @@ def get_logical_devices(): if WINDOWS: try: result = exec_command( - ["wmic", "logicaldisk", "get", - "name,VolumeName"]).get("out", "") + ["wmic", "logicaldisk", "get", "name,VolumeName"] + ).get("out", "") devicenamere = re.compile(r"^([A-Z]{1}\:)\s*(\S+)?") for line in result.split("\n"): match = devicenamere.match(line.strip()) if not match: continue - items.append({ - "path": match.group(1) + "\\", - "name": match.group(2) - }) + items.append({"path": match.group(1) + "\\", "name": match.group(2)}) return items except WindowsError: # pylint: disable=undefined-variable pass @@ -192,10 +185,7 @@ def get_logical_devices(): match = devicenamere.match(line.strip()) if not match: continue - items.append({ - "path": match.group(1), - "name": os.path.basename(match.group(1)) - }) + items.append({"path": match.group(1), "name": os.path.basename(match.group(1))}) return items @@ -205,22 +195,20 @@ def get_mdns_services(): except ImportError: from site import addsitedir from platformio.managers.core import get_core_package_dir + contrib_pysite_dir = get_core_package_dir("contrib-pysite") addsitedir(contrib_pysite_dir) sys.path.insert(0, contrib_pysite_dir) import zeroconf class mDNSListener(object): - def __init__(self): - self._zc = zeroconf.Zeroconf( - interfaces=zeroconf.InterfaceChoice.All) + self._zc = zeroconf.Zeroconf(interfaces=zeroconf.InterfaceChoice.All) self._found_types = [] self._found_services = [] def __enter__(self): - zeroconf.ServiceBrowser(self._zc, "_services._dns-sd._udp.local.", - self) + zeroconf.ServiceBrowser(self._zc, "_services._dns-sd._udp.local.", self) return self def __exit__(self, etype, value, traceback): @@ -233,8 +221,7 @@ def get_mdns_services(): try: assert zeroconf.service_type_name(name) assert str(name) - except (AssertionError, UnicodeError, - zeroconf.BadTypeInNameException): + except (AssertionError, UnicodeError, zeroconf.BadTypeInNameException): return if name not in self._found_types: self._found_types.append(name) @@ -255,29 +242,29 @@ def get_mdns_services(): if service.properties: try: properties = { - k.decode("utf8"): - v.decode("utf8") if isinstance(v, bytes) else v + k.decode("utf8"): v.decode("utf8") + if isinstance(v, bytes) + else v for k, v in service.properties.items() } json.dumps(properties) except UnicodeDecodeError: properties = None - items.append({ - "type": - service.type, - "name": - service.name, - "ip": - ".".join([ - str(c if isinstance(c, int) else ord(c)) - for c in service.address - ]), - "port": - service.port, - "properties": - properties - }) + items.append( + { + "type": service.type, + "name": service.name, + "ip": ".".join( + [ + str(c if isinstance(c, int) else ord(c)) + for c in service.address + ] + ), + "port": service.port, + "properties": properties, + } + ) return items @@ -293,10 +280,8 @@ def _api_request_session(): @throttle(500) def _get_api_result( - url, # pylint: disable=too-many-branches - params=None, - data=None, - auth=None): + url, params=None, data=None, auth=None # pylint: disable=too-many-branches +): from platformio.app import get_setting result = {} @@ -311,30 +296,29 @@ def _get_api_result( try: if data: - r = _api_request_session().post(url, - params=params, - data=data, - headers=headers, - auth=auth, - verify=verify_ssl) + r = _api_request_session().post( + url, + params=params, + data=data, + headers=headers, + auth=auth, + verify=verify_ssl, + ) else: - r = _api_request_session().get(url, - params=params, - headers=headers, - auth=auth, - verify=verify_ssl) + r = _api_request_session().get( + url, params=params, headers=headers, auth=auth, verify=verify_ssl + ) result = r.json() r.raise_for_status() return r.text except requests.exceptions.HTTPError as e: if result and "message" in result: - raise exception.APIRequestError(result['message']) + raise exception.APIRequestError(result["message"]) if result and "errors" in result: - raise exception.APIRequestError(result['errors'][0]['title']) + raise exception.APIRequestError(result["errors"][0]["title"]) raise exception.APIRequestError(e) except ValueError: - raise exception.APIRequestError("Invalid response: %s" % - r.text.encode("utf-8")) + raise exception.APIRequestError("Invalid response: %s" % r.text.encode("utf-8")) finally: if r: r.close() @@ -343,10 +327,12 @@ def _get_api_result( def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): from platformio.app import ContentCache + total = 0 max_retries = 5 - cache_key = (ContentCache.key_from_args(url, params, data, auth) - if cache_valid else None) + cache_key = ( + ContentCache.key_from_args(url, params, data, auth) if cache_valid else None + ) while total < max_retries: try: with ContentCache() as cc: @@ -363,24 +349,24 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): with ContentCache() as cc: cc.set(cache_key, result, cache_valid) return json.loads(result) - except (requests.exceptions.ConnectionError, - requests.exceptions.Timeout) as e: + except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: total += 1 if not PlatformioCLI.in_silence(): click.secho( "[API] ConnectionError: {0} (incremented retry: max={1}, " "total={2})".format(e, max_retries, total), - fg="yellow") + fg="yellow", + ) time.sleep(2 * total) raise exception.APIRequestError( - "Could not connect to PlatformIO API Service. " - "Please try later.") + "Could not connect to PlatformIO API Service. " "Please try later." + ) PING_INTERNET_IPS = [ "192.30.253.113", # github.com - "193.222.52.25" # dl.platformio.org + "193.222.52.25", # dl.platformio.org ] @@ -391,12 +377,9 @@ def _internet_on(): for ip in PING_INTERNET_IPS: try: if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")): - requests.get("http://%s" % ip, - allow_redirects=False, - timeout=timeout) + requests.get("http://%s" % ip, allow_redirects=False, timeout=timeout) else: - socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect( - (ip, 80)) + socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((ip, 80)) return True except: # pylint: disable=bare-except pass @@ -438,8 +421,7 @@ def merge_dicts(d1, d2, path=None): if path is None: path = [] for key in d2: - if (key in d1 and isinstance(d1[key], dict) - and isinstance(d2[key], dict)): + if key in d1 and isinstance(d1[key], dict) and isinstance(d2[key], dict): merge_dicts(d1[key], d2[key], path + [str(key)]) else: d1[key] = d2[key] @@ -450,9 +432,7 @@ def print_labeled_bar(label, is_error=False, fg=None): terminal_width, _ = click.get_terminal_size() width = len(click.unstyle(label)) half_line = "=" * int((terminal_width - width - 2) / 2) - click.secho("%s %s %s" % (half_line, label, half_line), - fg=fg, - err=is_error) + click.secho("%s %s %s" % (half_line, label, half_line), fg=fg, err=is_error) def humanize_duration_time(duration): diff --git a/platformio/vcsclient.py b/platformio/vcsclient.py index 51f454d0..468a4d0a 100644 --- a/platformio/vcsclient.py +++ b/platformio/vcsclient.py @@ -27,7 +27,6 @@ except ImportError: class VCSClientFactory(object): - @staticmethod def newClient(src_dir, remote_url, silent=False): result = urlparse(remote_url) @@ -38,15 +37,14 @@ class VCSClientFactory(object): remote_url = remote_url[4:] elif "+" in result.scheme: type_, _ = result.scheme.split("+", 1) - remote_url = remote_url[len(type_) + 1:] + remote_url = remote_url[len(type_) + 1 :] if "#" in remote_url: remote_url, tag = remote_url.rsplit("#", 1) if not type_: - raise PlatformioException("VCS: Unknown repository type %s" % - remote_url) - obj = getattr(modules[__name__], - "%sClient" % type_.title())(src_dir, remote_url, tag, - silent) + raise PlatformioException("VCS: Unknown repository type %s" % remote_url) + obj = getattr(modules[__name__], "%sClient" % type_.title())( + src_dir, remote_url, tag, silent + ) assert isinstance(obj, VCSClientBase) return obj @@ -71,8 +69,8 @@ class VCSClientBase(object): assert self.run_cmd(["--version"]) except (AssertionError, OSError, PlatformioException): raise UserSideException( - "VCS: `%s` client is not installed in your system" % - self.command) + "VCS: `%s` client is not installed in your system" % self.command + ) return True @property @@ -98,24 +96,23 @@ class VCSClientBase(object): def run_cmd(self, args, **kwargs): args = [self.command] + args if "cwd" not in kwargs: - kwargs['cwd'] = self.src_dir + kwargs["cwd"] = self.src_dir try: check_call(args, **kwargs) return True except CalledProcessError as e: - raise PlatformioException("VCS: Could not process command %s" % - e.cmd) + raise PlatformioException("VCS: Could not process command %s" % e.cmd) def get_cmd_output(self, args, **kwargs): args = [self.command] + args if "cwd" not in kwargs: - kwargs['cwd'] = self.src_dir + kwargs["cwd"] = self.src_dir result = exec_command(args, **kwargs) - if result['returncode'] == 0: - return result['out'].strip() + if result["returncode"] == 0: + return result["out"].strip() raise PlatformioException( - "VCS: Could not receive an output from `%s` command (%s)" % - (args, result)) + "VCS: Could not receive an output from `%s` command (%s)" % (args, result) + ) class GitClient(VCSClientBase): @@ -127,7 +124,8 @@ class GitClient(VCSClientBase): return VCSClientBase.check_client(self) except UserSideException: raise UserSideException( - "Please install Git client from https://git-scm.com/downloads") + "Please install Git client from https://git-scm.com/downloads" + ) def get_branches(self): output = self.get_cmd_output(["branch"]) @@ -232,7 +230,8 @@ class SvnClient(VCSClientBase): def get_current_revision(self): output = self.get_cmd_output( - ["info", "--non-interactive", "--trust-server-cert", "-r", "HEAD"]) + ["info", "--non-interactive", "--trust-server-cert", "-r", "HEAD"] + ) for line in output.split("\n"): line = line.strip() if line.startswith("Revision:"): diff --git a/tests/commands/test_boards.py b/tests/commands/test_boards.py index cd0041c5..bcb1f280 100644 --- a/tests/commands/test_boards.py +++ b/tests/commands/test_boards.py @@ -23,7 +23,7 @@ def test_board_json_output(clirunner, validate_cliresult): validate_cliresult(result) boards = json.loads(result.output) assert isinstance(boards, list) - assert any(["mbed" in b['frameworks'] for b in boards]) + assert any(["mbed" in b["frameworks"] for b in boards]) def test_board_raw_output(clirunner, validate_cliresult): @@ -33,8 +33,7 @@ def test_board_raw_output(clirunner, validate_cliresult): def test_board_options(clirunner, validate_cliresult): - required_opts = set( - ["fcpu", "frameworks", "id", "mcu", "name", "platform"]) + required_opts = set(["fcpu", "frameworks", "id", "mcu", "name", "platform"]) # fetch available platforms result = clirunner.invoke(cmd_platform_search, ["--json-output"]) @@ -42,7 +41,7 @@ def test_board_options(clirunner, validate_cliresult): search_result = json.loads(result.output) assert isinstance(search_result, list) assert len(search_result) - platforms = [item['name'] for item in search_result] + platforms = [item["name"] for item in search_result] result = clirunner.invoke(cmd_boards, ["mbed", "--json-output"]) validate_cliresult(result) @@ -50,4 +49,4 @@ def test_board_options(clirunner, validate_cliresult): for board in boards: assert required_opts.issubset(set(board)) - assert board['platform'] in platforms + assert board["platform"] in platforms diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index 29a377ab..c509a1f9 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -90,47 +90,43 @@ def test_check_cli_output(clirunner, check_dir): errors, warnings, style = count_defects(result.output) - assert (result.exit_code != 0) - assert (errors + warnings + style == EXPECTED_DEFECTS) + assert result.exit_code != 0 + assert errors + warnings + style == EXPECTED_DEFECTS def test_check_json_output(clirunner, check_dir): result = clirunner.invoke( - cmd_check, - ["--project-dir", str(check_dir), "--json-output"]) + cmd_check, ["--project-dir", str(check_dir), "--json-output"] + ) output = json.loads(result.stdout.strip()) assert isinstance(output, list) - assert (len(output[0].get("defects", [])) == EXPECTED_DEFECTS) + assert len(output[0].get("defects", [])) == EXPECTED_DEFECTS def test_check_tool_defines_passed(clirunner, check_dir): - result = clirunner.invoke( - cmd_check, - ["--project-dir", str(check_dir), "--verbose"]) + result = clirunner.invoke(cmd_check, ["--project-dir", str(check_dir), "--verbose"]) output = result.output - assert ("PLATFORMIO=" in output) - assert ("__GNUC__" in output) + assert "PLATFORMIO=" in output + assert "__GNUC__" in output def test_check_severity_threshold(clirunner, check_dir): result = clirunner.invoke( - cmd_check, - ["--project-dir", str(check_dir), "--severity=high"]) + cmd_check, ["--project-dir", str(check_dir), "--severity=high"] + ) errors, warnings, style = count_defects(result.output) - assert (result.exit_code != 0) - assert (errors == EXPECTED_ERRORS) - assert (warnings == 0) - assert (style == 0) + assert result.exit_code != 0 + assert errors == EXPECTED_ERRORS + assert warnings == 0 + assert style == 0 def test_check_includes_passed(clirunner, check_dir): - result = clirunner.invoke( - cmd_check, - ["--project-dir", str(check_dir), "--verbose"]) + result = clirunner.invoke(cmd_check, ["--project-dir", str(check_dir), "--verbose"]) output = result.output inc_count = 0 @@ -139,13 +135,11 @@ def test_check_includes_passed(clirunner, check_dir): inc_count = l.count("-I") # at least 1 include path for default mode - assert (inc_count > 1) + assert inc_count > 1 def test_check_silent_mode(clirunner, check_dir): - result = clirunner.invoke( - cmd_check, - ["--project-dir", str(check_dir), "--silent"]) + result = clirunner.invoke(cmd_check, ["--project-dir", str(check_dir), "--silent"]) errors, warnings, style = count_defects(result.output) @@ -159,9 +153,8 @@ def test_check_filter_sources(clirunner, check_dir): check_dir.mkdir(join("src", "app")).join("additional.cpp").write(TEST_CODE) result = clirunner.invoke( - cmd_check, - ["--project-dir", - str(check_dir), "--filter=-<*> +"]) + cmd_check, ["--project-dir", str(check_dir), "--filter=-<*> +"] + ) errors, warnings, style = count_defects(result.output) @@ -187,8 +180,8 @@ def test_check_failed_if_no_source_files(clirunner, tmpdir): def test_check_failed_if_bad_flag_passed(clirunner, check_dir): result = clirunner.invoke( - cmd_check, ["--project-dir", - str(check_dir), '"--flags=--UNKNOWN"']) + cmd_check, ["--project-dir", str(check_dir), '"--flags=--UNKNOWN"'] + ) errors, warnings, style = count_defects(result.output) @@ -200,7 +193,8 @@ def test_check_failed_if_bad_flag_passed(clirunner, check_dir): def test_check_success_if_no_errors(clirunner, tmpdir): tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) - tmpdir.mkdir("src").join("main.c").write(""" + tmpdir.mkdir("src").join("main.c").write( + """ #include void unused_functin(){ @@ -211,7 +205,8 @@ void unused_functin(){ int main() { } -""") +""" + ) result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) @@ -243,14 +238,17 @@ def test_check_individual_flags_passed(clirunner, tmpdir): def test_check_cppcheck_misra_addon(clirunner, check_dir): - check_dir.join("misra.json").write(""" + check_dir.join("misra.json").write( + """ { "script": "addons/misra.py", "args": ["--rule-texts=rules.txt"] } -""") +""" + ) - check_dir.join("rules.txt").write(""" + check_dir.join("rules.txt").write( + """ Appendix A Summary of guidelines Rule 3.1 Required R3.1 text. @@ -272,12 +270,12 @@ Rule 21.3 Required R21.3 Found MISRA defect Rule 21.4 R21.4 text. -""") +""" + ) result = clirunner.invoke( - cmd_check, - ["--project-dir", - str(check_dir), "--flags=--addon=misra.json"]) + cmd_check, ["--project-dir", str(check_dir), "--flags=--addon=misra.json"] + ) assert result.exit_code != 0 assert "R21.3 Found MISRA defect" in result.output diff --git a/tests/commands/test_ci.py b/tests/commands/test_ci.py index bce43700..0f7aceed 100644 --- a/tests/commands/test_ci.py +++ b/tests/commands/test_ci.py @@ -25,37 +25,63 @@ def test_ci_empty(clirunner): def test_ci_boards(clirunner, validate_cliresult): - result = clirunner.invoke(cmd_ci, [ - join("examples", "wiring-blink", "src", "main.cpp"), "-b", "uno", "-b", - "leonardo" - ]) + result = clirunner.invoke( + cmd_ci, + [ + join("examples", "wiring-blink", "src", "main.cpp"), + "-b", + "uno", + "-b", + "leonardo", + ], + ) validate_cliresult(result) def test_ci_build_dir(clirunner, tmpdir_factory, validate_cliresult): build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) - result = clirunner.invoke(cmd_ci, [ - join("examples", "wiring-blink", "src", "main.cpp"), "-b", "uno", - "--build-dir", build_dir - ]) + result = clirunner.invoke( + cmd_ci, + [ + join("examples", "wiring-blink", "src", "main.cpp"), + "-b", + "uno", + "--build-dir", + build_dir, + ], + ) validate_cliresult(result) assert not isfile(join(build_dir, "platformio.ini")) def test_ci_keep_build_dir(clirunner, tmpdir_factory, validate_cliresult): build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) - result = clirunner.invoke(cmd_ci, [ - join("examples", "wiring-blink", "src", "main.cpp"), "-b", "uno", - "--build-dir", build_dir, "--keep-build-dir" - ]) + result = clirunner.invoke( + cmd_ci, + [ + join("examples", "wiring-blink", "src", "main.cpp"), + "-b", + "uno", + "--build-dir", + build_dir, + "--keep-build-dir", + ], + ) validate_cliresult(result) assert isfile(join(build_dir, "platformio.ini")) # 2nd attempt - result = clirunner.invoke(cmd_ci, [ - join("examples", "wiring-blink", "src", "main.cpp"), "-b", "metro", - "--build-dir", build_dir, "--keep-build-dir" - ]) + result = clirunner.invoke( + cmd_ci, + [ + join("examples", "wiring-blink", "src", "main.cpp"), + "-b", + "metro", + "--build-dir", + build_dir, + "--keep-build-dir", + ], + ) validate_cliresult(result) assert "board: uno" in result.output @@ -64,10 +90,14 @@ def test_ci_keep_build_dir(clirunner, tmpdir_factory, validate_cliresult): def test_ci_project_conf(clirunner, validate_cliresult): project_dir = join("examples", "wiring-blink") - result = clirunner.invoke(cmd_ci, [ - join(project_dir, "src", "main.cpp"), "--project-conf", - join(project_dir, "platformio.ini") - ]) + result = clirunner.invoke( + cmd_ci, + [ + join(project_dir, "src", "main.cpp"), + "--project-conf", + join(project_dir, "platformio.ini"), + ], + ) validate_cliresult(result) assert "uno" in result.output @@ -75,12 +105,24 @@ def test_ci_project_conf(clirunner, validate_cliresult): def test_ci_lib_and_board(clirunner, tmpdir_factory, validate_cliresult): storage_dir = str(tmpdir_factory.mktemp("lib")) result = clirunner.invoke( - cmd_lib, ["--storage-dir", storage_dir, "install", "1@2.3.2"]) + cmd_lib, ["--storage-dir", storage_dir, "install", "1@2.3.2"] + ) validate_cliresult(result) - result = clirunner.invoke(cmd_ci, [ - join(storage_dir, "OneWire_ID1", "examples", "DS2408_Switch", - "DS2408_Switch.pde"), "-l", - join(storage_dir, "OneWire_ID1"), "-b", "uno" - ]) + result = clirunner.invoke( + cmd_ci, + [ + join( + storage_dir, + "OneWire_ID1", + "examples", + "DS2408_Switch", + "DS2408_Switch.pde", + ), + "-l", + join(storage_dir, "OneWire_ID1"), + "-b", + "uno", + ], + ) validate_cliresult(result) diff --git a/tests/commands/test_init.py b/tests/commands/test_init.py index 7c2e4a48..309b47c3 100644 --- a/tests/commands/test_init.py +++ b/tests/commands/test_init.py @@ -25,8 +25,7 @@ from platformio.project.config import ProjectConfig def validate_pioproject(pioproject_dir): pioconf_path = join(pioproject_dir, "platformio.ini") assert isfile(pioconf_path) and getsize(pioconf_path) > 0 - assert isdir(join(pioproject_dir, "src")) and isdir( - join(pioproject_dir, "lib")) + assert isdir(join(pioproject_dir, "src")) and isdir(join(pioproject_dir, "lib")) def test_init_default(clirunner, validate_cliresult): @@ -66,18 +65,17 @@ def test_init_ide_without_board(clirunner, tmpdir): def test_init_ide_atom(clirunner, validate_cliresult, tmpdir): with tmpdir.as_cwd(): result = clirunner.invoke( - cmd_init, ["--ide", "atom", "-b", "uno", "-b", "teensy31"]) + cmd_init, ["--ide", "atom", "-b", "uno", "-b", "teensy31"] + ) validate_cliresult(result) validate_pioproject(str(tmpdir)) - assert all([ - tmpdir.join(f).check() - for f in (".clang_complete", ".gcc-flags.json") - ]) + assert all( + [tmpdir.join(f).check() for f in (".clang_complete", ".gcc-flags.json")] + ) assert "arduinoavr" in tmpdir.join(".clang_complete").read() # switch to NodeMCU - result = clirunner.invoke(cmd_init, - ["--ide", "atom", "-b", "nodemcuv2"]) + result = clirunner.invoke(cmd_init, ["--ide", "atom", "-b", "nodemcuv2"]) validate_cliresult(result) validate_pioproject(str(tmpdir)) assert "arduinoespressif" in tmpdir.join(".clang_complete").read() @@ -110,46 +108,49 @@ def test_init_special_board(clirunner, validate_cliresult): config = ProjectConfig(join(getcwd(), "platformio.ini")) config.validate() - expected_result = dict(platform=str(boards[0]['platform']), - board="uno", - framework=[str(boards[0]['frameworks'][0])]) + expected_result = dict( + platform=str(boards[0]["platform"]), + board="uno", + framework=[str(boards[0]["frameworks"][0])], + ) assert config.has_section("env:uno") assert sorted(config.items(env="uno", as_dict=True).items()) == sorted( - expected_result.items()) + expected_result.items() + ) def test_init_enable_auto_uploading(clirunner, validate_cliresult): with clirunner.isolated_filesystem(): result = clirunner.invoke( - cmd_init, ["-b", "uno", "--project-option", "targets=upload"]) + cmd_init, ["-b", "uno", "--project-option", "targets=upload"] + ) validate_cliresult(result) validate_pioproject(getcwd()) config = ProjectConfig(join(getcwd(), "platformio.ini")) config.validate() - expected_result = dict(targets=["upload"], - platform="atmelavr", - board="uno", - framework=["arduino"]) + expected_result = dict( + targets=["upload"], platform="atmelavr", board="uno", framework=["arduino"] + ) assert config.has_section("env:uno") assert sorted(config.items(env="uno", as_dict=True).items()) == sorted( - expected_result.items()) + expected_result.items() + ) def test_init_custom_framework(clirunner, validate_cliresult): with clirunner.isolated_filesystem(): result = clirunner.invoke( - cmd_init, ["-b", "teensy31", "--project-option", "framework=mbed"]) + cmd_init, ["-b", "teensy31", "--project-option", "framework=mbed"] + ) validate_cliresult(result) validate_pioproject(getcwd()) config = ProjectConfig(join(getcwd(), "platformio.ini")) config.validate() - expected_result = dict(platform="teensy", - board="teensy31", - framework=["mbed"]) + expected_result = dict(platform="teensy", board="teensy31", framework=["mbed"]) assert config.has_section("env:teensy31") - assert sorted(config.items(env="teensy31", - as_dict=True).items()) == sorted( - expected_result.items()) + assert sorted(config.items(env="teensy31", as_dict=True).items()) == sorted( + expected_result.items() + ) def test_init_incorrect_board(clirunner): diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index 4b75d81b..b40935ee 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -28,19 +28,25 @@ def test_search(clirunner, validate_cliresult): match = re.search(r"Found\s+(\d+)\slibraries:", result.output) assert int(match.group(1)) > 2 - result = clirunner.invoke(cmd_lib, - ["search", "DHT22", "--platform=timsp430"]) + result = clirunner.invoke(cmd_lib, ["search", "DHT22", "--platform=timsp430"]) validate_cliresult(result) match = re.search(r"Found\s+(\d+)\slibraries:", result.output) assert int(match.group(1)) > 1 -def test_global_install_registry(clirunner, validate_cliresult, - isolated_pio_home): - result = clirunner.invoke(cmd_lib, [ - "-g", "install", "64", "ArduinoJson@~5.10.0", "547@2.2.4", - "AsyncMqttClient@<=0.8.2", "999@77d4eb3f8a" - ]) +def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_home): + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "64", + "ArduinoJson@~5.10.0", + "547@2.2.4", + "AsyncMqttClient@<=0.8.2", + "999@77d4eb3f8a", + ], + ) validate_cliresult(result) # install unknown library @@ -50,29 +56,40 @@ def test_global_install_registry(clirunner, validate_cliresult, items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] items2 = [ - "ArduinoJson_ID64", "ArduinoJson_ID64@5.10.1", "NeoPixelBus_ID547", - "AsyncMqttClient_ID346", "ESPAsyncTCP_ID305", "AsyncTCP_ID1826", - "RFcontrol_ID999" + "ArduinoJson_ID64", + "ArduinoJson_ID64@5.10.1", + "NeoPixelBus_ID547", + "AsyncMqttClient_ID346", + "ESPAsyncTCP_ID305", + "AsyncTCP_ID1826", + "RFcontrol_ID999", ] assert set(items1) == set(items2) -def test_global_install_archive(clirunner, validate_cliresult, - isolated_pio_home): - result = clirunner.invoke(cmd_lib, [ - "-g", "install", - "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip", - "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@5.8.2", - "SomeLib=http://dl.platformio.org/libraries/archives/0/9540.tar.gz", - "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip" - ]) +def test_global_install_archive(clirunner, validate_cliresult, isolated_pio_home): + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip", + "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@5.8.2", + "SomeLib=http://dl.platformio.org/libraries/archives/0/9540.tar.gz", + "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", + ], + ) validate_cliresult(result) # incorrect requirements - result = clirunner.invoke(cmd_lib, [ - "-g", "install", - "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@1.2.3" - ]) + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@1.2.3", + ], + ) assert result.exit_code != 0 items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] @@ -80,8 +97,7 @@ def test_global_install_archive(clirunner, validate_cliresult, assert set(items1) >= set(items2) -def test_global_install_repository(clirunner, validate_cliresult, - isolated_pio_home): +def test_global_install_repository(clirunner, validate_cliresult, isolated_pio_home): result = clirunner.invoke( cmd_lib, [ @@ -93,24 +109,28 @@ def test_global_install_repository(clirunner, validate_cliresult, "https://gitlab.com/ivankravets/rs485-nodeproto.git", "https://github.com/platformio/platformio-libmirror.git", # "https://developer.mbed.org/users/simon/code/TextLCD/", - "knolleary/pubsubclient#bef58148582f956dfa772687db80c44e2279a163" - ]) + "knolleary/pubsubclient#bef58148582f956dfa772687db80c44e2279a163", + ], + ) validate_cliresult(result) items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] items2 = [ - "PJON", "PJON@src-79de467ebe19de18287becff0a1fb42d", - "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", "rs485-nodeproto", - "platformio-libmirror", "PubSubClient" + "PJON", + "PJON@src-79de467ebe19de18287becff0a1fb42d", + "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", + "rs485-nodeproto", + "platformio-libmirror", + "PubSubClient", ] assert set(items1) >= set(items2) def test_install_duplicates(clirunner, validate_cliresult, without_internet): # registry - result = clirunner.invoke(cmd_lib, [ - "-g", "install", - "http://dl.platformio.org/libraries/archives/0/9540.tar.gz" - ]) + result = clirunner.invoke( + cmd_lib, + ["-g", "install", "http://dl.platformio.org/libraries/archives/0/9540.tar.gz"], + ) validate_cliresult(result) assert "is already installed" in result.output @@ -120,18 +140,22 @@ def test_install_duplicates(clirunner, validate_cliresult, without_internet): assert "is already installed" in result.output # archive - result = clirunner.invoke(cmd_lib, [ - "-g", "install", - "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip" - ]) + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", + ], + ) validate_cliresult(result) assert "is already installed" in result.output # repository - result = clirunner.invoke(cmd_lib, [ - "-g", "install", - "https://github.com/platformio/platformio-libmirror.git" - ]) + result = clirunner.invoke( + cmd_lib, + ["-g", "install", "https://github.com/platformio/platformio-libmirror.git"], + ) validate_cliresult(result) assert "is already installed" in result.output @@ -139,27 +163,48 @@ def test_install_duplicates(clirunner, validate_cliresult, without_internet): def test_global_lib_list(clirunner, validate_cliresult): result = clirunner.invoke(cmd_lib, ["-g", "list"]) validate_cliresult(result) - assert all([ - n in result.output for n in - ("Source: https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", - "Version: 5.10.1", - "Source: git+https://github.com/gioblu/PJON.git#3.0", - "Version: 1fb26fd") - ]) + assert all( + [ + n in result.output + for n in ( + "Source: https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", + "Version: 5.10.1", + "Source: git+https://github.com/gioblu/PJON.git#3.0", + "Version: 1fb26fd", + ) + ] + ) result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) - assert all([ - n in result.output for n in - ("__pkg_dir", - '"__src_url": "git+https://gitlab.com/ivankravets/rs485-nodeproto.git"', - '"version": "5.10.1"') - ]) - items1 = [i['name'] for i in json.loads(result.output)] + assert all( + [ + n in result.output + for n in ( + "__pkg_dir", + '"__src_url": "git+https://gitlab.com/ivankravets/rs485-nodeproto.git"', + '"version": "5.10.1"', + ) + ] + ) + items1 = [i["name"] for i in json.loads(result.output)] items2 = [ - "ESP32WebServer", "ArduinoJson", "ArduinoJson", "ArduinoJson", - "ArduinoJson", "AsyncMqttClient", "AsyncTCP", "SomeLib", "ESPAsyncTCP", - "NeoPixelBus", "OneWire", "PJON", "PJON", "PubSubClient", "RFcontrol", - "platformio-libmirror", "rs485-nodeproto" + "ESP32WebServer", + "ArduinoJson", + "ArduinoJson", + "ArduinoJson", + "ArduinoJson", + "AsyncMqttClient", + "AsyncTCP", + "SomeLib", + "ESPAsyncTCP", + "NeoPixelBus", + "OneWire", + "PJON", + "PJON", + "PubSubClient", + "RFcontrol", + "platformio-libmirror", + "rs485-nodeproto", ] assert sorted(items1) == sorted(items2) @@ -167,33 +212,37 @@ def test_global_lib_list(clirunner, validate_cliresult): "{name}@{version}".format(**item) for item in json.loads(result.output) ] versions2 = [ - "ArduinoJson@5.8.2", "ArduinoJson@5.10.1", "AsyncMqttClient@0.8.2", - "NeoPixelBus@2.2.4", "PJON@07fe9aa", "PJON@1fb26fd", - "PubSubClient@bef5814", "RFcontrol@77d4eb3f8a" + "ArduinoJson@5.8.2", + "ArduinoJson@5.10.1", + "AsyncMqttClient@0.8.2", + "NeoPixelBus@2.2.4", + "PJON@07fe9aa", + "PJON@1fb26fd", + "PubSubClient@bef5814", + "RFcontrol@77d4eb3f8a", ] assert set(versions1) >= set(versions2) def test_global_lib_update_check(clirunner, validate_cliresult): result = clirunner.invoke( - cmd_lib, ["-g", "update", "--only-check", "--json-output"]) + cmd_lib, ["-g", "update", "--only-check", "--json-output"] + ) validate_cliresult(result) output = json.loads(result.output) - assert set(["RFcontrol", - "NeoPixelBus"]) == set([l['name'] for l in output]) + assert set(["RFcontrol", "NeoPixelBus"]) == set([l["name"] for l in output]) def test_global_lib_update(clirunner, validate_cliresult): # update library using package directory result = clirunner.invoke( - cmd_lib, - ["-g", "update", "NeoPixelBus", "--only-check", "--json-output"]) + cmd_lib, ["-g", "update", "NeoPixelBus", "--only-check", "--json-output"] + ) validate_cliresult(result) oudated = json.loads(result.output) assert len(oudated) == 1 assert "__pkg_dir" in oudated[0] - result = clirunner.invoke(cmd_lib, - ["-g", "update", oudated[0]['__pkg_dir']]) + result = clirunner.invoke(cmd_lib, ["-g", "update", oudated[0]["__pkg_dir"]]) validate_cliresult(result) assert "Uninstalling NeoPixelBus @ 2.2.4" in result.output @@ -210,31 +259,43 @@ def test_global_lib_update(clirunner, validate_cliresult): assert isinstance(result.exception, exception.UnknownPackage) -def test_global_lib_uninstall(clirunner, validate_cliresult, - isolated_pio_home): +def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_home): # uninstall using package directory result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) validate_cliresult(result) items = json.loads(result.output) - result = clirunner.invoke(cmd_lib, - ["-g", "uninstall", items[5]['__pkg_dir']]) + result = clirunner.invoke(cmd_lib, ["-g", "uninstall", items[5]["__pkg_dir"]]) validate_cliresult(result) assert "Uninstalling AsyncTCP" in result.output # uninstall the rest libraries - result = clirunner.invoke(cmd_lib, [ - "-g", "uninstall", "1", "https://github.com/bblanchon/ArduinoJson.git", - "ArduinoJson@!=5.6.7", "RFcontrol" - ]) + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "uninstall", + "1", + "https://github.com/bblanchon/ArduinoJson.git", + "ArduinoJson@!=5.6.7", + "RFcontrol", + ], + ) validate_cliresult(result) items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] items2 = [ - "rs485-nodeproto", "platformio-libmirror", - "PubSubClient", "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", - "ESPAsyncTCP_ID305", "SomeLib_ID54", "NeoPixelBus_ID547", "PJON", - "AsyncMqttClient_ID346", "ArduinoJson_ID64", - "PJON@src-79de467ebe19de18287becff0a1fb42d", "ESP32WebServer" + "rs485-nodeproto", + "platformio-libmirror", + "PubSubClient", + "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", + "ESPAsyncTCP_ID305", + "SomeLib_ID54", + "NeoPixelBus_ID547", + "PJON", + "AsyncMqttClient_ID346", + "ArduinoJson_ID64", + "PJON@src-79de467ebe19de18287becff0a1fb42d", + "ESP32WebServer", ] assert set(items1) == set(items2) @@ -247,8 +308,7 @@ def test_global_lib_uninstall(clirunner, validate_cliresult, def test_lib_show(clirunner, validate_cliresult): result = clirunner.invoke(cmd_lib, ["show", "64"]) validate_cliresult(result) - assert all( - [s in result.output for s in ("ArduinoJson", "Arduino", "Atmel AVR")]) + assert all([s in result.output for s in ("ArduinoJson", "Arduino", "Atmel AVR")]) result = clirunner.invoke(cmd_lib, ["show", "OneWire", "--json-output"]) validate_cliresult(result) assert "OneWire" in result.output @@ -264,14 +324,23 @@ def test_lib_builtin(clirunner, validate_cliresult): def test_lib_stats(clirunner, validate_cliresult): result = clirunner.invoke(cmd_lib, ["stats"]) validate_cliresult(result) - assert all([ - s in result.output - for s in ("UPDATED", "POPULAR", "https://platformio.org/lib/show") - ]) + assert all( + [ + s in result.output + for s in ("UPDATED", "POPULAR", "https://platformio.org/lib/show") + ] + ) result = clirunner.invoke(cmd_lib, ["stats", "--json-output"]) validate_cliresult(result) - assert set([ - "dlweek", "added", "updated", "topkeywords", "dlmonth", "dlday", - "lastkeywords" - ]) == set(json.loads(result.output).keys()) + assert set( + [ + "dlweek", + "added", + "updated", + "topkeywords", + "dlmonth", + "dlday", + "lastkeywords", + ] + ) == set(json.loads(result.output).keys()) diff --git a/tests/commands/test_platform.py b/tests/commands/test_platform.py index 72941574..ff30dabb 100644 --- a/tests/commands/test_platform.py +++ b/tests/commands/test_platform.py @@ -19,13 +19,14 @@ from platformio.commands import platform as cli_platform def test_search_json_output(clirunner, validate_cliresult, isolated_pio_home): - result = clirunner.invoke(cli_platform.platform_search, - ["arduino", "--json-output"]) + result = clirunner.invoke( + cli_platform.platform_search, ["arduino", "--json-output"] + ) validate_cliresult(result) search_result = json.loads(result.output) assert isinstance(search_result, list) assert search_result - platforms = [item['name'] for item in search_result] + platforms = [item["name"] for item in search_result] assert "atmelsam" in platforms @@ -36,25 +37,22 @@ def test_search_raw_output(clirunner, validate_cliresult): def test_install_unknown_version(clirunner): - result = clirunner.invoke(cli_platform.platform_install, - ["atmelavr@99.99.99"]) + result = clirunner.invoke(cli_platform.platform_install, ["atmelavr@99.99.99"]) assert result.exit_code != 0 assert isinstance(result.exception, exception.UndefinedPackageVersion) def test_install_unknown_from_registry(clirunner): - result = clirunner.invoke(cli_platform.platform_install, - ["unknown-platform"]) + result = clirunner.invoke(cli_platform.platform_install, ["unknown-platform"]) assert result.exit_code != 0 assert isinstance(result.exception, exception.UnknownPackage) -def test_install_known_version(clirunner, validate_cliresult, - isolated_pio_home): - result = clirunner.invoke(cli_platform.platform_install, [ - "atmelavr@1.2.0", "--skip-default-package", "--with-package", - "tool-avrdude" - ]) +def test_install_known_version(clirunner, validate_cliresult, isolated_pio_home): + result = clirunner.invoke( + cli_platform.platform_install, + ["atmelavr@1.2.0", "--skip-default-package", "--with-package", "tool-avrdude"], + ) validate_cliresult(result) assert "atmelavr @ 1.2.0" in result.output assert "Installing tool-avrdude @" in result.output @@ -62,10 +60,13 @@ def test_install_known_version(clirunner, validate_cliresult, def test_install_from_vcs(clirunner, validate_cliresult, isolated_pio_home): - result = clirunner.invoke(cli_platform.platform_install, [ - "https://github.com/platformio/" - "platform-espressif8266.git#feature/stage", "--skip-default-package" - ]) + result = clirunner.invoke( + cli_platform.platform_install, + [ + "https://github.com/platformio/" "platform-espressif8266.git#feature/stage", + "--skip-default-package", + ], + ) validate_cliresult(result) assert "espressif8266" in result.output assert len(isolated_pio_home.join("packages").listdir()) == 1 @@ -77,24 +78,24 @@ def test_list_json_output(clirunner, validate_cliresult): list_result = json.loads(result.output) assert isinstance(list_result, list) assert list_result - platforms = [item['name'] for item in list_result] + platforms = [item["name"] for item in list_result] assert set(["atmelavr", "espressif8266"]) == set(platforms) def test_list_raw_output(clirunner, validate_cliresult): result = clirunner.invoke(cli_platform.platform_list) validate_cliresult(result) - assert all( - [s in result.output for s in ("atmelavr", "espressif8266")]) + assert all([s in result.output for s in ("atmelavr", "espressif8266")]) def test_update_check(clirunner, validate_cliresult, isolated_pio_home): - result = clirunner.invoke(cli_platform.platform_update, - ["--only-check", "--json-output"]) + result = clirunner.invoke( + cli_platform.platform_update, ["--only-check", "--json-output"] + ) validate_cliresult(result) output = json.loads(result.output) assert len(output) == 1 - assert output[0]['name'] == "atmelavr" + assert output[0]["name"] == "atmelavr" assert len(isolated_pio_home.join("packages").listdir()) == 1 @@ -107,7 +108,8 @@ def test_update_raw(clirunner, validate_cliresult, isolated_pio_home): def test_uninstall(clirunner, validate_cliresult, isolated_pio_home): - result = clirunner.invoke(cli_platform.platform_uninstall, - ["atmelavr", "espressif8266"]) + result = clirunner.invoke( + cli_platform.platform_uninstall, ["atmelavr", "espressif8266"] + ) validate_cliresult(result) assert not isolated_pio_home.join("platforms").listdir() diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index ca7ce99d..a201e723 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -20,11 +20,18 @@ from platformio import util def test_local_env(): - result = util.exec_command([ - "platformio", "test", "-d", - join("examples", "unit-testing", "calculator"), "-e", "native" - ]) - if result['returncode'] != 1: + result = util.exec_command( + [ + "platformio", + "test", + "-d", + join("examples", "unit-testing", "calculator"), + "-e", + "native", + ] + ) + if result["returncode"] != 1: pytest.fail(result) - assert all([s in result['err'] - for s in ("PASSED", "IGNORED", "FAILED")]), result['out'] + assert all([s in result["err"] for s in ("PASSED", "IGNORED", "FAILED")]), result[ + "out" + ] diff --git a/tests/conftest.py b/tests/conftest.py index c61166bc..a01bfa8c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,12 +22,9 @@ from platformio import util @pytest.fixture(scope="session") def validate_cliresult(): - def decorator(result): - assert result.exit_code == 0, "{} => {}".format( - result.exception, result.output) - assert not result.exception, "{} => {}".format(result.exception, - result.output) + assert result.exit_code == 0, "{} => {}".format(result.exception, result.output) + assert not result.exception, "{} => {}".format(result.exception, result.output) return decorator @@ -40,10 +37,10 @@ def clirunner(): @pytest.fixture(scope="module") def isolated_pio_home(request, tmpdir_factory): home_dir = tmpdir_factory.mktemp(".platformio") - os.environ['PLATFORMIO_CORE_DIR'] = str(home_dir) + os.environ["PLATFORMIO_CORE_DIR"] = str(home_dir) def fin(): - del os.environ['PLATFORMIO_CORE_DIR'] + del os.environ["PLATFORMIO_CORE_DIR"] request.addfinalizer(fin) return home_dir diff --git a/tests/test_builder.py b/tests/test_builder.py index 882c8662..d4f8012f 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -16,27 +16,34 @@ from platformio.commands.run import cli as cmd_run def test_build_flags(clirunner, validate_cliresult, tmpdir): - build_flags = [("-D TEST_INT=13", "-DTEST_INT=13"), - ("-DTEST_SINGLE_MACRO", "-DTEST_SINGLE_MACRO"), - ('-DTEST_STR_SPACE="Andrew Smith"', - '"-DTEST_STR_SPACE=Andrew Smith"')] + build_flags = [ + ("-D TEST_INT=13", "-DTEST_INT=13"), + ("-DTEST_SINGLE_MACRO", "-DTEST_SINGLE_MACRO"), + ('-DTEST_STR_SPACE="Andrew Smith"', '"-DTEST_STR_SPACE=Andrew Smith"'), + ] - tmpdir.join("platformio.ini").write(""" + tmpdir.join("platformio.ini").write( + """ [env:native] platform = native extra_scripts = extra.py build_flags = ; -DCOMMENTED_MACRO %s ; inline comment - """ % " ".join([f[0] for f in build_flags])) + """ + % " ".join([f[0] for f in build_flags]) + ) - tmpdir.join("extra.py").write(""" + tmpdir.join("extra.py").write( + """ Import("projenv") projenv.Append(CPPDEFINES="POST_SCRIPT_MACRO") - """) + """ + ) - tmpdir.mkdir("src").join("main.cpp").write(""" + tmpdir.mkdir("src").join("main.cpp").write( + """ #if !defined(TEST_INT) || TEST_INT != 13 #error "TEST_INT" #endif @@ -55,26 +62,28 @@ projenv.Append(CPPDEFINES="POST_SCRIPT_MACRO") int main() { } -""") +""" + ) - result = clirunner.invoke( - cmd_run, ["--project-dir", str(tmpdir), "--verbose"]) + result = clirunner.invoke(cmd_run, ["--project-dir", str(tmpdir), "--verbose"]) validate_cliresult(result) - build_output = result.output[result.output.find( - "Scanning dependencies..."):] + build_output = result.output[result.output.find("Scanning dependencies...") :] for flag in build_flags: assert flag[1] in build_output, flag def test_build_unflags(clirunner, validate_cliresult, tmpdir): - tmpdir.join("platformio.ini").write(""" + tmpdir.join("platformio.ini").write( + """ [env:native] platform = native build_unflags = -DTMP_MACRO1=45 -I. -DNON_EXISTING_MACRO -lunknownLib -Os extra_scripts = pre:extra.py -""") +""" + ) - tmpdir.join("extra.py").write(""" + tmpdir.join("extra.py").write( + """ Import("env") env.Append(CPPPATH="%s") env.Append(CPPDEFINES="TMP_MACRO1") @@ -82,22 +91,24 @@ env.Append(CPPDEFINES=["TMP_MACRO2"]) env.Append(CPPDEFINES=("TMP_MACRO3", 13)) env.Append(CCFLAGS=["-Os"]) env.Append(LIBS=["unknownLib"]) - """ % str(tmpdir)) + """ + % str(tmpdir) + ) - tmpdir.mkdir("src").join("main.c").write(""" + tmpdir.mkdir("src").join("main.c").write( + """ #ifdef TMP_MACRO1 #error "TMP_MACRO1 should be removed" #endif int main() { } -""") +""" + ) - result = clirunner.invoke( - cmd_run, ["--project-dir", str(tmpdir), "--verbose"]) + result = clirunner.invoke(cmd_run, ["--project-dir", str(tmpdir), "--verbose"]) validate_cliresult(result) - build_output = result.output[result.output.find( - "Scanning dependencies..."):] + build_output = result.output[result.output.find("Scanning dependencies...") :] assert "-DTMP_MACRO1" not in build_output assert "-Os" not in build_output assert str(tmpdir) not in build_output diff --git a/tests/test_examples.py b/tests/test_examples.py index 3cb365d6..013ecb26 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -35,12 +35,12 @@ def pytest_generate_tests(metafunc): # dev/platforms for manifest in PlatformManager().get_installed(): - p = PlatformFactory.newPlatform(manifest['__pkg_dir']) + p = PlatformFactory.newPlatform(manifest["__pkg_dir"]) ignore_conds = [ not p.is_embedded(), p.name == "ststm8", # issue with "version `CXXABI_1.3.9' not found (required by sdcc)" - "linux" in util.get_systype() and p.name == "intel_mcs51" + "linux" in util.get_systype() and p.name == "intel_mcs51", ] if any(ignore_conds): continue @@ -61,10 +61,9 @@ def pytest_generate_tests(metafunc): candidates[group] = [] candidates[group].append(root) - project_dirs.extend([ - random.choice(examples) for examples in candidates.values() - if examples - ]) + project_dirs.extend( + [random.choice(examples) for examples in candidates.values() if examples] + ) metafunc.parametrize("pioproject_dir", sorted(project_dirs)) @@ -76,12 +75,11 @@ def test_run(pioproject_dir): if isdir(build_dir): util.rmtree_(build_dir) - env_names = ProjectConfig(join(pioproject_dir, - "platformio.ini")).envs() + env_names = ProjectConfig(join(pioproject_dir, "platformio.ini")).envs() result = util.exec_command( - ["platformio", "run", "-e", - random.choice(env_names)]) - if result['returncode'] != 0: + ["platformio", "run", "-e", random.choice(env_names)] + ) + if result["returncode"] != 0: pytest.fail(str(result)) assert isdir(build_dir) diff --git a/tests/test_ino2cpp.py b/tests/test_ino2cpp.py index 0ef6e194..d1434df7 100644 --- a/tests/test_ino2cpp.py +++ b/tests/test_ino2cpp.py @@ -37,14 +37,10 @@ def test_example(clirunner, validate_cliresult, piotest_dir): def test_warning_line(clirunner, validate_cliresult): - result = clirunner.invoke(cmd_ci, - [join(INOTEST_DIR, "basic"), "-b", "uno"]) + result = clirunner.invoke(cmd_ci, [join(INOTEST_DIR, "basic"), "-b", "uno"]) validate_cliresult(result) - assert ('basic.ino:16:14: warning: #warning "Line number is 16"' in - result.output) - assert ('basic.ino:46:2: warning: #warning "Line number is 46"' in - result.output) - result = clirunner.invoke( - cmd_ci, [join(INOTEST_DIR, "strmultilines"), "-b", "uno"]) + assert 'basic.ino:16:14: warning: #warning "Line number is 16"' in result.output + assert 'basic.ino:46:2: warning: #warning "Line number is 46"' in result.output + result = clirunner.invoke(cmd_ci, [join(INOTEST_DIR, "strmultilines"), "-b", "uno"]) validate_cliresult(result) - assert ('main.ino:75:2: warning: #warning "Line 75"' in result.output) + assert 'main.ino:75:2: warning: #warning "Line 75"' in result.output diff --git a/tests/test_maintenance.py b/tests/test_maintenance.py index d7b4dea0..a1298211 100644 --- a/tests/test_maintenance.py +++ b/tests/test_maintenance.py @@ -23,7 +23,6 @@ from platformio.managers.platform import PlatformManager def test_check_pio_upgrade(clirunner, isolated_pio_home, validate_cliresult): - def _patch_pio_version(version): maintenance.__version__ = version cmd_upgrade.VERSION = version.split(".", 3) @@ -54,8 +53,7 @@ def test_check_pio_upgrade(clirunner, isolated_pio_home, validate_cliresult): def test_check_lib_updates(clirunner, isolated_pio_home, validate_cliresult): # install obsolete library - result = clirunner.invoke(cli_pio, - ["lib", "-g", "install", "ArduinoJson@<5.7"]) + result = clirunner.invoke(cli_pio, ["lib", "-g", "install", "ArduinoJson@<5.7"]) validate_cliresult(result) # reset check time @@ -65,15 +63,14 @@ def test_check_lib_updates(clirunner, isolated_pio_home, validate_cliresult): result = clirunner.invoke(cli_pio, ["lib", "-g", "list"]) validate_cliresult(result) - assert ("There are the new updates for libraries (ArduinoJson)" in - result.output) + assert "There are the new updates for libraries (ArduinoJson)" in result.output -def test_check_and_update_libraries(clirunner, isolated_pio_home, - validate_cliresult): +def test_check_and_update_libraries(clirunner, isolated_pio_home, validate_cliresult): # enable library auto-updates result = clirunner.invoke( - cli_pio, ["settings", "set", "auto_update_libraries", "Yes"]) + cli_pio, ["settings", "set", "auto_update_libraries", "Yes"] + ) # reset check time interval = int(app.get_setting("check_libraries_interval")) * 3600 * 24 @@ -89,27 +86,23 @@ def test_check_and_update_libraries(clirunner, isolated_pio_home, # initiate auto-updating result = clirunner.invoke(cli_pio, ["lib", "-g", "show", "ArduinoJson"]) validate_cliresult(result) - assert ("There are the new updates for libraries (ArduinoJson)" in - result.output) + assert "There are the new updates for libraries (ArduinoJson)" in result.output assert "Please wait while updating libraries" in result.output - assert re.search(r"Updating ArduinoJson\s+@ 5.6.7\s+\[[\d\.]+\]", - result.output) + assert re.search(r"Updating ArduinoJson\s+@ 5.6.7\s+\[[\d\.]+\]", result.output) # check updated version result = clirunner.invoke(cli_pio, ["lib", "-g", "list", "--json-output"]) validate_cliresult(result) - assert prev_data[0]['version'] != json.loads(result.output)[0]['version'] + assert prev_data[0]["version"] != json.loads(result.output)[0]["version"] -def test_check_platform_updates(clirunner, isolated_pio_home, - validate_cliresult): +def test_check_platform_updates(clirunner, isolated_pio_home, validate_cliresult): # install obsolete platform result = clirunner.invoke(cli_pio, ["platform", "install", "native"]) validate_cliresult(result) - manifest_path = isolated_pio_home.join("platforms", "native", - "platform.json") + manifest_path = isolated_pio_home.join("platforms", "native", "platform.json") manifest = json.loads(manifest_path.read()) - manifest['version'] = "0.0.0" + manifest["version"] = "0.0.0" manifest_path.write(json.dumps(manifest)) # reset cached manifests PlatformManager().cache_reset() @@ -124,11 +117,11 @@ def test_check_platform_updates(clirunner, isolated_pio_home, assert "There are the new updates for platforms (native)" in result.output -def test_check_and_update_platforms(clirunner, isolated_pio_home, - validate_cliresult): +def test_check_and_update_platforms(clirunner, isolated_pio_home, validate_cliresult): # enable library auto-updates result = clirunner.invoke( - cli_pio, ["settings", "set", "auto_update_platforms", "Yes"]) + cli_pio, ["settings", "set", "auto_update_platforms", "Yes"] + ) # reset check time interval = int(app.get_setting("check_platforms_interval")) * 3600 * 24 @@ -151,4 +144,4 @@ def test_check_and_update_platforms(clirunner, isolated_pio_home, # check updated version result = clirunner.invoke(cli_pio, ["platform", "list", "--json-output"]) validate_cliresult(result) - assert prev_data[0]['version'] != json.loads(result.output)[0]['version'] + assert prev_data[0]["version"] != json.loads(result.output)[0]["version"] diff --git a/tests/test_managers.py b/tests/test_managers.py index f2946f12..f4ab2ed8 100644 --- a/tests/test_managers.py +++ b/tests/test_managers.py @@ -29,126 +29,134 @@ def test_pkg_input_parser(): ["id=13@~1.2.3", ("id=13", "~1.2.3", None)], [ get_project_core_dir(), - (".platformio", None, "file://" + get_project_core_dir()) + (".platformio", None, "file://" + get_project_core_dir()), ], [ "LocalName=" + get_project_core_dir(), - ("LocalName", None, "file://" + get_project_core_dir()) + ("LocalName", None, "file://" + get_project_core_dir()), ], [ "LocalName=%s@>2.3.0" % get_project_core_dir(), - ("LocalName", ">2.3.0", "file://" + get_project_core_dir()) + ("LocalName", ">2.3.0", "file://" + get_project_core_dir()), ], [ "https://github.com/user/package.git", - ("package", None, "git+https://github.com/user/package.git") + ("package", None, "git+https://github.com/user/package.git"), ], [ "MyPackage=https://gitlab.com/user/package.git", - ("MyPackage", None, "git+https://gitlab.com/user/package.git") + ("MyPackage", None, "git+https://gitlab.com/user/package.git"), ], [ "MyPackage=https://gitlab.com/user/package.git@3.2.1,!=2", - ("MyPackage", "3.2.1,!=2", - "git+https://gitlab.com/user/package.git") + ("MyPackage", "3.2.1,!=2", "git+https://gitlab.com/user/package.git"), ], [ "https://somedomain.com/path/LibraryName-1.2.3.zip", - ("LibraryName-1.2.3", None, - "https://somedomain.com/path/LibraryName-1.2.3.zip") + ( + "LibraryName-1.2.3", + None, + "https://somedomain.com/path/LibraryName-1.2.3.zip", + ), ], [ "https://github.com/user/package/archive/branch.zip", - ("branch", None, - "https://github.com/user/package/archive/branch.zip") + ("branch", None, "https://github.com/user/package/archive/branch.zip"), ], [ "https://github.com/user/package/archive/branch.zip@~1.2.3", - ("branch", "~1.2.3", - "https://github.com/user/package/archive/branch.zip") + ("branch", "~1.2.3", "https://github.com/user/package/archive/branch.zip"), ], [ "https://github.com/user/package/archive/branch.tar.gz", - ("branch.tar", None, - "https://github.com/user/package/archive/branch.tar.gz") + ( + "branch.tar", + None, + "https://github.com/user/package/archive/branch.tar.gz", + ), ], [ "https://github.com/user/package/archive/branch.tar.gz@!=5", - ("branch.tar", "!=5", - "https://github.com/user/package/archive/branch.tar.gz") + ( + "branch.tar", + "!=5", + "https://github.com/user/package/archive/branch.tar.gz", + ), ], [ "https://developer.mbed.org/users/user/code/package/", - ("package", None, - "hg+https://developer.mbed.org/users/user/code/package/") + ("package", None, "hg+https://developer.mbed.org/users/user/code/package/"), ], [ "https://os.mbed.com/users/user/code/package/", - ("package", None, - "hg+https://os.mbed.com/users/user/code/package/") + ("package", None, "hg+https://os.mbed.com/users/user/code/package/"), ], [ "https://github.com/user/package#v1.2.3", - ("package", None, "git+https://github.com/user/package#v1.2.3") + ("package", None, "git+https://github.com/user/package#v1.2.3"), ], [ "https://github.com/user/package.git#branch", - ("package", None, "git+https://github.com/user/package.git#branch") + ("package", None, "git+https://github.com/user/package.git#branch"), ], [ "PkgName=https://github.com/user/package.git#a13d344fg56", - ("PkgName", None, - "git+https://github.com/user/package.git#a13d344fg56") - ], - [ - "user/package", - ("package", None, "git+https://github.com/user/package") + ("PkgName", None, "git+https://github.com/user/package.git#a13d344fg56"), ], + ["user/package", ("package", None, "git+https://github.com/user/package")], [ "PkgName=user/package", - ("PkgName", None, "git+https://github.com/user/package") + ("PkgName", None, "git+https://github.com/user/package"), ], [ "PkgName=user/package#master", - ("PkgName", None, "git+https://github.com/user/package#master") + ("PkgName", None, "git+https://github.com/user/package#master"), ], [ "git+https://github.com/user/package", - ("package", None, "git+https://github.com/user/package") + ("package", None, "git+https://github.com/user/package"), ], [ "hg+https://example.com/user/package", - ("package", None, "hg+https://example.com/user/package") + ("package", None, "hg+https://example.com/user/package"), ], [ "git@github.com:user/package.git", - ("package", None, "git+git@github.com:user/package.git") + ("package", None, "git+git@github.com:user/package.git"), ], [ "git@github.com:user/package.git#v1.2.0", - ("package", None, "git+git@github.com:user/package.git#v1.2.0") + ("package", None, "git+git@github.com:user/package.git#v1.2.0"), ], [ "LocalName=git@github.com:user/package.git#v1.2.0@~1.2.0", - ("LocalName", "~1.2.0", - "git+git@github.com:user/package.git#v1.2.0") + ("LocalName", "~1.2.0", "git+git@github.com:user/package.git#v1.2.0"), ], [ "git+ssh://git@gitlab.private-server.com/user/package#1.2.0", - ("package", None, - "git+ssh://git@gitlab.private-server.com/user/package#1.2.0") + ( + "package", + None, + "git+ssh://git@gitlab.private-server.com/user/package#1.2.0", + ), ], [ "git+ssh://user@gitlab.private-server.com:1234/package#1.2.0", - ("package", None, - "git+ssh://user@gitlab.private-server.com:1234/package#1.2.0") + ( + "package", + None, + "git+ssh://user@gitlab.private-server.com:1234/package#1.2.0", + ), ], [ "LocalName=git+ssh://user@gitlab.private-server.com:1234" "/package#1.2.0@!=13", - ("LocalName", "!=13", - "git+ssh://user@gitlab.private-server.com:1234/package#1.2.0") - ] + ( + "LocalName", + "!=13", + "git+ssh://user@gitlab.private-server.com:1234/package#1.2.0", + ), + ], ] for params, result in items: if isinstance(params, tuple): @@ -165,62 +173,55 @@ def test_install_packages(isolated_pio_home, tmpdir): dict(id=1, name="name_1", version="1.2"), dict(id=1, name="name_1", version="1.0.0"), dict(name="name_2", version="1.0.0"), - dict(name="name_2", - version="2.0.0", - __src_url="git+https://github.com"), - dict(name="name_2", - version="3.0.0", - __src_url="git+https://github2.com"), - dict(name="name_2", - version="4.0.0", - __src_url="git+https://github2.com") + dict(name="name_2", version="2.0.0", __src_url="git+https://github.com"), + dict(name="name_2", version="3.0.0", __src_url="git+https://github2.com"), + dict(name="name_2", version="4.0.0", __src_url="git+https://github2.com"), ] pm = PackageManager(join(get_project_core_dir(), "packages")) for package in packages: tmp_dir = tmpdir.mkdir("tmp-package") tmp_dir.join("package.json").write(json.dumps(package)) - pm._install_from_url(package['name'], "file://%s" % str(tmp_dir)) + pm._install_from_url(package["name"], "file://%s" % str(tmp_dir)) tmp_dir.remove(rec=1) assert len(pm.get_installed()) == len(packages) - 1 pkg_dirnames = [ - 'name_1_ID1', 'name_1_ID1@1.0.0', 'name_1_ID1@1.2', 'name_1_ID1@2.0.0', - 'name_1_ID1@shasum', 'name_2', - 'name_2@src-177cbce1f0705580d17790fda1cc2ef5', - 'name_2@src-f863b537ab00f4c7b5011fc44b120e1f' + "name_1_ID1", + "name_1_ID1@1.0.0", + "name_1_ID1@1.2", + "name_1_ID1@2.0.0", + "name_1_ID1@shasum", + "name_2", + "name_2@src-177cbce1f0705580d17790fda1cc2ef5", + "name_2@src-f863b537ab00f4c7b5011fc44b120e1f", ] - assert set([ - p.basename for p in isolated_pio_home.join("packages").listdir() - ]) == set(pkg_dirnames) + assert set( + [p.basename for p in isolated_pio_home.join("packages").listdir()] + ) == set(pkg_dirnames) def test_get_package(): tests = [ - [("unknown", ), None], - [("1", ), None], - [("id=1", "shasum"), - dict(id=1, name="name_1", version="shasum")], - [("id=1", "*"), - dict(id=1, name="name_1", version="2.1.0")], - [("id=1", "^1"), - dict(id=1, name="name_1", version="1.2")], - [("id=1", "^1"), - dict(id=1, name="name_1", version="1.2")], - [("name_1", "<2"), - dict(id=1, name="name_1", version="1.2")], + [("unknown",), None], + [("1",), None], + [("id=1", "shasum"), dict(id=1, name="name_1", version="shasum")], + [("id=1", "*"), dict(id=1, name="name_1", version="2.1.0")], + [("id=1", "^1"), dict(id=1, name="name_1", version="1.2")], + [("id=1", "^1"), dict(id=1, name="name_1", version="1.2")], + [("name_1", "<2"), dict(id=1, name="name_1", version="1.2")], [("name_1", ">2"), None], [("name_1", "2-0-0"), None], - [("name_2", ), dict(name="name_2", version="4.0.0")], - [("url_has_higher_priority", None, "git+https://github.com"), - dict(name="name_2", - version="2.0.0", - __src_url="git+https://github.com")], - [("name_2", None, "git+https://github.com"), - dict(name="name_2", - version="2.0.0", - __src_url="git+https://github.com")], + [("name_2",), dict(name="name_2", version="4.0.0")], + [ + ("url_has_higher_priority", None, "git+https://github.com"), + dict(name="name_2", version="2.0.0", __src_url="git+https://github.com"), + ], + [ + ("name_2", None, "git+https://github.com"), + dict(name="name_2", version="2.0.0", __src_url="git+https://github.com"), + ], ] pm = PackageManager(join(get_project_core_dir(), "packages")) diff --git a/tests/test_misc.py b/tests/test_misc.py index 8f6fccf2..1a1c7c32 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -20,8 +20,8 @@ from platformio import exception, util def test_platformio_cli(): result = util.exec_command(["pio", "--help"]) - assert result['returncode'] == 0 - assert "Usage: pio [OPTIONS] COMMAND [ARGS]..." in result['out'] + assert result["returncode"] == 0 + assert "Usage: pio [OPTIONS] COMMAND [ARGS]..." in result["out"] def test_ping_internet_ips(): @@ -38,5 +38,5 @@ def test_api_cache(monkeypatch, isolated_pio_home): api_kwargs = {"url": "/stats", "cache_valid": "10s"} result = util.get_api_result(**api_kwargs) assert result and "boards" in result - monkeypatch.setattr(util, '_internet_on', lambda: False) + monkeypatch.setattr(util, "_internet_on", lambda: False) assert util.get_api_result(**api_kwargs) == result diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index e0875a16..b2c1dc9c 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -18,14 +18,14 @@ import requests def validate_response(r): assert r.status_code == 200, r.url - assert int(r.headers['Content-Length']) > 0, r.url - assert r.headers['Content-Type'] in ("application/gzip", - "application/octet-stream") + assert int(r.headers["Content-Length"]) > 0, r.url + assert r.headers["Content-Type"] in ("application/gzip", "application/octet-stream") def test_packages(): pkgs_manifest = requests.get( - "https://dl.bintray.com/platformio/dl-packages/manifest.json").json() + "https://dl.bintray.com/platformio/dl-packages/manifest.json" + ).json() assert isinstance(pkgs_manifest, dict) items = [] for _, variants in pkgs_manifest.items(): @@ -33,12 +33,12 @@ def test_packages(): items.append(item) for item in items: - assert item['url'].endswith(".tar.gz"), item + assert item["url"].endswith(".tar.gz"), item - r = requests.head(item['url'], allow_redirects=True) + r = requests.head(item["url"], allow_redirects=True) validate_response(r) if "X-Checksum-Sha1" not in r.headers: return pytest.skip("X-Checksum-Sha1 is not provided") - assert item['sha1'] == r.headers.get("X-Checksum-Sha1")[0:40], item + assert item["sha1"] == r.headers.get("X-Checksum-Sha1")[0:40], item diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index 10bf4b2f..a5127dd0 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -117,8 +117,16 @@ def test_sections(config): config.getraw("unknown_section", "unknown_option") assert config.sections() == [ - "platformio", "env", "strict_ldf", "monitor_custom", "strict_settings", - "custom", "env:base", "env:test_extends", "env:extra_1", "env:extra_2" + "platformio", + "env", + "strict_ldf", + "monitor_custom", + "strict_settings", + "custom", + "env:base", + "env:test_extends", + "env:extra_1", + "env:extra_2", ] @@ -129,11 +137,20 @@ def test_envs(config): def test_options(config): assert config.options(env="base") == [ - "build_flags", "targets", "monitor_speed", "lib_deps", "lib_ignore" + "build_flags", + "targets", + "monitor_speed", + "lib_deps", + "lib_ignore", ] assert config.options(env="test_extends") == [ - "extends", "build_flags", "lib_ldf_mode", "lib_compat_mode", - "monitor_speed", "lib_deps", "lib_ignore" + "extends", + "build_flags", + "lib_ldf_mode", + "lib_compat_mode", + "monitor_speed", + "lib_deps", + "lib_ignore", ] @@ -154,15 +171,22 @@ def test_sysenv_options(config): os.environ["__PIO_TEST_CNF_EXTRA_FLAGS"] = "-L /usr/local/lib" assert config.get("custom", "extra_flags") == "-L /usr/local/lib" assert config.get("env:base", "build_flags") == [ - "-D DEBUG=1 -L /usr/local/lib", "-DSYSENVDEPS1 -DSYSENVDEPS2" + "-D DEBUG=1 -L /usr/local/lib", + "-DSYSENVDEPS1 -DSYSENVDEPS2", ] assert config.get("env:base", "upload_port") == "/dev/sysenv/port" assert config.get("env:extra_2", "upload_port") == "/dev/extra_2/port" # env var as option assert config.options(env="test_extends") == [ - "extends", "build_flags", "lib_ldf_mode", "lib_compat_mode", - "monitor_speed", "lib_deps", "lib_ignore", "upload_port" + "extends", + "build_flags", + "lib_ldf_mode", + "lib_compat_mode", + "monitor_speed", + "lib_deps", + "lib_ignore", + "upload_port", ] # sysenv @@ -208,7 +232,7 @@ def test_items(config): ("debug_flags", "-D DEBUG=1"), ("lib_flags", "-lc -lm"), ("extra_flags", None), - ("lib_ignore", "LibIgnoreCustom") + ("lib_ignore", "LibIgnoreCustom"), ] # yapf: disable assert config.items(env="base") == [ ("build_flags", ["-D DEBUG=1"]), @@ -228,7 +252,7 @@ def test_items(config): ("lib_ignore", ["LibIgnoreCustom", "Lib3"]), ("upload_port", "/dev/extra_2/port"), ("monitor_speed", "115200"), - ("lib_deps", ["Lib1", "Lib2"]) + ("lib_deps", ["Lib1", "Lib2"]), ] # yapf: disable assert config.items(env="test_extends") == [ ("extends", ["strict_settings"]), @@ -237,5 +261,5 @@ def test_items(config): ("lib_compat_mode", "strict"), ("monitor_speed", "9600"), ("lib_deps", ["Lib1", "Lib2"]), - ("lib_ignore", ["LibIgnoreCustom"]) + ("lib_ignore", ["LibIgnoreCustom"]), ] # yapf: disable diff --git a/tox.ini b/tox.ini index 597c7b8f..0c4de05d 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,7 @@ passenv = * usedevelop = True deps = isort - yapf + black pylint pytest pytest-xdist From 7b314b58a4f57826b1dd97135fc402dc8657170f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 23 Sep 2019 23:23:11 +0300 Subject: [PATCH 029/221] Move PIO Home to the root of source code --- platformio/commands/{home/command.py => home.py} | 16 ++++++++-------- .../{commands/home/rpc => home}/__init__.py | 0 platformio/{commands => }/home/helpers.py | 0 .../home/rpc/handlers => home/rpc}/__init__.py | 0 .../home => home/rpc/handlers}/__init__.py | 2 -- .../{commands => }/home/rpc/handlers/app.py | 0 .../{commands => }/home/rpc/handlers/ide.py | 0 .../{commands => }/home/rpc/handlers/misc.py | 2 +- .../{commands => }/home/rpc/handlers/os.py | 2 +- .../{commands => }/home/rpc/handlers/piocore.py | 2 +- .../{commands => }/home/rpc/handlers/project.py | 4 ++-- platformio/{commands => }/home/rpc/server.py | 0 platformio/{commands => }/home/web.py | 0 13 files changed, 13 insertions(+), 15 deletions(-) rename platformio/commands/{home/command.py => home.py} (86%) rename platformio/{commands/home/rpc => home}/__init__.py (100%) rename platformio/{commands => }/home/helpers.py (100%) rename platformio/{commands/home/rpc/handlers => home/rpc}/__init__.py (100%) rename platformio/{commands/home => home/rpc/handlers}/__init__.py (92%) rename platformio/{commands => }/home/rpc/handlers/app.py (100%) rename platformio/{commands => }/home/rpc/handlers/ide.py (100%) rename platformio/{commands => }/home/rpc/handlers/misc.py (97%) rename platformio/{commands => }/home/rpc/handlers/os.py (99%) rename platformio/{commands => }/home/rpc/handlers/piocore.py (99%) rename platformio/{commands => }/home/rpc/handlers/project.py (98%) rename platformio/{commands => }/home/rpc/server.py (100%) rename platformio/{commands => }/home/web.py (100%) diff --git a/platformio/commands/home/command.py b/platformio/commands/home.py similarity index 86% rename from platformio/commands/home/command.py rename to platformio/commands/home.py index f3c748a9..38b371bb 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home.py @@ -44,14 +44,14 @@ def cli(port, host, no_open): from twisted.web import server # pylint: enable=import-error - from platformio.commands.home.rpc.handlers.app import AppRPC - from platformio.commands.home.rpc.handlers.ide import IDERPC - from platformio.commands.home.rpc.handlers.misc import MiscRPC - from platformio.commands.home.rpc.handlers.os import OSRPC - from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC - from platformio.commands.home.rpc.handlers.project import ProjectRPC - from platformio.commands.home.rpc.server import JSONRPCServerFactory - from platformio.commands.home.web import WebRoot + from platformio.home.rpc.handlers.app import AppRPC + from platformio.home.rpc.handlers.ide import IDERPC + from platformio.home.rpc.handlers.misc import MiscRPC + from platformio.home.rpc.handlers.os import OSRPC + from platformio.home.rpc.handlers.piocore import PIOCoreRPC + from platformio.home.rpc.handlers.project import ProjectRPC + from platformio.home.rpc.server import JSONRPCServerFactory + from platformio.home.web import WebRoot factory = JSONRPCServerFactory() factory.addHandler(AppRPC(), namespace="app") diff --git a/platformio/commands/home/rpc/__init__.py b/platformio/home/__init__.py similarity index 100% rename from platformio/commands/home/rpc/__init__.py rename to platformio/home/__init__.py diff --git a/platformio/commands/home/helpers.py b/platformio/home/helpers.py similarity index 100% rename from platformio/commands/home/helpers.py rename to platformio/home/helpers.py diff --git a/platformio/commands/home/rpc/handlers/__init__.py b/platformio/home/rpc/__init__.py similarity index 100% rename from platformio/commands/home/rpc/handlers/__init__.py rename to platformio/home/rpc/__init__.py diff --git a/platformio/commands/home/__init__.py b/platformio/home/rpc/handlers/__init__.py similarity index 92% rename from platformio/commands/home/__init__.py rename to platformio/home/rpc/handlers/__init__.py index a889291e..b0514903 100644 --- a/platformio/commands/home/__init__.py +++ b/platformio/home/rpc/handlers/__init__.py @@ -11,5 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from platformio.commands.home.command import cli diff --git a/platformio/commands/home/rpc/handlers/app.py b/platformio/home/rpc/handlers/app.py similarity index 100% rename from platformio/commands/home/rpc/handlers/app.py rename to platformio/home/rpc/handlers/app.py diff --git a/platformio/commands/home/rpc/handlers/ide.py b/platformio/home/rpc/handlers/ide.py similarity index 100% rename from platformio/commands/home/rpc/handlers/ide.py rename to platformio/home/rpc/handlers/ide.py diff --git a/platformio/commands/home/rpc/handlers/misc.py b/platformio/home/rpc/handlers/misc.py similarity index 97% rename from platformio/commands/home/rpc/handlers/misc.py rename to platformio/home/rpc/handlers/misc.py index 004c14a3..5503cf8c 100644 --- a/platformio/commands/home/rpc/handlers/misc.py +++ b/platformio/home/rpc/handlers/misc.py @@ -18,7 +18,7 @@ import time from twisted.internet import defer, reactor # pylint: disable=import-error from platformio import app -from platformio.commands.home.rpc.handlers.os import OSRPC +from platformio.home.rpc.handlers.os import OSRPC class MiscRPC(object): diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/home/rpc/handlers/os.py similarity index 99% rename from platformio/commands/home/rpc/handlers/os.py rename to platformio/home/rpc/handlers/os.py index 960f04e6..7f5f5ece 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/home/rpc/handlers/os.py @@ -25,8 +25,8 @@ import click from twisted.internet import defer # pylint: disable=import-error from platformio import app, util -from platformio.commands.home import helpers from platformio.compat import PY2, get_filesystem_encoding +from platformio.home import helpers class OSRPC(object): diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/home/rpc/handlers/piocore.py similarity index 99% rename from platformio/commands/home/rpc/handlers/piocore.py rename to platformio/home/rpc/handlers/piocore.py index 9ef39a03..f2a692d0 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/home/rpc/handlers/piocore.py @@ -26,8 +26,8 @@ from twisted.internet import threads # pylint: disable=import-error from twisted.internet import utils # pylint: disable=import-error from platformio import __main__, __version__, fs -from platformio.commands.home import helpers from platformio.compat import PY2, get_filesystem_encoding, is_bytes, string_types +from platformio.home import helpers try: from thread import get_ident as thread_get_ident diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/home/rpc/handlers/project.py similarity index 98% rename from platformio/commands/home/rpc/handlers/project.py rename to platformio/home/rpc/handlers/project.py index c6f6a89f..4a0acfb5 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/home/rpc/handlers/project.py @@ -22,9 +22,9 @@ from os.path import basename, expanduser, getmtime, isdir, isfile, join, realpat import jsonrpc # pylint: disable=import-error from platformio import exception, fs -from platformio.commands.home.rpc.handlers.app import AppRPC -from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC from platformio.compat import PY2, get_filesystem_encoding +from platformio.home.rpc.handlers.app import AppRPC +from platformio.home.rpc.handlers.piocore import PIOCoreRPC from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager from platformio.project.config import ProjectConfig diff --git a/platformio/commands/home/rpc/server.py b/platformio/home/rpc/server.py similarity index 100% rename from platformio/commands/home/rpc/server.py rename to platformio/home/rpc/server.py diff --git a/platformio/commands/home/web.py b/platformio/home/web.py similarity index 100% rename from platformio/commands/home/web.py rename to platformio/home/web.py From 5453df94e4bbac9d8a4c7c8ee0fd5ba1db69606b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 23 Sep 2019 23:27:55 +0300 Subject: [PATCH 030/221] Move PIO Unified Debugger to the root --- platformio/commands/{debug/command.py => debug.py} | 4 ++-- platformio/{commands => }/debug/__init__.py | 2 -- platformio/{commands => }/debug/client.py | 6 +++--- platformio/{commands => }/debug/helpers.py | 0 platformio/{commands => }/debug/initcfgs.py | 0 platformio/{commands => }/debug/process.py | 0 platformio/{commands => }/debug/server.py | 2 +- 7 files changed, 6 insertions(+), 8 deletions(-) rename platformio/commands/{debug/command.py => debug.py} (97%) rename platformio/{commands => }/debug/__init__.py (92%) rename platformio/{commands => }/debug/client.py (98%) rename platformio/{commands => }/debug/helpers.py (100%) rename platformio/{commands => }/debug/initcfgs.py (100%) rename platformio/{commands => }/debug/process.py (100%) rename platformio/{commands => }/debug/server.py (98%) diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug.py similarity index 97% rename from platformio/commands/debug/command.py rename to platformio/commands/debug.py index 64e62765..470fdd97 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug.py @@ -22,7 +22,7 @@ from os.path import isfile, join import click from platformio import exception, fs, proc, util -from platformio.commands.debug import helpers +from platformio.debug import helpers from platformio.managers.core import inject_contrib_pysite from platformio.project.config import ProjectConfig from platformio.project.helpers import is_platformio_project, load_project_ide_data @@ -137,7 +137,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro # run debugging client inject_contrib_pysite() - from platformio.commands.debug.client import GDBClient, reactor + from platformio.debug.client import GDBClient, reactor client = GDBClient(project_dir, __unprocessed, debug_options, env_options) client.spawn(configuration["gdb_path"], configuration["prog_path"]) diff --git a/platformio/commands/debug/__init__.py b/platformio/debug/__init__.py similarity index 92% rename from platformio/commands/debug/__init__.py rename to platformio/debug/__init__.py index 7fba44c2..b0514903 100644 --- a/platformio/commands/debug/__init__.py +++ b/platformio/debug/__init__.py @@ -11,5 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from platformio.commands.debug.command import cli diff --git a/platformio/commands/debug/client.py b/platformio/debug/client.py similarity index 98% rename from platformio/commands/debug/client.py rename to platformio/debug/client.py index 2334fbf9..63b07c5e 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/debug/client.py @@ -27,10 +27,10 @@ from twisted.internet import stdio # pylint: disable=import-error from twisted.internet import task # pylint: disable=import-error from platformio import app, exception, fs, proc, util -from platformio.commands.debug import helpers, initcfgs -from platformio.commands.debug.process import BaseProcess -from platformio.commands.debug.server import DebugServer from platformio.compat import hashlib_encode_data +from platformio.debug import helpers, initcfgs +from platformio.debug.process import BaseProcess +from platformio.debug.server import DebugServer from platformio.project.helpers import get_project_cache_dir from platformio.telemetry import MeasurementProtocol diff --git a/platformio/commands/debug/helpers.py b/platformio/debug/helpers.py similarity index 100% rename from platformio/commands/debug/helpers.py rename to platformio/debug/helpers.py diff --git a/platformio/commands/debug/initcfgs.py b/platformio/debug/initcfgs.py similarity index 100% rename from platformio/commands/debug/initcfgs.py rename to platformio/debug/initcfgs.py diff --git a/platformio/commands/debug/process.py b/platformio/debug/process.py similarity index 100% rename from platformio/commands/debug/process.py rename to platformio/debug/process.py diff --git a/platformio/commands/debug/server.py b/platformio/debug/server.py similarity index 98% rename from platformio/commands/debug/server.py rename to platformio/debug/server.py index 213c413e..fbce1268 100644 --- a/platformio/commands/debug/server.py +++ b/platformio/debug/server.py @@ -19,7 +19,7 @@ from twisted.internet import error # pylint: disable=import-error from twisted.internet import reactor # pylint: disable=import-error from platformio import exception, fs, util -from platformio.commands.debug.process import BaseProcess +from platformio.debug.process import BaseProcess from platformio.proc import where_is_program From b1f190a7f8b372f589f65094f22a516fc5309bc6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 23 Sep 2019 23:44:28 +0300 Subject: [PATCH 031/221] Move PIO Unit Testing to the root --- platformio/commands/check/__init__.py | 15 - platformio/commands/check/command.py | 292 ------------------ platformio/commands/check/defect.py | 95 ------ platformio/commands/check/tools/__init__.py | 30 -- platformio/commands/check/tools/base.py | 143 --------- platformio/commands/check/tools/clangtidy.py | 67 ---- platformio/commands/check/tools/cppcheck.py | 142 --------- platformio/commands/run/command.py | 2 +- platformio/commands/run/processor.py | 2 +- .../commands/{test/command.py => test.py} | 4 +- platformio/{commands => }/test/__init__.py | 2 - platformio/{commands => }/test/embedded.py | 2 +- platformio/{commands => }/test/native.py | 2 +- platformio/{commands => }/test/processor.py | 0 14 files changed, 6 insertions(+), 792 deletions(-) delete mode 100644 platformio/commands/check/__init__.py delete mode 100644 platformio/commands/check/command.py delete mode 100644 platformio/commands/check/defect.py delete mode 100644 platformio/commands/check/tools/__init__.py delete mode 100644 platformio/commands/check/tools/base.py delete mode 100644 platformio/commands/check/tools/clangtidy.py delete mode 100644 platformio/commands/check/tools/cppcheck.py rename platformio/commands/{test/command.py => test.py} (98%) rename platformio/{commands => }/test/__init__.py (92%) rename platformio/{commands => }/test/embedded.py (98%) rename platformio/{commands => }/test/native.py (96%) rename platformio/{commands => }/test/processor.py (100%) diff --git a/platformio/commands/check/__init__.py b/platformio/commands/check/__init__.py deleted file mode 100644 index 76307f21..00000000 --- a/platformio/commands/check/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (c) 2019-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from platformio.commands.check.command import cli diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py deleted file mode 100644 index 0bd0a1bd..00000000 --- a/platformio/commands/check/command.py +++ /dev/null @@ -1,292 +0,0 @@ -# Copyright (c) 2019-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pylint: disable=too-many-arguments,too-many-locals,too-many-branches -# pylint: disable=redefined-builtin,too-many-statements - -import os -from collections import Counter -from os.path import basename, dirname, isfile, join -from time import time - -import click -from tabulate import tabulate - -from platformio import exception, fs, util -from platformio.commands.check.defect import DefectItem -from platformio.commands.check.tools import CheckToolFactory -from platformio.compat import dump_json_to_unicode -from platformio.project.config import ProjectConfig -from platformio.project.helpers import ( - find_project_dir_above, - get_project_dir, - get_project_include_dir, - get_project_src_dir, -) - - -@click.command("check", short_help="Run a static analysis tool on code") -@click.option("-e", "--environment", multiple=True) -@click.option( - "-d", - "--project-dir", - default=os.getcwd, - type=click.Path( - exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True - ), -) -@click.option( - "-c", - "--project-conf", - type=click.Path( - exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True - ), -) -@click.option("--filter", multiple=True, help="Pattern: + -") -@click.option("--flags", multiple=True) -@click.option( - "--severity", multiple=True, type=click.Choice(DefectItem.SEVERITY_LABELS.values()) -) -@click.option("-s", "--silent", is_flag=True) -@click.option("-v", "--verbose", is_flag=True) -@click.option("--json-output", is_flag=True) -def cli( - environment, - project_dir, - project_conf, - filter, - flags, - severity, - silent, - verbose, - json_output, -): - # find project directory on upper level - if isfile(project_dir): - project_dir = find_project_dir_above(project_dir) - - results = [] - with fs.cd(project_dir): - config = ProjectConfig.get_instance( - project_conf or join(project_dir, "platformio.ini") - ) - config.validate(environment) - - default_envs = config.default_envs() - for envname in config.envs(): - skipenv = any( - [ - environment and envname not in environment, - not environment and default_envs and envname not in default_envs, - ] - ) - - env_options = config.items(env=envname, as_dict=True) - env_dump = [] - for k, v in env_options.items(): - if k not in ("platform", "framework", "board"): - continue - env_dump.append( - "%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v) - ) - - default_filter = [ - "+<%s/>" % basename(d) - for d in (get_project_src_dir(), get_project_include_dir()) - ] - - tool_options = dict( - verbose=verbose, - silent=silent, - filter=filter or env_options.get("check_filter", default_filter), - flags=flags or env_options.get("check_flags"), - severity=[DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_HIGH]] - if silent - else (severity or env_options.get("check_severity")), - ) - - for tool in env_options.get("check_tool", ["cppcheck"]): - if skipenv: - results.append({"env": envname, "tool": tool}) - continue - if not silent and not json_output: - print_processing_header(tool, envname, env_dump) - - ct = CheckToolFactory.new( - tool, project_dir, config, envname, tool_options - ) - - result = {"env": envname, "tool": tool, "duration": time()} - rc = ct.check( - on_defect_callback=None - if (json_output or verbose) - else lambda defect: click.echo(repr(defect)) - ) - - result["defects"] = ct.get_defects() - result["duration"] = time() - result["duration"] - result["succeeded"] = rc == 0 and not any( - d.severity == DefectItem.SEVERITY_HIGH for d in result["defects"] - ) - results.append(result) - - if verbose: - click.echo("\n".join(repr(d) for d in result["defects"])) - - if not json_output and not silent: - if not result["defects"]: - click.echo("No defects found") - print_processing_footer(result) - - if json_output: - click.echo(dump_json_to_unicode(results_to_json(results))) - elif not silent: - print_check_summary(results) - - command_failed = any(r.get("succeeded") is False for r in results) - if command_failed: - raise exception.ReturnErrorCode(1) - - -def results_to_json(raw): - results = [] - for item in raw: - item.update( - { - "ignored": item.get("succeeded") is None, - "succeeded": bool(item.get("succeeded")), - "defects": [d.to_json() for d in item.get("defects", [])], - } - ) - results.append(item) - - return results - - -def print_processing_header(tool, envname, envdump): - click.echo( - "Checking %s > %s (%s)" - % (click.style(envname, fg="cyan", bold=True), tool, "; ".join(envdump)) - ) - terminal_width, _ = click.get_terminal_size() - click.secho("-" * terminal_width, bold=True) - - -def print_processing_footer(result): - is_failed = not result.get("succeeded") - util.print_labeled_bar( - "[%s] Took %.2f seconds" - % ( - ( - click.style("FAILED", fg="red", bold=True) - if is_failed - else click.style("PASSED", fg="green", bold=True) - ), - result["duration"], - ), - is_error=is_failed, - ) - - -def print_defects_stats(results): - components = dict() - - def _append_defect(component, defect): - if not components.get(component): - components[component] = Counter() - components[component].update({DefectItem.SEVERITY_LABELS[defect.severity]: 1}) - - for result in results: - for defect in result.get("defects", []): - component = dirname(defect.file) or defect.file - _append_defect(component, defect) - - if component.startswith(get_project_dir()): - while os.sep in component: - component = dirname(component) - _append_defect(component, defect) - - if not components: - return - - severity_labels = list(DefectItem.SEVERITY_LABELS.values()) - severity_labels.reverse() - tabular_data = list() - for k, v in components.items(): - tool_defect = [v.get(s, 0) for s in severity_labels] - tabular_data.append([k] + tool_defect) - - total = ["Total"] + [sum(d) for d in list(zip(*tabular_data))[1:]] - tabular_data.sort() - tabular_data.append([]) # Empty line as delimeter - tabular_data.append(total) - - headers = ["Component"] - headers.extend([l.upper() for l in severity_labels]) - headers = [click.style(h, bold=True) for h in headers] - click.echo(tabulate(tabular_data, headers=headers, numalign="center")) - click.echo() - - -def print_check_summary(results): - click.echo() - - tabular_data = [] - succeeded_nums = 0 - failed_nums = 0 - duration = 0 - - print_defects_stats(results) - - for result in results: - duration += result.get("duration", 0) - if result.get("succeeded") is False: - failed_nums += 1 - status_str = click.style("FAILED", fg="red") - elif result.get("succeeded") is None: - status_str = "IGNORED" - else: - succeeded_nums += 1 - status_str = click.style("PASSED", fg="green") - - tabular_data.append( - ( - click.style(result["env"], fg="cyan"), - result["tool"], - status_str, - util.humanize_duration_time(result.get("duration")), - ) - ) - - click.echo( - tabulate( - tabular_data, - headers=[ - click.style(s, bold=True) - for s in ("Environment", "Tool", "Status", "Duration") - ], - ), - err=failed_nums, - ) - - util.print_labeled_bar( - "%s%d succeeded in %s" - % ( - "%d failed, " % failed_nums if failed_nums else "", - succeeded_nums, - util.humanize_duration_time(duration), - ), - is_error=failed_nums, - fg="red" if failed_nums else "green", - ) diff --git a/platformio/commands/check/defect.py b/platformio/commands/check/defect.py deleted file mode 100644 index 0b25084c..00000000 --- a/platformio/commands/check/defect.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright (c) 2019-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from os.path import relpath - -import click - -from platformio.project.helpers import get_project_dir - -# pylint: disable=too-many-instance-attributes, redefined-builtin -# pylint: disable=too-many-arguments - - -class DefectItem(object): - - SEVERITY_HIGH = 1 - SEVERITY_MEDIUM = 2 - SEVERITY_LOW = 4 - SEVERITY_LABELS = {4: "low", 2: "medium", 1: "high"} - - def __init__( - self, - severity, - category, - message, - file="unknown", - line=0, - column=0, - id=None, - callstack=None, - cwe=None, - ): - assert severity in (self.SEVERITY_HIGH, self.SEVERITY_MEDIUM, self.SEVERITY_LOW) - self.severity = severity - self.category = category - self.message = message - self.line = line - self.column = column - self.callstack = callstack - self.cwe = cwe - self.id = id - self.file = file - if file.startswith(get_project_dir()): - self.file = relpath(file, get_project_dir()) - - def __repr__(self): - defect_color = None - if self.severity == self.SEVERITY_HIGH: - defect_color = "red" - elif self.severity == self.SEVERITY_MEDIUM: - defect_color = "yellow" - - format_str = "{file}:{line}: [{severity}:{category}] {message} {id}" - return format_str.format( - severity=click.style(self.SEVERITY_LABELS[self.severity], fg=defect_color), - category=click.style(self.category.lower(), fg=defect_color), - file=click.style(self.file, bold=True), - message=self.message, - line=self.line, - id="%s" % "[%s]" % self.id if self.id else "", - ) - - def __or__(self, defect): - return self.severity | defect.severity - - @staticmethod - def severity_to_int(label): - for key, value in DefectItem.SEVERITY_LABELS.items(): - if label == value: - return key - raise Exception("Unknown severity label -> %s" % label) - - def to_json(self): - return { - "severity": self.SEVERITY_LABELS[self.severity], - "category": self.category, - "message": self.message, - "file": self.file, - "line": self.line, - "column": self.column, - "callstack": self.callstack, - "id": self.id, - "cwe": self.cwe, - } diff --git a/platformio/commands/check/tools/__init__.py b/platformio/commands/check/tools/__init__.py deleted file mode 100644 index e3c4b190..00000000 --- a/platformio/commands/check/tools/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) 2019-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from platformio import exception -from platformio.commands.check.tools.clangtidy import ClangtidyCheckTool -from platformio.commands.check.tools.cppcheck import CppcheckCheckTool - - -class CheckToolFactory(object): - @staticmethod - def new(tool, project_dir, config, envname, options): - cls = None - if tool == "cppcheck": - cls = CppcheckCheckTool - elif tool == "clangtidy": - cls = ClangtidyCheckTool - else: - raise exception.PlatformioException("Unknown check tool `%s`" % tool) - return cls(project_dir, config, envname, options) diff --git a/platformio/commands/check/tools/base.py b/platformio/commands/check/tools/base.py deleted file mode 100644 index d9454dd0..00000000 --- a/platformio/commands/check/tools/base.py +++ /dev/null @@ -1,143 +0,0 @@ -# Copyright (c) 2019-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import click - -from platformio import fs, proc -from platformio.commands.check.defect import DefectItem -from platformio.project.helpers import get_project_dir, load_project_ide_data - - -class CheckToolBase(object): # pylint: disable=too-many-instance-attributes - def __init__(self, project_dir, config, envname, options): - self.config = config - self.envname = envname - self.options = options - self.cpp_defines = [] - self.cpp_includes = [] - - self._defects = [] - self._on_defect_callback = None - self._bad_input = False - self._load_cpp_data(project_dir, envname) - - # detect all defects by default - if not self.options.get("severity"): - self.options["severity"] = [ - DefectItem.SEVERITY_LOW, - DefectItem.SEVERITY_MEDIUM, - DefectItem.SEVERITY_HIGH, - ] - # cast to severity by ids - self.options["severity"] = [ - s if isinstance(s, int) else DefectItem.severity_to_int(s) - for s in self.options["severity"] - ] - - def _load_cpp_data(self, project_dir, envname): - data = load_project_ide_data(project_dir, envname) - if not data: - return - self.cpp_includes = data.get("includes", []) - self.cpp_defines = data.get("defines", []) - self.cpp_defines.extend(self._get_toolchain_defines(data.get("cc_path"))) - - def get_flags(self, tool): - result = [] - flags = self.options.get("flags") or [] - for flag in flags: - if ":" not in flag: - result.extend([f for f in flag.split(" ") if f]) - elif flag.startswith("%s:" % tool): - result.extend([f for f in flag.split(":", 1)[1].split(" ") if f]) - - return result - - @staticmethod - def _get_toolchain_defines(cc_path): - defines = [] - result = proc.exec_command("echo | %s -dM -E -x c++ -" % cc_path, shell=True) - - for line in result["out"].split("\n"): - tokens = line.strip().split(" ", 2) - if not tokens or tokens[0] != "#define": - continue - if len(tokens) > 2: - defines.append("%s=%s" % (tokens[1], tokens[2])) - else: - defines.append(tokens[1]) - - return defines - - @staticmethod - def is_flag_set(flag, flags): - return any(flag in f for f in flags) - - def get_defects(self): - return self._defects - - def configure_command(self): - raise NotImplementedError - - def on_tool_output(self, line): - line = self.tool_output_filter(line) - if not line: - return - - defect = self.parse_defect(line) - - if not isinstance(defect, DefectItem): - if self.options.get("verbose"): - click.echo(line) - return - - if defect.severity not in self.options["severity"]: - return - - self._defects.append(defect) - if self._on_defect_callback: - self._on_defect_callback(defect) - - @staticmethod - def tool_output_filter(line): - return line - - @staticmethod - def parse_defect(raw_line): - return raw_line - - def clean_up(self): - pass - - def get_project_src_files(self): - file_extensions = ["h", "hpp", "c", "cc", "cpp", "ino"] - return fs.match_src_files( - get_project_dir(), self.options.get("filter"), file_extensions - ) - - def check(self, on_defect_callback=None): - self._on_defect_callback = on_defect_callback - cmd = self.configure_command() - if self.options.get("verbose"): - click.echo(" ".join(cmd)) - - proc.exec_command( - cmd, - stdout=proc.LineBufferedAsyncPipe(self.on_tool_output), - stderr=proc.LineBufferedAsyncPipe(self.on_tool_output), - ) - - self.clean_up() - - return self._bad_input diff --git a/platformio/commands/check/tools/clangtidy.py b/platformio/commands/check/tools/clangtidy.py deleted file mode 100644 index c4154c52..00000000 --- a/platformio/commands/check/tools/clangtidy.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright (c) 2019-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -from os.path import join - -from platformio.commands.check.defect import DefectItem -from platformio.commands.check.tools.base import CheckToolBase -from platformio.managers.core import get_core_package_dir - - -class ClangtidyCheckTool(CheckToolBase): - def tool_output_filter(self, line): - if not self.options.get("verbose") and "[clang-diagnostic-error]" in line: - return "" - - if "[CommonOptionsParser]" in line: - self._bad_input = True - return line - - if any(d in line for d in ("note: ", "error: ", "warning: ")): - return line - - return "" - - def parse_defect(self, raw_line): - match = re.match(r"^(.*):(\d+):(\d+):\s+([^:]+):\s(.+)\[([^]]+)\]$", raw_line) - if not match: - return raw_line - - file_, line, column, category, message, defect_id = match.groups() - - severity = DefectItem.SEVERITY_LOW - if category == "error": - severity = DefectItem.SEVERITY_HIGH - elif category == "warning": - severity = DefectItem.SEVERITY_MEDIUM - - return DefectItem(severity, category, message, file_, line, column, defect_id) - - def configure_command(self): - tool_path = join(get_core_package_dir("tool-clangtidy"), "clang-tidy") - - cmd = [tool_path, "--quiet"] - flags = self.get_flags("clangtidy") - if not self.is_flag_set("--checks", flags): - cmd.append("--checks=*") - - cmd.extend(flags) - cmd.extend(self.get_project_src_files()) - cmd.append("--") - - cmd.extend(["-D%s" % d for d in self.cpp_defines]) - cmd.extend(["-I%s" % inc for inc in self.cpp_includes]) - - return cmd diff --git a/platformio/commands/check/tools/cppcheck.py b/platformio/commands/check/tools/cppcheck.py deleted file mode 100644 index 3279a704..00000000 --- a/platformio/commands/check/tools/cppcheck.py +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright (c) 2019-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from os import remove -from os.path import isfile, join -from tempfile import NamedTemporaryFile - -from platformio.commands.check.defect import DefectItem -from platformio.commands.check.tools.base import CheckToolBase -from platformio.managers.core import get_core_package_dir -from platformio.project.helpers import get_project_core_dir - - -class CppcheckCheckTool(CheckToolBase): - def __init__(self, *args, **kwargs): - self._tmp_files = [] - self.defect_fields = [ - "severity", - "message", - "file", - "line", - "column", - "callstack", - "cwe", - "id", - ] - super(CppcheckCheckTool, self).__init__(*args, **kwargs) - - def tool_output_filter(self, line): - if ( - not self.options.get("verbose") - and "--suppress=unmatchedSuppression:" in line - ): - return "" - - if any( - msg in line - for msg in ( - "No C or C++ source files found", - "unrecognized command line option", - ) - ): - self._bad_input = True - - return line - - def parse_defect(self, raw_line): - if "<&PIO&>" not in raw_line or any( - f not in raw_line for f in self.defect_fields - ): - return None - - args = dict() - for field in raw_line.split("<&PIO&>"): - field = field.strip().replace('"', "") - name, value = field.split("=", 1) - args[name] = value - - args["category"] = args["severity"] - if args["severity"] == "error": - args["severity"] = DefectItem.SEVERITY_HIGH - elif args["severity"] == "warning": - args["severity"] = DefectItem.SEVERITY_MEDIUM - else: - args["severity"] = DefectItem.SEVERITY_LOW - - return DefectItem(**args) - - def configure_command(self): - tool_path = join(get_core_package_dir("tool-cppcheck"), "cppcheck") - - cmd = [ - tool_path, - "--error-exitcode=1", - "--verbose" if self.options.get("verbose") else "--quiet", - ] - - cmd.append( - '--template="%s"' - % "<&PIO&>".join(["{0}={{{0}}}".format(f) for f in self.defect_fields]) - ) - - flags = self.get_flags("cppcheck") - if not self.is_flag_set("--platform", flags): - cmd.append("--platform=unspecified") - if not self.is_flag_set("--enable", flags): - enabled_checks = [ - "warning", - "style", - "performance", - "portability", - "unusedFunction", - ] - cmd.append("--enable=%s" % ",".join(enabled_checks)) - - cmd.extend(["-D%s" % d for d in self.cpp_defines]) - cmd.extend(flags) - - cmd.append("--file-list=%s" % self._generate_src_file()) - cmd.append("--includes-file=%s" % self._generate_inc_file()) - - core_dir = get_project_core_dir() - cmd.append("--suppress=*:%s*" % core_dir) - cmd.append("--suppress=unmatchedSuppression:%s*" % core_dir) - - return cmd - - def _create_tmp_file(self, data): - with NamedTemporaryFile("w", delete=False) as fp: - fp.write(data) - self._tmp_files.append(fp.name) - return fp.name - - def _generate_src_file(self): - return self._create_tmp_file("\n".join(self.get_project_src_files())) - - def _generate_inc_file(self): - return self._create_tmp_file("\n".join(self.cpp_includes)) - - def clean_up(self): - for f in self._tmp_files: - if isfile(f): - remove(f) - - # delete temporary dump files generated by addons - if not self.is_flag_set("--addon", self.get_flags("cppcheck")): - return - for f in self.get_project_src_files(): - dump_file = f + ".dump" - if isfile(dump_file): - remove(dump_file) diff --git a/platformio/commands/run/command.py b/platformio/commands/run/command.py index fd53e5fc..0349ab03 100644 --- a/platformio/commands/run/command.py +++ b/platformio/commands/run/command.py @@ -24,9 +24,9 @@ from platformio import exception, fs, util from platformio.commands.device import device_monitor as cmd_device_monitor from platformio.commands.run.helpers import clean_build_dir, handle_legacy_libdeps from platformio.commands.run.processor import EnvironmentProcessor -from platformio.commands.test.processor import CTX_META_TEST_IS_RUNNING from platformio.project.config import ProjectConfig from platformio.project.helpers import find_project_dir_above, get_project_build_dir +from platformio.test.processor import CTX_META_TEST_IS_RUNNING # pylint: disable=too-many-arguments,too-many-locals,too-many-branches diff --git a/platformio/commands/run/processor.py b/platformio/commands/run/processor.py index 032286e5..219444d8 100644 --- a/platformio/commands/run/processor.py +++ b/platformio/commands/run/processor.py @@ -14,8 +14,8 @@ from platformio import exception, telemetry from platformio.commands.platform import platform_install as cmd_platform_install -from platformio.commands.test.processor import CTX_META_TEST_RUNNING_NAME from platformio.managers.platform import PlatformFactory +from platformio.test.processor import CTX_META_TEST_RUNNING_NAME # pylint: disable=too-many-instance-attributes diff --git a/platformio/commands/test/command.py b/platformio/commands/test.py similarity index 98% rename from platformio/commands/test/command.py rename to platformio/commands/test.py index 17aec86f..b46c6e42 100644 --- a/platformio/commands/test/command.py +++ b/platformio/commands/test.py @@ -23,10 +23,10 @@ import click from tabulate import tabulate from platformio import exception, fs, util -from platformio.commands.test.embedded import EmbeddedTestProcessor -from platformio.commands.test.native import NativeTestProcessor from platformio.project.config import ProjectConfig from platformio.project.helpers import get_project_test_dir +from platformio.test.embedded import EmbeddedTestProcessor +from platformio.test.native import NativeTestProcessor @click.command("test", short_help="Unit Testing") diff --git a/platformio/commands/test/__init__.py b/platformio/test/__init__.py similarity index 92% rename from platformio/commands/test/__init__.py rename to platformio/test/__init__.py index 6d4c3e2b..b0514903 100644 --- a/platformio/commands/test/__init__.py +++ b/platformio/test/__init__.py @@ -11,5 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from platformio.commands.test.command import cli diff --git a/platformio/commands/test/embedded.py b/platformio/test/embedded.py similarity index 98% rename from platformio/commands/test/embedded.py rename to platformio/test/embedded.py index dc0d9ef0..6c6726e2 100644 --- a/platformio/commands/test/embedded.py +++ b/platformio/test/embedded.py @@ -18,8 +18,8 @@ import click import serial from platformio import exception, util -from platformio.commands.test.processor import TestProcessorBase from platformio.managers.platform import PlatformFactory +from platformio.test.processor import TestProcessorBase class EmbeddedTestProcessor(TestProcessorBase): diff --git a/platformio/commands/test/native.py b/platformio/test/native.py similarity index 96% rename from platformio/commands/test/native.py rename to platformio/test/native.py index 75d4fcf8..73029e98 100644 --- a/platformio/commands/test/native.py +++ b/platformio/test/native.py @@ -15,9 +15,9 @@ from os.path import join from platformio import fs, proc -from platformio.commands.test.processor import TestProcessorBase from platformio.proc import LineBufferedAsyncPipe from platformio.project.helpers import get_project_build_dir +from platformio.test.processor import TestProcessorBase class NativeTestProcessor(TestProcessorBase): diff --git a/platformio/commands/test/processor.py b/platformio/test/processor.py similarity index 100% rename from platformio/commands/test/processor.py rename to platformio/test/processor.py From aa955819b077a79beb448b4aff42cb9846922cf9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 23 Sep 2019 23:44:42 +0300 Subject: [PATCH 032/221] Move PIO Check to the root --- platformio/check/__init__.py | 13 ++ platformio/check/defect.py | 95 +++++++++ platformio/check/tools/__init__.py | 30 +++ platformio/check/tools/base.py | 143 ++++++++++++++ platformio/check/tools/clangtidy.py | 67 +++++++ platformio/check/tools/cppcheck.py | 142 ++++++++++++++ platformio/commands/check.py | 292 ++++++++++++++++++++++++++++ 7 files changed, 782 insertions(+) create mode 100644 platformio/check/__init__.py create mode 100644 platformio/check/defect.py create mode 100644 platformio/check/tools/__init__.py create mode 100644 platformio/check/tools/base.py create mode 100644 platformio/check/tools/clangtidy.py create mode 100644 platformio/check/tools/cppcheck.py create mode 100644 platformio/commands/check.py diff --git a/platformio/check/__init__.py b/platformio/check/__init__.py new file mode 100644 index 00000000..895dc9b9 --- /dev/null +++ b/platformio/check/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/check/defect.py b/platformio/check/defect.py new file mode 100644 index 00000000..0b25084c --- /dev/null +++ b/platformio/check/defect.py @@ -0,0 +1,95 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os.path import relpath + +import click + +from platformio.project.helpers import get_project_dir + +# pylint: disable=too-many-instance-attributes, redefined-builtin +# pylint: disable=too-many-arguments + + +class DefectItem(object): + + SEVERITY_HIGH = 1 + SEVERITY_MEDIUM = 2 + SEVERITY_LOW = 4 + SEVERITY_LABELS = {4: "low", 2: "medium", 1: "high"} + + def __init__( + self, + severity, + category, + message, + file="unknown", + line=0, + column=0, + id=None, + callstack=None, + cwe=None, + ): + assert severity in (self.SEVERITY_HIGH, self.SEVERITY_MEDIUM, self.SEVERITY_LOW) + self.severity = severity + self.category = category + self.message = message + self.line = line + self.column = column + self.callstack = callstack + self.cwe = cwe + self.id = id + self.file = file + if file.startswith(get_project_dir()): + self.file = relpath(file, get_project_dir()) + + def __repr__(self): + defect_color = None + if self.severity == self.SEVERITY_HIGH: + defect_color = "red" + elif self.severity == self.SEVERITY_MEDIUM: + defect_color = "yellow" + + format_str = "{file}:{line}: [{severity}:{category}] {message} {id}" + return format_str.format( + severity=click.style(self.SEVERITY_LABELS[self.severity], fg=defect_color), + category=click.style(self.category.lower(), fg=defect_color), + file=click.style(self.file, bold=True), + message=self.message, + line=self.line, + id="%s" % "[%s]" % self.id if self.id else "", + ) + + def __or__(self, defect): + return self.severity | defect.severity + + @staticmethod + def severity_to_int(label): + for key, value in DefectItem.SEVERITY_LABELS.items(): + if label == value: + return key + raise Exception("Unknown severity label -> %s" % label) + + def to_json(self): + return { + "severity": self.SEVERITY_LABELS[self.severity], + "category": self.category, + "message": self.message, + "file": self.file, + "line": self.line, + "column": self.column, + "callstack": self.callstack, + "id": self.id, + "cwe": self.cwe, + } diff --git a/platformio/check/tools/__init__.py b/platformio/check/tools/__init__.py new file mode 100644 index 00000000..a6161fb2 --- /dev/null +++ b/platformio/check/tools/__init__.py @@ -0,0 +1,30 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio import exception +from platformio.check.tools.clangtidy import ClangtidyCheckTool +from platformio.check.tools.cppcheck import CppcheckCheckTool + + +class CheckToolFactory(object): + @staticmethod + def new(tool, project_dir, config, envname, options): + cls = None + if tool == "cppcheck": + cls = CppcheckCheckTool + elif tool == "clangtidy": + cls = ClangtidyCheckTool + else: + raise exception.PlatformioException("Unknown check tool `%s`" % tool) + return cls(project_dir, config, envname, options) diff --git a/platformio/check/tools/base.py b/platformio/check/tools/base.py new file mode 100644 index 00000000..578b72c0 --- /dev/null +++ b/platformio/check/tools/base.py @@ -0,0 +1,143 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio import fs, proc +from platformio.check.defect import DefectItem +from platformio.project.helpers import get_project_dir, load_project_ide_data + + +class CheckToolBase(object): # pylint: disable=too-many-instance-attributes + def __init__(self, project_dir, config, envname, options): + self.config = config + self.envname = envname + self.options = options + self.cpp_defines = [] + self.cpp_includes = [] + + self._defects = [] + self._on_defect_callback = None + self._bad_input = False + self._load_cpp_data(project_dir, envname) + + # detect all defects by default + if not self.options.get("severity"): + self.options["severity"] = [ + DefectItem.SEVERITY_LOW, + DefectItem.SEVERITY_MEDIUM, + DefectItem.SEVERITY_HIGH, + ] + # cast to severity by ids + self.options["severity"] = [ + s if isinstance(s, int) else DefectItem.severity_to_int(s) + for s in self.options["severity"] + ] + + def _load_cpp_data(self, project_dir, envname): + data = load_project_ide_data(project_dir, envname) + if not data: + return + self.cpp_includes = data.get("includes", []) + self.cpp_defines = data.get("defines", []) + self.cpp_defines.extend(self._get_toolchain_defines(data.get("cc_path"))) + + def get_flags(self, tool): + result = [] + flags = self.options.get("flags") or [] + for flag in flags: + if ":" not in flag: + result.extend([f for f in flag.split(" ") if f]) + elif flag.startswith("%s:" % tool): + result.extend([f for f in flag.split(":", 1)[1].split(" ") if f]) + + return result + + @staticmethod + def _get_toolchain_defines(cc_path): + defines = [] + result = proc.exec_command("echo | %s -dM -E -x c++ -" % cc_path, shell=True) + + for line in result["out"].split("\n"): + tokens = line.strip().split(" ", 2) + if not tokens or tokens[0] != "#define": + continue + if len(tokens) > 2: + defines.append("%s=%s" % (tokens[1], tokens[2])) + else: + defines.append(tokens[1]) + + return defines + + @staticmethod + def is_flag_set(flag, flags): + return any(flag in f for f in flags) + + def get_defects(self): + return self._defects + + def configure_command(self): + raise NotImplementedError + + def on_tool_output(self, line): + line = self.tool_output_filter(line) + if not line: + return + + defect = self.parse_defect(line) + + if not isinstance(defect, DefectItem): + if self.options.get("verbose"): + click.echo(line) + return + + if defect.severity not in self.options["severity"]: + return + + self._defects.append(defect) + if self._on_defect_callback: + self._on_defect_callback(defect) + + @staticmethod + def tool_output_filter(line): + return line + + @staticmethod + def parse_defect(raw_line): + return raw_line + + def clean_up(self): + pass + + def get_project_src_files(self): + file_extensions = ["h", "hpp", "c", "cc", "cpp", "ino"] + return fs.match_src_files( + get_project_dir(), self.options.get("filter"), file_extensions + ) + + def check(self, on_defect_callback=None): + self._on_defect_callback = on_defect_callback + cmd = self.configure_command() + if self.options.get("verbose"): + click.echo(" ".join(cmd)) + + proc.exec_command( + cmd, + stdout=proc.LineBufferedAsyncPipe(self.on_tool_output), + stderr=proc.LineBufferedAsyncPipe(self.on_tool_output), + ) + + self.clean_up() + + return self._bad_input diff --git a/platformio/check/tools/clangtidy.py b/platformio/check/tools/clangtidy.py new file mode 100644 index 00000000..407375c5 --- /dev/null +++ b/platformio/check/tools/clangtidy.py @@ -0,0 +1,67 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from os.path import join + +from platformio.check.defect import DefectItem +from platformio.check.tools.base import CheckToolBase +from platformio.managers.core import get_core_package_dir + + +class ClangtidyCheckTool(CheckToolBase): + def tool_output_filter(self, line): + if not self.options.get("verbose") and "[clang-diagnostic-error]" in line: + return "" + + if "[CommonOptionsParser]" in line: + self._bad_input = True + return line + + if any(d in line for d in ("note: ", "error: ", "warning: ")): + return line + + return "" + + def parse_defect(self, raw_line): + match = re.match(r"^(.*):(\d+):(\d+):\s+([^:]+):\s(.+)\[([^]]+)\]$", raw_line) + if not match: + return raw_line + + file_, line, column, category, message, defect_id = match.groups() + + severity = DefectItem.SEVERITY_LOW + if category == "error": + severity = DefectItem.SEVERITY_HIGH + elif category == "warning": + severity = DefectItem.SEVERITY_MEDIUM + + return DefectItem(severity, category, message, file_, line, column, defect_id) + + def configure_command(self): + tool_path = join(get_core_package_dir("tool-clangtidy"), "clang-tidy") + + cmd = [tool_path, "--quiet"] + flags = self.get_flags("clangtidy") + if not self.is_flag_set("--checks", flags): + cmd.append("--checks=*") + + cmd.extend(flags) + cmd.extend(self.get_project_src_files()) + cmd.append("--") + + cmd.extend(["-D%s" % d for d in self.cpp_defines]) + cmd.extend(["-I%s" % inc for inc in self.cpp_includes]) + + return cmd diff --git a/platformio/check/tools/cppcheck.py b/platformio/check/tools/cppcheck.py new file mode 100644 index 00000000..f26e7824 --- /dev/null +++ b/platformio/check/tools/cppcheck.py @@ -0,0 +1,142 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os import remove +from os.path import isfile, join +from tempfile import NamedTemporaryFile + +from platformio.check.defect import DefectItem +from platformio.check.tools.base import CheckToolBase +from platformio.managers.core import get_core_package_dir +from platformio.project.helpers import get_project_core_dir + + +class CppcheckCheckTool(CheckToolBase): + def __init__(self, *args, **kwargs): + self._tmp_files = [] + self.defect_fields = [ + "severity", + "message", + "file", + "line", + "column", + "callstack", + "cwe", + "id", + ] + super(CppcheckCheckTool, self).__init__(*args, **kwargs) + + def tool_output_filter(self, line): + if ( + not self.options.get("verbose") + and "--suppress=unmatchedSuppression:" in line + ): + return "" + + if any( + msg in line + for msg in ( + "No C or C++ source files found", + "unrecognized command line option", + ) + ): + self._bad_input = True + + return line + + def parse_defect(self, raw_line): + if "<&PIO&>" not in raw_line or any( + f not in raw_line for f in self.defect_fields + ): + return None + + args = dict() + for field in raw_line.split("<&PIO&>"): + field = field.strip().replace('"', "") + name, value = field.split("=", 1) + args[name] = value + + args["category"] = args["severity"] + if args["severity"] == "error": + args["severity"] = DefectItem.SEVERITY_HIGH + elif args["severity"] == "warning": + args["severity"] = DefectItem.SEVERITY_MEDIUM + else: + args["severity"] = DefectItem.SEVERITY_LOW + + return DefectItem(**args) + + def configure_command(self): + tool_path = join(get_core_package_dir("tool-cppcheck"), "cppcheck") + + cmd = [ + tool_path, + "--error-exitcode=1", + "--verbose" if self.options.get("verbose") else "--quiet", + ] + + cmd.append( + '--template="%s"' + % "<&PIO&>".join(["{0}={{{0}}}".format(f) for f in self.defect_fields]) + ) + + flags = self.get_flags("cppcheck") + if not self.is_flag_set("--platform", flags): + cmd.append("--platform=unspecified") + if not self.is_flag_set("--enable", flags): + enabled_checks = [ + "warning", + "style", + "performance", + "portability", + "unusedFunction", + ] + cmd.append("--enable=%s" % ",".join(enabled_checks)) + + cmd.extend(["-D%s" % d for d in self.cpp_defines]) + cmd.extend(flags) + + cmd.append("--file-list=%s" % self._generate_src_file()) + cmd.append("--includes-file=%s" % self._generate_inc_file()) + + core_dir = get_project_core_dir() + cmd.append("--suppress=*:%s*" % core_dir) + cmd.append("--suppress=unmatchedSuppression:%s*" % core_dir) + + return cmd + + def _create_tmp_file(self, data): + with NamedTemporaryFile("w", delete=False) as fp: + fp.write(data) + self._tmp_files.append(fp.name) + return fp.name + + def _generate_src_file(self): + return self._create_tmp_file("\n".join(self.get_project_src_files())) + + def _generate_inc_file(self): + return self._create_tmp_file("\n".join(self.cpp_includes)) + + def clean_up(self): + for f in self._tmp_files: + if isfile(f): + remove(f) + + # delete temporary dump files generated by addons + if not self.is_flag_set("--addon", self.get_flags("cppcheck")): + return + for f in self.get_project_src_files(): + dump_file = f + ".dump" + if isfile(dump_file): + remove(dump_file) diff --git a/platformio/commands/check.py b/platformio/commands/check.py new file mode 100644 index 00000000..1c46640f --- /dev/null +++ b/platformio/commands/check.py @@ -0,0 +1,292 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=too-many-arguments,too-many-locals,too-many-branches +# pylint: disable=redefined-builtin,too-many-statements + +import os +from collections import Counter +from os.path import basename, dirname, isfile, join +from time import time + +import click +from tabulate import tabulate + +from platformio import exception, fs, util +from platformio.check.defect import DefectItem +from platformio.check.tools import CheckToolFactory +from platformio.compat import dump_json_to_unicode +from platformio.project.config import ProjectConfig +from platformio.project.helpers import ( + find_project_dir_above, + get_project_dir, + get_project_include_dir, + get_project_src_dir, +) + + +@click.command("check", short_help="Run a static analysis tool on code") +@click.option("-e", "--environment", multiple=True) +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path( + exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True + ), +) +@click.option( + "-c", + "--project-conf", + type=click.Path( + exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True + ), +) +@click.option("--filter", multiple=True, help="Pattern: + -") +@click.option("--flags", multiple=True) +@click.option( + "--severity", multiple=True, type=click.Choice(DefectItem.SEVERITY_LABELS.values()) +) +@click.option("-s", "--silent", is_flag=True) +@click.option("-v", "--verbose", is_flag=True) +@click.option("--json-output", is_flag=True) +def cli( + environment, + project_dir, + project_conf, + filter, + flags, + severity, + silent, + verbose, + json_output, +): + # find project directory on upper level + if isfile(project_dir): + project_dir = find_project_dir_above(project_dir) + + results = [] + with fs.cd(project_dir): + config = ProjectConfig.get_instance( + project_conf or join(project_dir, "platformio.ini") + ) + config.validate(environment) + + default_envs = config.default_envs() + for envname in config.envs(): + skipenv = any( + [ + environment and envname not in environment, + not environment and default_envs and envname not in default_envs, + ] + ) + + env_options = config.items(env=envname, as_dict=True) + env_dump = [] + for k, v in env_options.items(): + if k not in ("platform", "framework", "board"): + continue + env_dump.append( + "%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v) + ) + + default_filter = [ + "+<%s/>" % basename(d) + for d in (get_project_src_dir(), get_project_include_dir()) + ] + + tool_options = dict( + verbose=verbose, + silent=silent, + filter=filter or env_options.get("check_filter", default_filter), + flags=flags or env_options.get("check_flags"), + severity=[DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_HIGH]] + if silent + else (severity or env_options.get("check_severity")), + ) + + for tool in env_options.get("check_tool", ["cppcheck"]): + if skipenv: + results.append({"env": envname, "tool": tool}) + continue + if not silent and not json_output: + print_processing_header(tool, envname, env_dump) + + ct = CheckToolFactory.new( + tool, project_dir, config, envname, tool_options + ) + + result = {"env": envname, "tool": tool, "duration": time()} + rc = ct.check( + on_defect_callback=None + if (json_output or verbose) + else lambda defect: click.echo(repr(defect)) + ) + + result["defects"] = ct.get_defects() + result["duration"] = time() - result["duration"] + result["succeeded"] = rc == 0 and not any( + d.severity == DefectItem.SEVERITY_HIGH for d in result["defects"] + ) + results.append(result) + + if verbose: + click.echo("\n".join(repr(d) for d in result["defects"])) + + if not json_output and not silent: + if not result["defects"]: + click.echo("No defects found") + print_processing_footer(result) + + if json_output: + click.echo(dump_json_to_unicode(results_to_json(results))) + elif not silent: + print_check_summary(results) + + command_failed = any(r.get("succeeded") is False for r in results) + if command_failed: + raise exception.ReturnErrorCode(1) + + +def results_to_json(raw): + results = [] + for item in raw: + item.update( + { + "ignored": item.get("succeeded") is None, + "succeeded": bool(item.get("succeeded")), + "defects": [d.to_json() for d in item.get("defects", [])], + } + ) + results.append(item) + + return results + + +def print_processing_header(tool, envname, envdump): + click.echo( + "Checking %s > %s (%s)" + % (click.style(envname, fg="cyan", bold=True), tool, "; ".join(envdump)) + ) + terminal_width, _ = click.get_terminal_size() + click.secho("-" * terminal_width, bold=True) + + +def print_processing_footer(result): + is_failed = not result.get("succeeded") + util.print_labeled_bar( + "[%s] Took %.2f seconds" + % ( + ( + click.style("FAILED", fg="red", bold=True) + if is_failed + else click.style("PASSED", fg="green", bold=True) + ), + result["duration"], + ), + is_error=is_failed, + ) + + +def print_defects_stats(results): + components = dict() + + def _append_defect(component, defect): + if not components.get(component): + components[component] = Counter() + components[component].update({DefectItem.SEVERITY_LABELS[defect.severity]: 1}) + + for result in results: + for defect in result.get("defects", []): + component = dirname(defect.file) or defect.file + _append_defect(component, defect) + + if component.startswith(get_project_dir()): + while os.sep in component: + component = dirname(component) + _append_defect(component, defect) + + if not components: + return + + severity_labels = list(DefectItem.SEVERITY_LABELS.values()) + severity_labels.reverse() + tabular_data = list() + for k, v in components.items(): + tool_defect = [v.get(s, 0) for s in severity_labels] + tabular_data.append([k] + tool_defect) + + total = ["Total"] + [sum(d) for d in list(zip(*tabular_data))[1:]] + tabular_data.sort() + tabular_data.append([]) # Empty line as delimeter + tabular_data.append(total) + + headers = ["Component"] + headers.extend([l.upper() for l in severity_labels]) + headers = [click.style(h, bold=True) for h in headers] + click.echo(tabulate(tabular_data, headers=headers, numalign="center")) + click.echo() + + +def print_check_summary(results): + click.echo() + + tabular_data = [] + succeeded_nums = 0 + failed_nums = 0 + duration = 0 + + print_defects_stats(results) + + for result in results: + duration += result.get("duration", 0) + if result.get("succeeded") is False: + failed_nums += 1 + status_str = click.style("FAILED", fg="red") + elif result.get("succeeded") is None: + status_str = "IGNORED" + else: + succeeded_nums += 1 + status_str = click.style("PASSED", fg="green") + + tabular_data.append( + ( + click.style(result["env"], fg="cyan"), + result["tool"], + status_str, + util.humanize_duration_time(result.get("duration")), + ) + ) + + click.echo( + tabulate( + tabular_data, + headers=[ + click.style(s, bold=True) + for s in ("Environment", "Tool", "Status", "Duration") + ], + ), + err=failed_nums, + ) + + util.print_labeled_bar( + "%s%d succeeded in %s" + % ( + "%d failed, " % failed_nums if failed_nums else "", + succeeded_nums, + util.humanize_duration_time(duration), + ), + is_error=failed_nums, + fg="red" if failed_nums else "green", + ) From 392fe1cbd09e07ea6c378ce09e9c7fb87c837c37 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 24 Sep 2019 00:12:21 +0300 Subject: [PATCH 033/221] Move Run to the root --- platformio/commands/{run/command.py => run.py} | 4 ++-- platformio/{commands => }/run/__init__.py | 2 -- platformio/{commands => }/run/helpers.py | 0 platformio/{commands => }/run/processor.py | 0 4 files changed, 2 insertions(+), 4 deletions(-) rename platformio/commands/{run/command.py => run.py} (98%) rename platformio/{commands => }/run/__init__.py (92%) rename platformio/{commands => }/run/helpers.py (100%) rename platformio/{commands => }/run/processor.py (100%) diff --git a/platformio/commands/run/command.py b/platformio/commands/run.py similarity index 98% rename from platformio/commands/run/command.py rename to platformio/commands/run.py index 0349ab03..b9cf3459 100644 --- a/platformio/commands/run/command.py +++ b/platformio/commands/run.py @@ -22,10 +22,10 @@ from tabulate import tabulate from platformio import exception, fs, util from platformio.commands.device import device_monitor as cmd_device_monitor -from platformio.commands.run.helpers import clean_build_dir, handle_legacy_libdeps -from platformio.commands.run.processor import EnvironmentProcessor from platformio.project.config import ProjectConfig from platformio.project.helpers import find_project_dir_above, get_project_build_dir +from platformio.run.helpers import clean_build_dir, handle_legacy_libdeps +from platformio.run.processor import EnvironmentProcessor from platformio.test.processor import CTX_META_TEST_IS_RUNNING # pylint: disable=too-many-arguments,too-many-locals,too-many-branches diff --git a/platformio/commands/run/__init__.py b/platformio/run/__init__.py similarity index 92% rename from platformio/commands/run/__init__.py rename to platformio/run/__init__.py index 05d4d370..b0514903 100644 --- a/platformio/commands/run/__init__.py +++ b/platformio/run/__init__.py @@ -11,5 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from platformio.commands.run.command import cli diff --git a/platformio/commands/run/helpers.py b/platformio/run/helpers.py similarity index 100% rename from platformio/commands/run/helpers.py rename to platformio/run/helpers.py diff --git a/platformio/commands/run/processor.py b/platformio/run/processor.py similarity index 100% rename from platformio/commands/run/processor.py rename to platformio/run/processor.py From ca29b4e370ebc7f2bad38c283cb1296c9124b283 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 24 Sep 2019 00:17:08 +0300 Subject: [PATCH 034/221] Fixed "DeprecationWarning: the imp module is deprecated in favour of importlib" PY2/PY3 --- platformio/compat.py | 12 ++++++++++++ platformio/managers/platform.py | 7 ++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/platformio/compat.py b/platformio/compat.py index 712f65bc..1359bdad 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -29,6 +29,8 @@ def get_filesystem_encoding(): if PY2: + import imp + # pylint: disable=undefined-variable string_types = (str, unicode) @@ -76,8 +78,12 @@ if PY2: pathname = _magic_check.sub(r"[\1]", pathname) return drive + pathname + def load_python_module(name, pathname): + return imp.load_source(name, pathname) + else: + import importlib.util from glob import escape as glob_escape # pylint: disable=no-name-in-module string_types = (str,) @@ -107,3 +113,9 @@ else: if isinstance(obj, string_types): return obj return json.dumps(obj, ensure_ascii=False, sort_keys=True) + + def load_python_module(name, pathname): + spec = importlib.util.spec_from_file_location(name, pathname) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 4ab20b8d..124a4245 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -16,14 +16,13 @@ import base64 import os import re import sys -from imp import load_source from os.path import basename, dirname, isdir, isfile, join import click import semantic_version from platformio import __version__, app, exception, fs, util -from platformio.compat import PY2, hashlib_encode_data, is_bytes +from platformio.compat import PY2, hashlib_encode_data, is_bytes, load_python_module from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager from platformio.proc import ( @@ -230,12 +229,10 @@ class PlatformFactory(object): @staticmethod def load_module(name, path): - module = None try: - module = load_source("platformio.managers.platform.%s" % name, path) + return load_python_module("platformio.managers.platform.%s" % name, path) except ImportError: raise exception.UnknownPlatform(name) - return module @classmethod def newPlatform(cls, name, requirements=None): From 9a7e5d86fcb0d2be4aa753e630630ea701c00003 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 24 Sep 2019 00:21:16 +0300 Subject: [PATCH 035/221] Install Black only for Python 3.6+ --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 0c4de05d..3512bfb6 100644 --- a/tox.ini +++ b/tox.ini @@ -19,8 +19,8 @@ envlist = py27, py35, py36, py37, docs passenv = * usedevelop = True deps = + py36,py37: black isort - black pylint pytest pytest-xdist From 3d5c1411c05cce892f65ab59183258bdc1c05e06 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 24 Sep 2019 00:28:23 +0300 Subject: [PATCH 036/221] Fix PyLint for PY2 --- platformio/compat.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/platformio/compat.py b/platformio/compat.py index 1359bdad..64d437a1 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=unused-import +# pylint: disable=unused-import, no-name-in-module, import-error, +# pylint: disable=no-member, undefined-variable import json import os @@ -31,7 +32,6 @@ def get_filesystem_encoding(): if PY2: import imp - # pylint: disable=undefined-variable string_types = (str, unicode) def is_bytes(x): @@ -84,7 +84,7 @@ if PY2: else: import importlib.util - from glob import escape as glob_escape # pylint: disable=no-name-in-module + from glob import escape as glob_escape string_types = (str,) From 94f8afec38fc8d35db1055368f5fbe4e67c89e7e Mon Sep 17 00:00:00 2001 From: Florian Knodt Date: Tue, 24 Sep 2019 10:02:27 +0200 Subject: [PATCH 037/221] udev: Add GD32V DFU Bootloader (#3032) --- scripts/99-platformio-udev.rules | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/99-platformio-udev.rules b/scripts/99-platformio-udev.rules index 41ca21fa..b04b946f 100644 --- a/scripts/99-platformio-udev.rules +++ b/scripts/99-platformio-udev.rules @@ -67,6 +67,8 @@ ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="0666", ENV{ID_MM_DEVICE #TI MSP430 Launchpad ATTRS{idVendor}=="0451", ATTRS{idProduct}=="f432", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" +#GD32V DFU Bootloader +ATTRS{idVendor}=="28e9", ATTRS{idProduct}=="0189", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # # Debuggers From d2abac9b18ff47cd2ab32312995ba52e78cfd62a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 27 Sep 2019 14:13:53 +0300 Subject: [PATCH 038/221] Fixed an issue when configuration file options partly ignored when ``--project-conf`` // Resolve #3034 (#3055) * Fixed an issue when configuration file options partly ignored when using custom ``--project-conf`` // Resolve #3034 * Py2 compatible makedirs * Fix circle dependency * Fix broken import in test examples * Fix history * Remove YAPF markers * PyLint fix * Fix invalid project conf path * Move PIO Core to the root on Windows, issue with long CPPPATHs * Respect global PLATFORMIO_BUILD_CACHE_DIR env var * Fix Appveyor paths * Minor changes --- .appveyor.yml | 5 +- .pylintrc | 3 +- HISTORY.rst | 1 + platformio/__main__.py | 2 +- platformio/app.py | 31 +++-- platformio/builder/main.py | 57 +++++---- platformio/builder/tools/pioide.py | 2 +- platformio/builder/tools/piolib.py | 15 ++- platformio/builder/tools/piomisc.py | 14 +-- platformio/builder/tools/platformio.py | 12 +- platformio/check/tools/cppcheck.py | 3 +- platformio/commands/check.py | 22 ++-- platformio/commands/debug.py | 10 +- platformio/commands/device.py | 10 +- platformio/commands/init.py | 19 ++- platformio/commands/lib.py | 18 ++- platformio/commands/platform.py | 2 +- platformio/commands/run.py | 18 +-- platformio/commands/test.py | 15 ++- platformio/commands/upgrade.py | 2 +- platformio/debug/client.py | 10 +- platformio/debug/server.py | 2 +- platformio/exception.py | 2 +- platformio/home/rpc/handlers/project.py | 27 +++-- platformio/ide/projectgenerator.py | 19 ++- platformio/managers/core.py | 6 +- platformio/managers/lib.py | 9 +- platformio/managers/package.py | 2 +- platformio/managers/platform.py | 34 +++--- platformio/project/config.py | 126 ++++++++++++++++---- platformio/project/helpers.py | 147 ++++-------------------- platformio/project/options.py | 88 +++++++++++--- platformio/run/helpers.py | 10 +- platformio/run/processor.py | 8 +- platformio/test/native.py | 6 +- platformio/test/processor.py | 7 +- platformio/util.py | 6 +- tests/commands/test_check.py | 4 +- tests/test_examples.py | 6 +- tests/test_projectconf.py | 9 ++ 40 files changed, 411 insertions(+), 378 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index c9703e56..12d10edf 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -6,14 +6,15 @@ platform: environment: matrix: - TOXENV: "py27" - PLATFORMIO_BUILD_CACHE_DIR: C:/Temp/PIO_Build_Cache_P2_{build} + PLATFORMIO_BUILD_CACHE_DIR: C:\Temp\PIO_Build_Cache_P2_{build} - TOXENV: "py36" - PLATFORMIO_BUILD_CACHE_DIR: C:/Temp/PIO_Build_Cache_P3_{build} + PLATFORMIO_BUILD_CACHE_DIR: C:\Temp\PIO_Build_Cache_P3_{build} install: - cmd: git submodule update --init --recursive - cmd: SET PATH=C:\MinGW\bin;%PATH% + - cmd: SET PLATFORMIO_CORE_DIR=C:\.pio - cmd: pip install --force-reinstall tox test_script: diff --git a/.pylintrc b/.pylintrc index e4203273..67ce4ef7 100644 --- a/.pylintrc +++ b/.pylintrc @@ -11,4 +11,5 @@ disable= too-few-public-methods, useless-object-inheritance, useless-import-alias, - fixme + fixme, + bad-option-value diff --git a/HISTORY.rst b/HISTORY.rst index d381bdd3..213c54f5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -23,6 +23,7 @@ PlatformIO Core 4.0 * Added ``--no-ansi`` flag for `PIO Core `__ to disable ANSI control characters * Fixed an issue with project generator for `CLion IDE `__ when 2 environments were used (`issue #2824 `_) * Fixed default PIO Unified Debugger configuration for `J-Link probe `__ +* Fixed an issue when configuration file options partly ignored when using custom ``--project-conf`` (`issue #3034 `_) 4.0.3 (2019-08-30) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/__main__.py b/platformio/__main__.py index dbf66524..043befbd 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -70,7 +70,7 @@ def configure(): # https://urllib3.readthedocs.org # /en/latest/security.html#insecureplatformwarning try: - import urllib3 + import urllib3 # pylint: disable=import-outside-toplevel urllib3.disable_warnings() except (AttributeError, ImportError): diff --git a/platformio/app.py b/platformio/app.py index 2858b37a..42dac9c0 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -17,7 +17,7 @@ import hashlib import os import uuid from os import environ, getenv, listdir, remove -from os.path import abspath, dirname, expanduser, isdir, isfile, join +from os.path import abspath, dirname, isdir, isfile, join from time import time import requests @@ -25,21 +25,11 @@ import requests from platformio import exception, fs, lockfile from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data from platformio.proc import is_ci -from platformio.project.helpers import get_project_cache_dir, get_project_core_dir - - -def get_default_projects_dir(): - docs_dir = join(expanduser("~"), "Documents") - try: - assert WINDOWS - import ctypes.wintypes - - buf = ctypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH) - ctypes.windll.shell32.SHGetFolderPathW(None, 5, None, 0, buf) - docs_dir = buf.value - except: # pylint: disable=bare-except - pass - return join(docs_dir, "PlatformIO", "Projects") +from platformio.project.helpers import ( + get_default_projects_dir, + get_project_cache_dir, + get_project_core_dir, +) def projects_dir_validate(projects_dir): @@ -88,7 +78,12 @@ DEFAULT_SETTINGS = { }, } -SESSION_VARS = {"command_ctx": None, "force_option": False, "caller_id": None} +SESSION_VARS = { + "command_ctx": None, + "force_option": False, + "caller_id": None, + "custom_project_conf": None, +} class State(object): @@ -415,6 +410,6 @@ def get_cid(): uid = uuid.getnode() cid = uuid.UUID(bytes=hashlib.md5(hashlib_encode_data(uid)).digest()) cid = str(cid) - if WINDOWS or os.getuid() > 0: # yapf: disable pylint: disable=no-member + if WINDOWS or os.getuid() > 0: # pylint: disable=no-member set_state_item("cid", cid) return cid diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 52adb5a3..0fef773b 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -31,7 +31,7 @@ from platformio import fs from platformio.compat import PY2, dump_json_to_unicode from platformio.managers.platform import PlatformBase from platformio.proc import get_pythonexe_path -from platformio.project import helpers as project_helpers +from platformio.project.helpers import get_project_dir AllowSubstExceptions(NameError) @@ -44,7 +44,7 @@ clivars.AddVariables( ("PIOENV",), ("PIOTEST_RUNNING_NAME",), ("UPLOAD_PORT",), -) # yapf: disable +) DEFAULT_ENV_OPTIONS = dict( tools=[ @@ -67,26 +67,10 @@ DEFAULT_ENV_OPTIONS = dict( # Propagating External Environment ENV=environ, UNIX_TIME=int(time()), - PROJECT_DIR=project_helpers.get_project_dir(), - PROJECTCORE_DIR=project_helpers.get_project_core_dir(), - PROJECTPACKAGES_DIR=project_helpers.get_project_packages_dir(), - PROJECTWORKSPACE_DIR=project_helpers.get_project_workspace_dir(), - PROJECTLIBDEPS_DIR=project_helpers.get_project_libdeps_dir(), - PROJECTINCLUDE_DIR=project_helpers.get_project_include_dir(), - PROJECTSRC_DIR=project_helpers.get_project_src_dir(), - PROJECTTEST_DIR=project_helpers.get_project_test_dir(), - PROJECTDATA_DIR=project_helpers.get_project_data_dir(), - PROJECTBUILD_DIR=project_helpers.get_project_build_dir(), - BUILDCACHE_DIR=project_helpers.get_project_optional_dir("build_cache_dir"), - BUILD_DIR=join("$PROJECTBUILD_DIR", "$PIOENV"), - BUILDSRC_DIR=join("$BUILD_DIR", "src"), - BUILDTEST_DIR=join("$BUILD_DIR", "test"), + BUILD_DIR=join("$PROJECT_BUILD_DIR", "$PIOENV"), + BUILD_SRC_DIR=join("$BUILD_DIR", "src"), + BUILD_TEST_DIR=join("$BUILD_DIR", "test"), LIBPATH=["$BUILD_DIR"], - LIBSOURCE_DIRS=[ - project_helpers.get_project_lib_dir(), - join("$PROJECTLIBDEPS_DIR", "$PIOENV"), - project_helpers.get_project_global_lib_dir(), - ], PROGNAME="program", PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"), PYTHONEXE=get_pythonexe_path(), @@ -110,10 +94,33 @@ env.Replace( } ) -if env.subst("$BUILDCACHE_DIR"): - if not isdir(env.subst("$BUILDCACHE_DIR")): - makedirs(env.subst("$BUILDCACHE_DIR")) - env.CacheDir("$BUILDCACHE_DIR") +# Setup project optional directories +config = env.GetProjectConfig() +env.Replace( + PROJECT_DIR=get_project_dir(), + PROJECT_CORE_DIR=config.get_optional_dir("core"), + PROJECT_PACKAGES_DIR=config.get_optional_dir("packages"), + PROJECT_WORKSPACE_DIR=config.get_optional_dir("workspace"), + PROJECT_LIBDEPS_DIR=config.get_optional_dir("libdeps"), + PROJECT_INCLUDE_DIR=config.get_optional_dir("include"), + PROJECT_SRC_DIR=config.get_optional_dir("src"), + PROJECTSRC_DIR=config.get_optional_dir("src"), # legacy for dev/platform + PROJECT_TEST_DIR=config.get_optional_dir("test"), + PROJECT_DATA_DIR=config.get_optional_dir("data"), + PROJECTDATA_DIR=config.get_optional_dir("data"), # legacy for dev/platform + PROJECT_BUILD_DIR=config.get_optional_dir("build"), + BUILD_CACHE_DIR=config.get_optional_dir("build_cache"), + LIBSOURCE_DIRS=[ + config.get_optional_dir("lib"), + join("$PROJECT_LIBDEPS_DIR", "$PIOENV"), + config.get_optional_dir("globallib"), + ], +) + +if env.subst("$BUILD_CACHE_DIR"): + if not isdir(env.subst("$BUILD_CACHE_DIR")): + makedirs(env.subst("$BUILD_CACHE_DIR")) + env.CacheDir("$BUILD_CACHE_DIR") if int(ARGUMENTS.get("ISATTY", 0)): # pylint: disable=protected-access diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 39d388a4..606a3bb0 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -54,7 +54,7 @@ def _dump_includes(env, projenv): if unity_dir: includes.append(unity_dir) - includes.extend([env.subst("$PROJECTINCLUDE_DIR"), env.subst("$PROJECTSRC_DIR")]) + includes.extend([env.subst("$PROJECT_INCLUDE_DIR"), env.subst("$PROJECT_SRC_DIR")]) # remove duplicates result = [] diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index e67b9f3b..5ca4b2a2 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -601,8 +601,7 @@ class MbedLibBuilder(LibBuilderBase): mbed_config_path = join(self.env.subst(p), "mbed_config.h") if isfile(mbed_config_path): break - else: - mbed_config_path = None + mbed_config_path = None if not mbed_config_path: return None @@ -821,16 +820,16 @@ class ProjectAsLibBuilder(LibBuilderBase): @property def include_dir(self): - include_dir = self.env.subst("$PROJECTINCLUDE_DIR") + include_dir = self.env.subst("$PROJECT_INCLUDE_DIR") return include_dir if isdir(include_dir) else None @property def src_dir(self): - return self.env.subst("$PROJECTSRC_DIR") + return self.env.subst("$PROJECT_SRC_DIR") def get_include_dirs(self): include_dirs = [] - project_include_dir = self.env.subst("$PROJECTINCLUDE_DIR") + project_include_dir = self.env.subst("$PROJECT_INCLUDE_DIR") if isdir(project_include_dir): include_dirs.append(project_include_dir) for include_dir in LibBuilderBase.get_include_dirs(self): @@ -845,9 +844,9 @@ class ProjectAsLibBuilder(LibBuilderBase): if "__test" in COMMAND_LINE_TARGETS: items.extend( [ - join("$PROJECTTEST_DIR", item) + join("$PROJECT_TEST_DIR", item) for item in self.env.MatchSourceFiles( - "$PROJECTTEST_DIR", "$PIOTEST_SRC_FILTER" + "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER" ) ] ) @@ -896,7 +895,7 @@ class ProjectAsLibBuilder(LibBuilderBase): not_found_uri.append(uri) did_install = False - lm = LibraryManager(self.env.subst(join("$PROJECTLIBDEPS_DIR", "$PIOENV"))) + lm = LibraryManager(self.env.subst(join("$PROJECT_LIBDEPS_DIR", "$PIOENV"))) for uri in not_found_uri: try: lm.install(uri) diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 78295ea3..a1b564b9 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -117,7 +117,7 @@ class InoToCPPConverter(object): stropen = True newlines.append(line[:-1]) continue - elif stropen: + if stropen: newlines[len(newlines) - 1] += line[:-1] continue elif stropen and line.endswith(('",', '";')): @@ -199,7 +199,7 @@ class InoToCPPConverter(object): def ConvertInoToCpp(env): - src_dir = glob_escape(env.subst("$PROJECTSRC_DIR")) + src_dir = glob_escape(env.subst("$PROJECT_SRC_DIR")) ino_nodes = env.Glob(join(src_dir, "*.ino")) + env.Glob(join(src_dir, "*.pde")) if not ino_nodes: return @@ -256,7 +256,7 @@ def GetActualLDScript(env): if f == "-T": script_in_next = True continue - elif script_in_next: + if script_in_next: script_in_next = False raw_script = f elif f.startswith("-Wl,-T"): @@ -309,7 +309,7 @@ def PioClean(env, clean_dir): env.Exit(0) -def ProcessDebug(env): +def ConfigureDebugTarget(env): if not env.subst("$PIODEBUGFLAGS"): env.Replace(PIODEBUGFLAGS=["-Og", "-g3", "-ggdb3"]) env.Append( @@ -322,7 +322,7 @@ def ProcessDebug(env): env.Append(BUILD_UNFLAGS=unflags) -def ProcessTest(env): +def ConfigureTestTarget(env): env.Append( CPPDEFINES=["UNIT_TEST", "UNITY_INCLUDE_CONFIG_H"], CPPPATH=[join("$BUILD_DIR", "UnityTestLib")], @@ -361,7 +361,7 @@ def generate(env): env.AddMethod(GetActualLDScript) env.AddMethod(VerboseAction) env.AddMethod(PioClean) - env.AddMethod(ProcessDebug) - env.AddMethod(ProcessTest) + env.AddMethod(ConfigureDebugTarget) + env.AddMethod(ConfigureTestTarget) env.AddMethod(GetExtraScripts) return env diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index a33dafb3..e197b0fb 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -67,15 +67,17 @@ def _build_project_deps(env): is_test = "__test" in COMMAND_LINE_TARGETS if is_test: projenv.BuildSources( - "$BUILDTEST_DIR", "$PROJECTTEST_DIR", "$PIOTEST_SRC_FILTER" + "$BUILD_TEST_DIR", "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER" ) if not is_test or env.GetProjectOption("test_build_project_src", False): - projenv.BuildSources("$BUILDSRC_DIR", "$PROJECTSRC_DIR", env.get("SRC_FILTER")) + projenv.BuildSources( + "$BUILD_SRC_DIR", "$PROJECT_SRC_DIR", env.get("SRC_FILTER") + ) if not env.get("PIOBUILDFILES") and not COMMAND_LINE_TARGETS: sys.stderr.write( "Error: Nothing to build. Please put your source code files " - "to '%s' folder\n" % env.subst("$PROJECTSRC_DIR") + "to '%s' folder\n" % env.subst("$PROJECT_SRC_DIR") ) env.Exit(1) @@ -102,7 +104,7 @@ def BuildProgram(env): env.Replace(AS="$CC", ASCOM="$ASPPCOM") if "debug" in COMMAND_LINE_TARGETS or env.GetProjectOption("build_type") == "debug": - env.ProcessDebug() + env.ConfigureDebugTarget() # process extra flags from board if "BOARD" in env and "build.extra_flags" in env.BoardConfig(): @@ -121,7 +123,7 @@ def BuildProgram(env): env.ProcessUnFlags(env.get("BUILD_UNFLAGS")) if "__test" in COMMAND_LINE_TARGETS: - env.ProcessTest() + env.ConfigureTestTarget() # build project with dependencies _build_project_deps(env) diff --git a/platformio/check/tools/cppcheck.py b/platformio/check/tools/cppcheck.py index f26e7824..d0f1d6eb 100644 --- a/platformio/check/tools/cppcheck.py +++ b/platformio/check/tools/cppcheck.py @@ -19,7 +19,6 @@ from tempfile import NamedTemporaryFile from platformio.check.defect import DefectItem from platformio.check.tools.base import CheckToolBase from platformio.managers.core import get_core_package_dir -from platformio.project.helpers import get_project_core_dir class CppcheckCheckTool(CheckToolBase): @@ -110,7 +109,7 @@ class CppcheckCheckTool(CheckToolBase): cmd.append("--file-list=%s" % self._generate_src_file()) cmd.append("--includes-file=%s" % self._generate_inc_file()) - core_dir = get_project_core_dir() + core_dir = self.config.get_optional_dir("core") cmd.append("--suppress=*:%s*" % core_dir) cmd.append("--suppress=unmatchedSuppression:%s*" % core_dir) diff --git a/platformio/commands/check.py b/platformio/commands/check.py index 1c46640f..392e618d 100644 --- a/platformio/commands/check.py +++ b/platformio/commands/check.py @@ -17,23 +17,18 @@ import os from collections import Counter -from os.path import basename, dirname, isfile, join +from os.path import basename, dirname, isfile from time import time import click from tabulate import tabulate -from platformio import exception, fs, util +from platformio import app, exception, fs, util from platformio.check.defect import DefectItem from platformio.check.tools import CheckToolFactory from platformio.compat import dump_json_to_unicode from platformio.project.config import ProjectConfig -from platformio.project.helpers import ( - find_project_dir_above, - get_project_dir, - get_project_include_dir, - get_project_src_dir, -) +from platformio.project.helpers import find_project_dir_above, get_project_dir @click.command("check", short_help="Run a static analysis tool on code") @@ -72,15 +67,15 @@ def cli( verbose, json_output, ): + app.set_session_var("custom_project_conf", project_conf) + # find project directory on upper level if isfile(project_dir): project_dir = find_project_dir_above(project_dir) results = [] with fs.cd(project_dir): - config = ProjectConfig.get_instance( - project_conf or join(project_dir, "platformio.ini") - ) + config = ProjectConfig.get_instance(project_conf) config.validate(environment) default_envs = config.default_envs() @@ -103,7 +98,10 @@ def cli( default_filter = [ "+<%s/>" % basename(d) - for d in (get_project_src_dir(), get_project_include_dir()) + for d in ( + config.get_optional_dir("src"), + config.get_optional_dir("include"), + ) ] tool_options = dict( diff --git a/platformio/commands/debug.py b/platformio/commands/debug.py index 470fdd97..21b0bfd8 100644 --- a/platformio/commands/debug.py +++ b/platformio/commands/debug.py @@ -17,11 +17,11 @@ import os import signal -from os.path import isfile, join +from os.path import isfile import click -from platformio import exception, fs, proc, util +from platformio import app, exception, fs, proc, util from platformio.debug import helpers from platformio.managers.core import inject_contrib_pysite from platformio.project.config import ProjectConfig @@ -54,6 +54,8 @@ from platformio.project.helpers import is_platformio_project, load_project_ide_d @click.argument("__unprocessed", nargs=-1, type=click.UNPROCESSED) @click.pass_context def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unprocessed): + app.set_session_var("custom_project_conf", project_conf) + # use env variables from Eclipse or CLion for sysenv in ("CWD", "PWD", "PLATFORMIO_PROJECT_DIR"): if is_platformio_project(project_dir): @@ -62,9 +64,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro project_dir = os.getenv(sysenv) with fs.cd(project_dir): - config = ProjectConfig.get_instance( - project_conf or join(project_dir, "platformio.ini") - ) + config = ProjectConfig.get_instance(project_conf) config.validate(envs=[environment] if environment else None) env_name = environment or helpers.get_default_debug_env(config) diff --git a/platformio/commands/device.py b/platformio/commands/device.py index 351ba641..45f94ae7 100644 --- a/platformio/commands/device.py +++ b/platformio/commands/device.py @@ -15,12 +15,11 @@ import sys from fnmatch import fnmatch from os import getcwd -from os.path import join import click from serial.tools import miniterm -from platformio import exception, util +from platformio import exception, fs, util from platformio.compat import dump_json_to_unicode from platformio.project.config import ProjectConfig @@ -175,7 +174,8 @@ def device_list( # pylint: disable=too-many-branches def device_monitor(**kwargs): # pylint: disable=too-many-branches env_options = {} try: - env_options = get_project_options(kwargs["project_dir"], kwargs["environment"]) + with fs.cd(kwargs["project_dir"]): + env_options = get_project_options(kwargs["environment"]) for k in ("port", "speed", "rts", "dtr"): k2 = "monitor_%s" % k if k == "speed": @@ -225,8 +225,8 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches raise exception.MinitermException(e) -def get_project_options(project_dir, environment=None): - config = ProjectConfig.get_instance(join(project_dir, "platformio.ini")) +def get_project_options(environment=None): + config = ProjectConfig.get_instance() config.validate(envs=[environment] if environment else None) if not environment: default_envs = config.default_envs() diff --git a/platformio/commands/init.py b/platformio/commands/init.py index 03a08fab..ebf8a3f6 100644 --- a/platformio/commands/init.py +++ b/platformio/commands/init.py @@ -24,13 +24,7 @@ from platformio.commands.platform import platform_install as cli_platform_instal from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager from platformio.project.config import ProjectConfig -from platformio.project.helpers import ( - get_project_include_dir, - get_project_lib_dir, - get_project_src_dir, - get_project_test_dir, - is_platformio_project, -) +from platformio.project.helpers import is_platformio_project def validate_boards(ctx, param, value): # pylint: disable=W0613 @@ -133,13 +127,14 @@ def cli( def init_base_project(project_dir): - ProjectConfig(join(project_dir, "platformio.ini")).save() with fs.cd(project_dir): + config = ProjectConfig() + config.save() dir_to_readme = [ - (get_project_src_dir(), None), - (get_project_include_dir(), init_include_readme), - (get_project_lib_dir(), init_lib_readme), - (get_project_test_dir(), init_test_readme), + (config.get_optional_dir("src"), None), + (config.get_optional_dir("include"), init_include_readme), + (config.get_optional_dir("lib"), init_lib_readme), + (config.get_optional_dir("test"), init_test_readme), ] for (path, cb) in dir_to_readme: if isdir(path): diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 6244da99..83e6cff4 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -21,18 +21,13 @@ import click import semantic_version from tabulate import tabulate -from platformio import exception, fs, util +from platformio import exception, util from platformio.commands import PlatformioCLI from platformio.compat import dump_json_to_unicode from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib from platformio.proc import is_ci from platformio.project.config import ProjectConfig -from platformio.project.helpers import ( - get_project_dir, - get_project_global_lib_dir, - get_project_libdeps_dir, - is_platformio_project, -) +from platformio.project.helpers import get_project_dir, is_platformio_project try: from urllib.parse import quote @@ -45,6 +40,10 @@ CTX_META_STORAGE_DIRS_KEY = __name__ + ".storage_dirs" CTX_META_STORAGE_LIBDEPS_KEY = __name__ + ".storage_lib_deps" +def get_project_global_lib_dir(): + return ProjectConfig.get_instance().get_optional_dir("globallib") + + @click.group(short_help="Library Manager") @click.option( "-d", @@ -105,10 +104,9 @@ def cli(ctx, **options): if not is_platformio_project(storage_dir): ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) continue - with fs.cd(storage_dir): - libdeps_dir = get_project_libdeps_dir() config = ProjectConfig.get_instance(join(storage_dir, "platformio.ini")) config.validate(options["environment"], silent=in_silence) + libdeps_dir = config.get_optional_dir("libdeps") for env in config.envs(): if options["environment"] and env not in options["environment"]: continue @@ -336,7 +334,7 @@ def lib_search(query, json_output, page, noninteractive, **filters): click.secho(" *", fg="green") click.secho("For example: DS*, PCA*, DHT* and etc.\n", fg="yellow") click.echo( - "For more examples and advanced search syntax, " "please use documentation:" + "For more examples and advanced search syntax, please use documentation:" ) click.secho( "https://docs.platformio.org/page/userguide/lib/cmd_search.html\n", diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 30afd2c9..968f978b 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -331,7 +331,7 @@ def platform_uninstall(platforms): for platform in platforms: if pm.uninstall(platform): click.secho( - "The platform '%s' has been successfully " "uninstalled!" % platform, + "The platform '%s' has been successfully uninstalled!" % platform, fg="green", ) diff --git a/platformio/commands/run.py b/platformio/commands/run.py index b9cf3459..e378c528 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -14,16 +14,16 @@ from multiprocessing import cpu_count from os import getcwd -from os.path import isfile, join +from os.path import isfile from time import time import click from tabulate import tabulate -from platformio import exception, fs, util +from platformio import app, exception, fs, util from platformio.commands.device import device_monitor as cmd_device_monitor from platformio.project.config import ProjectConfig -from platformio.project.helpers import find_project_dir_above, get_project_build_dir +from platformio.project.helpers import find_project_dir_above from platformio.run.helpers import clean_build_dir, handle_legacy_libdeps from platformio.run.processor import EnvironmentProcessor from platformio.test.processor import CTX_META_TEST_IS_RUNNING @@ -81,6 +81,8 @@ def cli( verbose, disable_auto_clean, ): + app.set_session_var("custom_project_conf", project_conf) + # find project directory on upper level if isfile(project_dir): project_dir = find_project_dir_above(project_dir) @@ -88,20 +90,18 @@ def cli( is_test_running = CTX_META_TEST_IS_RUNNING in ctx.meta with fs.cd(project_dir): - config = ProjectConfig.get_instance( - project_conf or join(project_dir, "platformio.ini") - ) + config = ProjectConfig.get_instance(project_conf) config.validate(environment) # clean obsolete build dir if not disable_auto_clean: + build_dir = config.get_optional_dir("build") try: - clean_build_dir(get_project_build_dir(), config) + clean_build_dir(build_dir, config) except: # pylint: disable=bare-except click.secho( "Can not remove temporary directory `%s`. Please remove " - "it manually to avoid build issues" - % get_project_build_dir(force=True), + "it manually to avoid build issues" % build_dir, fg="yellow", ) diff --git a/platformio/commands/test.py b/platformio/commands/test.py index b46c6e42..49b188df 100644 --- a/platformio/commands/test.py +++ b/platformio/commands/test.py @@ -22,9 +22,8 @@ from time import time import click from tabulate import tabulate -from platformio import exception, fs, util +from platformio import app, exception, fs, util from platformio.project.config import ProjectConfig -from platformio.project.helpers import get_project_test_dir from platformio.test.embedded import EmbeddedTestProcessor from platformio.test.native import NativeTestProcessor @@ -97,17 +96,17 @@ def cli( # pylint: disable=redefined-builtin monitor_dtr, verbose, ): + app.set_session_var("custom_project_conf", project_conf) + with fs.cd(project_dir): - test_dir = get_project_test_dir() + config = ProjectConfig.get_instance(project_conf) + config.validate(envs=environment) + + test_dir = config.get_optional_dir("test") if not isdir(test_dir): raise exception.TestDirNotExists(test_dir) test_names = get_test_names(test_dir) - config = ProjectConfig.get_instance( - project_conf or join(project_dir, "platformio.ini") - ) - config.validate(envs=environment) - click.echo("Verbose mode can be enabled via `-v, --verbose` option") click.secho("Collected %d items" % len(test_names), bold=True) diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index 8c0d3496..a933ab7f 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -92,7 +92,7 @@ WARNING! Don't use `sudo` for the rest PlatformIO commands. def get_pip_package(to_develop): if not to_develop: return "platformio" - dl_url = "https://github.com/platformio/" "platformio-core/archive/develop.zip" + dl_url = "https://github.com/platformio/platformio-core/archive/develop.zip" cache_dir = get_project_cache_dir() if not os.path.isdir(cache_dir): os.makedirs(cache_dir) diff --git a/platformio/debug/client.py b/platformio/debug/client.py index 63b07c5e..eca409e5 100644 --- a/platformio/debug/client.py +++ b/platformio/debug/client.py @@ -91,7 +91,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes self.project_dir, "-l", "10", - ] # yapf: disable + ] args.extend(self.args) if not gdb_path: raise exception.DebugInvalidOptions("GDB client is not configured") @@ -139,14 +139,14 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes " echo Warning! Undefined pio_reset_target command\\n", " mon reset", "end", - ] + commands # yapf: disable + ] + commands if not any("define pio_reset_halt_target" in cmd for cmd in commands): commands = [ "define pio_reset_halt_target", " echo Warning! Undefined pio_reset_halt_target command\\n", " mon reset halt", "end", - ] + commands # yapf: disable + ] + commands if not any("define pio_restart_target" in cmd for cmd in commands): commands += [ "define pio_restart_target", @@ -154,7 +154,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes " $INIT_BREAK", " %s" % ("continue" if patterns["INIT_BREAK"] else "next"), "end", - ] # yapf: disable + ] banner = [ "echo PlatformIO Unified Debugger -> http://bit.ly/pio-debug\\n", @@ -243,7 +243,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes % self.debug_options["init_break"] ) self.console_log( - "PlatformIO: More configuration options -> " "http://bit.ly/pio-debug" + "PlatformIO: More configuration options -> http://bit.ly/pio-debug" ) self.transport.write( b"0-exec-continue\n" if helpers.is_mi_mode(self.args) else b"continue\n" diff --git a/platformio/debug/server.py b/platformio/debug/server.py index fbce1268..49d730a0 100644 --- a/platformio/debug/server.py +++ b/platformio/debug/server.py @@ -64,7 +64,7 @@ class DebugServer(BaseProcess): self._debug_port = ":3333" openocd_pipe_allowed = all( [not self.debug_options["port"], "openocd" in server_executable] - ) # yapf: disable + ) if openocd_pipe_allowed: args = [] if server["cwd"]: diff --git a/platformio/exception.py b/platformio/exception.py index c9f3f509..6e5910f8 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -148,7 +148,7 @@ class FDSizeMismatch(PlatformIOPackageException): class FDSHASumMismatch(PlatformIOPackageException): MESSAGE = ( - "The 'sha1' sum '{0}' of downloaded file '{1}' " "is not equal to remote '{2}'" + "The 'sha1' sum '{0}' of downloaded file '{1}' is not equal to remote '{2}'" ) diff --git a/platformio/home/rpc/handlers/project.py b/platformio/home/rpc/handlers/project.py index 4a0acfb5..097f7e45 100644 --- a/platformio/home/rpc/handlers/project.py +++ b/platformio/home/rpc/handlers/project.py @@ -28,20 +28,16 @@ from platformio.home.rpc.handlers.piocore import PIOCoreRPC from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager from platformio.project.config import ProjectConfig -from platformio.project.helpers import ( - get_project_libdeps_dir, - get_project_src_dir, - is_platformio_project, -) +from platformio.project.helpers import is_platformio_project class ProjectRPC(object): @staticmethod def _get_projects(project_dirs=None): - def _get_project_data(project_dir): + def _get_project_data(): data = {"boards": [], "envLibdepsDirs": [], "libExtraDirs": []} - config = ProjectConfig(join(project_dir, "platformio.ini")) - libdeps_dir = get_project_libdeps_dir() + config = ProjectConfig() + libdeps_dir = config.get_optional_dir("libdeps") data["libExtraDirs"].extend(config.get("platformio", "lib_extra_dirs", [])) @@ -76,7 +72,7 @@ class ProjectRPC(object): boards = [] try: with fs.cd(project_dir): - data = _get_project_data(project_dir) + data = _get_project_data() except exception.PlatformIOProjectException: continue @@ -178,9 +174,10 @@ class ProjectRPC(object): "", "void loop() {", " // put your main code here, to run repeatedly:", - "}" "", + "}", + "", ] - ) # yapf: disable + ) elif framework == "mbed": main_content = "\n".join( [ @@ -196,11 +193,12 @@ class ProjectRPC(object): "}", "", ] - ) # yapf: disable + ) if not main_content: return project_dir with fs.cd(project_dir): - src_dir = get_project_src_dir() + config = ProjectConfig() + src_dir = config.get_optional_dir("src") main_path = join(src_dir, "main.cpp") if isfile(main_path): return project_dir @@ -258,7 +256,8 @@ class ProjectRPC(object): @staticmethod def _finalize_arduino_import(_, project_dir, arduino_project_dir): with fs.cd(project_dir): - src_dir = get_project_src_dir() + config = ProjectConfig() + src_dir = config.get_optional_dir("src") if isdir(src_dir): fs.rmtree(src_dir) shutil.copytree(arduino_project_dir, src_dir) diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index 75252866..defeaafa 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -23,12 +23,7 @@ from platformio import fs, util from platformio.compat import get_file_contents from platformio.proc import where_is_program from platformio.project.config import ProjectConfig -from platformio.project.helpers import ( - get_project_lib_dir, - get_project_libdeps_dir, - get_project_src_dir, - load_project_ide_data, -) +from platformio.project.helpers import load_project_ide_data class ProjectGenerator(object): @@ -76,7 +71,7 @@ class ProjectGenerator(object): else where_is_program("platformio"), "env_path": os.getenv("PATH"), "env_pathsep": os.pathsep, - } # yapf: disable + } # default env configuration tpl_vars.update(self.config.items(env=self.env_name, as_dict=True)) @@ -87,13 +82,13 @@ class ProjectGenerator(object): tpl_vars.update( { "src_files": self.get_src_files(), - "project_src_dir": get_project_src_dir(), - "project_lib_dir": get_project_lib_dir(), + "project_src_dir": self.config.get_optional_dir("src"), + "project_lib_dir": self.config.get_optional_dir("lib"), "project_libdeps_dir": join( - get_project_libdeps_dir(), self.env_name + self.config.get_optional_dir("libdeps"), self.env_name ), } - ) # yapf: disable + ) for key, value in tpl_vars.items(): if key.endswith(("_path", "_dir")): @@ -109,7 +104,7 @@ class ProjectGenerator(object): def get_src_files(self): result = [] with fs.cd(self.project_dir): - for root, _, files in os.walk(get_project_src_dir()): + for root, _, files in os.walk(self.config.get_optional_dir("src")): for f in files: result.append(relpath(join(root, f))) return result diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 9d1487d6..ee59e36b 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -21,7 +21,7 @@ from platformio import __version__, exception, fs from platformio.compat import PY2, WINDOWS from platformio.managers.package import PackageManager from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path -from platformio.project.helpers import get_project_packages_dir +from platformio.project.config import ProjectConfig CORE_PACKAGES = { "contrib-piohome": "^2.3.2", @@ -40,8 +40,10 @@ PIOPLUS_AUTO_UPDATES_MAX = 100 class CorePackageManager(PackageManager): def __init__(self): + config = ProjectConfig.get_instance() + packages_dir = config.get_optional_dir("packages") super(CorePackageManager, self).__init__( - get_project_packages_dir(), + packages_dir, [ "https://dl.bintray.com/platformio/dl-packages/manifest.json", "http%s://dl.platformio.org/packages/manifest.json" diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index 943126d7..85cf4b29 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -27,7 +27,7 @@ from platformio import app, exception, util from platformio.compat import glob_escape, string_types from platformio.managers.package import BasePkgManager from platformio.managers.platform import PlatformFactory, PlatformManager -from platformio.project.helpers import get_project_global_lib_dir +from platformio.project.config import ProjectConfig class LibraryManager(BasePkgManager): @@ -35,9 +35,10 @@ class LibraryManager(BasePkgManager): FILE_CACHE_VALID = "30d" # 1 month def __init__(self, package_dir=None): - if not package_dir: - package_dir = get_project_global_lib_dir() - super(LibraryManager, self).__init__(package_dir) + self.config = ProjectConfig.get_instance() + super(LibraryManager, self).__init__( + package_dir or self.config.get_optional_dir("globallib") + ) @property def manifest_names(self): diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 01cf5254..f940767b 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -405,7 +405,7 @@ class PkgInstallerMixin(object): self.parse_semver_version(manifest["version"], raise_exception=True) ): continue - elif not best or ( + if not best or ( self.parse_semver_version(manifest["version"], raise_exception=True) > self.parse_semver_version(best["version"], raise_exception=True) ): diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 124a4245..2536386a 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=too-many-public-methods, too-many-instance-attributes + import base64 import os import re @@ -32,12 +34,6 @@ from platformio.proc import ( get_pythonexe_path, ) from platformio.project.config import ProjectConfig -from platformio.project.helpers import ( - get_project_boards_dir, - get_project_core_dir, - get_project_packages_dir, - get_project_platforms_dir, -) try: from urllib.parse import quote @@ -54,8 +50,9 @@ class PlatformManager(BasePkgManager): "https" if app.get_setting("strict_ssl") else "http" ), ] + self.config = ProjectConfig.get_instance() BasePkgManager.__init__( - self, package_dir or get_project_platforms_dir(), repositories + self, package_dir or self.config.get_optional_dir("platforms"), repositories ) @property @@ -164,7 +161,7 @@ class PlatformManager(BasePkgManager): deppkgs[pkgname] = set() deppkgs[pkgname].add(pkgmanifest["version"]) - pm = PackageManager(get_project_packages_dir()) + pm = PackageManager(self.config.get_optional_dir("packages")) for manifest in pm.get_installed(): if manifest["name"] not in names: continue @@ -290,7 +287,7 @@ class PlatformPackagesMixin(object): version = opts.get("version", "") if name in without_packages: continue - elif name in with_packages or not ( + if name in with_packages or not ( skip_default_package or opts.get("optional", False) ): if ":" in version: @@ -385,8 +382,7 @@ class PlatformRunMixin(object): assert isinstance(variables, dict) assert isinstance(targets, list) - config = ProjectConfig.get_instance(variables["project_config"]) - options = config.items(env=variables["pioenv"], as_dict=True) + options = self.config.items(env=variables["pioenv"], as_dict=True) if "framework" in options: # support PIO Core 3.0 dev/platforms options["pioframework"] = options["framework"] @@ -421,7 +417,7 @@ class PlatformRunMixin(object): str(jobs), "--sconstruct", join(fs.get_source_dir(), "builder", "main.py"), - ] # yapf: disable + ] args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0)) # pylint: disable=protected-access args.append("ISATTY=%d" % (1 if click._compat.isatty(sys.stdout) else 0)) @@ -503,9 +499,7 @@ class PlatformRunMixin(object): click.echo(banner, err=True) -class PlatformBase( # pylint: disable=too-many-public-methods - PlatformPackagesMixin, PlatformRunMixin -): +class PlatformBase(PlatformPackagesMixin, PlatformRunMixin): PIO_VERSION = semantic_version.Version(util.pepver_to_semver(__version__)) _BOARDS_CACHE = {} @@ -519,8 +513,10 @@ class PlatformBase( # pylint: disable=too-many-public-methods self._manifest = fs.load_json(manifest_path) self._custom_packages = None - self.pm = PackageManager(get_project_packages_dir(), self.package_repositories) - + self.config = ProjectConfig.get_instance() + self.pm = PackageManager( + self.config.get_optional_dir("packages"), self.package_repositories + ) # if self.engines and "platformio" in self.engines: # if self.PIO_VERSION not in semantic_version.SimpleSpec( # self.engines['platformio']): @@ -619,8 +615,8 @@ class PlatformBase( # pylint: disable=too-many-public-methods self._BOARDS_CACHE[board_id] = config bdirs = [ - get_project_boards_dir(), - join(get_project_core_dir(), "boards"), + self.config.get_optional_dir("boards"), + join(self.config.get_optional_dir("core"), "boards"), join(self.get_dir(), "boards"), ] diff --git a/platformio/project/config.py b/platformio/project/config.py index 28e4c955..d0abf25c 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -16,11 +16,12 @@ import glob import json import os import re -from os.path import expanduser, getmtime, isfile +from hashlib import sha1 import click from platformio import exception +from platformio.compat import WINDOWS, hashlib_encode_data from platformio.project.options import ProjectOptions try: @@ -41,7 +42,7 @@ CONFIG_HEADER = """;PlatformIO Project Configuration File """ -class ProjectConfig(object): +class ProjectConfigBase(object): INLINE_COMMENT_RE = re.compile(r"\s+;.*$") VARTPL_RE = re.compile(r"\$\{([^\.\}]+)\.([^\}]+)\}") @@ -49,7 +50,6 @@ class ProjectConfig(object): expand_interpolations = True warnings = [] - _instances = {} _parser = None _parsed = [] @@ -66,33 +66,28 @@ class ProjectConfig(object): if not item or item.startswith((";", "#")): continue if ";" in item: - item = ProjectConfig.INLINE_COMMENT_RE.sub("", item).strip() + item = ProjectConfigBase.INLINE_COMMENT_RE.sub("", item).strip() result.append(item) return result @staticmethod - def get_instance(path): - mtime = getmtime(path) if isfile(path) else 0 - instance = ProjectConfig._instances.get(path) - if instance and instance["mtime"] != mtime: - instance = None - if not instance: - instance = {"mtime": mtime, "config": ProjectConfig(path)} - ProjectConfig._instances[path] = instance - return instance["config"] + def get_default_path(): + from platformio import app - def __init__(self, path, parse_extra=True, expand_interpolations=True): + return app.get_session_var("custom_project_conf") or os.path.join( + os.getcwd(), "platformio.ini" + ) + + def __init__(self, path=None, parse_extra=True, expand_interpolations=True): + path = self.get_default_path() if path is None else path self.path = path self.expand_interpolations = expand_interpolations self.warnings = [] self._parsed = [] self._parser = ConfigParser.ConfigParser() - if isfile(path): + if path and os.path.isfile(path): self.read(path, parse_extra) - def __repr__(self): - return "" % (self.path or "in-memory") - def __getattr__(self, name): return getattr(self._parser, name) @@ -111,7 +106,7 @@ class ProjectConfig(object): # load extra configs for pattern in self.get("platformio", "extra_configs", []): if pattern.startswith("~"): - pattern = expanduser(pattern) + pattern = os.path.expanduser(pattern) for item in glob.glob(pattern): self.read(item) @@ -165,7 +160,7 @@ class ProjectConfig(object): unknown_conditions = [ ("%s.%s" % (scope, option)) not in ProjectOptions, scope != "env" or not option.startswith(("custom_", "board_")), - ] # yapf: disable + ] if all(unknown_conditions): self.warnings.append( "Ignore unknown configuration option `%s` " @@ -288,7 +283,7 @@ class ProjectConfig(object): # option is not specified by user if value is None: - return default + return default if default is not None else option_meta.default try: return self._cast_to(value, option_meta.type) @@ -313,7 +308,7 @@ class ProjectConfig(object): return self.get("platformio", "default_envs", []) def validate(self, envs=None, silent=False): - if not isfile(self.path): + if not os.path.isfile(self.path): raise exception.NotPlatformIOProject(self.path) # check envs known = set(self.envs()) @@ -327,6 +322,93 @@ class ProjectConfig(object): click.secho("Warning! %s" % warning, fg="yellow") return True + +class ProjectConfigDirsMixin(object): + def _get_core_dir(self, exists=False): + default = ProjectOptions["platformio.core_dir"].default + core_dir = self.get("platformio", "core_dir") + win_core_dir = None + if WINDOWS and core_dir == default: + win_core_dir = os.path.splitdrive(core_dir)[0] + "\\.platformio" + if os.path.isdir(win_core_dir): + core_dir = win_core_dir + + if exists and not os.path.isdir(core_dir): + try: + os.makedirs(core_dir) + except OSError as e: + if win_core_dir: + os.makedirs(win_core_dir) + core_dir = win_core_dir + else: + raise e + + return core_dir + + def get_optional_dir(self, name, exists=False): + if not ProjectOptions.get("platformio.%s_dir" % name): + raise ValueError("Unknown optional directory -> " + name) + + if name == "core": + result = self._get_core_dir(exists) + else: + result = self.get("platformio", name + "_dir") + + if result is None: + return None + + project_dir = os.getcwd() + + # patterns + if "$PROJECT_HASH" in result: + result = result.replace( + "$PROJECT_HASH", + "%s-%s" + % ( + os.path.basename(project_dir), + sha1(hashlib_encode_data(project_dir)).hexdigest()[:10], + ), + ) + + if "$PROJECT_DIR" in result: + result = result.replace("$PROJECT_DIR", project_dir) + if "$PROJECT_CORE_DIR" in result: + result = result.replace("$PROJECT_CORE_DIR", self.get_optional_dir("core")) + if "$PROJECT_WORKSPACE_DIR" in result: + result = result.replace( + "$PROJECT_WORKSPACE_DIR", self.get_optional_dir("workspace") + ) + + if result.startswith("~"): + result = os.path.expanduser(result) + + result = os.path.realpath(result) + + if exists and not os.path.isdir(result): + os.makedirs(result) + + return result + + +class ProjectConfig(ProjectConfigBase, ProjectConfigDirsMixin): + + _instances = {} + + @staticmethod + def get_instance(path=None): + path = ProjectConfig.get_default_path() if path is None else path + mtime = os.path.getmtime(path) if os.path.isfile(path) else 0 + instance = ProjectConfig._instances.get(path) + if instance and instance["mtime"] != mtime: + instance = None + if not instance: + instance = {"mtime": mtime, "config": ProjectConfig(path)} + ProjectConfig._instances[path] = instance + return instance["config"] + + def __repr__(self): + return "" % (self.path or "in-memory") + def to_json(self): result = {} for section in self.sections(): diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index f1e6702e..e28c6643 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -16,16 +16,7 @@ import json import os from hashlib import sha1 from os import walk -from os.path import ( - basename, - dirname, - expanduser, - isdir, - isfile, - join, - realpath, - splitdrive, -) +from os.path import dirname, expanduser, isdir, isfile, join from click.testing import CliRunner @@ -54,126 +45,32 @@ def find_project_dir_above(path): return None -def get_project_optional_dir(name, default=None): - project_dir = get_project_dir() - config = ProjectConfig.get_instance(join(project_dir, "platformio.ini")) - optional_dir = config.get("platformio", name) - - if not optional_dir: - return default - - if "$PROJECT_HASH" in optional_dir: - optional_dir = optional_dir.replace( - "$PROJECT_HASH", - "%s-%s" - % ( - basename(project_dir), - sha1(hashlib_encode_data(project_dir)).hexdigest()[:10], - ), - ) - - if optional_dir.startswith("~"): - optional_dir = expanduser(optional_dir) - - return realpath(optional_dir) - - def get_project_core_dir(): - default = join(expanduser("~"), ".platformio") - core_dir = get_project_optional_dir( - "core_dir", get_project_optional_dir("home_dir", default) - ) - win_core_dir = None - if WINDOWS and core_dir == default: - win_core_dir = splitdrive(core_dir)[0] + "\\.platformio" - if isdir(win_core_dir): - core_dir = win_core_dir - - if not isdir(core_dir): - try: - os.makedirs(core_dir) - except OSError as e: - if win_core_dir: - os.makedirs(win_core_dir) - core_dir = win_core_dir - else: - raise e - - assert isdir(core_dir) - return core_dir - - -def get_project_global_lib_dir(): - return get_project_optional_dir( - "globallib_dir", join(get_project_core_dir(), "lib") - ) - - -def get_project_platforms_dir(): - return get_project_optional_dir( - "platforms_dir", join(get_project_core_dir(), "platforms") - ) - - -def get_project_packages_dir(): - return get_project_optional_dir( - "packages_dir", join(get_project_core_dir(), "packages") - ) + """ Deprecated, use ProjectConfig.get_optional_dir("core") instead """ + return ProjectConfig.get_instance( + join(get_project_dir(), "platformio.ini") + ).get_optional_dir("core", exists=True) def get_project_cache_dir(): - return get_project_optional_dir("cache_dir", join(get_project_core_dir(), ".cache")) + """ Deprecated, use ProjectConfig.get_optional_dir("cache") instead """ + return ProjectConfig.get_instance( + join(get_project_dir(), "platformio.ini") + ).get_optional_dir("cache") -def get_project_workspace_dir(): - return get_project_optional_dir("workspace_dir", join(get_project_dir(), ".pio")) - - -def get_project_build_dir(force=False): - path = get_project_optional_dir( - "build_dir", join(get_project_workspace_dir(), "build") - ) +def get_default_projects_dir(): + docs_dir = join(expanduser("~"), "Documents") try: - if not isdir(path): - os.makedirs(path) - except Exception as e: # pylint: disable=broad-except - if not force: - raise Exception(e) - return path + assert WINDOWS + import ctypes.wintypes # pylint: disable=import-outside-toplevel - -def get_project_libdeps_dir(): - return get_project_optional_dir( - "libdeps_dir", join(get_project_workspace_dir(), "libdeps") - ) - - -def get_project_lib_dir(): - return get_project_optional_dir("lib_dir", join(get_project_dir(), "lib")) - - -def get_project_include_dir(): - return get_project_optional_dir("include_dir", join(get_project_dir(), "include")) - - -def get_project_src_dir(): - return get_project_optional_dir("src_dir", join(get_project_dir(), "src")) - - -def get_project_test_dir(): - return get_project_optional_dir("test_dir", join(get_project_dir(), "test")) - - -def get_project_boards_dir(): - return get_project_optional_dir("boards_dir", join(get_project_dir(), "boards")) - - -def get_project_data_dir(): - return get_project_optional_dir("data_dir", join(get_project_dir(), "data")) - - -def get_project_shared_dir(): - return get_project_optional_dir("shared_dir", join(get_project_dir(), "shared")) + buf = ctypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH) + ctypes.windll.shell32.SHGetFolderPathW(None, 5, None, 0, buf) + docs_dir = buf.value + except: # pylint: disable=bare-except + pass + return join(docs_dir, "PlatformIO", "Projects") def compute_project_checksum(config): @@ -185,7 +82,11 @@ def compute_project_checksum(config): # project file structure check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S") - for d in (get_project_include_dir(), get_project_src_dir(), get_project_lib_dir()): + for d in ( + config.get_optional_dir("include"), + config.get_optional_dir("src"), + config.get_optional_dir("lib"), + ): if not isdir(d): continue chunks = [] diff --git a/platformio/project/options.py b/platformio/project/options.py index 0dc66343..f7294c40 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -14,13 +14,23 @@ # pylint: disable=redefined-builtin, too-many-arguments +import os from collections import OrderedDict, namedtuple import click ConfigOptionClass = namedtuple( "ConfigOption", - ["scope", "name", "type", "multiple", "sysenvvar", "buildenvvar", "oldnames"], + [ + "scope", + "name", + "type", + "multiple", + "sysenvvar", + "buildenvvar", + "oldnames", + "default", + ], ) @@ -32,9 +42,10 @@ def ConfigOption( sysenvvar=None, buildenvvar=None, oldnames=None, + default=None, ): return ConfigOptionClass( - scope, name, type, multiple, sysenvvar, buildenvvar, oldnames + scope, name, type, multiple, sysenvvar, buildenvvar, oldnames, default ) @@ -63,40 +74,83 @@ ProjectOptions = OrderedDict( ConfigPlatformioOption(name="extra_configs", multiple=True), # Dirs ConfigPlatformioOption( - name="core_dir", oldnames=["home_dir"], sysenvvar="PLATFORMIO_CORE_DIR" + name="core_dir", + oldnames=["home_dir"], + sysenvvar="PLATFORMIO_CORE_DIR", + default=os.path.join(os.path.expanduser("~"), ".platformio"), ), ConfigPlatformioOption( - name="globallib_dir", sysenvvar="PLATFORMIO_GLOBALLIB_DIR" + name="globallib_dir", + sysenvvar="PLATFORMIO_GLOBALLIB_DIR", + default=os.path.join("$PROJECT_CORE_DIR", "lib"), ), ConfigPlatformioOption( - name="platforms_dir", sysenvvar="PLATFORMIO_PLATFORMS_DIR" + name="platforms_dir", + sysenvvar="PLATFORMIO_PLATFORMS_DIR", + default=os.path.join("$PROJECT_CORE_DIR", "platforms"), ), ConfigPlatformioOption( - name="packages_dir", sysenvvar="PLATFORMIO_PACKAGES_DIR" + name="packages_dir", + sysenvvar="PLATFORMIO_PACKAGES_DIR", + default=os.path.join("$PROJECT_CORE_DIR", "packages"), + ), + ConfigPlatformioOption( + name="cache_dir", + sysenvvar="PLATFORMIO_CACHE_DIR", + default=os.path.join("$PROJECT_CORE_DIR", ".cache"), ), - ConfigPlatformioOption(name="cache_dir", sysenvvar="PLATFORMIO_CACHE_DIR"), ConfigPlatformioOption( name="build_cache_dir", sysenvvar="PLATFORMIO_BUILD_CACHE_DIR" ), ConfigPlatformioOption( - name="workspace_dir", sysenvvar="PLATFORMIO_WORKSPACE_DIR" + name="workspace_dir", + sysenvvar="PLATFORMIO_WORKSPACE_DIR", + default=os.path.join("$PROJECT_DIR", ".pio"), ), - ConfigPlatformioOption(name="build_dir", sysenvvar="PLATFORMIO_BUILD_DIR"), ConfigPlatformioOption( - name="libdeps_dir", sysenvvar="PLATFORMIO_LIBDEPS_DIR" + name="build_dir", + sysenvvar="PLATFORMIO_BUILD_DIR", + default=os.path.join("$PROJECT_WORKSPACE_DIR", "build"), ), - ConfigPlatformioOption(name="lib_dir", sysenvvar="PLATFORMIO_LIB_DIR"), ConfigPlatformioOption( - name="include_dir", sysenvvar="PLATFORMIO_INCLUDE_DIR" + name="libdeps_dir", + sysenvvar="PLATFORMIO_LIBDEPS_DIR", + default=os.path.join("$PROJECT_WORKSPACE_DIR", "libdeps"), ), - ConfigPlatformioOption(name="src_dir", sysenvvar="PLATFORMIO_SRC_DIR"), - ConfigPlatformioOption(name="test_dir", sysenvvar="PLATFORMIO_TEST_DIR"), ConfigPlatformioOption( - name="boards_dir", sysenvvar="PLATFORMIO_BOARDS_DIR" + name="lib_dir", + sysenvvar="PLATFORMIO_LIB_DIR", + default=os.path.join("$PROJECT_DIR", "lib"), ), - ConfigPlatformioOption(name="data_dir", sysenvvar="PLATFORMIO_DATA_DIR"), ConfigPlatformioOption( - name="shared_dir", sysenvvar="PLATFORMIO_SHARED_DIR" + name="include_dir", + sysenvvar="PLATFORMIO_INCLUDE_DIR", + default=os.path.join("$PROJECT_DIR", "include"), + ), + ConfigPlatformioOption( + name="src_dir", + sysenvvar="PLATFORMIO_SRC_DIR", + default=os.path.join("$PROJECT_DIR", "src"), + ), + ConfigPlatformioOption( + name="test_dir", + sysenvvar="PLATFORMIO_TEST_DIR", + default=os.path.join("$PROJECT_DIR", "test"), + ), + ConfigPlatformioOption( + name="boards_dir", + sysenvvar="PLATFORMIO_BOARDS_DIR", + default=os.path.join("$PROJECT_DIR", "boards"), + ), + ConfigPlatformioOption( + name="data_dir", + sysenvvar="PLATFORMIO_DATA_DIR", + default=os.path.join("$PROJECT_DIR", "data"), + ), + ConfigPlatformioOption( + name="shared_dir", + sysenvvar="PLATFORMIO_SHARED_DIR", + default=os.path.join("$PROJECT_DIR", "shared"), ), # # [env] diff --git a/platformio/run/helpers.py b/platformio/run/helpers.py index d9481433..a4c79e83 100644 --- a/platformio/run/helpers.py +++ b/platformio/run/helpers.py @@ -18,16 +18,14 @@ from os.path import isdir, isfile, join import click from platformio import fs -from platformio.project.helpers import ( - compute_project_checksum, - get_project_dir, - get_project_libdeps_dir, -) +from platformio.project.helpers import compute_project_checksum, get_project_dir def handle_legacy_libdeps(project_dir, config): legacy_libdeps_dir = join(project_dir, ".piolibdeps") - if not isdir(legacy_libdeps_dir) or legacy_libdeps_dir == get_project_libdeps_dir(): + if not isdir(legacy_libdeps_dir) or legacy_libdeps_dir == config.get_optional_dir( + "libdeps" + ): return if not config.has_section("env"): config.add_section("env") diff --git a/platformio/run/processor.py b/platformio/run/processor.py index 219444d8..490ae497 100644 --- a/platformio/run/processor.py +++ b/platformio/run/processor.py @@ -48,9 +48,11 @@ class EnvironmentProcessor(object): return variables def get_build_targets(self): - if self.targets: - return [t for t in self.targets] - return self.config.get("env:" + self.name, "targets", []) + return ( + self.targets + if self.targets + else self.config.get("env:" + self.name, "targets", []) + ) def process(self): if "platform" not in self.options: diff --git a/platformio/test/native.py b/platformio/test/native.py index 73029e98..f028e4f3 100644 --- a/platformio/test/native.py +++ b/platformio/test/native.py @@ -14,9 +14,8 @@ from os.path import join -from platformio import fs, proc +from platformio import proc from platformio.proc import LineBufferedAsyncPipe -from platformio.project.helpers import get_project_build_dir from platformio.test.processor import TestProcessorBase @@ -32,8 +31,7 @@ class NativeTestProcessor(TestProcessorBase): return self.run() def run(self): - with fs.cd(self.options["project_dir"]): - build_dir = get_project_build_dir() + build_dir = self.options["project_config"].get_optional_dir("build") result = proc.exec_command( [join(build_dir, self.env_name, "program")], stdout=LineBufferedAsyncPipe(self.on_run_out), diff --git a/platformio/test/processor.py b/platformio/test/processor.py index b04a2789..e228a8c0 100644 --- a/platformio/test/processor.py +++ b/platformio/test/processor.py @@ -20,7 +20,6 @@ from string import Template import click from platformio import exception -from platformio.project.helpers import get_project_test_dir TRANSPORT_OPTIONS = { "arduino": { @@ -104,7 +103,9 @@ class TestProcessorBase(object): def build_or_upload(self, target): if not self._outputcpp_generated: - self.generate_outputcpp(get_project_test_dir()) + self.generate_outputcpp( + self.options["project_config"].get_optional_dir("test") + ) self._outputcpp_generated = True if self.test_name != "*": @@ -175,7 +176,7 @@ class TestProcessorBase(object): " $end;", "}", ] - ) # yapf: disable + ) def delete_tmptest_file(file_): try: diff --git a/platformio/util.py b/platformio/util.py index bfd4abfb..cff14277 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -191,7 +191,7 @@ def get_logical_devices(): def get_mdns_services(): try: - import zeroconf + import zeroconf # pylint: disable=import-outside-toplevel except ImportError: from site import addsitedir from platformio.managers.core import get_core_package_dir @@ -199,7 +199,7 @@ def get_mdns_services(): contrib_pysite_dir = get_core_package_dir("contrib-pysite") addsitedir(contrib_pysite_dir) sys.path.insert(0, contrib_pysite_dir) - import zeroconf + import zeroconf # pylint: disable=import-outside-toplevel class mDNSListener(object): def __init__(self): @@ -360,7 +360,7 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): time.sleep(2 * total) raise exception.APIRequestError( - "Could not connect to PlatformIO API Service. " "Please try later." + "Could not connect to PlatformIO API Service. Please try later." ) diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index c509a1f9..e4bb6db9 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -164,7 +164,7 @@ def test_check_filter_sources(clirunner, check_dir): assert style == EXPECTED_STYLE -def test_check_failed_if_no_source_files(clirunner, tmpdir): +def test_check_no_source_files(clirunner, tmpdir): tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) tmpdir.mkdir("src") @@ -178,7 +178,7 @@ def test_check_failed_if_no_source_files(clirunner, tmpdir): assert style == 0 -def test_check_failed_if_bad_flag_passed(clirunner, check_dir): +def test_check_bad_flag_passed(clirunner, check_dir): result = clirunner.invoke( cmd_check, ["--project-dir", str(check_dir), '"--flags=--UNKNOWN"'] ) diff --git a/tests/test_examples.py b/tests/test_examples.py index 013ecb26..eac51469 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -22,7 +22,6 @@ import pytest from platformio import util from platformio.managers.platform import PlatformFactory, PlatformManager from platformio.project.config import ProjectConfig -from platformio.project.helpers import get_project_build_dir def pytest_generate_tests(metafunc): @@ -71,11 +70,12 @@ def pytest_generate_tests(metafunc): @pytest.mark.examples def test_run(pioproject_dir): with util.cd(pioproject_dir): - build_dir = get_project_build_dir() + config = ProjectConfig() + build_dir = config.get_optional_dir("build") if isdir(build_dir): util.rmtree_(build_dir) - env_names = ProjectConfig(join(pioproject_dir, "platformio.ini")).envs() + env_names = config.envs() result = util.exec_command( ["platformio", "run", "-e", random.choice(env_names)] ) diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index a5127dd0..2ec24521 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -112,6 +112,15 @@ def test_warnings(config): config.validate(["non-existing-env"]) +def test_defaults(config): + assert config.get_optional_dir("core") == os.path.join( + os.path.expanduser("~"), ".platformio" + ) + assert config.get_optional_dir("build_cache") == os.environ.get( + "PLATFORMIO_BUILD_CACHE_DIR" + ) + + def test_sections(config): with pytest.raises(ConfigParser.NoSectionError): config.getraw("unknown_section", "unknown_option") From 536a9566da2a7d049c4499316cbb3f71f43a85d0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 27 Sep 2019 14:18:35 +0300 Subject: [PATCH 039/221] Feature/pio size data (#3056) * Add initial support for detailed memory usage report * Tidy up sizedata target * Add toolchain to environment paths * Make sizedata target a bit more readable --- platformio/builder/main.py | 13 ++ platformio/builder/tools/piosize.py | 197 +++++++++++++++++++++++++ platformio/builder/tools/platformio.py | 3 + 3 files changed, 213 insertions(+) create mode 100644 platformio/builder/tools/piosize.py diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 0fef773b..95099b98 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -61,6 +61,7 @@ DEFAULT_ENV_OPTIONS = dict( "pioupload", "piomisc", "pioide", + "piosize", ], toolpath=[join(fs.get_source_dir(), "builder", "tools")], variables=clivars, @@ -189,3 +190,15 @@ if "idedata" in COMMAND_LINE_TARGETS: ) ) env.Exit(0) + +if "sizedata" in COMMAND_LINE_TARGETS: + AlwaysBuild( + env.Alias( + "sizedata", DEFAULT_TARGETS, + env.VerboseAction( + env.DumpSizeData, "Generating memory usage report...", + ) + ) + ) + + Default("sizedata") diff --git a/platformio/builder/tools/piosize.py b/platformio/builder/tools/piosize.py new file mode 100644 index 00000000..5a9a6e65 --- /dev/null +++ b/platformio/builder/tools/piosize.py @@ -0,0 +1,197 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from os import environ +from os.path import join + +from elftools.elf.descriptions import describe_sh_flags +from elftools.elf.elffile import ELFFile + +from platformio.compat import dump_json_to_unicode +from platformio.proc import exec_command + + +def _get_file_location(env, elf_path, addr, sysenv): + cmd = [env.subst("$CC").replace( + "-gcc", "-addr2line"), "-e", elf_path, hex(addr)] + result = exec_command(cmd, env=sysenv) + return result['out'].strip().replace("\\", "/") + + +def _determine_section(sections, symbol_addr): + for section, info in sections.items(): + if symbol_addr in range(info['start_addr'], + info['start_addr'] + info['size']): + return section + return "unknown" + + +def _demangle_cpp_name(env, symbol_name, sysenv): + cmd = [env.subst("$CC").replace("-gcc", "-c++filt"), symbol_name] + result = exec_command(cmd, env=sysenv) + demangled_name = result['out'].strip() + if "(" in demangled_name: + demangled_name = demangled_name[0:demangled_name.find("(")] + return demangled_name + + +def _is_ram_section(section): + return section.get("type", "") in ( + "SHT_NOBITS", "SHT_PROGBITS") and section.get("flags", "") == "WA" + + +def _is_flash_section(section): + return section.get("type") == "SHT_PROGBITS" and "A" in section.get( + "flags") + + +def _is_valid_symbol(symbol_name, symbol_type, symbol_address): + return symbol_name and symbol_address != 0 and symbol_type != "STT_NOTYPE" + + +def _collect_sections_info(elffile): + sections = {} + for section in elffile.iter_sections(): + if section.is_null(): + continue + + section_type = section['sh_type'] + section_flags = describe_sh_flags(section['sh_flags']) + section_size = section.data_size + + sections[section.name] = { + "size": section_size, + "start_addr": section['sh_addr'], + "type": section_type, + "flags": section_flags + } + + return sections + + +def _collect_symbols_info(env, elffile, elf_path, sections): + symbols = [] + + symbol_section = elffile.get_section_by_name('.symtab') + if symbol_section.is_null(): + sys.stderr.write("Couldn't find symbol table. Is ELF file stripped?") + env.Exit(1) + + sysenv = environ.copy() + sysenv["PATH"] = str(env["ENV"]["PATH"]) + + for s in symbol_section.iter_symbols(): + symbol_info = s.entry['st_info'] + symbol_addr = s['st_value'] + symbol_size = s['st_size'] + symbol_type = symbol_info['type'] + + if not _is_valid_symbol(s.name, symbol_type, symbol_addr): + continue + + symbol = { + "addr": symbol_addr, + "bind": symbol_info['bind'], + "location": _get_file_location(env, elf_path, symbol_addr, sysenv), + "name": s.name, + "type": symbol_type, + "size": symbol_size, + "section": _determine_section(sections, symbol_addr) + } + + if s.name.startswith("_Z"): + symbol['demangled_name'] = _demangle_cpp_name(env, s.name, sysenv) + + symbols.append(symbol) + + return symbols + + +def _calculate_firmware_size(sections): + flash_size = ram_size = 0 + for section_info in sections.values(): + if _is_flash_section(section_info): + flash_size += section_info.get("size", 0) + if _is_ram_section(section_info): + ram_size += section_info.get("size", 0) + + return ram_size, flash_size + + +def DumpSizeData(_, target, source, env): + data = { + "memory": {}, + "version": 1 + } + + elf_path = env.subst("$PIOMAINPROG") + + with open(elf_path, "rb") as fp: + elffile = ELFFile(fp) + + if not elffile.has_dwarf_info(): + sys.stderr.write("Elf file doesn't contain DWARF information") + env.Exit(1) + + sections = _collect_sections_info(elffile) + firmware_ram, firmware_flash = _calculate_firmware_size(sections) + data['memory']['total'] = { + "ram_size": firmware_ram, + "flash_size": firmware_flash, + "sections": sections + } + + files = dict() + for symbol in _collect_symbols_info(env, elffile, elf_path, sections): + file_path, _ = symbol.get("location").rsplit(":", 1) + if not file_path or file_path.startswith("?"): + file_path = "unknown" + + if not files.get(file_path, {}): + files[file_path] = { + "symbols": [], + "ram_size": 0, + "flash_size": 0 + } + + symbol_size = symbol.get("size", 0) + section = sections.get(symbol.get("section", ""), {}) + if _is_ram_section(section): + files[file_path]['ram_size'] += symbol_size + if _is_flash_section(section): + files[file_path]['flash_size'] += symbol_size + + files[file_path]['symbols'].append(symbol) + + data['memory']['files'] = files + + with open(join(env.subst("$BUILD_DIR"), "sizedata.json"), "w") as fp: + fp.write(dump_json_to_unicode(data)) + + +def ConfigureSizeDataTarget(env): + for flags_section in ("ASFLAGS", "CCFLAGS", "LINKFLAGS"): + if not any("-g" in f for f in env.get(flags_section, [])): + env.Prepend(**{flags_section: ["-g"]}) + + +def exists(_): + return True + + +def generate(env): + env.AddMethod(DumpSizeData) + env.AddMethod(ConfigureSizeDataTarget) + return env diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index e197b0fb..2f689b92 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -125,6 +125,9 @@ def BuildProgram(env): if "__test" in COMMAND_LINE_TARGETS: env.ConfigureTestTarget() + if "sizedata" in COMMAND_LINE_TARGETS: + env.ConfigureSizeDataTarget() + # build project with dependencies _build_project_deps(env) From cd2a4ea53523547b433918aaeb6d610b118beae4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 27 Sep 2019 17:21:35 +0300 Subject: [PATCH 040/221] Update copyrights --- platformio/builder/tools/piosize.py | 2 +- platformio/check/__init__.py | 2 +- platformio/check/defect.py | 2 +- platformio/check/tools/__init__.py | 2 +- platformio/check/tools/base.py | 2 +- platformio/check/tools/clangtidy.py | 2 +- platformio/check/tools/cppcheck.py | 2 +- platformio/commands/check.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/platformio/builder/tools/piosize.py b/platformio/builder/tools/piosize.py index 5a9a6e65..b797509a 100644 --- a/platformio/builder/tools/piosize.py +++ b/platformio/builder/tools/piosize.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-present PlatformIO +# Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/platformio/check/__init__.py b/platformio/check/__init__.py index 895dc9b9..b0514903 100644 --- a/platformio/check/__init__.py +++ b/platformio/check/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-present PlatformIO +# Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/platformio/check/defect.py b/platformio/check/defect.py index 0b25084c..adef3666 100644 --- a/platformio/check/defect.py +++ b/platformio/check/defect.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-present PlatformIO +# Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/platformio/check/tools/__init__.py b/platformio/check/tools/__init__.py index a6161fb2..0649e81d 100644 --- a/platformio/check/tools/__init__.py +++ b/platformio/check/tools/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-present PlatformIO +# Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/platformio/check/tools/base.py b/platformio/check/tools/base.py index 578b72c0..3306dd3c 100644 --- a/platformio/check/tools/base.py +++ b/platformio/check/tools/base.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-present PlatformIO +# Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/platformio/check/tools/clangtidy.py b/platformio/check/tools/clangtidy.py index 407375c5..36c00c26 100644 --- a/platformio/check/tools/clangtidy.py +++ b/platformio/check/tools/clangtidy.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-present PlatformIO +# Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/platformio/check/tools/cppcheck.py b/platformio/check/tools/cppcheck.py index d0f1d6eb..aacb24f8 100644 --- a/platformio/check/tools/cppcheck.py +++ b/platformio/check/tools/cppcheck.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-present PlatformIO +# Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/platformio/commands/check.py b/platformio/commands/check.py index 392e618d..0bf33012 100644 --- a/platformio/commands/check.py +++ b/platformio/commands/check.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-present PlatformIO +# Copyright (c) 2014-present PlatformIO # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 1b74f380a616c3f5025056a60c577d30faad8264 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 27 Sep 2019 17:22:21 +0300 Subject: [PATCH 041/221] Refactor appending of debugging flags --- .isort.cfg | 2 +- platformio/builder/main.py | 7 +-- platformio/builder/tools/piomisc.py | 36 +++++++----- platformio/builder/tools/piosize.py | 77 +++++++++++--------------- platformio/builder/tools/platformio.py | 13 ++--- setup.py | 54 ++++++++++++------ 6 files changed, 99 insertions(+), 90 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index 7e5d6e51..2270008c 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,3 +1,3 @@ [settings] line_length=88 -known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial,twisted,autobahn,jsonrpc,tabulate +known_third_party=SCons, twisted, autobahn, jsonrpc diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 95099b98..e0179bf4 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -194,10 +194,9 @@ if "idedata" in COMMAND_LINE_TARGETS: if "sizedata" in COMMAND_LINE_TARGETS: AlwaysBuild( env.Alias( - "sizedata", DEFAULT_TARGETS, - env.VerboseAction( - env.DumpSizeData, "Generating memory usage report...", - ) + "sizedata", + DEFAULT_TARGETS, + env.VerboseAction(env.DumpSizeData, "Generating memory usage report..."), ) ) diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index a1b564b9..5c3e414c 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -23,6 +23,7 @@ from tempfile import mkstemp from SCons.Action import Action # pylint: disable=import-error from SCons.Script import ARGUMENTS # pylint: disable=import-error +from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from platformio import fs, util from platformio.compat import get_file_contents, glob_escape @@ -294,32 +295,37 @@ def VerboseAction(_, act, actstr): def PioClean(env, clean_dir): if not isdir(clean_dir): - print ("Build environment is clean") + print("Build environment is clean") env.Exit(0) clean_rel_path = relpath(clean_dir) for root, _, files in walk(clean_dir): for f in files: dst = join(root, f) remove(dst) - print ( + print( "Removed %s" % (dst if clean_rel_path.startswith(".") else relpath(dst)) ) - print ("Done cleaning") + print("Done cleaning") fs.rmtree(clean_dir) env.Exit(0) -def ConfigureDebugTarget(env): - if not env.subst("$PIODEBUGFLAGS"): - env.Replace(PIODEBUGFLAGS=["-Og", "-g3", "-ggdb3"]) - env.Append( - BUILD_FLAGS=list(env["PIODEBUGFLAGS"]) + ["-D__PLATFORMIO_BUILD_DEBUG__"] - ) - unflags = ["-Os"] - for level in [0, 1, 2]: - for flag in ("O", "g", "ggdb"): - unflags.append("-%s%d" % (flag, level)) - env.Append(BUILD_UNFLAGS=unflags) +def ConfigureDebugFlags(env): + def _cleanup_debug_flags(scope): + if scope not in env: + return + unflags = ["-Os", "-g"] + for level in [0, 1, 2]: + for flag in ("O", "g", "ggdb"): + unflags.append("-%s%d" % (flag, level)) + env[scope] = [f for f in env.get(scope, []) if f not in unflags] + + env.Append(CPPDEFINES=["__PLATFORMIO_BUILD_DEBUG__"]) + + debug_flags = ["-Og", "-g3", "-ggdb3"] + for scope in ("ASFLAGS", "CCFLAGS", "LINKFLAGS"): + _cleanup_debug_flags(scope) + env.Append(**{scope: debug_flags}) def ConfigureTestTarget(env): @@ -361,7 +367,7 @@ def generate(env): env.AddMethod(GetActualLDScript) env.AddMethod(VerboseAction) env.AddMethod(PioClean) - env.AddMethod(ConfigureDebugTarget) + env.AddMethod(ConfigureDebugFlags) env.AddMethod(ConfigureTestTarget) env.AddMethod(GetExtraScripts) return env diff --git a/platformio/builder/tools/piosize.py b/platformio/builder/tools/piosize.py index b797509a..12303747 100644 --- a/platformio/builder/tools/piosize.py +++ b/platformio/builder/tools/piosize.py @@ -24,16 +24,14 @@ from platformio.proc import exec_command def _get_file_location(env, elf_path, addr, sysenv): - cmd = [env.subst("$CC").replace( - "-gcc", "-addr2line"), "-e", elf_path, hex(addr)] + cmd = [env.subst("$CC").replace("-gcc", "-addr2line"), "-e", elf_path, hex(addr)] result = exec_command(cmd, env=sysenv) - return result['out'].strip().replace("\\", "/") + return result["out"].strip().replace("\\", "/") def _determine_section(sections, symbol_addr): for section, info in sections.items(): - if symbol_addr in range(info['start_addr'], - info['start_addr'] + info['size']): + if symbol_addr in range(info["start_addr"], info["start_addr"] + info["size"]): return section return "unknown" @@ -41,20 +39,21 @@ def _determine_section(sections, symbol_addr): def _demangle_cpp_name(env, symbol_name, sysenv): cmd = [env.subst("$CC").replace("-gcc", "-c++filt"), symbol_name] result = exec_command(cmd, env=sysenv) - demangled_name = result['out'].strip() + demangled_name = result["out"].strip() if "(" in demangled_name: - demangled_name = demangled_name[0:demangled_name.find("(")] + demangled_name = demangled_name[0 : demangled_name.find("(")] return demangled_name def _is_ram_section(section): - return section.get("type", "") in ( - "SHT_NOBITS", "SHT_PROGBITS") and section.get("flags", "") == "WA" + return ( + section.get("type", "") in ("SHT_NOBITS", "SHT_PROGBITS") + and section.get("flags", "") == "WA" + ) def _is_flash_section(section): - return section.get("type") == "SHT_PROGBITS" and "A" in section.get( - "flags") + return section.get("type") == "SHT_PROGBITS" and "A" in section.get("flags") def _is_valid_symbol(symbol_name, symbol_type, symbol_address): @@ -67,15 +66,15 @@ def _collect_sections_info(elffile): if section.is_null(): continue - section_type = section['sh_type'] - section_flags = describe_sh_flags(section['sh_flags']) + section_type = section["sh_type"] + section_flags = describe_sh_flags(section["sh_flags"]) section_size = section.data_size sections[section.name] = { "size": section_size, - "start_addr": section['sh_addr'], + "start_addr": section["sh_addr"], "type": section_type, - "flags": section_flags + "flags": section_flags, } return sections @@ -84,7 +83,7 @@ def _collect_sections_info(elffile): def _collect_symbols_info(env, elffile, elf_path, sections): symbols = [] - symbol_section = elffile.get_section_by_name('.symtab') + symbol_section = elffile.get_section_by_name(".symtab") if symbol_section.is_null(): sys.stderr.write("Couldn't find symbol table. Is ELF file stripped?") env.Exit(1) @@ -93,26 +92,26 @@ def _collect_symbols_info(env, elffile, elf_path, sections): sysenv["PATH"] = str(env["ENV"]["PATH"]) for s in symbol_section.iter_symbols(): - symbol_info = s.entry['st_info'] - symbol_addr = s['st_value'] - symbol_size = s['st_size'] - symbol_type = symbol_info['type'] + symbol_info = s.entry["st_info"] + symbol_addr = s["st_value"] + symbol_size = s["st_size"] + symbol_type = symbol_info["type"] if not _is_valid_symbol(s.name, symbol_type, symbol_addr): continue symbol = { "addr": symbol_addr, - "bind": symbol_info['bind'], + "bind": symbol_info["bind"], "location": _get_file_location(env, elf_path, symbol_addr, sysenv), "name": s.name, "type": symbol_type, "size": symbol_size, - "section": _determine_section(sections, symbol_addr) + "section": _determine_section(sections, symbol_addr), } if s.name.startswith("_Z"): - symbol['demangled_name'] = _demangle_cpp_name(env, s.name, sysenv) + symbol["demangled_name"] = _demangle_cpp_name(env, s.name, sysenv) symbols.append(symbol) @@ -130,11 +129,8 @@ def _calculate_firmware_size(sections): return ram_size, flash_size -def DumpSizeData(_, target, source, env): - data = { - "memory": {}, - "version": 1 - } +def DumpSizeData(_, target, source, env): # pylint: disable=unused-argument + data = {"memory": {}, "version": 1} elf_path = env.subst("$PIOMAINPROG") @@ -147,10 +143,10 @@ def DumpSizeData(_, target, source, env): sections = _collect_sections_info(elffile) firmware_ram, firmware_flash = _calculate_firmware_size(sections) - data['memory']['total'] = { + data["memory"]["total"] = { "ram_size": firmware_ram, "flash_size": firmware_flash, - "sections": sections + "sections": sections, } files = dict() @@ -160,38 +156,27 @@ def DumpSizeData(_, target, source, env): file_path = "unknown" if not files.get(file_path, {}): - files[file_path] = { - "symbols": [], - "ram_size": 0, - "flash_size": 0 - } + files[file_path] = {"symbols": [], "ram_size": 0, "flash_size": 0} symbol_size = symbol.get("size", 0) section = sections.get(symbol.get("section", ""), {}) if _is_ram_section(section): - files[file_path]['ram_size'] += symbol_size + files[file_path]["ram_size"] += symbol_size if _is_flash_section(section): - files[file_path]['flash_size'] += symbol_size + files[file_path]["flash_size"] += symbol_size - files[file_path]['symbols'].append(symbol) + files[file_path]["symbols"].append(symbol) - data['memory']['files'] = files + data["memory"]["files"] = files with open(join(env.subst("$BUILD_DIR"), "sizedata.json"), "w") as fp: fp.write(dump_json_to_unicode(data)) -def ConfigureSizeDataTarget(env): - for flags_section in ("ASFLAGS", "CCFLAGS", "LINKFLAGS"): - if not any("-g" in f for f in env.get(flags_section, [])): - env.Prepend(**{flags_section: ["-g"]}) - - def exists(_): return True def generate(env): env.AddMethod(DumpSizeData) - env.AddMethod(ConfigureSizeDataTarget) return env diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 2f689b92..f032be94 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -103,9 +103,6 @@ def BuildProgram(env): if not Util.case_sensitive_suffixes(".s", ".S"): env.Replace(AS="$CC", ASCOM="$ASPPCOM") - if "debug" in COMMAND_LINE_TARGETS or env.GetProjectOption("build_type") == "debug": - env.ConfigureDebugTarget() - # process extra flags from board if "BOARD" in env and "build.extra_flags" in env.BoardConfig(): env.ProcessFlags(env.BoardConfig().get("build.extra_flags")) @@ -116,8 +113,11 @@ def BuildProgram(env): # process framework scripts env.BuildFrameworks(env.get("PIOFRAMEWORK")) - # restore PIO macros if it was deleted by framework - _append_pio_macros() + if ( + set(["debug", "sizedata"]) & set(COMMAND_LINE_TARGETS) + or env.GetProjectOption("build_type") == "debug" + ): + env.ConfigureDebugFlags() # remove specified flags env.ProcessUnFlags(env.get("BUILD_UNFLAGS")) @@ -125,9 +125,6 @@ def BuildProgram(env): if "__test" in COMMAND_LINE_TARGETS: env.ConfigureTestTarget() - if "sizedata" in COMMAND_LINE_TARGETS: - env.ConfigureSizeDataTarget() - # build project with dependencies _build_project_deps(env) diff --git a/setup.py b/setup.py index 7d37ad42..60b0a115 100644 --- a/setup.py +++ b/setup.py @@ -14,8 +14,15 @@ from setuptools import find_packages, setup -from platformio import (__author__, __description__, __email__, __license__, - __title__, __url__, __version__) +from platformio import ( + __author__, + __description__, + __email__, + __license__, + __title__, + __url__, + __version__, +) install_requires = [ "bottle<0.13", @@ -24,7 +31,8 @@ install_requires = [ "pyserial>=3,<4,!=3.3", "requests>=2.4.0,<3", "semantic_version>=2.8.1,<3", - "tabulate>=0.8.3,<1" + "tabulate>=0.8.3,<1", + "pyelftools>=0.25,<1", ] setup( @@ -36,8 +44,9 @@ setup( author_email=__email__, url=__url__, license=__license__, - python_requires=", ".join([ - ">=2.7", "!=3.0.*", "!=3.1.*", "!=3.2.*", "!=3.3.*", "!=3.4.*"]), + python_requires=", ".join( + [">=2.7", "!=3.0.*", "!=3.1.*", "!=3.2.*", "!=3.3.*", "!=3.4.*"] + ), install_requires=install_requires, packages=find_packages() + ["scripts"], package_data={ @@ -45,17 +54,15 @@ setup( "ide/tpls/*/.*.tpl", "ide/tpls/*/*.tpl", "ide/tpls/*/*/*.tpl", - "ide/tpls/*/.*/*.tpl" + "ide/tpls/*/.*/*.tpl", ], - "scripts": [ - "99-platformio-udev.rules" - ] + "scripts": ["99-platformio-udev.rules"], }, entry_points={ "console_scripts": [ + "platformio = platformio.__main__:main", "pio = platformio.__main__:main", "piodebuggdb = platformio.__main__:debug_gdb_main", - "platformio = platformio.__main__:main" ] }, classifiers=[ @@ -70,11 +77,26 @@ setup( "Programming Language :: Python :: 3", "Topic :: Software Development", "Topic :: Software Development :: Build Tools", - "Topic :: Software Development :: Compilers" + "Topic :: Software Development :: Compilers", ], keywords=[ - "iot", "embedded", "arduino", "mbed", "esp8266", "esp32", "fpga", - "firmware", "continuous-integration", "cloud-ide", "avr", "arm", - "ide", "unit-testing", "hardware", "verilog", "microcontroller", - "debug" - ]) + "iot", + "embedded", + "arduino", + "mbed", + "esp8266", + "esp32", + "fpga", + "firmware", + "continuous-integration", + "cloud-ide", + "avr", + "arm", + "ide", + "unit-testing", + "hardware", + "verilog", + "microcontroller", + "debug", + ], +) From 9a3dcd3daa067fd306c1f43019ce5d982399f554 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 27 Sep 2019 18:53:58 +0300 Subject: [PATCH 042/221] PY2 fix with absolute import --- platformio/builder/tools/piomisc.py | 7 +++---- platformio/builder/tools/piosize.py | 2 ++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 5c3e414c..0719e498 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -23,7 +23,6 @@ from tempfile import mkstemp from SCons.Action import Action # pylint: disable=import-error from SCons.Script import ARGUMENTS # pylint: disable=import-error -from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from platformio import fs, util from platformio.compat import get_file_contents, glob_escape @@ -295,17 +294,17 @@ def VerboseAction(_, act, actstr): def PioClean(env, clean_dir): if not isdir(clean_dir): - print("Build environment is clean") + print ("Build environment is clean") env.Exit(0) clean_rel_path = relpath(clean_dir) for root, _, files in walk(clean_dir): for f in files: dst = join(root, f) remove(dst) - print( + print ( "Removed %s" % (dst if clean_rel_path.startswith(".") else relpath(dst)) ) - print("Done cleaning") + print ("Done cleaning") fs.rmtree(clean_dir) env.Exit(0) diff --git a/platformio/builder/tools/piosize.py b/platformio/builder/tools/piosize.py index 12303747..7bb538c5 100644 --- a/platformio/builder/tools/piosize.py +++ b/platformio/builder/tools/piosize.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import absolute_import + import sys from os import environ from os.path import join From 3dcf1784fb1db0a0dee20ec50b63df73e41b5f01 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 27 Sep 2019 19:36:49 +0300 Subject: [PATCH 043/221] Update PIO Remote to 2.5.5 --- platformio/managers/core.py | 2 +- platformio/util.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/platformio/managers/core.py b/platformio/managers/core.py index ee59e36b..8c6227fa 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -26,7 +26,7 @@ from platformio.project.config import ProjectConfig CORE_PACKAGES = { "contrib-piohome": "^2.3.2", "contrib-pysite": "~2.%d%d.190418" % (sys.version_info[0], sys.version_info[1]), - "tool-pioplus": "^2.5.2", + "tool-pioplus": "^2.5.5", "tool-unity": "~1.20403.0", "tool-scons": "~2.20501.7" if PY2 else "~3.30101.0", "tool-cppcheck": "~1.189.0", diff --git a/platformio/util.py b/platformio/util.py index cff14277..48f679c2 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -366,6 +366,7 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): PING_INTERNET_IPS = [ "192.30.253.113", # github.com + "31.28.1.238", # dl.platformio.org "193.222.52.25", # dl.platformio.org ] From 2137eb17942a13bdd79a692a764da426dfa78f7e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 30 Sep 2019 13:27:34 +0300 Subject: [PATCH 044/221] Do not append debug flags to linker stage --- platformio/builder/tools/piomisc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 0719e498..91184381 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -322,7 +322,7 @@ def ConfigureDebugFlags(env): env.Append(CPPDEFINES=["__PLATFORMIO_BUILD_DEBUG__"]) debug_flags = ["-Og", "-g3", "-ggdb3"] - for scope in ("ASFLAGS", "CCFLAGS", "LINKFLAGS"): + for scope in ("ASFLAGS", "CCFLAGS",): _cleanup_debug_flags(scope) env.Append(**{scope: debug_flags}) From 5f55c183734c9dcb3734875b6f9c2b2dc3e1f55b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 30 Sep 2019 17:59:06 +0300 Subject: [PATCH 045/221] Introduce DataModel, package manifest parser and base manifest model --- platformio/builder/tools/piomisc.py | 2 +- platformio/commands/lib.py | 14 +- platformio/compat.py | 10 + platformio/datamodel.py | 154 ++++++++++ platformio/package/__init__.py | 13 + platformio/package/manifest/__init__.py | 13 + platformio/package/manifest/model.py | 57 ++++ platformio/package/manifest/parser.py | 369 ++++++++++++++++++++++++ tests/test_pkgmanifest.py | 206 +++++++++++-- 9 files changed, 811 insertions(+), 27 deletions(-) create mode 100644 platformio/datamodel.py create mode 100644 platformio/package/__init__.py create mode 100644 platformio/package/manifest/__init__.py create mode 100644 platformio/package/manifest/model.py create mode 100644 platformio/package/manifest/parser.py diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 91184381..bea63a31 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -322,7 +322,7 @@ def ConfigureDebugFlags(env): env.Append(CPPDEFINES=["__PLATFORMIO_BUILD_DEBUG__"]) debug_flags = ["-Og", "-g3", "-ggdb3"] - for scope in ("ASFLAGS", "CCFLAGS",): + for scope in ("ASFLAGS", "CCFLAGS"): _cleanup_debug_flags(scope) env.Append(**{scope: debug_flags}) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 83e6cff4..8f61769f 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -14,8 +14,8 @@ # pylint: disable=too-many-branches, too-many-locals +import os import time -from os.path import isdir, join import click import semantic_version @@ -25,6 +25,7 @@ from platformio import exception, util from platformio.commands import PlatformioCLI from platformio.compat import dump_json_to_unicode from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib +from platformio.package.manifest.parser import ManifestFactory from platformio.proc import is_ci from platformio.project.config import ProjectConfig from platformio.project.helpers import get_project_dir, is_platformio_project @@ -104,13 +105,13 @@ def cli(ctx, **options): if not is_platformio_project(storage_dir): ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) continue - config = ProjectConfig.get_instance(join(storage_dir, "platformio.ini")) + config = ProjectConfig.get_instance(os.path.join(storage_dir, "platformio.ini")) config.validate(options["environment"], silent=in_silence) libdeps_dir = config.get_optional_dir("libdeps") for env in config.envs(): if options["environment"] and env not in options["environment"]: continue - storage_dir = join(libdeps_dir, env) + storage_dir = os.path.join(libdeps_dir, env) ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY][storage_dir] = config.get( "env:" + env, "lib_deps", [] @@ -169,7 +170,7 @@ def lib_install( # pylint: disable=too-many-arguments input_dirs = ctx.meta.get(CTX_META_INPUT_DIRS_KEY, []) project_environments = ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] for input_dir in input_dirs: - config = ProjectConfig.get_instance(join(input_dir, "platformio.ini")) + config = ProjectConfig.get_instance(os.path.join(input_dir, "platformio.ini")) config.validate(project_environments) for env in config.envs(): if project_environments and env not in project_environments: @@ -231,7 +232,7 @@ def lib_update(ctx, libraries, only_check, dry_run, json_output): if only_check and json_output: result = [] for library in _libraries: - pkg_dir = library if isdir(library) else None + pkg_dir = library if os.path.isdir(library) else None requirements = None url = None if not pkg_dir: @@ -492,6 +493,9 @@ def lib_register(config_url): if not config_url.startswith("http://") and not config_url.startswith("https://"): raise exception.InvalidLibConfURL(config_url) + manifest = ManifestFactory.new_from_url(config_url) + assert set(["name", "version"]) & set(list(manifest.as_dict())) + result = util.get_api_result("/lib/register", data=dict(config_url=config_url)) if "message" in result and result["message"]: click.secho( diff --git a/platformio/compat.py b/platformio/compat.py index 64d437a1..15942556 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -15,6 +15,7 @@ # pylint: disable=unused-import, no-name-in-module, import-error, # pylint: disable=no-member, undefined-variable +import inspect import json import os import re @@ -29,6 +30,15 @@ def get_filesystem_encoding(): return sys.getfilesystemencoding() or sys.getdefaultencoding() +def get_class_attributes(cls): + attributes = inspect.getmembers(cls, lambda a: not (inspect.isroutine(a))) + return { + a[0]: a[1] + for a in attributes + if not (a[0].startswith("__") and a[0].endswith("__")) + } + + if PY2: import imp diff --git a/platformio/datamodel.py b/platformio/datamodel.py new file mode 100644 index 00000000..39547d59 --- /dev/null +++ b/platformio/datamodel.py @@ -0,0 +1,154 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import re + +from platformio.compat import get_class_attributes, string_types +from platformio.exception import PlatformioException + +# pylint: disable=too-many-instance-attributes +# pylint: disable=redefined-builtin, too-many-arguments + + +class DataModelException(PlatformioException): + pass + + +class DataField(object): + def __init__( + self, + default=None, + type=str, + required=False, + min_length=None, + max_length=None, + regex=None, + validate_factory=None, + title=None, + ): + self.default = default + self.type = type + self.required = required + self.min_length = min_length + self.max_length = max_length + self.regex = regex + self.validate_factory = validate_factory + self.title = title + + self._value = None + + def __repr__(self): + return '' % ( + self.title, + self.default if self._value is None else self._value, + ) + + def validate(self, value, parent, attr): + if self.title is None: + self.title = attr.title() + try: + if self.required and value is None: + raise ValueError("Required field, value is None") + if self.validate_factory is not None: + value = self.validate_factory(value) + if value is None: + return self.default + if issubclass(self.type, (str, list, bool)): + return getattr(self, "_validate_%s_value" % self.type.__name__)(value) + except (AssertionError, ValueError) as e: + raise DataModelException( + "%s for %s.%s" % (str(e), parent.__class__.__name__, attr) + ) + return value + + def _validate_str_value(self, value): + if not isinstance(value, string_types): + value = str(value) + assert self.min_length is None or len(value) >= self.min_length, ( + "Minimum allowed length is %d characters" % self.min_length + ) + assert self.max_length is None or len(value) <= self.max_length, ( + "Maximum allowed length is %d characters" % self.max_length + ) + assert self.regex is None or re.match( + self.regex, value + ), "Value `%s` does not match RegExp `%s` pattern" % (value, self.regex) + return value + + @staticmethod + def _validate_bool_value(value): + if isinstance(value, bool): + return value + return str(value).lower() in ("true", "yes", "1") + + +class DataModel(object): + __PRIVATE_ATTRIBUTES__ = ("__PRIVATE_ATTRIBUTES__", "_init_type", "as_dict") + + def __init__(self, data=None): + data = data or {} + assert isinstance(data, dict) + + for attr, scheme_or_model in get_class_attributes(self).items(): + if attr in self.__PRIVATE_ATTRIBUTES__: + continue + if isinstance(scheme_or_model, list): + assert len(scheme_or_model) == 1 + if data.get(attr) is None: + setattr(self, attr, None) + continue + + if not isinstance(data.get(attr), list): + raise DataModelException("Value should be a list for %s" % (attr)) + setattr( + self, + attr, + [ + self._init_type(scheme_or_model[0], v, attr) + for v in data.get(attr) + ], + ) + else: + setattr( + self, attr, self._init_type(scheme_or_model, data.get(attr), attr) + ) + + def __repr__(self): + attrs = [] + for name, value in get_class_attributes(self).items(): + if name in self.__PRIVATE_ATTRIBUTES__: + continue + attrs.append('%s="%s"' % (name, value)) + return "<%s %s>" % (self.__class__.__name__, " ".join(attrs)) + + def _init_type(self, type_, value, attr): + if inspect.isclass(type_) and issubclass(type_, DataModel): + return type_(value) + if isinstance(type_, DataField): + return type_.validate(value, parent=self, attr=attr) + raise DataModelException("Undeclared or unknown data type for %s" % attr) + + def as_dict(self): + result = {} + for name, value in get_class_attributes(self).items(): + if name in self.__PRIVATE_ATTRIBUTES__: + continue + if isinstance(value, DataModel): + result[name] = value.as_dict() + elif value and isinstance(value, list) and isinstance(value[0], DataModel): + result[name] = value[0].as_dict() + else: + result[name] = value + return result diff --git a/platformio/package/__init__.py b/platformio/package/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/package/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/package/manifest/__init__.py b/platformio/package/manifest/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/package/manifest/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/package/manifest/model.py b/platformio/package/manifest/model.py new file mode 100644 index 00000000..63bad52e --- /dev/null +++ b/platformio/package/manifest/model.py @@ -0,0 +1,57 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import semantic_version + +from platformio.datamodel import DataField, DataModel + + +class AuthorModel(DataModel): + name = DataField(max_length=50, required=True) + email = DataField(max_length=50) + maintainer = DataField(default=False, type=bool) + url = DataField(max_length=255) + + +class RepositoryModel(DataModel): + type = DataField(max_length=3, required=True) + url = DataField(max_length=255, required=True) + branch = DataField(max_length=50) + + +class ExportModel(DataModel): + include = [DataField()] + exclude = [DataField()] + + +class ManifestModel(DataModel): + + name = DataField(max_length=100, required=True) + version = DataField( + required=True, + max_length=50, + validate_factory=lambda v: v if semantic_version.Version.coerce(v) else None, + ) + + description = DataField(max_length=1000) + keywords = [DataField(max_length=255, regex=r"^[a-z][a-z\d\- ]*[a-z]$")] + authors = [AuthorModel] + + homepage = DataField(max_length=255) + license = DataField(max_length=255) + platforms = [DataField(max_length=50, regex=r"^[a-z\d\-_\*]+$")] + frameworks = [DataField(max_length=50, regex=r"^[a-z\d\-_\*]+$")] + + repository = RepositoryModel + export = ExportModel diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py new file mode 100644 index 00000000..faf576c1 --- /dev/null +++ b/platformio/package/manifest/parser.py @@ -0,0 +1,369 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import re + +import requests + +from platformio.compat import get_class_attributes, string_types +from platformio.exception import PlatformioException +from platformio.fs import get_file_contents +from platformio.package.manifest.model import ManifestModel + +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + + +class ManifestException(PlatformioException): + pass + + +class ManifestParserException(ManifestException): + pass + + +class ManifestFileType(object): + PLATFORM_JSON = "platform.json" + LIBRARY_JSON = "library.json" + LIBRARY_PROPERTIES = "library.properties" + MODULE_JSON = "module.json" + PACKAGE_JSON = "package.json" + + @classmethod + def from_uri(cls, uri): + if uri.endswith(".properties"): + return ManifestFileType.LIBRARY_PROPERTIES + if uri.endswith("platform.json"): + return ManifestFileType.PLATFORM_JSON + if uri.endswith("module.json"): + return ManifestFileType.MODULE_JSON + if uri.endswith("package.json"): + return ManifestFileType.PACKAGE_JSON + return ManifestFileType.LIBRARY_JSON + + +class ManifestFactory(object): + @staticmethod + def type_to_clsname(type_): + type_ = type_.replace(".", " ") + type_ = type_.title() + return "%sManifestParser" % type_.replace(" ", "") + + @staticmethod + def new_from_file(path): + if not path or not os.path.isfile(path): + raise ManifestException("Manifest file does not exist %s" % path) + for type_ in get_class_attributes(ManifestFileType).values(): + if path.endswith(type_): + return ManifestFactory.new(get_file_contents(path), type_) + raise ManifestException("Unknown manifest file type %s" % path) + + @staticmethod + def new_from_url(remote_url): + r = requests.get(remote_url) + r.raise_for_status() + return ManifestFactory.new( + r.text, ManifestFileType.from_uri(remote_url), remote_url + ) + + @staticmethod + def new(contents, type_, remote_url=None): + clsname = ManifestFactory.type_to_clsname(type_) + if clsname not in globals(): + raise ManifestException("Unknown manifest file type %s" % clsname) + mp = globals()[clsname](contents, remote_url) + return ManifestModel(mp.as_dict()) + + +class BaseManifestParser(object): + def __init__(self, contents, remote_url=None): + self.remote_url = remote_url + self._data = self.parse(contents) + + def parse(self, contents): + raise NotImplementedError + + def as_dict(self): + return self._data + + @staticmethod + def _cleanup_author(author): + if author.get("email"): + author["email"] = re.sub(r"\s+[aA][tT]\s+", "@", author["email"]) + return author + + @staticmethod + def parse_author_name_and_email(raw): + if raw == "None" or "://" in raw: + return (None, None) + name = raw + email = None + for ldel, rdel in [("<", ">"), ("(", ")")]: + if ldel in raw and rdel in raw: + name = raw[: raw.index(ldel)] + email = raw[raw.index(ldel) + 1 : raw.index(rdel)] + return (name.strip(), email.strip() if email else None) + + +class LibraryJsonManifestParser(BaseManifestParser): + def parse(self, contents): + data = json.loads(contents) + data = self._process_renamed_fields(data) + + # normalize Union[str, list] fields + for k in ("keywords", "platforms", "frameworks"): + if k in data: + data[k] = self._str_to_list(data[k], sep=",") + + if "authors" in data: + data["authors"] = self._parse_authors(data["authors"]) + if "platforms" in data: + data["platforms"] = self._parse_platforms(data["platforms"]) or None + + return data + + @staticmethod + def _str_to_list(value, sep=",", lowercase=True): + if isinstance(value, string_types): + value = value.split(sep) + assert isinstance(value, list) + result = [] + for item in value: + item = item.strip() + if not item: + continue + if lowercase: + item = item.lower() + result.append(item) + return result + + @staticmethod + def _process_renamed_fields(data): + if "url" in data: + data["homepage"] = data["url"] + del data["url"] + + for key in ("include", "exclude"): + if key not in data: + continue + if "export" not in data: + data["export"] = {} + data["export"][key] = ( + data[key] if isinstance(data[key], list) else [data[key]] + ) + del data[key] + + return data + + def _parse_authors(self, raw): + if not raw: + return None + # normalize Union[dict, list] fields + if not isinstance(raw, list): + raw = [raw] + return [self._cleanup_author(author) for author in raw] + + @staticmethod + def _parse_platforms(raw): + assert isinstance(raw, list) + result = [] + # renamed platforms + for item in raw: + if item == "espressif": + item = "espressif8266" + result.append(item) + return result + + +class ModuleJsonManifestParser(BaseManifestParser): + def parse(self, contents): + data = json.loads(contents) + return dict( + name=data["name"], + version=data["version"], + keywords=data.get("keywords"), + description=data["description"], + frameworks=["mbed"], + platforms=["*"], + homepage=data.get("homepage"), + export={"exclude": ["tests", "test", "*.doxyfile", "*.pdf"]}, + authors=self._parse_authors(data.get("author")), + license=self._parse_license(data.get("licenses")), + ) + + def _parse_authors(self, raw): + if not raw: + return None + result = [] + for author in raw.split(","): + name, email = self.parse_author_name_and_email(author) + if not name: + continue + result.append( + self._cleanup_author(dict(name=name, email=email, maintainer=False)) + ) + return result + + @staticmethod + def _parse_license(raw): + if not raw or not isinstance(raw, list): + return None + return raw[0].get("type") + + +class LibraryPropertiesManifestParser(BaseManifestParser): + def parse(self, contents): + properties = self._parse_properties(contents) + repository = self._parse_repository(properties) + homepage = properties.get("url") + if repository and repository["url"] == homepage: + homepage = None + return dict( + name=properties["name"], + version=properties["version"], + description=properties["sentence"], + frameworks=["arduino"], + platforms=self._process_platforms(properties) or ["*"], + keywords=self._parse_keywords(properties), + authors=self._parse_authors(properties) or None, + homepage=homepage, + repository=repository or None, + export=self._parse_export(), + ) + + @staticmethod + def _parse_properties(contents): + data = {} + for line in contents.splitlines(): + line = line.strip() + if not line or "=" not in line: + continue + # skip comments + if line.startswith("#"): + continue + key, value = line.split("=", 1) + data[key.strip()] = value.strip() + + required_fields = set(["name", "version", "author", "sentence"]) + if not set(data.keys()) >= required_fields: + raise ManifestParserException( + "Missing fields: " + ",".join(required_fields - set(data.keys())) + ) + return data + + @staticmethod + def _parse_keywords(properties): + result = [] + for item in re.split(r"[\s/]+", properties.get("category", "uncategorized")): + item = item.strip() + if not item: + continue + result.append(item.lower()) + return result + + @staticmethod + def _process_platforms(properties): + result = [] + platforms_map = { + "avr": "atmelavr", + "sam": "atmelsam", + "samd": "atmelsam", + "esp8266": "espressif8266", + "esp32": "espressif32", + "arc32": "intel_arc32", + "stm32": "ststm32", + } + for arch in properties.get("architectures", "").split(","): + if "particle-" in arch: + raise ManifestParserException("Particle is not supported yet") + arch = arch.strip() + if not arch: + continue + if arch == "*": + return ["*"] + if arch in platforms_map: + result.append(platforms_map[arch]) + return result + + def _parse_authors(self, properties): + authors = [] + for author in properties["author"].split(","): + name, email = self.parse_author_name_and_email(author) + if not name: + continue + authors.append( + self._cleanup_author(dict(name=name, email=email, maintainer=False)) + ) + for author in properties.get("maintainer", "").split(","): + name, email = self.parse_author_name_and_email(author) + if not name: + continue + found = False + for item in authors: + if item["name"].lower() != name.lower(): + continue + found = True + item["maintainer"] = True + if not item["email"]: + item["email"] = email + if not found: + authors.append( + self._cleanup_author(dict(name=name, email=email, maintainer=True)) + ) + return authors + + def _parse_repository(self, properties): + if self.remote_url: + repo_parse = urlparse(self.remote_url) + repo_path_tokens = repo_parse.path[1:].split("/")[:-1] + if "github" in repo_parse.netloc: + return dict( + type="git", + url="%s://github.com/%s" + % (repo_parse.scheme, "/".join(repo_path_tokens[:2])), + ) + if "raw" in repo_path_tokens: + return dict( + type="git", + url="%s://%s/%s" + % ( + repo_parse.scheme, + repo_parse.netloc, + "/".join(repo_path_tokens[: repo_path_tokens.index("raw")]), + ), + ) + if properties.get("url", "").startswith("https://github.com"): + return dict(type="git", url=properties["url"]) + return None + + def _parse_export(self): + include = None + if self.remote_url: + repo_parse = urlparse(self.remote_url) + repo_path_tokens = repo_parse.path[1:].split("/")[:-1] + if "github" in repo_parse.netloc: + include = "/".join(repo_path_tokens[3:]) or None + elif "raw" in repo_path_tokens: + include = ( + "/".join(repo_path_tokens[repo_path_tokens.index("raw") + 2 :]) + or None + ) + return { + "include": include, + "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"], + } diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index b2c1dc9c..8ca36add 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -13,32 +13,196 @@ # limitations under the License. import pytest -import requests + +from platformio.package.manifest import parser -def validate_response(r): - assert r.status_code == 200, r.url - assert int(r.headers["Content-Length"]) > 0, r.url - assert r.headers["Content-Type"] in ("application/gzip", "application/octet-stream") +def test_library_json_parser(): + contents = """ +{ + "name": "TestPackage", + "keywords": "kw1, KW2, kw3", + "platforms": ["atmelavr", "espressif"], + "url": "http://old.url.format", + "exclude": [".gitignore", "tests"], + "include": "mylib" +} +""" + mp = parser.LibraryJsonManifestParser(contents) + assert sorted(mp.as_dict().items()) == sorted( + { + "name": "TestPackage", + "platforms": ["atmelavr", "espressif8266"], + "export": {"exclude": [".gitignore", "tests"], "include": ["mylib"]}, + "keywords": ["kw1", "kw2", "kw3"], + "homepage": "http://old.url.format", + }.items() + ) -def test_packages(): - pkgs_manifest = requests.get( - "https://dl.bintray.com/platformio/dl-packages/manifest.json" - ).json() - assert isinstance(pkgs_manifest, dict) - items = [] - for _, variants in pkgs_manifest.items(): - for item in variants: - items.append(item) +def test_module_json_parser(): + contents = """ +{ + "author": "Name Surname ", + "description": "This is Yotta library", + "homepage": "https://yottabuild.org", + "keywords": [ + "mbed", + "Yotta" + ], + "licenses": [ + { + "type": "Apache-2.0", + "url": "https://spdx.org/licenses/Apache-2.0" + } + ], + "name": "YottaLibrary", + "repository": { + "type": "git", + "url": "git@github.com:username/repo.git" + }, + "version": "1.2.3" +} +""" + mp = parser.ModuleJsonManifestParser(contents) + assert sorted(mp.as_dict().items()) == sorted( + { + "name": "YottaLibrary", + "description": "This is Yotta library", + "homepage": "https://yottabuild.org", + "keywords": ["mbed", "Yotta"], + "license": "Apache-2.0", + "platforms": ["*"], + "frameworks": ["mbed"], + "export": {"exclude": ["tests", "test", "*.doxyfile", "*.pdf"]}, + "authors": [ + { + "maintainer": False, + "email": "name@surname.com", + "name": "Name Surname", + } + ], + "version": "1.2.3", + }.items() + ) - for item in items: - assert item["url"].endswith(".tar.gz"), item - r = requests.head(item["url"], allow_redirects=True) - validate_response(r) +def test_library_properties_parser(): + # test missed fields + with pytest.raises(parser.ManifestParserException): + parser.LibraryPropertiesManifestParser("name=TestPackage") - if "X-Checksum-Sha1" not in r.headers: - return pytest.skip("X-Checksum-Sha1 is not provided") + # Base + contents = """ +name=TestPackage +version=1.2.3 +author=SomeAuthor +sentence=This is Arduino library +""" + mp = parser.LibraryPropertiesManifestParser(contents) + assert sorted(mp.as_dict().items()) == sorted( + { + "name": "TestPackage", + "version": "1.2.3", + "description": "This is Arduino library", + "repository": None, + "platforms": ["*"], + "frameworks": ["arduino"], + "export": { + "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"], + "include": None, + }, + "authors": [ + {"maintainer": False, "email": "info@author.com", "name": "SomeAuthor"} + ], + "keywords": ["uncategorized"], + "homepage": None, + }.items() + ) - assert item["sha1"] == r.headers.get("X-Checksum-Sha1")[0:40], item + # Platforms ALL + mp = parser.LibraryPropertiesManifestParser("architectures=*\n" + contents) + assert mp.as_dict()["platforms"] == ["*"] + # Platforms specific + mp = parser.LibraryPropertiesManifestParser("architectures=avr, esp32\n" + contents) + assert mp.as_dict()["platforms"] == ["atmelavr", "espressif32"] + + # Remote URL + mp = parser.LibraryPropertiesManifestParser( + contents, + remote_url=( + "https://raw.githubusercontent.com/username/reponame/master/" + "libraries/TestPackage/library.properties" + ), + ) + assert mp.as_dict()["export"] == { + "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"], + "include": "libraries/TestPackage", + } + + # Hope page + mp = parser.LibraryPropertiesManifestParser( + "url=https://github.com/username/reponame.git\n" + contents + ) + assert mp.as_dict()["homepage"] is None + assert mp.as_dict()["repository"] == { + "type": "git", + "url": "https://github.com/username/reponame.git", + } + + +def test_library_json_model(): + contents = """ +{ + "name": "ArduinoJson", + "keywords": "JSON, rest, http, web", + "description": "An elegant and efficient JSON library for embedded systems", + "homepage": "https://arduinojson.org", + "repository": { + "type": "git", + "url": "https://github.com/bblanchon/ArduinoJson.git" + }, + "version": "6.12.0", + "authors": { + "name": "Benoit Blanchon", + "url": "https://blog.benoitblanchon.fr" + }, + "exclude": [ + "fuzzing", + "scripts", + "test", + "third-party" + ], + "frameworks": "arduino", + "platforms": "*", + "license": "MIT" +} +""" + model = parser.ManifestFactory.new(contents, parser.ManifestFileType.LIBRARY_JSON) + assert sorted(model.as_dict().items()) == sorted( + { + "name": "ArduinoJson", + "keywords": ["json", "rest", "http", "web"], + "description": "An elegant and efficient JSON library for embedded systems", + "homepage": "https://arduinojson.org", + "repository": { + "url": "https://github.com/bblanchon/ArduinoJson.git", + "type": "git", + "branch": None, + }, + "version": "6.12.0", + "authors": { + "url": "https://blog.benoitblanchon.fr", + "maintainer": False, + "email": None, + "name": "Benoit Blanchon", + }, + "export": { + "include": None, + "exclude": ["fuzzing", "scripts", "test", "third-party"], + }, + "frameworks": ["arduino"], + "platforms": ["*"], + "license": "MIT", + }.items() + ) From 744881da59f6d586cf7512350b68206f7d5a24ff Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 30 Sep 2019 19:44:03 +0300 Subject: [PATCH 046/221] Refactor DataModel with a strict type declaration --- platformio/commands/lib.py | 5 +- platformio/datamodel.py | 125 ++++++++++++-------------- platformio/package/manifest/model.py | 35 +++++--- platformio/package/manifest/parser.py | 18 ++-- tests/test_pkgmanifest.py | 27 +++--- 5 files changed, 100 insertions(+), 110 deletions(-) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 8f61769f..897a8a39 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -25,6 +25,7 @@ from platformio import exception, util from platformio.commands import PlatformioCLI from platformio.compat import dump_json_to_unicode from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib +from platformio.package.manifest.model import ManifestModel from platformio.package.manifest.parser import ManifestFactory from platformio.proc import is_ci from platformio.project.config import ProjectConfig @@ -493,8 +494,8 @@ def lib_register(config_url): if not config_url.startswith("http://") and not config_url.startswith("https://"): raise exception.InvalidLibConfURL(config_url) - manifest = ManifestFactory.new_from_url(config_url) - assert set(["name", "version"]) & set(list(manifest.as_dict())) + model = ManifestModel(**ManifestFactory.new_from_url(config_url).as_dict()) + assert set(["name", "version"]) & set(list(model.as_dict())) result = util.get_api_result("/lib/register", data=dict(config_url=config_url)) if "message" in result and result["message"]: diff --git a/platformio/datamodel.py b/platformio/datamodel.py index 39547d59..2a754e6d 100644 --- a/platformio/datamodel.py +++ b/platformio/datamodel.py @@ -26,6 +26,11 @@ class DataModelException(PlatformioException): pass +class ListOfType(object): + def __init__(self, type): + self.type = type + + class DataField(object): def __init__( self, @@ -47,6 +52,8 @@ class DataField(object): self.validate_factory = validate_factory self.title = title + self._parent = None + self._name = None self._value = None def __repr__(self): @@ -55,36 +62,53 @@ class DataField(object): self.default if self._value is None else self._value, ) - def validate(self, value, parent, attr): - if self.title is None: - self.title = attr.title() + def validate(self, parent, name, value): + self._parent = parent + self._name = name + self.title = self.title or name.title() + try: if self.required and value is None: - raise ValueError("Required field, value is None") + raise ValueError("Required field `%s` is None" % name) if self.validate_factory is not None: - value = self.validate_factory(value) + return self.validate_factory(self, value) or self.default if value is None: return self.default - if issubclass(self.type, (str, list, bool)): + if inspect.isclass(self.type) and issubclass(self.type, DataModel): + return self.type(**value).as_dict() + if isinstance(self.type, ListOfType): + return self._validate_list_of_type(self.type.type, value) + if issubclass(self.type, (str, bool)): return getattr(self, "_validate_%s_value" % self.type.__name__)(value) - except (AssertionError, ValueError) as e: + except ValueError as e: raise DataModelException( - "%s for %s.%s" % (str(e), parent.__class__.__name__, attr) + "%s for %s.%s" % (str(e), parent.__class__.__name__, name) ) return value + def _validate_list_of_type(self, list_of_type, value): + if not isinstance(value, list): + raise ValueError("Value should be a list") + if isinstance(list_of_type, DataField): + return [list_of_type.validate(self._parent, self._name, v) for v in value] + assert issubclass(list_of_type, DataModel) + return [list_of_type(**v).as_dict() for v in value] + def _validate_str_value(self, value): if not isinstance(value, string_types): value = str(value) - assert self.min_length is None or len(value) >= self.min_length, ( - "Minimum allowed length is %d characters" % self.min_length - ) - assert self.max_length is None or len(value) <= self.max_length, ( - "Maximum allowed length is %d characters" % self.max_length - ) - assert self.regex is None or re.match( - self.regex, value - ), "Value `%s` does not match RegExp `%s` pattern" % (value, self.regex) + if self.min_length and len(value) < self.min_length: + raise ValueError( + "Minimum allowed length is %d characters" % self.min_length + ) + if self.max_length and len(value) > self.max_length: + raise ValueError( + "Maximum allowed length is %d characters" % self.max_length + ) + if self.regex and not re.match(self.regex, value): + raise ValueError( + "Value `%s` does not match RegExp `%s` pattern" % (value, self.regex) + ) return value @staticmethod @@ -95,60 +119,21 @@ class DataField(object): class DataModel(object): - __PRIVATE_ATTRIBUTES__ = ("__PRIVATE_ATTRIBUTES__", "_init_type", "as_dict") - - def __init__(self, data=None): - data = data or {} - assert isinstance(data, dict) - - for attr, scheme_or_model in get_class_attributes(self).items(): - if attr in self.__PRIVATE_ATTRIBUTES__: + def __init__(self, **kwargs): + self._known_attributes = [] + for name, field in get_class_attributes(self).items(): + if not isinstance(field, DataField): continue - if isinstance(scheme_or_model, list): - assert len(scheme_or_model) == 1 - if data.get(attr) is None: - setattr(self, attr, None) - continue + self._known_attributes.append(name) + setattr(self, name, field.validate(self, name, kwargs.get(name))) - if not isinstance(data.get(attr), list): - raise DataModelException("Value should be a list for %s" % (attr)) - setattr( - self, - attr, - [ - self._init_type(scheme_or_model[0], v, attr) - for v in data.get(attr) - ], - ) - else: - setattr( - self, attr, self._init_type(scheme_or_model, data.get(attr), attr) - ) - - def __repr__(self): - attrs = [] - for name, value in get_class_attributes(self).items(): - if name in self.__PRIVATE_ATTRIBUTES__: - continue - attrs.append('%s="%s"' % (name, value)) - return "<%s %s>" % (self.__class__.__name__, " ".join(attrs)) - - def _init_type(self, type_, value, attr): - if inspect.isclass(type_) and issubclass(type_, DataModel): - return type_(value) - if isinstance(type_, DataField): - return type_.validate(value, parent=self, attr=attr) - raise DataModelException("Undeclared or unknown data type for %s" % attr) + # def __repr__(self): + # attrs = [] + # for name, value in get_class_attributes(self).items(): + # if name in self.__PRIVATE_ATTRIBUTES__: + # continue + # attrs.append('%s="%s"' % (name, value)) + # return "<%s %s>" % (self.__class__.__name__, " ".join(attrs)) def as_dict(self): - result = {} - for name, value in get_class_attributes(self).items(): - if name in self.__PRIVATE_ATTRIBUTES__: - continue - if isinstance(value, DataModel): - result[name] = value.as_dict() - elif value and isinstance(value, list) and isinstance(value[0], DataModel): - result[name] = value[0].as_dict() - else: - result[name] = value - return result + return {name: getattr(self, name) for name in self._known_attributes} diff --git a/platformio/package/manifest/model.py b/platformio/package/manifest/model.py index 63bad52e..3e3fd4ea 100644 --- a/platformio/package/manifest/model.py +++ b/platformio/package/manifest/model.py @@ -14,7 +14,7 @@ import semantic_version -from platformio.datamodel import DataField, DataModel +from platformio.datamodel import DataField, DataModel, ListOfType class AuthorModel(DataModel): @@ -31,27 +31,36 @@ class RepositoryModel(DataModel): class ExportModel(DataModel): - include = [DataField()] - exclude = [DataField()] + include = DataField(type=ListOfType(DataField())) + exclude = DataField(type=ListOfType(DataField())) class ManifestModel(DataModel): + # Required fields name = DataField(max_length=100, required=True) version = DataField( - required=True, max_length=50, - validate_factory=lambda v: v if semantic_version.Version.coerce(v) else None, + validate_factory=lambda field, value: value + if semantic_version.Version.coerce(value) + else None, + required=True, ) - - description = DataField(max_length=1000) - keywords = [DataField(max_length=255, regex=r"^[a-z][a-z\d\- ]*[a-z]$")] - authors = [AuthorModel] + description = DataField(max_length=1000, required=True) + keywords = DataField( + type=ListOfType(DataField(max_length=255, regex=r"^[a-z][a-z\d\- ]*[a-z]$")), + required=True, + ) + authors = DataField(type=ListOfType(AuthorModel), required=True) homepage = DataField(max_length=255) license = DataField(max_length=255) - platforms = [DataField(max_length=50, regex=r"^[a-z\d\-_\*]+$")] - frameworks = [DataField(max_length=50, regex=r"^[a-z\d\-_\*]+$")] + platforms = DataField( + type=ListOfType(DataField(max_length=50, regex=r"^([a-z\d\-_]+|\*)$")) + ) + frameworks = DataField( + type=ListOfType(DataField(max_length=50, regex=r"^([a-z\d\-_\*]+|\*)$")) + ) - repository = RepositoryModel - export = ExportModel + repository = DataField(type=RepositoryModel) + export = DataField(type=ExportModel) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index faf576c1..580a9465 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -21,7 +21,6 @@ import requests from platformio.compat import get_class_attributes, string_types from platformio.exception import PlatformioException from platformio.fs import get_file_contents -from platformio.package.manifest.model import ManifestModel try: from urllib.parse import urlparse @@ -86,8 +85,7 @@ class ManifestFactory(object): clsname = ManifestFactory.type_to_clsname(type_) if clsname not in globals(): raise ManifestException("Unknown manifest file type %s" % clsname) - mp = globals()[clsname](contents, remote_url) - return ManifestModel(mp.as_dict()) + return globals()[clsname](contents, remote_url) class BaseManifestParser(object): @@ -234,9 +232,9 @@ class LibraryPropertiesManifestParser(BaseManifestParser): if repository and repository["url"] == homepage: homepage = None return dict( - name=properties["name"], - version=properties["version"], - description=properties["sentence"], + name=properties.get("name"), + version=properties.get("version"), + description=properties.get("sentence"), frameworks=["arduino"], platforms=self._process_platforms(properties) or ["*"], keywords=self._parse_keywords(properties), @@ -258,12 +256,6 @@ class LibraryPropertiesManifestParser(BaseManifestParser): continue key, value = line.split("=", 1) data[key.strip()] = value.strip() - - required_fields = set(["name", "version", "author", "sentence"]) - if not set(data.keys()) >= required_fields: - raise ManifestParserException( - "Missing fields: " + ",".join(required_fields - set(data.keys())) - ) return data @staticmethod @@ -301,6 +293,8 @@ class LibraryPropertiesManifestParser(BaseManifestParser): return result def _parse_authors(self, properties): + if "author" not in properties: + return None authors = [] for author in properties["author"].split(","): name, email = self.parse_author_name_and_email(author) diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 8ca36add..1a09852a 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -14,7 +14,9 @@ import pytest +from platformio.datamodel import DataModelException from platformio.package.manifest import parser +from platformio.package.manifest.model import ManifestModel def test_library_json_parser(): @@ -88,10 +90,6 @@ def test_module_json_parser(): def test_library_properties_parser(): - # test missed fields - with pytest.raises(parser.ManifestParserException): - parser.LibraryPropertiesManifestParser("name=TestPackage") - # Base contents = """ name=TestPackage @@ -151,7 +149,7 @@ sentence=This is Arduino library } -def test_library_json_model(): +def test_library_json_valid_model(): contents = """ { "name": "ArduinoJson", @@ -178,7 +176,8 @@ def test_library_json_model(): "license": "MIT" } """ - model = parser.ManifestFactory.new(contents, parser.ManifestFileType.LIBRARY_JSON) + data = parser.ManifestFactory.new(contents, parser.ManifestFileType.LIBRARY_JSON) + model = ManifestModel(**data.as_dict()) assert sorted(model.as_dict().items()) == sorted( { "name": "ArduinoJson", @@ -191,15 +190,17 @@ def test_library_json_model(): "branch": None, }, "version": "6.12.0", - "authors": { - "url": "https://blog.benoitblanchon.fr", - "maintainer": False, - "email": None, - "name": "Benoit Blanchon", - }, + "authors": [ + { + "url": "https://blog.benoitblanchon.fr", + "maintainer": False, + "email": None, + "name": "Benoit Blanchon", + } + ], "export": { - "include": None, "exclude": ["fuzzing", "scripts", "test", "third-party"], + "include": None, }, "frameworks": ["arduino"], "platforms": ["*"], From 703912fdc9a7928eab0032b2667bd2ce77170189 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 30 Sep 2019 23:45:03 +0300 Subject: [PATCH 047/221] Strict manifest validation when submitting to Registry, more tests for manifest model --- platformio/commands/lib.py | 6 +-- platformio/datamodel.py | 16 +++--- platformio/package/manifest/model.py | 27 ++++++---- platformio/package/manifest/parser.py | 24 ++++++--- tests/test_pkgmanifest.py | 77 ++++++++++++++++++++++++++- 5 files changed, 121 insertions(+), 29 deletions(-) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 897a8a39..18ec9da2 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -25,7 +25,7 @@ from platformio import exception, util from platformio.commands import PlatformioCLI from platformio.compat import dump_json_to_unicode from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib -from platformio.package.manifest.model import ManifestModel +from platformio.package.manifest.model import StrictManifestModel from platformio.package.manifest.parser import ManifestFactory from platformio.proc import is_ci from platformio.project.config import ProjectConfig @@ -494,8 +494,8 @@ def lib_register(config_url): if not config_url.startswith("http://") and not config_url.startswith("https://"): raise exception.InvalidLibConfURL(config_url) - model = ManifestModel(**ManifestFactory.new_from_url(config_url).as_dict()) - assert set(["name", "version"]) & set(list(model.as_dict())) + # Validate manifest + StrictManifestModel(**ManifestFactory.new_from_url(config_url).as_dict()) result = util.get_api_result("/lib/register", data=dict(config_url=config_url)) if "message" in result and result["message"]: diff --git a/platformio/datamodel.py b/platformio/datamodel.py index 2a754e6d..0231913b 100644 --- a/platformio/datamodel.py +++ b/platformio/datamodel.py @@ -69,7 +69,7 @@ class DataField(object): try: if self.required and value is None: - raise ValueError("Required field `%s` is None" % name) + raise ValueError("Missed value") if self.validate_factory is not None: return self.validate_factory(self, value) or self.default if value is None: @@ -82,7 +82,7 @@ class DataField(object): return getattr(self, "_validate_%s_value" % self.type.__name__)(value) except ValueError as e: raise DataModelException( - "%s for %s.%s" % (str(e), parent.__class__.__name__, name) + "%s for `%s.%s` field" % (str(e), parent.__class__.__name__, name) ) return value @@ -127,13 +127,11 @@ class DataModel(object): self._known_attributes.append(name) setattr(self, name, field.validate(self, name, kwargs.get(name))) - # def __repr__(self): - # attrs = [] - # for name, value in get_class_attributes(self).items(): - # if name in self.__PRIVATE_ATTRIBUTES__: - # continue - # attrs.append('%s="%s"' % (name, value)) - # return "<%s %s>" % (self.__class__.__name__, " ".join(attrs)) + def __repr__(self): + fields = [] + for name in self._known_attributes: + fields.append('%s="%s"' % (name, getattr(self, name))) + return "<%s %s>" % (self.__class__.__name__, " ".join(fields)) def as_dict(self): return {name: getattr(self, name) for name in self._known_attributes} diff --git a/platformio/package/manifest/model.py b/platformio/package/manifest/model.py index 3e3fd4ea..b55795d3 100644 --- a/platformio/package/manifest/model.py +++ b/platformio/package/manifest/model.py @@ -17,6 +17,12 @@ import semantic_version from platformio.datamodel import DataField, DataModel, ListOfType +def validate_semver_field(_, value): + if "." not in value: + raise ValueError("Invalid semantic versioning format") + return value if semantic_version.Version.coerce(value) else None + + class AuthorModel(DataModel): name = DataField(max_length=50, required=True) email = DataField(max_length=50) @@ -40,18 +46,14 @@ class ManifestModel(DataModel): # Required fields name = DataField(max_length=100, required=True) version = DataField( - max_length=50, - validate_factory=lambda field, value: value - if semantic_version.Version.coerce(value) - else None, - required=True, + max_length=50, validate_factory=validate_semver_field, required=True ) - description = DataField(max_length=1000, required=True) + + description = DataField(max_length=1000) keywords = DataField( - type=ListOfType(DataField(max_length=255, regex=r"^[a-z][a-z\d\- ]*[a-z]$")), - required=True, + type=ListOfType(DataField(max_length=255, regex=r"^[a-z][a-z\d\- ]*[a-z]$")) ) - authors = DataField(type=ListOfType(AuthorModel), required=True) + authors = DataField(type=ListOfType(AuthorModel)) homepage = DataField(max_length=255) license = DataField(max_length=255) @@ -64,3 +66,10 @@ class ManifestModel(DataModel): repository = DataField(type=RepositoryModel) export = DataField(type=ExportModel) + + +class StrictManifestModel(ManifestModel): + def __init__(self, *args, **kwargs): + for name in ("description", "keywords", "authors"): + getattr(self, name).required = True + super(StrictManifestModel, self).__init__(*args, **kwargs) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 580a9465..90be3a9b 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -232,15 +232,15 @@ class LibraryPropertiesManifestParser(BaseManifestParser): if repository and repository["url"] == homepage: homepage = None return dict( - name=properties.get("name"), - version=properties.get("version"), - description=properties.get("sentence"), frameworks=["arduino"], - platforms=self._process_platforms(properties) or ["*"], - keywords=self._parse_keywords(properties), - authors=self._parse_authors(properties) or None, homepage=homepage, repository=repository or None, + name=properties.get("name"), + version=properties.get("version"), + description=self._parse_description(properties), + platforms=self._parse_platforms(properties) or ["*"], + keywords=self._parse_keywords(properties), + authors=self._parse_authors(properties) or None, export=self._parse_export(), ) @@ -258,6 +258,16 @@ class LibraryPropertiesManifestParser(BaseManifestParser): data[key.strip()] = value.strip() return data + @staticmethod + def _parse_description(properties): + lines = [] + for k in ("sentence", "paragraph"): + if k in properties and properties[k] not in lines: + lines.append(properties[k]) + if len(lines) == 2 and not lines[0].endswith("."): + lines[0] += "." + return " ".join(lines) + @staticmethod def _parse_keywords(properties): result = [] @@ -269,7 +279,7 @@ class LibraryPropertiesManifestParser(BaseManifestParser): return result @staticmethod - def _process_platforms(properties): + def _parse_platforms(properties): result = [] platforms_map = { "avr": "atmelavr", diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 1a09852a..e631af94 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -16,7 +16,7 @@ import pytest from platformio.datamodel import DataModelException from platformio.package.manifest import parser -from platformio.package.manifest.model import ManifestModel +from platformio.package.manifest.model import ManifestModel, StrictManifestModel def test_library_json_parser(): @@ -207,3 +207,78 @@ def test_library_json_valid_model(): "license": "MIT", }.items() ) + + +def test_library_properties_valid_model(): + contents = """ +name=U8glib +version=1.19.1 +author=oliver +maintainer=oliver +sentence=A library for monochrome TFTs and OLEDs +paragraph=Supported display controller: SSD1306, SSD1309, SSD1322, SSD1325 +category=Display +url=https://github.com/olikraus/u8glib +architectures=avr,sam +""" + data = parser.ManifestFactory.new( + contents, parser.ManifestFileType.LIBRARY_PROPERTIES + ) + model = ManifestModel(**data.as_dict()) + assert sorted(model.as_dict().items()) == sorted( + { + "license": None, + "description": ( + "A library for monochrome TFTs and OLEDs. Supported display " + "controller: SSD1306, SSD1309, SSD1322, SSD1325" + ), + "repository": { + "url": "https://github.com/olikraus/u8glib", + "type": "git", + "branch": None, + }, + "frameworks": ["arduino"], + "platforms": ["atmelavr", "atmelsam"], + "version": "1.19.1", + "export": { + "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"], + "include": None, + }, + "authors": [ + { + "url": None, + "maintainer": True, + "email": "olikraus@gmail.com", + "name": "oliver", + } + ], + "keywords": ["display"], + "homepage": None, + "name": "U8glib", + }.items() + ) + + +def test_broken_model(): + # "version" field is required + with pytest.raises( + DataModelException, match="Missed value for `ManifestModel.version` field" + ): + assert ManifestModel(name="MyPackage") + + # broken SemVer + with pytest.raises( + DataModelException, + match="Invalid semantic versioning format for `ManifestModel.version` field", + ): + assert ManifestModel(name="MyPackage", version="broken_version") + + # the only name and version fields are required for base ManifestModel + assert ManifestModel(name="MyPackage", version="1.0") + + # check strict model + with pytest.raises( + DataModelException, + match="Missed value for `StrictManifestModel.description` field", + ): + assert StrictManifestModel(name="MyPackage", version="1.0") From af1a0f35870050633aed53700c4cf96a1e329bae Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 1 Oct 2019 00:11:31 +0300 Subject: [PATCH 048/221] Allow to build a manifest parser from directory --- platformio/commands/lib.py | 4 +-- platformio/package/manifest/parser.py | 43 +++++++++++++++++++-------- tests/test_pkgmanifest.py | 6 ++-- 3 files changed, 37 insertions(+), 16 deletions(-) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 18ec9da2..3f52cb8b 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -26,7 +26,7 @@ from platformio.commands import PlatformioCLI from platformio.compat import dump_json_to_unicode from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib from platformio.package.manifest.model import StrictManifestModel -from platformio.package.manifest.parser import ManifestFactory +from platformio.package.manifest.parser import ManifestParserFactory from platformio.proc import is_ci from platformio.project.config import ProjectConfig from platformio.project.helpers import get_project_dir, is_platformio_project @@ -495,7 +495,7 @@ def lib_register(config_url): raise exception.InvalidLibConfURL(config_url) # Validate manifest - StrictManifestModel(**ManifestFactory.new_from_url(config_url).as_dict()) + StrictManifestModel(**ManifestParserFactory.new_from_url(config_url).as_dict()) result = util.get_api_result("/lib/register", data=dict(config_url=config_url)) if "message" in result and result["message"]: diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 90be3a9b..ec2e713c 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -56,33 +56,52 @@ class ManifestFileType(object): return ManifestFileType.LIBRARY_JSON -class ManifestFactory(object): +class ManifestParserFactory(object): @staticmethod - def type_to_clsname(type_): - type_ = type_.replace(".", " ") - type_ = type_.title() - return "%sManifestParser" % type_.replace(" ", "") + def type_to_clsname(t): + t = t.replace(".", " ") + t = t.title() + return "%sManifestParser" % t.replace(" ", "") @staticmethod - def new_from_file(path): + def new_from_file(path, remote_url=False): if not path or not os.path.isfile(path): raise ManifestException("Manifest file does not exist %s" % path) - for type_ in get_class_attributes(ManifestFileType).values(): - if path.endswith(type_): - return ManifestFactory.new(get_file_contents(path), type_) + for t in get_class_attributes(ManifestFileType).values(): + if path.endswith(t): + return ManifestParserFactory.new(get_file_contents(path), t, remote_url) raise ManifestException("Unknown manifest file type %s" % path) + @staticmethod + def new_from_dir(path, remote_url=None): + assert os.path.isdir(path), "Invalid directory %s" % path + file_order = [ + ManifestFileType.PLATFORM_JSON, + ManifestFileType.LIBRARY_JSON, + ManifestFileType.LIBRARY_PROPERTIES, + ManifestFileType.MODULE_JSON, + ManifestFileType.PACKAGE_JSON, + ] + for t in file_order: + if not os.path.isfile(os.path.join(path, t)): + continue + return ManifestParserFactory.new( + get_file_contents(os.path.join(path, t)), t, remote_url + ) + raise ManifestException("Unknown manifest file type in %s directory" % path) + @staticmethod def new_from_url(remote_url): r = requests.get(remote_url) r.raise_for_status() - return ManifestFactory.new( + return ManifestParserFactory.new( r.text, ManifestFileType.from_uri(remote_url), remote_url ) @staticmethod - def new(contents, type_, remote_url=None): - clsname = ManifestFactory.type_to_clsname(type_) + def new(contents, type, remote_url=None): + # pylint: disable=redefined-builtin + clsname = ManifestParserFactory.type_to_clsname(type) if clsname not in globals(): raise ManifestException("Unknown manifest file type %s" % clsname) return globals()[clsname](contents, remote_url) diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index e631af94..9a71d384 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -176,7 +176,9 @@ def test_library_json_valid_model(): "license": "MIT" } """ - data = parser.ManifestFactory.new(contents, parser.ManifestFileType.LIBRARY_JSON) + data = parser.ManifestParserFactory.new( + contents, parser.ManifestFileType.LIBRARY_JSON + ) model = ManifestModel(**data.as_dict()) assert sorted(model.as_dict().items()) == sorted( { @@ -221,7 +223,7 @@ category=Display url=https://github.com/olikraus/u8glib architectures=avr,sam """ - data = parser.ManifestFactory.new( + data = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.LIBRARY_PROPERTIES ) model = ManifestModel(**data.as_dict()) From 39c8996093c2a379ea90c2b553192703d9b8de53 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 1 Oct 2019 16:11:55 +0300 Subject: [PATCH 049/221] Fix docs typos --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 52c3dc5a..f8f3c977 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 52c3dc5a884d81f85d79b912739f4300d1c7d80d +Subproject commit f8f3c97743af99e71c4f91ecddb594e1a79e2677 From df6a8da2908be2a49394fc4f802d6ce837633003 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 1 Oct 2019 16:13:25 +0300 Subject: [PATCH 050/221] DataModel: add support for silent validation and "get_exceptions" API --- platformio/datamodel.py | 76 +++++++++++++++++++++------ platformio/package/manifest/model.py | 17 +++--- platformio/package/manifest/parser.py | 17 ++++-- tests/test_pkgmanifest.py | 71 ++++++++++++++++--------- 4 files changed, 128 insertions(+), 53 deletions(-) diff --git a/platformio/datamodel.py b/platformio/datamodel.py index 0231913b..5390484c 100644 --- a/platformio/datamodel.py +++ b/platformio/datamodel.py @@ -26,6 +26,20 @@ class DataModelException(PlatformioException): pass +class DataFieldException(DataModelException): + def __init__(self, field, message): + self.field = field + self.message = message + super(DataFieldException, self).__init__() + + def __str__(self): + return "%s for `%s.%s` field" % ( + self.message, + self.field.parent.__class__.__name__, + self.field.name, + ) + + class ListOfType(object): def __init__(self, type): self.type = type @@ -52,19 +66,19 @@ class DataField(object): self.validate_factory = validate_factory self.title = title - self._parent = None - self._name = None - self._value = None + self.parent = None + self.name = None + self.value = None def __repr__(self): return '' % ( self.title, - self.default if self._value is None else self._value, + self.default if self.value is None else self.value, ) def validate(self, parent, name, value): - self._parent = parent - self._name = name + self.parent = parent + self.name = name self.title = self.title or name.title() try: @@ -81,16 +95,14 @@ class DataField(object): if issubclass(self.type, (str, bool)): return getattr(self, "_validate_%s_value" % self.type.__name__)(value) except ValueError as e: - raise DataModelException( - "%s for `%s.%s` field" % (str(e), parent.__class__.__name__, name) - ) + raise DataFieldException(self, str(e)) return value def _validate_list_of_type(self, list_of_type, value): if not isinstance(value, list): raise ValueError("Value should be a list") if isinstance(list_of_type, DataField): - return [list_of_type.validate(self._parent, self._name, v) for v in value] + return [list_of_type.validate(self.parent, self.name, v) for v in value] assert issubclass(list_of_type, DataModel) return [list_of_type(**v).as_dict() for v in value] @@ -119,19 +131,53 @@ class DataField(object): class DataModel(object): + + _field_names = None + _exceptions = None + def __init__(self, **kwargs): - self._known_attributes = [] + self._field_names = [] + self._exceptions = [] for name, field in get_class_attributes(self).items(): if not isinstance(field, DataField): continue - self._known_attributes.append(name) - setattr(self, name, field.validate(self, name, kwargs.get(name))) + self._field_names.append(name) + value = None + try: + value = field.validate(self, name, kwargs.get(name)) + except DataFieldException as e: + self._exceptions.append(e) + if isinstance(self, StrictDataModel): + raise e + finally: + setattr(self, name, value) + + def __eq__(self, other): + assert isinstance(other, DataModel) + if self.get_field_names() != other.get_field_names(): + return False + if self.get_exceptions() != other.get_exceptions(): + return False + for name in self._field_names: + if getattr(self, name) != getattr(other, name): + return False + return True def __repr__(self): fields = [] - for name in self._known_attributes: + for name in self._field_names: fields.append('%s="%s"' % (name, getattr(self, name))) return "<%s %s>" % (self.__class__.__name__, " ".join(fields)) + def get_field_names(self): + return self._field_names + + def get_exceptions(self): + return self._exceptions + def as_dict(self): - return {name: getattr(self, name) for name in self._known_attributes} + return {name: getattr(self, name) for name in self._field_names} + + +class StrictDataModel(DataModel): + pass diff --git a/platformio/package/manifest/model.py b/platformio/package/manifest/model.py index b55795d3..c87f3417 100644 --- a/platformio/package/manifest/model.py +++ b/platformio/package/manifest/model.py @@ -14,7 +14,7 @@ import semantic_version -from platformio.datamodel import DataField, DataModel, ListOfType +from platformio.datamodel import DataField, DataModel, ListOfType, StrictDataModel def validate_semver_field(_, value): @@ -48,12 +48,12 @@ class ManifestModel(DataModel): version = DataField( max_length=50, validate_factory=validate_semver_field, required=True ) - - description = DataField(max_length=1000) + description = DataField(max_length=1000, required=True) keywords = DataField( - type=ListOfType(DataField(max_length=255, regex=r"^[a-z][a-z\d\- ]*[a-z]$")) + type=ListOfType(DataField(max_length=255, regex=r"^[a-z\d\- ]+$")), + required=True, ) - authors = DataField(type=ListOfType(AuthorModel)) + authors = DataField(type=ListOfType(AuthorModel), required=True) homepage = DataField(max_length=255) license = DataField(max_length=255) @@ -68,8 +68,5 @@ class ManifestModel(DataModel): export = DataField(type=ExportModel) -class StrictManifestModel(ManifestModel): - def __init__(self, *args, **kwargs): - for name in ("description", "keywords", "authors"): - getattr(self, name).required = True - super(StrictManifestModel, self).__init__(*args, **kwargs) +class StrictManifestModel(ManifestModel, StrictDataModel): + pass diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index ec2e713c..c2489a97 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -151,6 +151,8 @@ class LibraryJsonManifestParser(BaseManifestParser): data["authors"] = self._parse_authors(data["authors"]) if "platforms" in data: data["platforms"] = self._parse_platforms(data["platforms"]) or None + if "export" in data: + data["export"] = self._parse_export(data["export"]) return data @@ -180,9 +182,7 @@ class LibraryJsonManifestParser(BaseManifestParser): continue if "export" not in data: data["export"] = {} - data["export"][key] = ( - data[key] if isinstance(data[key], list) else [data[key]] - ) + data["export"][key] = data[key] del data[key] return data @@ -206,6 +206,17 @@ class LibraryJsonManifestParser(BaseManifestParser): result.append(item) return result + @staticmethod + def _parse_export(raw): + if not isinstance(raw, dict): + return None + result = {} + for k in ("include", "exclude"): + if k not in raw: + continue + result[k] = raw[k] if isinstance(raw[k], list) else [raw[k]] + return result + class ModuleJsonManifestParser(BaseManifestParser): def parse(self, contents): diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 9a71d384..6bfe93e0 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -14,7 +14,7 @@ import pytest -from platformio.datamodel import DataModelException +from platformio.datamodel import DataFieldException from platformio.package.manifest import parser from platformio.package.manifest.model import ManifestModel, StrictManifestModel @@ -41,6 +41,26 @@ def test_library_json_parser(): }.items() ) + contents = """ +{ + "keywords": ["sound", "audio", "music", "SD", "card", "playback"], + "frameworks": "arduino", + "platforms": "atmelavr", + "export": { + "exclude": "audio_samples" + } +} +""" + mp = parser.LibraryJsonManifestParser(contents) + assert sorted(mp.as_dict().items()) == sorted( + { + "keywords": ["sound", "audio", "music", "sd", "card", "playback"], + "frameworks": ["arduino"], + "export": {"exclude": ["audio_samples"]}, + "platforms": ["atmelavr"], + }.items() + ) + def test_module_json_parser(): contents = """ @@ -180,8 +200,8 @@ def test_library_json_valid_model(): contents, parser.ManifestFileType.LIBRARY_JSON ) model = ManifestModel(**data.as_dict()) - assert sorted(model.as_dict().items()) == sorted( - { + assert model == ManifestModel( + **{ "name": "ArduinoJson", "keywords": ["json", "rest", "http", "web"], "description": "An elegant and efficient JSON library for embedded systems", @@ -207,7 +227,7 @@ def test_library_json_valid_model(): "frameworks": ["arduino"], "platforms": ["*"], "license": "MIT", - }.items() + } ) @@ -227,8 +247,9 @@ architectures=avr,sam contents, parser.ManifestFileType.LIBRARY_PROPERTIES ) model = ManifestModel(**data.as_dict()) - assert sorted(model.as_dict().items()) == sorted( - { + assert not model.get_exceptions() + assert model == ManifestModel( + **{ "license": None, "description": ( "A library for monochrome TFTs and OLEDs. Supported display " @@ -257,30 +278,30 @@ architectures=avr,sam "keywords": ["display"], "homepage": None, "name": "U8glib", - }.items() + } ) def test_broken_model(): - # "version" field is required - with pytest.raises( - DataModelException, match="Missed value for `ManifestModel.version` field" - ): - assert ManifestModel(name="MyPackage") + # non-strict mode + assert len(ManifestModel(name="MyPackage").get_exceptions()) == 4 + assert ManifestModel(name="MyPackage", version="broken_version").version is None + + # strict mode + + with pytest.raises(DataFieldException) as excinfo: + assert StrictManifestModel(name="MyPackage") + assert excinfo.match(r"Missed value for `StrictManifestModel.[a-z]+` field") # broken SemVer with pytest.raises( - DataModelException, - match="Invalid semantic versioning format for `ManifestModel.version` field", + DataFieldException, + match="Invalid semantic versioning format for `StrictManifestModel.version` field", ): - assert ManifestModel(name="MyPackage", version="broken_version") - - # the only name and version fields are required for base ManifestModel - assert ManifestModel(name="MyPackage", version="1.0") - - # check strict model - with pytest.raises( - DataModelException, - match="Missed value for `StrictManifestModel.description` field", - ): - assert StrictManifestModel(name="MyPackage", version="1.0") + assert StrictManifestModel( + name="MyPackage", + description="MyDescription", + keywords=["a", "b"], + authors=[{"name": "Author"}], + version="broken_version", + ) From a82c4666d490c1d6d7c7a35bee1674d4462cc5cd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 1 Oct 2019 17:37:11 +0300 Subject: [PATCH 051/221] DataModel: add support for DictOfType, extend base manifest with ExampelsModel --- platformio/datamodel.py | 16 ++++++++++++++++ platformio/package/manifest/model.py | 14 +++++++++++++- tests/test_pkgmanifest.py | 28 ++++++++++++++++++++++++---- 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/platformio/datamodel.py b/platformio/datamodel.py index 5390484c..63281858 100644 --- a/platformio/datamodel.py +++ b/platformio/datamodel.py @@ -39,12 +39,20 @@ class DataFieldException(DataModelException): self.field.name, ) + def __repr__(self): + return str(self) + class ListOfType(object): def __init__(self, type): self.type = type +class DictOfType(object): + def __init__(self, type): + self.type = type + + class DataField(object): def __init__( self, @@ -92,6 +100,8 @@ class DataField(object): return self.type(**value).as_dict() if isinstance(self.type, ListOfType): return self._validate_list_of_type(self.type.type, value) + if isinstance(self.type, DictOfType): + return self._validate_dict_of_type(self.type.type, value) if issubclass(self.type, (str, bool)): return getattr(self, "_validate_%s_value" % self.type.__name__)(value) except ValueError as e: @@ -106,6 +116,12 @@ class DataField(object): assert issubclass(list_of_type, DataModel) return [list_of_type(**v).as_dict() for v in value] + def _validate_dict_of_type(self, dict_of_type, value): + if not isinstance(value, dict): + raise ValueError("Value should be a dict") + assert issubclass(dict_of_type, DataModel) + return {k: dict_of_type(**v).as_dict() for k, v in value.items()} + def _validate_str_value(self, value): if not isinstance(value, string_types): value = str(value) diff --git a/platformio/package/manifest/model.py b/platformio/package/manifest/model.py index c87f3417..bfbbfa24 100644 --- a/platformio/package/manifest/model.py +++ b/platformio/package/manifest/model.py @@ -14,7 +14,13 @@ import semantic_version -from platformio.datamodel import DataField, DataModel, ListOfType, StrictDataModel +from platformio.datamodel import ( + DataField, + DataModel, + DictOfType, + ListOfType, + StrictDataModel, +) def validate_semver_field(_, value): @@ -41,6 +47,11 @@ class ExportModel(DataModel): exclude = DataField(type=ListOfType(DataField())) +class ExampleModel(DataModel): + base = DataField(required=True) + files = DataField(type=ListOfType(DataField())) + + class ManifestModel(DataModel): # Required fields @@ -66,6 +77,7 @@ class ManifestModel(DataModel): repository = DataField(type=RepositoryModel) export = DataField(type=ExportModel) + examples = DataField(type=DictOfType(ExampleModel)) class StrictManifestModel(ManifestModel, StrictDataModel): diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 6bfe93e0..0262b26a 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -169,7 +169,7 @@ sentence=This is Arduino library } -def test_library_json_valid_model(): +def test_library_json_model(): contents = """ { "name": "ArduinoJson", @@ -193,7 +193,17 @@ def test_library_json_valid_model(): ], "frameworks": "arduino", "platforms": "*", - "license": "MIT" + "license": "MIT", + "examples": { + "JsonConfigFile": { + "base": "examples/JsonConfigFile", + "files": ["JsonConfigFile.ino"] + }, + "JsonHttpClient": { + "base": "examples/JsonHttpClient", + "files": ["JsonHttpClient.ino"] + } + } } """ data = parser.ManifestParserFactory.new( @@ -227,11 +237,21 @@ def test_library_json_valid_model(): "frameworks": ["arduino"], "platforms": ["*"], "license": "MIT", + "examples": { + "JsonConfigFile": { + "base": "examples/JsonConfigFile", + "files": ["JsonConfigFile.ino"], + }, + "JsonHttpClient": { + "base": "examples/JsonHttpClient", + "files": ["JsonHttpClient.ino"], + }, + }, } ) -def test_library_properties_valid_model(): +def library_properties_model(): contents = """ name=U8glib version=1.19.1 @@ -282,7 +302,7 @@ architectures=avr,sam ) -def test_broken_model(): +def test_broken_models(): # non-strict mode assert len(ManifestModel(name="MyPackage").get_exceptions()) == 4 assert ManifestModel(name="MyPackage", version="broken_version").version is None From 5b77adccb150d3611bda0ef4928624ab8db4cb56 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 1 Oct 2019 18:10:48 +0300 Subject: [PATCH 052/221] DataModels: fix issue when traversing using model fields --- platformio/datamodel.py | 38 +++++++++++++++++++++++--------------- tests/test_pkgmanifest.py | 2 ++ 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/platformio/datamodel.py b/platformio/datamodel.py index 63281858..92121114 100644 --- a/platformio/datamodel.py +++ b/platformio/datamodel.py @@ -84,7 +84,9 @@ class DataField(object): self.default if self.value is None else self.value, ) - def validate(self, parent, name, value): + def validate( + self, parent, name, value + ): # pylint: disable=too-many-return-statements self.parent = parent self.name = name self.title = self.title or name.title() @@ -97,7 +99,7 @@ class DataField(object): if value is None: return self.default if inspect.isclass(self.type) and issubclass(self.type, DataModel): - return self.type(**value).as_dict() + return self.type(**value) if isinstance(self.type, ListOfType): return self._validate_list_of_type(self.type.type, value) if isinstance(self.type, DictOfType): @@ -114,13 +116,14 @@ class DataField(object): if isinstance(list_of_type, DataField): return [list_of_type.validate(self.parent, self.name, v) for v in value] assert issubclass(list_of_type, DataModel) - return [list_of_type(**v).as_dict() for v in value] + return [list_of_type(**v) for v in value] - def _validate_dict_of_type(self, dict_of_type, value): + @staticmethod + def _validate_dict_of_type(dict_of_type, value): if not isinstance(value, dict): raise ValueError("Value should be a dict") assert issubclass(dict_of_type, DataModel) - return {k: dict_of_type(**v).as_dict() for k, v in value.items()} + return {k: dict_of_type(**v) for k, v in value.items()} def _validate_str_value(self, value): if not isinstance(value, string_types): @@ -169,15 +172,7 @@ class DataModel(object): setattr(self, name, value) def __eq__(self, other): - assert isinstance(other, DataModel) - if self.get_field_names() != other.get_field_names(): - return False - if self.get_exceptions() != other.get_exceptions(): - return False - for name in self._field_names: - if getattr(self, name) != getattr(other, name): - return False - return True + raise NotImplementedError def __repr__(self): fields = [] @@ -192,7 +187,20 @@ class DataModel(object): return self._exceptions def as_dict(self): - return {name: getattr(self, name) for name in self._field_names} + result = {} + for name in self._field_names: + value = getattr(self, name) + if isinstance(value, DataModel): + value = getattr(self, name).as_dict() + if isinstance(value, dict): + for k, v in value.items(): + if not isinstance(v, DataModel): + continue + value[k] = v.as_dict() + elif isinstance(value, list): + value = [v.as_dict() if isinstance(v, DataModel) else v for v in value] + result[name] = value + return result class StrictDataModel(DataModel): diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 0262b26a..a465f8c3 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -210,6 +210,8 @@ def test_library_json_model(): contents, parser.ManifestFileType.LIBRARY_JSON ) model = ManifestModel(**data.as_dict()) + assert model.repository.url == "https://github.com/bblanchon/ArduinoJson.git" + assert model.examples["JsonHttpClient"].files == ["JsonHttpClient.ino"] assert model == ManifestModel( **{ "name": "ArduinoJson", From fec19849b53085e5dfa314436fadbcce7e8ec79c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 1 Oct 2019 22:02:55 +0300 Subject: [PATCH 053/221] Docs: Add info about ignoring individual parts of mbed framework --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index f8f3c977..03572d65 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit f8f3c97743af99e71c4f91ecddb594e1a79e2677 +Subproject commit 03572d6534edb456e9cd026420e7cf15161e97ff From dee2d2c538c005c07968d8f35bfde72e53a6a6aa Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 1 Oct 2019 22:03:23 +0300 Subject: [PATCH 054/221] Add manifest parsers for platform.json and package.json --- platformio/datamodel.py | 7 +- platformio/package/manifest/model.py | 8 ++ platformio/package/manifest/parser.py | 41 +++++++++ tests/test_pkgmanifest.py | 128 ++++++++++++++++++++++++-- 4 files changed, 176 insertions(+), 8 deletions(-) diff --git a/platformio/datamodel.py b/platformio/datamodel.py index 92121114..4583646a 100644 --- a/platformio/datamodel.py +++ b/platformio/datamodel.py @@ -172,7 +172,12 @@ class DataModel(object): setattr(self, name, value) def __eq__(self, other): - raise NotImplementedError + assert isinstance(other, DataModel) + if self.get_field_names() != other.get_field_names(): + return False + if len(self.get_exceptions()) != len(other.get_exceptions()): + return False + return self.as_dict() == other.as_dict() def __repr__(self): fields = [] diff --git a/platformio/package/manifest/model.py b/platformio/package/manifest/model.py index bfbbfa24..5045725d 100644 --- a/platformio/package/manifest/model.py +++ b/platformio/package/manifest/model.py @@ -79,6 +79,14 @@ class ManifestModel(DataModel): export = DataField(type=ExportModel) examples = DataField(type=DictOfType(ExampleModel)) + # platform.json specific + title = DataField(max_length=100) + + # package.json specific + system = DataField( + type=ListOfType(DataField(max_length=50, regex=r"^[a-z\d\-_]+$")) + ) + class StrictManifestModel(ManifestModel, StrictDataModel): pass diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index c2489a97..291fbae6 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -401,3 +401,44 @@ class LibraryPropertiesManifestParser(BaseManifestParser): "include": include, "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"], } + + +class PlatformJsonManifestParser(BaseManifestParser): + def parse(self, contents): + data = json.loads(contents) + if "frameworks" in data: + data["frameworks"] = self._parse_frameworks(data["frameworks"]) + return data + + @staticmethod + def _parse_frameworks(raw): + if not isinstance(raw, dict): + return None + return [name.lower() for name in raw.keys()] + + +class PackageJsonManifestParser(BaseManifestParser): + def parse(self, contents): + data = json.loads(contents) + data = self._parse_system(data) + data = self._parse_homepage(data) + return data + + @staticmethod + def _parse_system(data): + if "system" not in data: + return data + if data["system"] in ("*", ["*"]): + del data["system"] + return data + if not isinstance(data["system"], list): + data["system"] = [data["system"]] + data["system"] = [s.strip().lower() for s in data["system"]] + return data + + @staticmethod + def _parse_homepage(data): + if "url" in data: + data["homepage"] = data["url"] + del data["url"] + return data diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index a465f8c3..453ef9a9 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -206,13 +206,13 @@ def test_library_json_model(): } } """ - data = parser.ManifestParserFactory.new( + mp = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.LIBRARY_JSON ) - model = ManifestModel(**data.as_dict()) + model = StrictManifestModel(**mp.as_dict()) assert model.repository.url == "https://github.com/bblanchon/ArduinoJson.git" assert model.examples["JsonHttpClient"].files == ["JsonHttpClient.ino"] - assert model == ManifestModel( + assert model == StrictManifestModel( **{ "name": "ArduinoJson", "keywords": ["json", "rest", "http", "web"], @@ -226,10 +226,10 @@ def test_library_json_model(): "version": "6.12.0", "authors": [ { + "name": "Benoit Blanchon", "url": "https://blog.benoitblanchon.fr", "maintainer": False, "email": None, - "name": "Benoit Blanchon", } ], "export": { @@ -265,12 +265,12 @@ category=Display url=https://github.com/olikraus/u8glib architectures=avr,sam """ - data = parser.ManifestParserFactory.new( + mp = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.LIBRARY_PROPERTIES ) - model = ManifestModel(**data.as_dict()) + model = StrictManifestModel(**mp.as_dict()) assert not model.get_exceptions() - assert model == ManifestModel( + assert model == StrictManifestModel( **{ "license": None, "description": ( @@ -304,6 +304,120 @@ architectures=avr,sam ) +def test_platform_json_model(): + contents = """ +{ + "name": "atmelavr", + "title": "Atmel AVR", + "description": "Atmel AVR 8- and 32-bit MCUs deliver a unique combination of performance, power efficiency and design flexibility. Optimized to speed time to market-and easily adapt to new ones-they are based on the industrys most code-efficient architecture for C and assembly programming.", + "url": "http://www.atmel.com/products/microcontrollers/avr/default.aspx", + "homepage": "http://platformio.org/platforms/atmelavr", + "license": "Apache-2.0", + "engines": { + "platformio": "<5" + }, + "repository": { + "type": "git", + "url": "https://github.com/platformio/platform-atmelavr.git" + }, + "version": "1.15.0", + "frameworks": { + "arduino": { + "package": "framework-arduinoavr", + "script": "builder/frameworks/arduino.py" + }, + "simba": { + "package": "framework-simba", + "script": "builder/frameworks/simba.py" + } + }, + "packages": { + "toolchain-atmelavr": { + "type": "toolchain", + "version": "~1.50400.0" + }, + "framework-arduinoavr": { + "type": "framework", + "optional": true, + "version": "~4.2.0" + }, + "framework-simba": { + "type": "framework", + "optional": true, + "version": ">=7.0.0" + }, + "tool-avrdude": { + "type": "uploader", + "optional": true, + "version": "~1.60300.0" + } + } +} +""" + mp = parser.ManifestParserFactory.new( + contents, parser.ManifestFileType.PLATFORM_JSON + ) + model = ManifestModel(**mp.as_dict()) + assert sorted(model.frameworks) == sorted(["arduino", "simba"]) + assert model == ManifestModel( + **{ + "name": "atmelavr", + "title": "Atmel AVR", + "description": ( + "Atmel AVR 8- and 32-bit MCUs deliver a unique combination of " + "performance, power efficiency and design flexibility. Optimized to " + "speed time to market-and easily adapt to new ones-they are based " + "on the industrys most code-efficient architecture for C and " + "assembly programming." + ), + "homepage": "http://platformio.org/platforms/atmelavr", + "license": "Apache-2.0", + "repository": { + "url": "https://github.com/platformio/platform-atmelavr.git", + "type": "git", + "branch": None, + }, + "frameworks": ["simba", "arduino"], + "version": "1.15.0", + } + ) + + +def test_package_json_model(): + contents = """ +{ + "name": "tool-scons", + "description": "SCons software construction tool", + "url": "http://www.scons.org", + "version": "3.30101.0" +} +""" + mp = parser.ManifestParserFactory.new( + contents, parser.ManifestFileType.PACKAGE_JSON + ) + model = ManifestModel(**mp.as_dict()) + assert model.system is None + assert model.homepage == "http://www.scons.org" + assert model == ManifestModel( + **{ + "name": "tool-scons", + "description": "SCons software construction tool", + "homepage": "http://www.scons.org", + "version": "3.30101.0", + } + ) + + mp = parser.ManifestParserFactory.new( + '{"system": "*"}', parser.ManifestFileType.PACKAGE_JSON + ) + assert "system" not in mp.as_dict() + + mp = parser.ManifestParserFactory.new( + '{"system": "darwin_x86_64"}', parser.ManifestFileType.PACKAGE_JSON + ) + assert mp.as_dict()["system"] == ["darwin_x86_64"] + + def test_broken_models(): # non-strict mode assert len(ManifestModel(name="MyPackage").get_exceptions()) == 4 From a2213a1aa4e5695af6ff1dc1320e74b4ca5d0d19 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 2 Oct 2019 11:04:29 +0300 Subject: [PATCH 055/221] Change "examples" field in package manifest to ListOf(ExampleModel) --- platformio/package/manifest/model.py | 11 +++-------- tests/test_pkgmanifest.py | 29 +++++++++++++++++----------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/platformio/package/manifest/model.py b/platformio/package/manifest/model.py index 5045725d..7fdb66e5 100644 --- a/platformio/package/manifest/model.py +++ b/platformio/package/manifest/model.py @@ -14,13 +14,7 @@ import semantic_version -from platformio.datamodel import ( - DataField, - DataModel, - DictOfType, - ListOfType, - StrictDataModel, -) +from platformio.datamodel import DataField, DataModel, ListOfType, StrictDataModel def validate_semver_field(_, value): @@ -48,6 +42,7 @@ class ExportModel(DataModel): class ExampleModel(DataModel): + name = DataField(max_length=100, regex=r"^[a-zA-Z\d\-\_ ]+$", required=True) base = DataField(required=True) files = DataField(type=ListOfType(DataField())) @@ -77,7 +72,7 @@ class ManifestModel(DataModel): repository = DataField(type=RepositoryModel) export = DataField(type=ExportModel) - examples = DataField(type=DictOfType(ExampleModel)) + examples = DataField(type=ListOfType(ExampleModel)) # platform.json specific title = DataField(max_length=100) diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 453ef9a9..72712674 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -194,16 +194,18 @@ def test_library_json_model(): "frameworks": "arduino", "platforms": "*", "license": "MIT", - "examples": { - "JsonConfigFile": { + "examples": [ + { + "name": "JsonConfigFile", "base": "examples/JsonConfigFile", "files": ["JsonConfigFile.ino"] }, - "JsonHttpClient": { + { + "name": "JsonHttpClient", "base": "examples/JsonHttpClient", "files": ["JsonHttpClient.ino"] } - } + ] } """ mp = parser.ManifestParserFactory.new( @@ -211,7 +213,8 @@ def test_library_json_model(): ) model = StrictManifestModel(**mp.as_dict()) assert model.repository.url == "https://github.com/bblanchon/ArduinoJson.git" - assert model.examples["JsonHttpClient"].files == ["JsonHttpClient.ino"] + assert model.examples[1].base == "examples/JsonHttpClient" + assert model.examples[1].files == ["JsonHttpClient.ino"] assert model == StrictManifestModel( **{ "name": "ArduinoJson", @@ -239,16 +242,18 @@ def test_library_json_model(): "frameworks": ["arduino"], "platforms": ["*"], "license": "MIT", - "examples": { - "JsonConfigFile": { + "examples": [ + { + "name": "JsonConfigFile", "base": "examples/JsonConfigFile", "files": ["JsonConfigFile.ino"], }, - "JsonHttpClient": { + { + "name": "JsonHttpClient", "base": "examples/JsonHttpClient", "files": ["JsonHttpClient.ino"], }, - }, + ], } ) @@ -357,8 +362,10 @@ def test_platform_json_model(): mp = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.PLATFORM_JSON ) + data = mp.as_dict() + data["frameworks"] = sorted(data["frameworks"]) model = ManifestModel(**mp.as_dict()) - assert sorted(model.frameworks) == sorted(["arduino", "simba"]) + assert model.frameworks == ["arduino", "simba"] assert model == ManifestModel( **{ "name": "atmelavr", @@ -377,7 +384,7 @@ def test_platform_json_model(): "type": "git", "branch": None, }, - "frameworks": ["simba", "arduino"], + "frameworks": ["arduino", "simba"], "version": "1.15.0", } ) From d38c843574ee5221a8a11766e348354f203da938 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 2 Oct 2019 11:52:14 +0300 Subject: [PATCH 056/221] Fixed an issue when installing a package using custom Git tag and submodules were not updated correctly // Resolve #3060 --- HISTORY.rst | 1 + platformio/vcsclient.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 213c54f5..62d41eaa 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -24,6 +24,7 @@ PlatformIO Core 4.0 * Fixed an issue with project generator for `CLion IDE `__ when 2 environments were used (`issue #2824 `_) * Fixed default PIO Unified Debugger configuration for `J-Link probe `__ * Fixed an issue when configuration file options partly ignored when using custom ``--project-conf`` (`issue #3034 `_) +* Fixed an issue when installing a package using custom Git tag and submodules were not updated correctly (`issue #3060 `_) 4.0.3 (2019-08-30) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/vcsclient.py b/platformio/vcsclient.py index 468a4d0a..4267ef3c 100644 --- a/platformio/vcsclient.py +++ b/platformio/vcsclient.py @@ -164,7 +164,8 @@ class GitClient(VCSClientBase): args += [self.remote_url, self.src_dir] assert self.run_cmd(args) if is_commit: - return self.run_cmd(["reset", "--hard", self.tag]) + assert self.run_cmd(["reset", "--hard", self.tag]) + return self.run_cmd(["submodule", "update", "--init", "--recursive"]) return True def update(self): From c2970631a5c4365efc2c5d4d3c40cd5feb905a88 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 2 Oct 2019 12:34:20 +0300 Subject: [PATCH 057/221] Add "--force" for git update // Issue #3060 --- platformio/vcsclient.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/platformio/vcsclient.py b/platformio/vcsclient.py index 4267ef3c..56291966 100644 --- a/platformio/vcsclient.py +++ b/platformio/vcsclient.py @@ -165,7 +165,9 @@ class GitClient(VCSClientBase): assert self.run_cmd(args) if is_commit: assert self.run_cmd(["reset", "--hard", self.tag]) - return self.run_cmd(["submodule", "update", "--init", "--recursive"]) + return self.run_cmd( + ["submodule", "update", "--init", "--recursive", "--force"] + ) return True def update(self): From a1ed99962c903c2c90c4fc426e85053309fd69af Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 2 Oct 2019 12:34:50 +0300 Subject: [PATCH 058/221] Better handling of non-dict values passed to DataModel --- platformio/datamodel.py | 20 +++++++++++++------- tests/test_pkgmanifest.py | 17 ++++++++++++++++- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/platformio/datamodel.py b/platformio/datamodel.py index 4583646a..3dc4111f 100644 --- a/platformio/datamodel.py +++ b/platformio/datamodel.py @@ -99,7 +99,7 @@ class DataField(object): if value is None: return self.default if inspect.isclass(self.type) and issubclass(self.type, DataModel): - return self.type(**value) + return self.type(**self._ensure_value_is_dict(value)) if isinstance(self.type, ListOfType): return self._validate_list_of_type(self.type.type, value) if isinstance(self.type, DictOfType): @@ -110,20 +110,26 @@ class DataField(object): raise DataFieldException(self, str(e)) return value + @staticmethod + def _ensure_value_is_dict(value): + if not isinstance(value, dict): + raise ValueError("Value should be type of dict, not `%s`" % type(value)) + return value + def _validate_list_of_type(self, list_of_type, value): if not isinstance(value, list): raise ValueError("Value should be a list") if isinstance(list_of_type, DataField): return [list_of_type.validate(self.parent, self.name, v) for v in value] assert issubclass(list_of_type, DataModel) - return [list_of_type(**v) for v in value] + return [list_of_type(**self._ensure_value_is_dict(v)) for v in value] - @staticmethod - def _validate_dict_of_type(dict_of_type, value): - if not isinstance(value, dict): - raise ValueError("Value should be a dict") + def _validate_dict_of_type(self, dict_of_type, value): assert issubclass(dict_of_type, DataModel) - return {k: dict_of_type(**v) for k, v in value.items()} + value = self._ensure_value_is_dict(value) + return { + k: dict_of_type(**self._ensure_value_is_dict(v)) for k, v in value.items() + } def _validate_str_value(self, value): if not isinstance(value, string_types): diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 72712674..ec19b29d 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -439,7 +439,10 @@ def test_broken_models(): # broken SemVer with pytest.raises( DataFieldException, - match="Invalid semantic versioning format for `StrictManifestModel.version` field", + match=( + "Invalid semantic versioning format for " + "`StrictManifestModel.version` field" + ), ): assert StrictManifestModel( name="MyPackage", @@ -448,3 +451,15 @@ def test_broken_models(): authors=[{"name": "Author"}], version="broken_version", ) + + # broken value for DataModel + with pytest.raises( + DataFieldException, match="Value should be type of dict, not `" + ): + assert StrictManifestModel( + name="MyPackage", + description="MyDescription", + keywords=["a", "b"], + authors=["should be dict here"], + version="1.2.3", + ) From 7ba2a7cd3d32f674e3544ada10514569694f6dff Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 2 Oct 2019 13:33:12 +0300 Subject: [PATCH 059/221] Bump version to 4.1.0b2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index e4672201..a997299b 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, "0b1") +VERSION = (4, 1, "0b2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From bbd694c5ea34315b5acc67be5e7dd6e9259a6b1a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 2 Oct 2019 17:54:59 +0300 Subject: [PATCH 060/221] ManifestParse: automatically generate examples from package dir --- platformio/package/manifest/parser.py | 92 ++++++++++++++++++++++++--- tests/test_pkgmanifest.py | 92 ++++++++++++++++++++++++++- 2 files changed, 172 insertions(+), 12 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 291fbae6..46895ecd 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -21,6 +21,7 @@ import requests from platformio.compat import get_class_attributes, string_types from platformio.exception import PlatformioException from platformio.fs import get_file_contents +from platformio.project.helpers import is_platformio_project try: from urllib.parse import urlparse @@ -86,7 +87,10 @@ class ManifestParserFactory(object): if not os.path.isfile(os.path.join(path, t)): continue return ManifestParserFactory.new( - get_file_contents(os.path.join(path, t)), t, remote_url + get_file_contents(os.path.join(path, t)), + t, + remote_url=remote_url, + package_dir=path, ) raise ManifestException("Unknown manifest file type in %s directory" % path) @@ -99,18 +103,20 @@ class ManifestParserFactory(object): ) @staticmethod - def new(contents, type, remote_url=None): + def new(contents, type, remote_url=None, package_dir=None): # pylint: disable=redefined-builtin clsname = ManifestParserFactory.type_to_clsname(type) if clsname not in globals(): raise ManifestException("Unknown manifest file type %s" % clsname) - return globals()[clsname](contents, remote_url) + return globals()[clsname](contents, remote_url, package_dir) class BaseManifestParser(object): - def __init__(self, contents, remote_url=None): + def __init__(self, contents, remote_url=None, package_dir=None): self.remote_url = remote_url + self.package_dir = package_dir self._data = self.parse(contents) + self._data = self.parse_examples(self._data) def parse(self, contents): raise NotImplementedError @@ -119,7 +125,7 @@ class BaseManifestParser(object): return self._data @staticmethod - def _cleanup_author(author): + def cleanup_author(author): if author.get("email"): author["email"] = re.sub(r"\s+[aA][tT]\s+", "@", author["email"]) return author @@ -136,6 +142,74 @@ class BaseManifestParser(object): email = raw[raw.index(ldel) + 1 : raw.index(rdel)] return (name.strip(), email.strip() if email else None) + def parse_examples(self, data): + examples = data.get("examples") + if ( + not examples + or not isinstance(examples, list) + or not all(isinstance(v, dict) for v in examples) + ): + data["examples"] = None + if not examples and self.package_dir: + data["examples"] = self.parse_examples_from_dir(self.package_dir) + if "examples" in data and not data["examples"]: + del data["examples"] + return data + + @staticmethod + def parse_examples_from_dir(package_dir): + assert os.path.isdir(package_dir) + examples_dir = os.path.join(package_dir, "examples") + if not os.path.isdir(examples_dir): + return None + + allowed_exts = ( + ".c", + ".cc", + ".cpp", + ".h", + ".hpp", + ".asm", + ".ASM", + ".s", + ".S", + ".ino", + ".pde", + ) + + result = {} + last_pio_project = None + for root, _, files in os.walk(examples_dir): + if is_platformio_project(root): + last_pio_project = root + result[last_pio_project] = dict( + name=os.path.relpath(root, examples_dir), + base=os.path.relpath(root, package_dir), + files=files, + ) + continue + if last_pio_project: + if root.startswith(last_pio_project): + result[last_pio_project]["files"].extend( + [ + os.path.relpath(os.path.join(root, f), last_pio_project) + for f in files + ] + ) + continue + last_pio_project = None + + matched_files = [f for f in files if f.endswith(allowed_exts)] + if not matched_files: + continue + result[root] = dict( + name=os.path.relpath(root, examples_dir), + base=os.path.relpath(root, package_dir), + files=matched_files, + ) + + return list(result.values()) or None + class LibraryJsonManifestParser(BaseManifestParser): def parse(self, contents): @@ -193,7 +267,7 @@ class LibraryJsonManifestParser(BaseManifestParser): # normalize Union[dict, list] fields if not isinstance(raw, list): raw = [raw] - return [self._cleanup_author(author) for author in raw] + return [self.cleanup_author(author) for author in raw] @staticmethod def _parse_platforms(raw): @@ -243,7 +317,7 @@ class ModuleJsonManifestParser(BaseManifestParser): if not name: continue result.append( - self._cleanup_author(dict(name=name, email=email, maintainer=False)) + self.cleanup_author(dict(name=name, email=email, maintainer=False)) ) return result @@ -341,7 +415,7 @@ class LibraryPropertiesManifestParser(BaseManifestParser): if not name: continue authors.append( - self._cleanup_author(dict(name=name, email=email, maintainer=False)) + self.cleanup_author(dict(name=name, email=email, maintainer=False)) ) for author in properties.get("maintainer", "").split(","): name, email = self.parse_author_name_and_email(author) @@ -357,7 +431,7 @@ class LibraryPropertiesManifestParser(BaseManifestParser): item["email"] = email if not found: authors.append( - self._cleanup_author(dict(name=name, email=email, maintainer=True)) + self.cleanup_author(dict(name=name, email=email, maintainer=True)) ) return authors diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index ec19b29d..64e6b88e 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -425,6 +425,94 @@ def test_package_json_model(): assert mp.as_dict()["system"] == ["darwin_x86_64"] +def test_examples_from_dir(tmpdir_factory): + package_dir = tmpdir_factory.mktemp("project") + package_dir.join("library.json").write('{"name": "pkg", "version": "1.0.0"}') + examples_dir = package_dir.mkdir("examples") + + # PlatformIO project #1 + pio_dir = examples_dir.mkdir("PlatformIO").mkdir("hello") + pio_dir.join("platformio.ini").write("") + pio_dir.mkdir("include").join("main.h").write("") + pio_dir.mkdir("src").join("main.cpp").write("") + + # wiring examples + examples_dir.mkdir("SomeSketchIno").join("SomeSketchIno.ino").write("") + examples_dir.mkdir("SomeSketchPde").join("SomeSketchPde.pde").write("") + + # custom examples + demo_dir = examples_dir.mkdir("demo") + demo_dir.join("demo.cpp").write("") + demo_dir.join("demo.h").write("") + demo_dir.join("util.h").write("") + + # PlatformIO project #2 + pio_dir = examples_dir.mkdir("world") + pio_dir.join("platformio.ini").write("") + pio_dir.join("README").write("") + pio_dir.join("extra.py").write("") + pio_dir.mkdir("include").join("world.h").write("") + pio_dir.mkdir("src").join("world.c").write("") + + # invalid example + examples_dir.mkdir("invalid-example").join("hello.json") + + # Do testing + + data = parser.ManifestParserFactory.new_from_dir(str(package_dir)).as_dict() + assert isinstance(data["examples"], list) + assert len(data["examples"]) == 5 + + def _sort_examples(items): + for i, item in enumerate(items): + items[i]["files"] = sorted(item["files"]) + return sorted(items, key=lambda item: item["name"]) + + data["examples"] = _sort_examples(data["examples"]) + model = ManifestModel(**data) + assert model == ManifestModel( + **{ + "version": "1.0.0", + "name": "pkg", + "examples": _sort_examples( + [ + { + "name": "PlatformIO/hello", + "base": "examples/PlatformIO/hello", + "files": ["platformio.ini", "include/main.h", "src/main.cpp"], + }, + { + "name": "SomeSketchIno", + "base": "examples/SomeSketchIno", + "files": ["SomeSketchIno.ino"], + }, + { + "name": "SomeSketchPde", + "base": "examples/SomeSketchPde", + "files": ["SomeSketchPde.pde"], + }, + { + "name": "demo", + "base": "examples/demo", + "files": ["demo.h", "util.h", "demo.cpp"], + }, + { + "name": "world", + "base": "examples/world", + "files": [ + "platformio.ini", + "include/world.h", + "src/world.c", + "README", + "extra.py", + ], + }, + ] + ), + } + ) + + def test_broken_models(): # non-strict mode assert len(ManifestModel(name="MyPackage").get_exceptions()) == 4 @@ -453,9 +541,7 @@ def test_broken_models(): ) # broken value for DataModel - with pytest.raises( - DataFieldException, match="Value should be type of dict, not `" - ): + with pytest.raises(DataFieldException, match="Value should be type of dict"): assert StrictManifestModel( name="MyPackage", description="MyDescription", From 47ba1277338182b7618e55817f2dbe5fcb1b14c5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 2 Oct 2019 18:15:17 +0300 Subject: [PATCH 061/221] Add ReadTheDocs config --- .readthedocs.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .readthedocs.yml diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..5a302fa1 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,9 @@ +# See https://docs.readthedocs.io/en/stable/config-file/index.html + +version: 2 + +sphinx: + configuration: docs/conf.py + +formats: + - pdf From e6ea4cb6134417fb03a838f9c90b4e85cf1aa0e9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 2 Oct 2019 20:42:01 +0300 Subject: [PATCH 062/221] PackageManifest: Ignore hidden files for examples --- .readthedocs.yml | 3 --- platformio/package/manifest/model.py | 2 +- platformio/package/manifest/parser.py | 5 +++++ tests/test_pkgmanifest.py | 2 ++ 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 5a302fa1..91f1c27f 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -2,8 +2,5 @@ version: 2 -sphinx: - configuration: docs/conf.py - formats: - pdf diff --git a/platformio/package/manifest/model.py b/platformio/package/manifest/model.py index 7fdb66e5..47595bc5 100644 --- a/platformio/package/manifest/model.py +++ b/platformio/package/manifest/model.py @@ -42,7 +42,7 @@ class ExportModel(DataModel): class ExampleModel(DataModel): - name = DataField(max_length=100, regex=r"^[a-zA-Z\d\-\_ ]+$", required=True) + name = DataField(max_length=100, regex=r"^[a-zA-Z\d\-\_/ ]+$", required=True) base = DataField(required=True) files = DataField(type=ListOfType(DataField())) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 46895ecd..ad364963 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -180,6 +180,11 @@ class BaseManifestParser(object): result = {} last_pio_project = None for root, _, files in os.walk(examples_dir): + # skip hidden files and folders + files = [f for f in files if not f.startswith(".")] + if os.path.basename(root).startswith(".") or not files: + continue + if is_platformio_project(root): last_pio_project = root result[last_pio_project] = dict( diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 64e6b88e..d5dabb91 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -432,6 +432,7 @@ def test_examples_from_dir(tmpdir_factory): # PlatformIO project #1 pio_dir = examples_dir.mkdir("PlatformIO").mkdir("hello") + pio_dir.join(".vimrc").write("") pio_dir.join("platformio.ini").write("") pio_dir.mkdir("include").join("main.h").write("") pio_dir.mkdir("src").join("main.cpp").write("") @@ -470,6 +471,7 @@ def test_examples_from_dir(tmpdir_factory): data["examples"] = _sort_examples(data["examples"]) model = ManifestModel(**data) + assert model.examples[0].name == "PlatformIO/hello" assert model == ManifestModel( **{ "version": "1.0.0", From b68b9794ec0d7d7f1c6c4fe53ae4bcca147f858a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 2 Oct 2019 20:51:39 +0300 Subject: [PATCH 063/221] Fix docs with "htmlzip" format --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 91f1c27f..15ea892d 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -3,4 +3,4 @@ version: 2 formats: - - pdf + - htmlzip From 6d50aa2e2513cd71418da3af2d4d032339ab44e1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 2 Oct 2019 20:54:36 +0300 Subject: [PATCH 064/221] Remove RTD confs --- .readthedocs.yml | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .readthedocs.yml diff --git a/.readthedocs.yml b/.readthedocs.yml deleted file mode 100644 index 15ea892d..00000000 --- a/.readthedocs.yml +++ /dev/null @@ -1,6 +0,0 @@ -# See https://docs.readthedocs.io/en/stable/config-file/index.html - -version: 2 - -formats: - - htmlzip From d0ee0c2919c6eafc63a5a2b629881fb1a9122dea Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 2 Oct 2019 21:32:31 +0300 Subject: [PATCH 065/221] Sync docs --- .readthedocs.yml | 13 +++++++++++++ docs | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 .readthedocs.yml diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..93c2aa5a --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,13 @@ +# See https://docs.readthedocs.io/en/stable/config-file/index.html + +version: 2 + +sphinx: + configuration: docs/conf.py + +formats: + - pdf + +submodules: + - include: all + - recursive: true diff --git a/docs b/docs index 03572d65..d8441bba 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 03572d6534edb456e9cd026420e7cf15161e97ff +Subproject commit d8441bba8630ba719e5c8559089bcfd698b1084f From dc067642b22c6b7eeb1d1cebc469ced9db1a3292 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 2 Oct 2019 21:33:40 +0300 Subject: [PATCH 066/221] Fix RTD conf --- .readthedocs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 93c2aa5a..6fa2ea01 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,5 +9,5 @@ formats: - pdf submodules: - - include: all - - recursive: true + include: all + recursive: true From 77c591ce815a70fe830b83b7a3d94649fc61d7e1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 2 Oct 2019 21:35:13 +0300 Subject: [PATCH 067/221] Fix RTD conf --- .readthedocs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 6fa2ea01..42a9ba4f 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -10,4 +10,3 @@ formats: submodules: include: all - recursive: true From ab5650f84b1c36b8f0319fb7d88f69df2659a394 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 2 Oct 2019 23:46:42 +0300 Subject: [PATCH 068/221] Use max line length hooks for all systems --- platformio/builder/main.py | 2 +- .../builder/tools/{piowinhooks.py => piomaxlen.py} | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) rename platformio/builder/tools/{piowinhooks.py => piomaxlen.py} (94%) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index e0179bf4..39565c6b 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -56,7 +56,7 @@ DEFAULT_ENV_OPTIONS = dict( "platformio", "pioplatform", "pioproject", - "piowinhooks", + "piomaxlen", "piolib", "pioupload", "piomisc", diff --git a/platformio/builder/tools/piowinhooks.py b/platformio/builder/tools/piomaxlen.py similarity index 94% rename from platformio/builder/tools/piowinhooks.py rename to platformio/builder/tools/piomaxlen.py index a3187291..04f1304f 100644 --- a/platformio/builder/tools/piowinhooks.py +++ b/platformio/builder/tools/piomaxlen.py @@ -22,12 +22,12 @@ from platformio.compat import WINDOWS, hashlib_encode_data # Windows CLI has limit with command length to 8192 # Leave 2000 chars for flags and other options -MAX_SOURCES_LENGTH = 6000 +MAX_LINE_LENGTH = 6000 if WINDOWS else 128072 def long_sources_hook(env, sources): _sources = str(sources).replace("\\", "/") - if len(str(_sources)) < MAX_SOURCES_LENGTH: + if len(str(_sources)) < MAX_LINE_LENGTH: return sources # fix space in paths @@ -43,7 +43,7 @@ def long_sources_hook(env, sources): def long_incflags_hook(env, incflags): _incflags = env.subst(incflags).replace("\\", "/") - if len(_incflags) < MAX_SOURCES_LENGTH: + if len(_incflags) < MAX_LINE_LENGTH: return incflags # fix space in paths @@ -76,9 +76,6 @@ def exists(_): def generate(env): - if not WINDOWS: - return None - env.Replace(_long_sources_hook=long_sources_hook) env.Replace(_long_incflags_hook=long_incflags_hook) coms = {} From 85a814c21a356a8d003f1dbf9730f92c20b5500c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 3 Oct 2019 10:33:11 +0300 Subject: [PATCH 069/221] Allow dot in manifest example name --- platformio/package/manifest/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/manifest/model.py b/platformio/package/manifest/model.py index 47595bc5..08e7001b 100644 --- a/platformio/package/manifest/model.py +++ b/platformio/package/manifest/model.py @@ -42,7 +42,7 @@ class ExportModel(DataModel): class ExampleModel(DataModel): - name = DataField(max_length=100, regex=r"^[a-zA-Z\d\-\_/ ]+$", required=True) + name = DataField(max_length=100, regex=r"^[a-zA-Z\d\-\_/\. ]+$", required=True) base = DataField(required=True) files = DataField(type=ListOfType(DataField())) From 8febdc19eac39cb501c76a4d9f81fb0606793769 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 3 Oct 2019 12:47:41 +0300 Subject: [PATCH 070/221] ManifestParser: normalize example names --- platformio/package/manifest/model.py | 2 +- platformio/package/manifest/parser.py | 12 ++++++++++-- tests/test_pkgmanifest.py | 26 ++++++++++++++++++-------- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/platformio/package/manifest/model.py b/platformio/package/manifest/model.py index 08e7001b..27684eb1 100644 --- a/platformio/package/manifest/model.py +++ b/platformio/package/manifest/model.py @@ -42,7 +42,7 @@ class ExportModel(DataModel): class ExampleModel(DataModel): - name = DataField(max_length=100, regex=r"^[a-zA-Z\d\-\_/\. ]+$", required=True) + name = DataField(max_length=100, regex=r"^[a-zA-Z\d\-\_/]+$", required=True) base = DataField(required=True) files = DataField(type=ListOfType(DataField())) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index ad364963..799e1a58 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -208,12 +208,20 @@ class BaseManifestParser(object): if not matched_files: continue result[root] = dict( - name=os.path.relpath(root, examples_dir), + name="Examples" + if root == examples_dir + else os.path.relpath(root, examples_dir), base=os.path.relpath(root, package_dir), files=matched_files, ) - return list(result.values()) or None + result = list(result.values()) + + # normalize example names + for item in result: + item["name"] = re.sub(r"[^a-z\d\d\-\_/]+", "_", item["name"], flags=re.I) + + return result or None class LibraryJsonManifestParser(BaseManifestParser): diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index d5dabb91..0b2fa58d 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -438,8 +438,9 @@ def test_examples_from_dir(tmpdir_factory): pio_dir.mkdir("src").join("main.cpp").write("") # wiring examples - examples_dir.mkdir("SomeSketchIno").join("SomeSketchIno.ino").write("") - examples_dir.mkdir("SomeSketchPde").join("SomeSketchPde.pde").write("") + arduino_dir = examples_dir.mkdir("1. General") + arduino_dir.mkdir("SomeSketchIno").join("SomeSketchIno.ino").write("") + arduino_dir.mkdir("SomeSketchPde").join("SomeSketchPde.pde").write("") # custom examples demo_dir = examples_dir.mkdir("demo") @@ -455,6 +456,10 @@ def test_examples_from_dir(tmpdir_factory): pio_dir.mkdir("include").join("world.h").write("") pio_dir.mkdir("src").join("world.c").write("") + # example files in root + examples_dir.join("root.c").write("") + examples_dir.join("root.h").write("") + # invalid example examples_dir.mkdir("invalid-example").join("hello.json") @@ -462,7 +467,7 @@ def test_examples_from_dir(tmpdir_factory): data = parser.ManifestParserFactory.new_from_dir(str(package_dir)).as_dict() assert isinstance(data["examples"], list) - assert len(data["examples"]) == 5 + assert len(data["examples"]) == 6 def _sort_examples(items): for i, item in enumerate(items): @@ -471,7 +476,7 @@ def test_examples_from_dir(tmpdir_factory): data["examples"] = _sort_examples(data["examples"]) model = ManifestModel(**data) - assert model.examples[0].name == "PlatformIO/hello" + assert model.examples[3].name == "PlatformIO/hello" assert model == ManifestModel( **{ "version": "1.0.0", @@ -484,13 +489,13 @@ def test_examples_from_dir(tmpdir_factory): "files": ["platformio.ini", "include/main.h", "src/main.cpp"], }, { - "name": "SomeSketchIno", - "base": "examples/SomeSketchIno", + "name": "1_General/SomeSketchIno", + "base": "examples/1. General/SomeSketchIno", "files": ["SomeSketchIno.ino"], }, { - "name": "SomeSketchPde", - "base": "examples/SomeSketchPde", + "name": "1_General/SomeSketchPde", + "base": "examples/1. General/SomeSketchPde", "files": ["SomeSketchPde.pde"], }, { @@ -509,6 +514,11 @@ def test_examples_from_dir(tmpdir_factory): "extra.py", ], }, + { + "name": "Examples", + "base": "examples", + "files": ["root.c", "root.h"], + }, ] ), } From 76865a1730e02e7bd20cf458bf38be2342a6802a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 3 Oct 2019 14:55:04 +0300 Subject: [PATCH 071/221] ManifetPatrser fixes (#3080) * Skip broken examples declaration * Allow dots in keywords * Allow "+" in keywords --- platformio/package/manifest/model.py | 2 +- platformio/package/manifest/parser.py | 2 +- tests/test_pkgmanifest.py | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/platformio/package/manifest/model.py b/platformio/package/manifest/model.py index 27684eb1..f881a370 100644 --- a/platformio/package/manifest/model.py +++ b/platformio/package/manifest/model.py @@ -56,7 +56,7 @@ class ManifestModel(DataModel): ) description = DataField(max_length=1000, required=True) keywords = DataField( - type=ListOfType(DataField(max_length=255, regex=r"^[a-z\d\- ]+$")), + type=ListOfType(DataField(max_length=255, regex=r"^[a-z\d\-\+\. ]+$")), required=True, ) authors = DataField(type=ListOfType(AuthorModel), required=True) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 799e1a58..4e6399f3 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -149,7 +149,7 @@ class BaseManifestParser(object): or not isinstance(examples, list) or not all(isinstance(v, dict) for v in examples) ): - data["examples"] = None + examples = None if not examples and self.package_dir: data["examples"] = self.parse_examples_from_dir(self.package_dir) if "examples" in data and not data["examples"]: diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 0b2fa58d..e8b827d6 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -427,7 +427,9 @@ def test_package_json_model(): def test_examples_from_dir(tmpdir_factory): package_dir = tmpdir_factory.mktemp("project") - package_dir.join("library.json").write('{"name": "pkg", "version": "1.0.0"}') + package_dir.join("library.json").write( + '{"name": "pkg", "version": "1.0.0", "examples": ["examples/*/*.pde"]}' + ) examples_dir = package_dir.mkdir("examples") # PlatformIO project #1 From a7855ae6647b54f506a10a917c7ca96d8d6c4d64 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 3 Oct 2019 16:14:51 +0300 Subject: [PATCH 072/221] ManifestParser: init from dir using name of file in remote url if provided --- platformio/package/manifest/parser.py | 16 ++++++++++++++-- tests/test_pkgmanifest.py | 14 ++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 4e6399f3..f41d97e6 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -54,7 +54,7 @@ class ManifestFileType(object): return ManifestFileType.MODULE_JSON if uri.endswith("package.json"): return ManifestFileType.PACKAGE_JSON - return ManifestFileType.LIBRARY_JSON + return None class ManifestParserFactory(object): @@ -76,6 +76,16 @@ class ManifestParserFactory(object): @staticmethod def new_from_dir(path, remote_url=None): assert os.path.isdir(path), "Invalid directory %s" % path + + type_from_uri = ManifestFileType.from_uri(remote_url) if remote_url else None + if type_from_uri and os.path.isfile(os.path.join(path, type_from_uri)): + return ManifestParserFactory.new( + get_file_contents(os.path.join(path, type_from_uri)), + type_from_uri, + remote_url=remote_url, + package_dir=path, + ) + file_order = [ ManifestFileType.PLATFORM_JSON, ManifestFileType.LIBRARY_JSON, @@ -99,7 +109,9 @@ class ManifestParserFactory(object): r = requests.get(remote_url) r.raise_for_status() return ManifestParserFactory.new( - r.text, ManifestFileType.from_uri(remote_url), remote_url + r.text, + ManifestFileType.from_uri(remote_url) or ManifestFileType.LIBRARY_JSON, + remote_url, ) @staticmethod diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index e8b827d6..027733a1 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -425,6 +425,20 @@ def test_package_json_model(): assert mp.as_dict()["system"] == ["darwin_x86_64"] +def test_parser_from_dir(tmpdir_factory): + pkg_dir = tmpdir_factory.mktemp("package") + pkg_dir.join("library.json").write('{"name": "library.json"}') + pkg_dir.join("library.properties").write("name=library.properties") + + data = parser.ManifestParserFactory.new_from_dir(str(pkg_dir)).as_dict() + assert data["name"] == "library.json" + + data = parser.ManifestParserFactory.new_from_dir( + str(pkg_dir), remote_url="http://localhost/library.properties" + ).as_dict() + assert data["name"] == "library.properties" + + def test_examples_from_dir(tmpdir_factory): package_dir = tmpdir_factory.mktemp("project") package_dir.join("library.json").write( From 9954900a0eebd146bd9343d8dfb7f0a373705538 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 3 Oct 2019 18:16:55 +0300 Subject: [PATCH 073/221] Return back LINKFLAGS for debug mode --- platformio/builder/tools/piomisc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index bea63a31..0719e498 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -322,7 +322,7 @@ def ConfigureDebugFlags(env): env.Append(CPPDEFINES=["__PLATFORMIO_BUILD_DEBUG__"]) debug_flags = ["-Og", "-g3", "-ggdb3"] - for scope in ("ASFLAGS", "CCFLAGS"): + for scope in ("ASFLAGS", "CCFLAGS", "LINKFLAGS"): _cleanup_debug_flags(scope) env.Append(**{scope: debug_flags}) From 9ce19c7e831838c455f29ba5780f25986232ac52 Mon Sep 17 00:00:00 2001 From: valeros Date: Fri, 4 Oct 2019 10:52:55 +0300 Subject: [PATCH 074/221] Skip debug sections when calculating sizedata --- platformio/builder/tools/piosize.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/platformio/builder/tools/piosize.py b/platformio/builder/tools/piosize.py index 7bb538c5..814e7c42 100644 --- a/platformio/builder/tools/piosize.py +++ b/platformio/builder/tools/piosize.py @@ -33,6 +33,8 @@ def _get_file_location(env, elf_path, addr, sysenv): def _determine_section(sections, symbol_addr): for section, info in sections.items(): + if not _is_flash_section(info) and not _is_ram_section(info): + continue if symbol_addr in range(info["start_addr"], info["start_addr"] + info["size"]): return section return "unknown" @@ -55,7 +57,7 @@ def _is_ram_section(section): def _is_flash_section(section): - return section.get("type") == "SHT_PROGBITS" and "A" in section.get("flags") + return section.get("type", "") == "SHT_PROGBITS" and "A" in section.get("flags", "") def _is_valid_symbol(symbol_name, symbol_type, symbol_address): From 47e297fecb8a89c28cd2aa1897e1e5c3a9ac2b24 Mon Sep 17 00:00:00 2001 From: valeros Date: Fri, 4 Oct 2019 13:27:02 +0300 Subject: [PATCH 075/221] Use less verbose debug output --- platformio/builder/tools/piomisc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 0719e498..dd7cffd6 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -321,7 +321,7 @@ def ConfigureDebugFlags(env): env.Append(CPPDEFINES=["__PLATFORMIO_BUILD_DEBUG__"]) - debug_flags = ["-Og", "-g3", "-ggdb3"] + debug_flags = ["-Og", "-g2", "-ggdb2"] for scope in ("ASFLAGS", "CCFLAGS", "LINKFLAGS"): _cleanup_debug_flags(scope) env.Append(**{scope: debug_flags}) From 36acdd77976cddf24b8d88b6f1262744af5586e6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 4 Oct 2019 18:30:48 +0300 Subject: [PATCH 076/221] DataModel: allow valid values in non-strict mode for TypeOfList and TypeOfDict --- platformio/datamodel.py | 145 ++++++++++++++++++++------- platformio/package/manifest/model.py | 8 +- tests/test_pkgmanifest.py | 145 ++++++++++++++++++--------- 3 files changed, 213 insertions(+), 85 deletions(-) diff --git a/platformio/datamodel.py b/platformio/datamodel.py index 3dc4111f..53b99c29 100644 --- a/platformio/datamodel.py +++ b/platformio/datamodel.py @@ -74,8 +74,8 @@ class DataField(object): self.validate_factory = validate_factory self.title = title - self.parent = None - self.name = None + self._parent = None + self._name = None self.value = None def __repr__(self): @@ -84,13 +84,24 @@ class DataField(object): self.default if self.value is None else self.value, ) - def validate( - self, parent, name, value - ): # pylint: disable=too-many-return-statements - self.parent = parent - self.name = name - self.title = self.title or name.title() + @property + def parent(self): + return self._parent + @parent.setter + def parent(self, value): + self._parent = value + + @property + def name(self): + return self._name + + @name.setter + def name(self, value): + self._name = value + self.title = self.title or value.title() + + def validate(self, value): try: if self.required and value is None: raise ValueError("Missed value") @@ -99,38 +110,19 @@ class DataField(object): if value is None: return self.default if inspect.isclass(self.type) and issubclass(self.type, DataModel): - return self.type(**self._ensure_value_is_dict(value)) - if isinstance(self.type, ListOfType): - return self._validate_list_of_type(self.type.type, value) - if isinstance(self.type, DictOfType): - return self._validate_dict_of_type(self.type.type, value) - if issubclass(self.type, (str, bool)): + return self.type(**self.ensure_value_is_dict(value)) + if inspect.isclass(self.type) and issubclass(self.type, (str, bool)): return getattr(self, "_validate_%s_value" % self.type.__name__)(value) except ValueError as e: raise DataFieldException(self, str(e)) return value @staticmethod - def _ensure_value_is_dict(value): + def ensure_value_is_dict(value): if not isinstance(value, dict): raise ValueError("Value should be type of dict, not `%s`" % type(value)) return value - def _validate_list_of_type(self, list_of_type, value): - if not isinstance(value, list): - raise ValueError("Value should be a list") - if isinstance(list_of_type, DataField): - return [list_of_type.validate(self.parent, self.name, v) for v in value] - assert issubclass(list_of_type, DataModel) - return [list_of_type(**self._ensure_value_is_dict(v)) for v in value] - - def _validate_dict_of_type(self, dict_of_type, value): - assert issubclass(dict_of_type, DataModel) - value = self._ensure_value_is_dict(value) - return { - k: dict_of_type(**self._ensure_value_is_dict(v)) for k, v in value.items() - } - def _validate_str_value(self, value): if not isinstance(value, string_types): value = str(value) @@ -162,21 +154,94 @@ class DataModel(object): def __init__(self, **kwargs): self._field_names = [] - self._exceptions = [] + self._exceptions = set() for name, field in get_class_attributes(self).items(): if not isinstance(field, DataField): continue + field.parent = self + field.name = name self._field_names.append(name) + + raw_value = kwargs.get(name) value = None try: - value = field.validate(self, name, kwargs.get(name)) + if isinstance(field.type, ListOfType): + value = self._validate_list_of_type(field, name, raw_value) + elif isinstance(field.type, DictOfType): + value = self._validate_dict_of_type(field, name, raw_value) + else: + value = field.validate(raw_value) except DataFieldException as e: - self._exceptions.append(e) + self._exceptions.add(e) if isinstance(self, StrictDataModel): raise e finally: setattr(self, name, value) + def _validate_list_of_type(self, field, name, value): + data_type = field.type.type + # check if ListOfType is not required + value = field.validate(value) + if not value: + return None + if not isinstance(value, list): + raise DataFieldException(field, "Value should be a list") + + if isinstance(data_type, DataField): + result = [] + data_type.parent = self + data_type.name = name + for v in value: + try: + result.append(data_type.validate(v)) + except DataFieldException as e: + self._exceptions.add(e) + if isinstance(self, StrictDataModel): + raise e + return result + + assert issubclass(data_type, DataModel) + + result = [] + for v in value: + try: + if not isinstance(v, dict): + raise DataFieldException( + field, "Value `%s` should be type of dictionary" % v + ) + result.append(data_type(**v)) + except DataFieldException as e: + self._exceptions.add(e) + if isinstance(self, StrictDataModel): + raise e + return result + + def _validate_dict_of_type(self, field, _, value): + data_type = field.type.type + assert issubclass(data_type, DataModel) + + # check if DictOfType is not required + value = field.validate(value) + if not value: + return None + if not isinstance(value, dict): + raise DataFieldException( + field, "Value `%s` should be type of dictionary" % value + ) + result = {} + for k, v in value.items(): + try: + if not isinstance(v, dict): + raise DataFieldException( + field, "Value `%s` should be type of dictionary" % v + ) + result[k] = data_type(**v) + except DataFieldException as e: + self._exceptions.add(e) + if isinstance(self, StrictDataModel): + raise e + return result + def __eq__(self, other): assert isinstance(other, DataModel) if self.get_field_names() != other.get_field_names(): @@ -195,7 +260,19 @@ class DataModel(object): return self._field_names def get_exceptions(self): - return self._exceptions + result = list(self._exceptions) + for name in self._field_names: + value = getattr(self, name) + if isinstance(value, DataModel): + result.extend(value.get_exceptions()) + continue + if not isinstance(value, (dict, list)): + continue + for v in value.values() if isinstance(value, dict) else value: + if not isinstance(v, DataModel): + continue + result.extend(v.get_exceptions()) + return result def as_dict(self): result = {} diff --git a/platformio/package/manifest/model.py b/platformio/package/manifest/model.py index f881a370..23900777 100644 --- a/platformio/package/manifest/model.py +++ b/platformio/package/manifest/model.py @@ -30,6 +30,10 @@ class AuthorModel(DataModel): url = DataField(max_length=255) +class StrictAuthorModel(AuthorModel, StrictDataModel): + pass + + class RepositoryModel(DataModel): type = DataField(max_length=3, required=True) url = DataField(max_length=255, required=True) @@ -44,7 +48,7 @@ class ExportModel(DataModel): class ExampleModel(DataModel): name = DataField(max_length=100, regex=r"^[a-zA-Z\d\-\_/]+$", required=True) base = DataField(required=True) - files = DataField(type=ListOfType(DataField())) + files = DataField(type=ListOfType(DataField()), required=True) class ManifestModel(DataModel): @@ -84,4 +88,4 @@ class ManifestModel(DataModel): class StrictManifestModel(ManifestModel, StrictDataModel): - pass + authors = DataField(type=ListOfType(StrictAuthorModel), required=True) diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 027733a1..3bca641b 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -14,9 +14,8 @@ import pytest -from platformio.datamodel import DataFieldException -from platformio.package.manifest import parser -from platformio.package.manifest.model import ManifestModel, StrictManifestModel +from platformio import datamodel +from platformio.package.manifest import model, parser def test_library_json_parser(): @@ -139,31 +138,39 @@ sentence=This is Arduino library ) # Platforms ALL - mp = parser.LibraryPropertiesManifestParser("architectures=*\n" + contents) - assert mp.as_dict()["platforms"] == ["*"] + data = parser.LibraryPropertiesManifestParser( + "architectures=*\n" + contents + ).as_dict() + assert data["platforms"] == ["*"] # Platforms specific - mp = parser.LibraryPropertiesManifestParser("architectures=avr, esp32\n" + contents) - assert mp.as_dict()["platforms"] == ["atmelavr", "espressif32"] + data = parser.LibraryPropertiesManifestParser( + "architectures=avr, esp32\n" + contents + ).as_dict() + assert data["platforms"] == ["atmelavr", "espressif32"] # Remote URL - mp = parser.LibraryPropertiesManifestParser( + data = parser.LibraryPropertiesManifestParser( contents, remote_url=( "https://raw.githubusercontent.com/username/reponame/master/" "libraries/TestPackage/library.properties" ), - ) - assert mp.as_dict()["export"] == { + ).as_dict() + assert data["export"] == { "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"], "include": "libraries/TestPackage", } + assert data["repository"] == { + "url": "https://github.com/username/reponame", + "type": "git", + } # Hope page - mp = parser.LibraryPropertiesManifestParser( + data = parser.LibraryPropertiesManifestParser( "url=https://github.com/username/reponame.git\n" + contents - ) - assert mp.as_dict()["homepage"] is None - assert mp.as_dict()["repository"] == { + ).as_dict() + assert data["homepage"] is None + assert data["repository"] == { "type": "git", "url": "https://github.com/username/reponame.git", } @@ -208,14 +215,14 @@ def test_library_json_model(): ] } """ - mp = parser.ManifestParserFactory.new( + data = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.LIBRARY_JSON - ) - model = StrictManifestModel(**mp.as_dict()) - assert model.repository.url == "https://github.com/bblanchon/ArduinoJson.git" - assert model.examples[1].base == "examples/JsonHttpClient" - assert model.examples[1].files == ["JsonHttpClient.ino"] - assert model == StrictManifestModel( + ).as_dict() + m = model.StrictManifestModel(**data) + assert m.repository.url == "https://github.com/bblanchon/ArduinoJson.git" + assert m.examples[1].base == "examples/JsonHttpClient" + assert m.examples[1].files == ["JsonHttpClient.ino"] + assert m == model.StrictManifestModel( **{ "name": "ArduinoJson", "keywords": ["json", "rest", "http", "web"], @@ -270,12 +277,12 @@ category=Display url=https://github.com/olikraus/u8glib architectures=avr,sam """ - mp = parser.ManifestParserFactory.new( + data = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.LIBRARY_PROPERTIES - ) - model = StrictManifestModel(**mp.as_dict()) - assert not model.get_exceptions() - assert model == StrictManifestModel( + ).as_dict() + m = model.StrictManifestModel(**data) + assert not m.get_exceptions() + assert m == model.StrictManifestModel( **{ "license": None, "description": ( @@ -359,14 +366,13 @@ def test_platform_json_model(): } } """ - mp = parser.ManifestParserFactory.new( + data = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.PLATFORM_JSON - ) - data = mp.as_dict() + ).as_dict() data["frameworks"] = sorted(data["frameworks"]) - model = ManifestModel(**mp.as_dict()) - assert model.frameworks == ["arduino", "simba"] - assert model == ManifestModel( + m = model.ManifestModel(**data) + assert m.frameworks == ["arduino", "simba"] + assert m == model.ManifestModel( **{ "name": "atmelavr", "title": "Atmel AVR", @@ -399,13 +405,13 @@ def test_package_json_model(): "version": "3.30101.0" } """ - mp = parser.ManifestParserFactory.new( + data = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.PACKAGE_JSON - ) - model = ManifestModel(**mp.as_dict()) - assert model.system is None - assert model.homepage == "http://www.scons.org" - assert model == ManifestModel( + ).as_dict() + m = model.ManifestModel(**data) + assert m.system is None + assert m.homepage == "http://www.scons.org" + assert m == model.ManifestModel( **{ "name": "tool-scons", "description": "SCons software construction tool", @@ -491,9 +497,9 @@ def test_examples_from_dir(tmpdir_factory): return sorted(items, key=lambda item: item["name"]) data["examples"] = _sort_examples(data["examples"]) - model = ManifestModel(**data) - assert model.examples[3].name == "PlatformIO/hello" - assert model == ManifestModel( + m = model.ManifestModel(**data) + assert m.examples[3].name == "PlatformIO/hello" + assert m == model.ManifestModel( **{ "version": "1.0.0", "name": "pkg", @@ -541,26 +547,64 @@ def test_examples_from_dir(tmpdir_factory): ) +def test_dict_of_type(): + class TestModel(datamodel.DataModel): + examples = datamodel.DataField(type=datamodel.DictOfType(model.ExampleModel)) + + class StrictTestModel(TestModel, datamodel.StrictDataModel): + pass + + # valid + m = TestModel( + examples={ + "valid": dict(name="Valid", base="valid", files=["valid.h"]), + "invalid": "test", + } + ) + assert list(m.examples.keys()) == ["valid"] + + # invalid + with pytest.raises(datamodel.DataFieldException): + StrictTestModel(examples=[dict(name="Valid", base="valid", files=["valid.h"])]) + + with pytest.raises(datamodel.DataFieldException): + StrictTestModel( + examples={ + "valid": dict(name="Valid", base="valid", files=["valid.h"]), + "invalid": "test", + } + ) + + def test_broken_models(): # non-strict mode - assert len(ManifestModel(name="MyPackage").get_exceptions()) == 4 - assert ManifestModel(name="MyPackage", version="broken_version").version is None + assert len(model.ManifestModel(name="MyPackage").get_exceptions()) == 4 + assert ( + model.ManifestModel(name="MyPackage", version="broken_version").version is None + ) + + # invalid keywords + m = model.ManifestModel(keywords=["kw1", "*^[]"]) + assert any( + "Value `*^[]` does not match RegExp" in str(e) for e in m.get_exceptions() + ) + assert m.keywords == ["kw1"] # strict mode - with pytest.raises(DataFieldException) as excinfo: - assert StrictManifestModel(name="MyPackage") + with pytest.raises(datamodel.DataFieldException) as excinfo: + assert model.StrictManifestModel(name="MyPackage") assert excinfo.match(r"Missed value for `StrictManifestModel.[a-z]+` field") # broken SemVer with pytest.raises( - DataFieldException, + datamodel.DataFieldException, match=( "Invalid semantic versioning format for " "`StrictManifestModel.version` field" ), ): - assert StrictManifestModel( + assert model.StrictManifestModel( name="MyPackage", description="MyDescription", keywords=["a", "b"], @@ -569,8 +613,11 @@ def test_broken_models(): ) # broken value for DataModel - with pytest.raises(DataFieldException, match="Value should be type of dict"): - assert StrictManifestModel( + with pytest.raises( + datamodel.DataFieldException, + match=("Value `should be dict here` should be type of dictionary"), + ): + assert model.StrictManifestModel( name="MyPackage", description="MyDescription", keywords=["a", "b"], From dce26550049edff835f40869cdcbc3b0a2174105 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 4 Oct 2019 20:50:39 +0300 Subject: [PATCH 077/221] Fix broken serial monitor called via run target while uploading // Resolve #3081 --- platformio/run/processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/run/processor.py b/platformio/run/processor.py index 490ae497..6b7a9e20 100644 --- a/platformio/run/processor.py +++ b/platformio/run/processor.py @@ -59,7 +59,7 @@ class EnvironmentProcessor(object): raise exception.UndefinedEnvPlatform(self.name) build_vars = self.get_build_variables() - build_targets = self.get_build_targets() + build_targets = list(self.get_build_targets()) telemetry.on_run_environment(self.options, build_targets) From 46be56af43c113528ce2994fb7a846011caa8471 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 4 Oct 2019 20:51:05 +0300 Subject: [PATCH 078/221] Bump version to 4.1.0b3 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index a997299b..e6e63191 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, "0b2") +VERSION = (4, 1, "0b3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 0f7fe260d1e5fbfe44030b663f55007b73cbab6e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 4 Oct 2019 21:15:37 +0300 Subject: [PATCH 079/221] Docs: Sync ESP32 dev platform --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index d8441bba..0298fecd 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit d8441bba8630ba719e5c8559089bcfd698b1084f +Subproject commit 0298fecd5f117a472adaae23c9d9a7cfc43b5f6d From 5d7e7b17969f0834849cb0d2e94fa7528e46d74e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 4 Oct 2019 23:52:06 +0300 Subject: [PATCH 080/221] DataModel: capture exceptions from failed models in non-strict mode --- platformio/datamodel.py | 16 +++++++++--- tests/test_pkgmanifest.py | 54 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/platformio/datamodel.py b/platformio/datamodel.py index 53b99c29..600de738 100644 --- a/platformio/datamodel.py +++ b/platformio/datamodel.py @@ -209,7 +209,12 @@ class DataModel(object): raise DataFieldException( field, "Value `%s` should be type of dictionary" % v ) - result.append(data_type(**v)) + m = data_type(**v) + me = m.get_exceptions() + if not me: + result.append(m) + else: + self._exceptions |= set(me) except DataFieldException as e: self._exceptions.add(e) if isinstance(self, StrictDataModel): @@ -235,7 +240,12 @@ class DataModel(object): raise DataFieldException( field, "Value `%s` should be type of dictionary" % v ) - result[k] = data_type(**v) + m = data_type(**v) + me = m.get_exceptions() + if not me: + result[k] = m + else: + self._exceptions |= set(me) except DataFieldException as e: self._exceptions.add(e) if isinstance(self, StrictDataModel): @@ -246,8 +256,6 @@ class DataModel(object): assert isinstance(other, DataModel) if self.get_field_names() != other.get_field_names(): return False - if len(self.get_exceptions()) != len(other.get_exceptions()): - return False return self.as_dict() == other.as_dict() def __repr__(self): diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 3bca641b..cb3fc38c 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -265,7 +265,7 @@ def test_library_json_model(): ) -def library_properties_model(): +def test_library_properties_model(): contents = """ name=U8glib version=1.19.1 @@ -315,6 +315,58 @@ architectures=avr,sam } ) + # Broken fields + contents = """ +name=Mozzi +version=1.0.3 +author=Tim Barrass and contributors as documented in source, and at https://github.com/sensorium/Mozzi/graphs/contributors +maintainer=Tim Barrass +sentence=Sound synthesis library for Arduino +paragraph=With Mozzi, you can construct sounds using familiar synthesis units like oscillators, delays, filters and envelopes. +category=Signal Input/Output +url=https://sensorium.github.io/Mozzi/ +architectures=* +dot_a_linkage=false +includes=MozziGuts.h +""" + data = parser.ManifestParserFactory.new( + contents, + parser.ManifestFileType.LIBRARY_PROPERTIES, + remote_url=( + "https://raw.githubusercontent.com/sensorium/Mozzi/" + "master/library.properties" + ), + ).as_dict() + m = model.ManifestModel(**data) + assert m.get_exceptions() + assert m == model.ManifestModel( + **{ + "name": "Mozzi", + "version": "1.0.3", + "description": ( + "Sound synthesis library for Arduino. With Mozzi, you can construct " + "sounds using familiar synthesis units like oscillators, delays, " + "filters and envelopes." + ), + "repository": {"url": "https://github.com/sensorium/Mozzi", "type": "git"}, + "platforms": ["*"], + "frameworks": ["arduino"], + "export": { + "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"], + "include": None, + }, + "authors": [ + { + "maintainer": True, + "email": "faveflave@gmail.com", + "name": "Tim Barrass", + } + ], + "keywords": ["signal", "input", "output"], + "homepage": "https://sensorium.github.io/Mozzi/", + } + ) + def test_platform_json_model(): contents = """ From 10bada0bccea0494aaa15ee8ecb2a4f445da77b5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 5 Oct 2019 20:21:39 +0300 Subject: [PATCH 081/221] ManifestPaser: handle examples from "Examples" folder --- platformio/package/manifest/parser.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index f41d97e6..53586a98 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -173,7 +173,9 @@ class BaseManifestParser(object): assert os.path.isdir(package_dir) examples_dir = os.path.join(package_dir, "examples") if not os.path.isdir(examples_dir): - return None + examples_dir = os.path.join(package_dir, "Examples") + if not os.path.isdir(examples_dir): + return None allowed_exts = ( ".c", From d1c8cc38f27da744639b14560d3a6e449ba1111d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 5 Oct 2019 23:40:27 +0300 Subject: [PATCH 082/221] Cast semver to string when validating version --- platformio/package/manifest/model.py | 1 + 1 file changed, 1 insertion(+) diff --git a/platformio/package/manifest/model.py b/platformio/package/manifest/model.py index 23900777..8dfb168b 100644 --- a/platformio/package/manifest/model.py +++ b/platformio/package/manifest/model.py @@ -18,6 +18,7 @@ from platformio.datamodel import DataField, DataModel, ListOfType, StrictDataMod def validate_semver_field(_, value): + value = str(value) if "." not in value: raise ValueError("Invalid semantic versioning format") return value if semantic_version.Version.coerce(value) else None From b66bf5f4c071b08e195c824cf6f9e419365bc076 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 7 Oct 2019 20:35:01 +0300 Subject: [PATCH 083/221] Ignore symbolic links for package examples --- platformio/package/manifest/parser.py | 6 +++++- tests/test_pkgmanifest.py | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 53586a98..0427d171 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -195,7 +195,11 @@ class BaseManifestParser(object): last_pio_project = None for root, _, files in os.walk(examples_dir): # skip hidden files and folders - files = [f for f in files if not f.startswith(".")] + files = [ + f + for f in files + if not f.startswith(".") and not os.path.islink(os.path.join(root, f)) + ] if os.path.basename(root).startswith(".") or not files: continue diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index cb3fc38c..675169d4 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -507,6 +507,9 @@ def test_examples_from_dir(tmpdir_factory): # PlatformIO project #1 pio_dir = examples_dir.mkdir("PlatformIO").mkdir("hello") pio_dir.join(".vimrc").write("") + pio_ini = pio_dir.join("platformio.ini") + pio_ini.write("") + pio_dir.join("platformio.ini.copy").mksymlinkto(pio_ini) pio_dir.join("platformio.ini").write("") pio_dir.mkdir("include").join("main.h").write("") pio_dir.mkdir("src").join("main.cpp").write("") From e5d50eb45c8bdfd567b10629a4b51b3962f7a4ac Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 8 Oct 2019 11:49:04 +0300 Subject: [PATCH 084/221] Docs: RV-LINK debug tool, sync GDV32 dev/platform --- docs | 2 +- platformio/package/manifest/parser.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 0298fecd..7cabe5bb 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 0298fecd5f117a472adaae23c9d9a7cfc43b5f6d +Subproject commit 7cabe5bb16a54b7f802c853fcb6f94ae8f86d904 diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 0427d171..265dd13a 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -194,7 +194,7 @@ class BaseManifestParser(object): result = {} last_pio_project = None for root, _, files in os.walk(examples_dir): - # skip hidden files and folders + # skip hidden files, symlinks, and folders files = [ f for f in files From eda02750ae489d6c4f7b978c98ac213da75aca03 Mon Sep 17 00:00:00 2001 From: valeros Date: Tue, 8 Oct 2019 13:45:36 +0300 Subject: [PATCH 085/221] Export files as list instead of dict for sizedata target --- platformio/builder/tools/piosize.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/platformio/builder/tools/piosize.py b/platformio/builder/tools/piosize.py index 814e7c42..dd6ec357 100644 --- a/platformio/builder/tools/piosize.py +++ b/platformio/builder/tools/piosize.py @@ -171,7 +171,11 @@ def DumpSizeData(_, target, source, env): # pylint: disable=unused-argument files[file_path]["symbols"].append(symbol) - data["memory"]["files"] = files + data["memory"]["files"] = list() + for k, v in files.items(): + file_data = {"path": k} + file_data.update(v) + data["memory"]["files"].append(file_data) with open(join(env.subst("$BUILD_DIR"), "sizedata.json"), "w") as fp: fp.write(dump_json_to_unicode(data)) From a785c238b1778d5907b3b6d106a2462069fc2cb0 Mon Sep 17 00:00:00 2001 From: valeros Date: Wed, 9 Oct 2019 00:39:57 +0300 Subject: [PATCH 086/221] Use OS-native directory separator in sizedata report --- platformio/builder/tools/piosize.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/platformio/builder/tools/piosize.py b/platformio/builder/tools/piosize.py index dd6ec357..533e22ab 100644 --- a/platformio/builder/tools/piosize.py +++ b/platformio/builder/tools/piosize.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=too-many-locals + from __future__ import absolute_import import sys @@ -23,12 +25,14 @@ from elftools.elf.elffile import ELFFile from platformio.compat import dump_json_to_unicode from platformio.proc import exec_command +from platformio.util import get_systype def _get_file_location(env, elf_path, addr, sysenv): cmd = [env.subst("$CC").replace("-gcc", "-addr2line"), "-e", elf_path, hex(addr)] result = exec_command(cmd, env=sysenv) - return result["out"].strip().replace("\\", "/") + path = result["out"].strip() + return path.replace("/", "\\") if "windows" in get_systype() else path def _determine_section(sections, symbol_addr): From 971eb8e35c7fbb54bd5c86e3f9a923e774d595f6 Mon Sep 17 00:00:00 2001 From: valeros Date: Wed, 9 Oct 2019 17:37:24 +0300 Subject: [PATCH 087/221] Revert back unix style directory separator in sizedata report --- platformio/builder/tools/piosize.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/platformio/builder/tools/piosize.py b/platformio/builder/tools/piosize.py index 533e22ab..db8f2b5e 100644 --- a/platformio/builder/tools/piosize.py +++ b/platformio/builder/tools/piosize.py @@ -25,14 +25,12 @@ from elftools.elf.elffile import ELFFile from platformio.compat import dump_json_to_unicode from platformio.proc import exec_command -from platformio.util import get_systype def _get_file_location(env, elf_path, addr, sysenv): cmd = [env.subst("$CC").replace("-gcc", "-addr2line"), "-e", elf_path, hex(addr)] result = exec_command(cmd, env=sysenv) - path = result["out"].strip() - return path.replace("/", "\\") if "windows" in get_systype() else path + return result["out"].strip().replace("\\", "/") def _determine_section(sections, symbol_addr): From 4366719ed2a93eda29717a9b185f652653653c11 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 10 Oct 2019 14:50:34 +0300 Subject: [PATCH 088/221] Restore missed project helpers needed by "platformio-node-helpers" --- platformio/project/helpers.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index e28c6643..5b921e9d 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -59,6 +59,36 @@ def get_project_cache_dir(): ).get_optional_dir("cache") +def get_project_global_lib_dir(): + """ + Deprecated, use ProjectConfig.get_optional_dir("globallib") instead + "platformio-node-helpers" depends on it + """ + return ProjectConfig.get_instance( + join(get_project_dir(), "platformio.ini") + ).get_optional_dir("globallib") + + +def get_project_lib_dir(): + """ + Deprecated, use ProjectConfig.get_optional_dir("lib") instead + "platformio-node-helpers" depends on it + """ + return ProjectConfig.get_instance( + join(get_project_dir(), "platformio.ini") + ).get_optional_dir("lib") + + +def get_project_libdeps_dir(): + """ + Deprecated, use ProjectConfig.get_optional_dir("libdeps") instead + "platformio-node-helpers" depends on it + """ + return ProjectConfig.get_instance( + join(get_project_dir(), "platformio.ini") + ).get_optional_dir("libdeps") + + def get_default_projects_dir(): docs_dir = join(expanduser("~"), "Documents") try: From c8354b100eac8cdba5e0a883bff34fdbe31f0d93 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 10 Oct 2019 14:51:14 +0300 Subject: [PATCH 089/221] Bump version to 4.1.0b4 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index e6e63191..28d0a201 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, "0b3") +VERSION = (4, 1, "0b4") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 82ec0164b0384c59dc434582cb65ca89b42ac6d4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 10 Oct 2019 23:35:59 +0300 Subject: [PATCH 090/221] Update docs on how to install Python 3.7 on Windows --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 7cabe5bb..5a4ce029 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 7cabe5bb16a54b7f802c853fcb6f94ae8f86d904 +Subproject commit 5a4ce029b90ed1015d93bf8a51671b1bcf6390a2 From 181adb277f26142bbebc1ff1c2039a8cbfbac495 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 12 Oct 2019 19:58:34 +0300 Subject: [PATCH 091/221] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 5a4ce029..9bd32976 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 5a4ce029b90ed1015d93bf8a51671b1bcf6390a2 +Subproject commit 9bd3297645386bb2b84bdbb9de793369ec76ce83 From 55d905a0d04a705d7e9267b83a29742d1039eeb9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 12 Oct 2019 20:00:12 +0300 Subject: [PATCH 092/221] Add a new RPC method "project.config_call" for Home API --- platformio/home/rpc/handlers/project.py | 4 ++++ platformio/project/config.py | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/platformio/home/rpc/handlers/project.py b/platformio/home/rpc/handlers/project.py index 097f7e45..f7a9b8f8 100644 --- a/platformio/home/rpc/handlers/project.py +++ b/platformio/home/rpc/handlers/project.py @@ -32,6 +32,10 @@ from platformio.project.helpers import is_platformio_project class ProjectRPC(object): + @staticmethod + def config_call(path, method, *args): + return getattr(ProjectConfig(path), method)(*args) + @staticmethod def _get_projects(project_dirs=None): def _get_project_data(): diff --git a/platformio/project/config.py b/platformio/project/config.py index d0abf25c..988b53c2 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -409,11 +409,11 @@ class ProjectConfig(ProjectConfigBase, ProjectConfigDirsMixin): def __repr__(self): return "" % (self.path or "in-memory") + def as_dict(self): + return {s: self.items(s, as_dict=True) for s in self.sections()} + def to_json(self): - result = {} - for section in self.sections(): - result[section] = self.items(section, as_dict=True) - return json.dumps(result) + return json.dumps(self.as_dict()) def save(self, path=None): path = path or self.path From 2e9b0066de246f4b1f1520d080b9d5fcf94c349d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 14 Oct 2019 23:36:15 +0300 Subject: [PATCH 093/221] Capture manifest parser exceptions --- platformio/package/manifest/parser.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 265dd13a..1d119fb0 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -127,7 +127,10 @@ class BaseManifestParser(object): def __init__(self, contents, remote_url=None, package_dir=None): self.remote_url = remote_url self.package_dir = package_dir - self._data = self.parse(contents) + try: + self._data = self.parse(contents) + except Exception as e: + raise ManifestParserException("Could not parse manifest -> %s" % e) self._data = self.parse_examples(self._data) def parse(self, contents): From 239befa4eeb4535854c215ce0817f7e57d2326f4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 15 Oct 2019 13:05:56 +0300 Subject: [PATCH 094/221] New Shakti dev/platform --- README.rst | 2 ++ docs | 2 +- examples | 2 +- tests/test_pkgmanifest.py | 5 +++-- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 1b539c67..abb3f5f6 100644 --- a/README.rst +++ b/README.rst @@ -93,6 +93,7 @@ Development Platforms * `RISC-V `_ * `RISC-V GAP `_ * `Samsung ARTIK `_ +* `Shakti `_ * `Silicon Labs EFM32 `_ * `ST STM32 `_ * `ST STM8 `_ @@ -117,6 +118,7 @@ Frameworks * `mbed `_ * `PULP OS `_ * `Pumbaa `_ +* `Shakti `_ * `Simba `_ * `SPL `_ * `STM32Cube `_ diff --git a/docs b/docs index 9bd32976..e755c715 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 9bd3297645386bb2b84bdbb9de793369ec76ce83 +Subproject commit e755c715fff103557eb4727f46c391216b378c08 diff --git a/examples b/examples index 7e80325b..bf351dbc 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 7e80325b974382034f1247372974bdfd2a6194ae +Subproject commit bf351dbce107e39850bbc95385a6986b3088c302 diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 675169d4..db531b93 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -15,6 +15,7 @@ import pytest from platformio import datamodel +from platformio.compat import WINDOWS from platformio.package.manifest import model, parser @@ -509,8 +510,8 @@ def test_examples_from_dir(tmpdir_factory): pio_dir.join(".vimrc").write("") pio_ini = pio_dir.join("platformio.ini") pio_ini.write("") - pio_dir.join("platformio.ini.copy").mksymlinkto(pio_ini) - pio_dir.join("platformio.ini").write("") + if not WINDOWS: + pio_dir.join("platformio.ini.copy").mksymlinkto(pio_ini) pio_dir.mkdir("include").join("main.h").write("") pio_dir.mkdir("src").join("main.cpp").write("") From e8692334f6b628be3c4a3a720f4c74fc2e6d8387 Mon Sep 17 00:00:00 2001 From: Matt McCartney Date: Tue, 15 Oct 2019 12:00:48 -0700 Subject: [PATCH 095/221] Replace deprecated in python3: iteritems with items and basestr with str (#3119) --- scripts/get-platformio.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/get-platformio.py b/scripts/get-platformio.py index ec201d62..988f6054 100644 --- a/scripts/get-platformio.py +++ b/scripts/get-platformio.py @@ -69,8 +69,8 @@ def exec_command(*args, **kwargs): result['out'], result['err'] = p.communicate() result['returncode'] = p.returncode - for k, v in result.iteritems(): - if v and isinstance(v, basestring): + for k, v in result.items(): + if v and isinstance(v, str): result[k].strip() return result From a481a5dedad8c45fa5fe362b47eb1e9876938bc5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 15 Oct 2019 23:30:02 +0300 Subject: [PATCH 096/221] Fix issue with "remote test" // Resolve #3127 --- platformio/managers/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 8c6227fa..4ff9e5da 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -26,7 +26,7 @@ from platformio.project.config import ProjectConfig CORE_PACKAGES = { "contrib-piohome": "^2.3.2", "contrib-pysite": "~2.%d%d.190418" % (sys.version_info[0], sys.version_info[1]), - "tool-pioplus": "^2.5.5", + "tool-pioplus": "^2.5.6", "tool-unity": "~1.20403.0", "tool-scons": "~2.20501.7" if PY2 else "~3.30101.0", "tool-cppcheck": "~1.189.0", From 9da19fbf54f973269606f1d587f99dd4b4b7e99c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 16 Oct 2019 12:09:53 +0300 Subject: [PATCH 097/221] Use isolated SCons sign DB per Python interpreter --- platformio/builder/main.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 39565c6b..be5a4e07 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import sys from os import environ, makedirs from os.path import isdir, join from time import time @@ -28,7 +29,7 @@ from SCons.Script import Import # pylint: disable=import-error from SCons.Script import Variables # pylint: disable=import-error from platformio import fs -from platformio.compat import PY2, dump_json_to_unicode +from platformio.compat import dump_json_to_unicode from platformio.managers.platform import PlatformBase from platformio.proc import get_pythonexe_path from platformio.project.helpers import get_project_dir @@ -137,7 +138,9 @@ env.LoadProjectOptions() env.LoadPioPlatform() env.SConscriptChdir(0) -env.SConsignFile(join("$BUILD_DIR", ".sconsign.dblite" if PY2 else ".sconsign3.dblite")) +env.SConsignFile( + join("$BUILD_DIR", ".sconsign.py%d%d" % (sys.version_info[0], sys.version_info[1])) +) for item in env.GetExtraScripts("pre"): env.SConscript(item, exports="env") From 9cfccc5cd4713ba3f9e05e562ee62fc2c807b116 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 16 Oct 2019 13:58:50 +0300 Subject: [PATCH 098/221] Minor fixes to manifest parser --- docs | 2 +- platformio/package/manifest/model.py | 2 +- platformio/package/manifest/parser.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs b/docs index e755c715..3c565d17 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit e755c715fff103557eb4727f46c391216b378c08 +Subproject commit 3c565d175d608db0428aeafbca41c380876590a6 diff --git a/platformio/package/manifest/model.py b/platformio/package/manifest/model.py index 8dfb168b..d5428ec7 100644 --- a/platformio/package/manifest/model.py +++ b/platformio/package/manifest/model.py @@ -72,7 +72,7 @@ class ManifestModel(DataModel): type=ListOfType(DataField(max_length=50, regex=r"^([a-z\d\-_]+|\*)$")) ) frameworks = DataField( - type=ListOfType(DataField(max_length=50, regex=r"^([a-z\d\-_\*]+|\*)$")) + type=ListOfType(DataField(max_length=50, regex=r"^([a-z\d\-_]+|\*)$")) ) repository = DataField(type=RepositoryModel) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 1d119fb0..7970fecd 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -240,6 +240,7 @@ class BaseManifestParser(object): # normalize example names for item in result: + item["name"] = item["name"].replace(os.path.sep, "/") item["name"] = re.sub(r"[^a-z\d\d\-\_/]+", "_", item["name"], flags=re.I) return result or None From 27fc19d6b39f0c9fd40fee802de9b80768fb4ad2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 17 Oct 2019 00:17:16 +0300 Subject: [PATCH 099/221] Switch to Marshmallow ODM framework --- docs | 2 +- platformio/commands/lib.py | 8 +- platformio/datamodel.py | 303 -------------------------- platformio/package/exception.py | 36 +++ platformio/package/manifest/model.py | 92 -------- platformio/package/manifest/parser.py | 51 ++--- platformio/package/manifest/schema.py | 181 +++++++++++++++ setup.py | 1 + tests/test_pkgmanifest.py | 268 +++++++++-------------- tox.ini | 1 + 10 files changed, 358 insertions(+), 585 deletions(-) delete mode 100644 platformio/datamodel.py create mode 100644 platformio/package/exception.py delete mode 100644 platformio/package/manifest/model.py create mode 100644 platformio/package/manifest/schema.py diff --git a/docs b/docs index 3c565d17..e8b93616 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 3c565d175d608db0428aeafbca41c380876590a6 +Subproject commit e8b9361615d00180d9cb01af0b401885da091df8 diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 3f52cb8b..58cb85ed 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -25,8 +25,8 @@ from platformio import exception, util from platformio.commands import PlatformioCLI from platformio.compat import dump_json_to_unicode from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib -from platformio.package.manifest.model import StrictManifestModel from platformio.package.manifest.parser import ManifestParserFactory +from platformio.package.manifest.schema import ManifestSchema, ManifestValidationError from platformio.proc import is_ci from platformio.project.config import ProjectConfig from platformio.project.helpers import get_project_dir, is_platformio_project @@ -495,7 +495,11 @@ def lib_register(config_url): raise exception.InvalidLibConfURL(config_url) # Validate manifest - StrictManifestModel(**ManifestParserFactory.new_from_url(config_url).as_dict()) + data, error = ManifestSchema(strict=False).load( + ManifestParserFactory.new_from_url(config_url).as_dict() + ) + if error: + raise ManifestValidationError(error, data) result = util.get_api_result("/lib/register", data=dict(config_url=config_url)) if "message" in result and result["message"]: diff --git a/platformio/datamodel.py b/platformio/datamodel.py deleted file mode 100644 index 600de738..00000000 --- a/platformio/datamodel.py +++ /dev/null @@ -1,303 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import inspect -import re - -from platformio.compat import get_class_attributes, string_types -from platformio.exception import PlatformioException - -# pylint: disable=too-many-instance-attributes -# pylint: disable=redefined-builtin, too-many-arguments - - -class DataModelException(PlatformioException): - pass - - -class DataFieldException(DataModelException): - def __init__(self, field, message): - self.field = field - self.message = message - super(DataFieldException, self).__init__() - - def __str__(self): - return "%s for `%s.%s` field" % ( - self.message, - self.field.parent.__class__.__name__, - self.field.name, - ) - - def __repr__(self): - return str(self) - - -class ListOfType(object): - def __init__(self, type): - self.type = type - - -class DictOfType(object): - def __init__(self, type): - self.type = type - - -class DataField(object): - def __init__( - self, - default=None, - type=str, - required=False, - min_length=None, - max_length=None, - regex=None, - validate_factory=None, - title=None, - ): - self.default = default - self.type = type - self.required = required - self.min_length = min_length - self.max_length = max_length - self.regex = regex - self.validate_factory = validate_factory - self.title = title - - self._parent = None - self._name = None - self.value = None - - def __repr__(self): - return '' % ( - self.title, - self.default if self.value is None else self.value, - ) - - @property - def parent(self): - return self._parent - - @parent.setter - def parent(self, value): - self._parent = value - - @property - def name(self): - return self._name - - @name.setter - def name(self, value): - self._name = value - self.title = self.title or value.title() - - def validate(self, value): - try: - if self.required and value is None: - raise ValueError("Missed value") - if self.validate_factory is not None: - return self.validate_factory(self, value) or self.default - if value is None: - return self.default - if inspect.isclass(self.type) and issubclass(self.type, DataModel): - return self.type(**self.ensure_value_is_dict(value)) - if inspect.isclass(self.type) and issubclass(self.type, (str, bool)): - return getattr(self, "_validate_%s_value" % self.type.__name__)(value) - except ValueError as e: - raise DataFieldException(self, str(e)) - return value - - @staticmethod - def ensure_value_is_dict(value): - if not isinstance(value, dict): - raise ValueError("Value should be type of dict, not `%s`" % type(value)) - return value - - def _validate_str_value(self, value): - if not isinstance(value, string_types): - value = str(value) - if self.min_length and len(value) < self.min_length: - raise ValueError( - "Minimum allowed length is %d characters" % self.min_length - ) - if self.max_length and len(value) > self.max_length: - raise ValueError( - "Maximum allowed length is %d characters" % self.max_length - ) - if self.regex and not re.match(self.regex, value): - raise ValueError( - "Value `%s` does not match RegExp `%s` pattern" % (value, self.regex) - ) - return value - - @staticmethod - def _validate_bool_value(value): - if isinstance(value, bool): - return value - return str(value).lower() in ("true", "yes", "1") - - -class DataModel(object): - - _field_names = None - _exceptions = None - - def __init__(self, **kwargs): - self._field_names = [] - self._exceptions = set() - for name, field in get_class_attributes(self).items(): - if not isinstance(field, DataField): - continue - field.parent = self - field.name = name - self._field_names.append(name) - - raw_value = kwargs.get(name) - value = None - try: - if isinstance(field.type, ListOfType): - value = self._validate_list_of_type(field, name, raw_value) - elif isinstance(field.type, DictOfType): - value = self._validate_dict_of_type(field, name, raw_value) - else: - value = field.validate(raw_value) - except DataFieldException as e: - self._exceptions.add(e) - if isinstance(self, StrictDataModel): - raise e - finally: - setattr(self, name, value) - - def _validate_list_of_type(self, field, name, value): - data_type = field.type.type - # check if ListOfType is not required - value = field.validate(value) - if not value: - return None - if not isinstance(value, list): - raise DataFieldException(field, "Value should be a list") - - if isinstance(data_type, DataField): - result = [] - data_type.parent = self - data_type.name = name - for v in value: - try: - result.append(data_type.validate(v)) - except DataFieldException as e: - self._exceptions.add(e) - if isinstance(self, StrictDataModel): - raise e - return result - - assert issubclass(data_type, DataModel) - - result = [] - for v in value: - try: - if not isinstance(v, dict): - raise DataFieldException( - field, "Value `%s` should be type of dictionary" % v - ) - m = data_type(**v) - me = m.get_exceptions() - if not me: - result.append(m) - else: - self._exceptions |= set(me) - except DataFieldException as e: - self._exceptions.add(e) - if isinstance(self, StrictDataModel): - raise e - return result - - def _validate_dict_of_type(self, field, _, value): - data_type = field.type.type - assert issubclass(data_type, DataModel) - - # check if DictOfType is not required - value = field.validate(value) - if not value: - return None - if not isinstance(value, dict): - raise DataFieldException( - field, "Value `%s` should be type of dictionary" % value - ) - result = {} - for k, v in value.items(): - try: - if not isinstance(v, dict): - raise DataFieldException( - field, "Value `%s` should be type of dictionary" % v - ) - m = data_type(**v) - me = m.get_exceptions() - if not me: - result[k] = m - else: - self._exceptions |= set(me) - except DataFieldException as e: - self._exceptions.add(e) - if isinstance(self, StrictDataModel): - raise e - return result - - def __eq__(self, other): - assert isinstance(other, DataModel) - if self.get_field_names() != other.get_field_names(): - return False - return self.as_dict() == other.as_dict() - - def __repr__(self): - fields = [] - for name in self._field_names: - fields.append('%s="%s"' % (name, getattr(self, name))) - return "<%s %s>" % (self.__class__.__name__, " ".join(fields)) - - def get_field_names(self): - return self._field_names - - def get_exceptions(self): - result = list(self._exceptions) - for name in self._field_names: - value = getattr(self, name) - if isinstance(value, DataModel): - result.extend(value.get_exceptions()) - continue - if not isinstance(value, (dict, list)): - continue - for v in value.values() if isinstance(value, dict) else value: - if not isinstance(v, DataModel): - continue - result.extend(v.get_exceptions()) - return result - - def as_dict(self): - result = {} - for name in self._field_names: - value = getattr(self, name) - if isinstance(value, DataModel): - value = getattr(self, name).as_dict() - if isinstance(value, dict): - for k, v in value.items(): - if not isinstance(v, DataModel): - continue - value[k] = v.as_dict() - elif isinstance(value, list): - value = [v.as_dict() if isinstance(v, DataModel) else v for v in value] - result[name] = value - return result - - -class StrictDataModel(DataModel): - pass diff --git a/platformio/package/exception.py b/platformio/package/exception.py new file mode 100644 index 00000000..81753ad1 --- /dev/null +++ b/platformio/package/exception.py @@ -0,0 +1,36 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.exception import PlatformioException + + +class ManifestException(PlatformioException): + pass + + +class ManifestParserError(ManifestException): + pass + + +class ManifestValidationError(ManifestException): + def __init__(self, error, data): + super(ManifestValidationError, self).__init__() + self.error = error + self.data = data + + def __str__(self): + return ( + "Invalid manifest fields: %s. \nPlease check specification -> " + "http://docs.platformio.org/page/librarymanager/config.html" % self.error + ) diff --git a/platformio/package/manifest/model.py b/platformio/package/manifest/model.py deleted file mode 100644 index d5428ec7..00000000 --- a/platformio/package/manifest/model.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import semantic_version - -from platformio.datamodel import DataField, DataModel, ListOfType, StrictDataModel - - -def validate_semver_field(_, value): - value = str(value) - if "." not in value: - raise ValueError("Invalid semantic versioning format") - return value if semantic_version.Version.coerce(value) else None - - -class AuthorModel(DataModel): - name = DataField(max_length=50, required=True) - email = DataField(max_length=50) - maintainer = DataField(default=False, type=bool) - url = DataField(max_length=255) - - -class StrictAuthorModel(AuthorModel, StrictDataModel): - pass - - -class RepositoryModel(DataModel): - type = DataField(max_length=3, required=True) - url = DataField(max_length=255, required=True) - branch = DataField(max_length=50) - - -class ExportModel(DataModel): - include = DataField(type=ListOfType(DataField())) - exclude = DataField(type=ListOfType(DataField())) - - -class ExampleModel(DataModel): - name = DataField(max_length=100, regex=r"^[a-zA-Z\d\-\_/]+$", required=True) - base = DataField(required=True) - files = DataField(type=ListOfType(DataField()), required=True) - - -class ManifestModel(DataModel): - - # Required fields - name = DataField(max_length=100, required=True) - version = DataField( - max_length=50, validate_factory=validate_semver_field, required=True - ) - description = DataField(max_length=1000, required=True) - keywords = DataField( - type=ListOfType(DataField(max_length=255, regex=r"^[a-z\d\-\+\. ]+$")), - required=True, - ) - authors = DataField(type=ListOfType(AuthorModel), required=True) - - homepage = DataField(max_length=255) - license = DataField(max_length=255) - platforms = DataField( - type=ListOfType(DataField(max_length=50, regex=r"^([a-z\d\-_]+|\*)$")) - ) - frameworks = DataField( - type=ListOfType(DataField(max_length=50, regex=r"^([a-z\d\-_]+|\*)$")) - ) - - repository = DataField(type=RepositoryModel) - export = DataField(type=ExportModel) - examples = DataField(type=ListOfType(ExampleModel)) - - # platform.json specific - title = DataField(max_length=100) - - # package.json specific - system = DataField( - type=ListOfType(DataField(max_length=50, regex=r"^[a-z\d\-_]+$")) - ) - - -class StrictManifestModel(ManifestModel, StrictDataModel): - authors = DataField(type=ListOfType(StrictAuthorModel), required=True) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 7970fecd..aa0e1c7e 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -19,8 +19,8 @@ import re import requests from platformio.compat import get_class_attributes, string_types -from platformio.exception import PlatformioException from platformio.fs import get_file_contents +from platformio.package.exception import ManifestParserError from platformio.project.helpers import is_platformio_project try: @@ -29,14 +29,6 @@ except ImportError: from urlparse import urlparse -class ManifestException(PlatformioException): - pass - - -class ManifestParserException(ManifestException): - pass - - class ManifestFileType(object): PLATFORM_JSON = "platform.json" LIBRARY_JSON = "library.json" @@ -67,11 +59,11 @@ class ManifestParserFactory(object): @staticmethod def new_from_file(path, remote_url=False): if not path or not os.path.isfile(path): - raise ManifestException("Manifest file does not exist %s" % path) + raise ManifestParserError("Manifest file does not exist %s" % path) for t in get_class_attributes(ManifestFileType).values(): if path.endswith(t): return ManifestParserFactory.new(get_file_contents(path), t, remote_url) - raise ManifestException("Unknown manifest file type %s" % path) + raise ManifestParserError("Unknown manifest file type %s" % path) @staticmethod def new_from_dir(path, remote_url=None): @@ -102,7 +94,7 @@ class ManifestParserFactory(object): remote_url=remote_url, package_dir=path, ) - raise ManifestException("Unknown manifest file type in %s directory" % path) + raise ManifestParserError("Unknown manifest file type in %s directory" % path) @staticmethod def new_from_url(remote_url): @@ -119,7 +111,7 @@ class ManifestParserFactory(object): # pylint: disable=redefined-builtin clsname = ManifestParserFactory.type_to_clsname(type) if clsname not in globals(): - raise ManifestException("Unknown manifest file type %s" % clsname) + raise ManifestParserError("Unknown manifest file type %s" % clsname) return globals()[clsname](contents, remote_url, package_dir) @@ -130,9 +122,14 @@ class BaseManifestParser(object): try: self._data = self.parse(contents) except Exception as e: - raise ManifestParserException("Could not parse manifest -> %s" % e) + raise ManifestParserError("Could not parse manifest -> %s" % e) self._data = self.parse_examples(self._data) + # remove None fields + for key in list(self._data.keys()): + if self._data[key] is None: + del self._data[key] + def parse(self, contents): raise NotImplementedError @@ -141,8 +138,12 @@ class BaseManifestParser(object): @staticmethod def cleanup_author(author): + assert isinstance(author, dict) if author.get("email"): author["email"] = re.sub(r"\s+[aA][tT]\s+", "@", author["email"]) + for key in list(author.keys()): + if author[key] is None: + del author[key] return author @staticmethod @@ -351,9 +352,7 @@ class ModuleJsonManifestParser(BaseManifestParser): name, email = self.parse_author_name_and_email(author) if not name: continue - result.append( - self.cleanup_author(dict(name=name, email=email, maintainer=False)) - ) + result.append(self.cleanup_author(dict(name=name, email=email))) return result @staticmethod @@ -431,7 +430,7 @@ class LibraryPropertiesManifestParser(BaseManifestParser): } for arch in properties.get("architectures", "").split(","): if "particle-" in arch: - raise ManifestParserException("Particle is not supported yet") + raise ManifestParserError("Particle is not supported yet") arch = arch.strip() if not arch: continue @@ -449,20 +448,18 @@ class LibraryPropertiesManifestParser(BaseManifestParser): name, email = self.parse_author_name_and_email(author) if not name: continue - authors.append( - self.cleanup_author(dict(name=name, email=email, maintainer=False)) - ) + authors.append(self.cleanup_author(dict(name=name, email=email))) for author in properties.get("maintainer", "").split(","): name, email = self.parse_author_name_and_email(author) if not name: continue found = False for item in authors: - if item["name"].lower() != name.lower(): + if item.get("name", "").lower() != name.lower(): continue found = True item["maintainer"] = True - if not item["email"]: + if not item.get("email"): item["email"] = email if not found: authors.append( @@ -495,6 +492,7 @@ class LibraryPropertiesManifestParser(BaseManifestParser): return None def _parse_export(self): + result = {"exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"]} include = None if self.remote_url: repo_parse = urlparse(self.remote_url) @@ -506,10 +504,9 @@ class LibraryPropertiesManifestParser(BaseManifestParser): "/".join(repo_path_tokens[repo_path_tokens.index("raw") + 2 :]) or None ) - return { - "include": include, - "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"], - } + if include: + result["include"] = include + return result class PlatformJsonManifestParser(BaseManifestParser): diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py new file mode 100644 index 00000000..75471323 --- /dev/null +++ b/platformio/package/manifest/schema.py @@ -0,0 +1,181 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import requests +import semantic_version +from marshmallow import Schema, ValidationError, fields, validate, validates + +from platformio.package.exception import ManifestValidationError +from platformio.util import memoized + + +class StrictSchema(Schema): + def handle_error(self, error, data): + # skip broken records + if self.many: + error.data = [ + item for idx, item in enumerate(data) if idx not in error.messages + ] + else: + error.data = None + raise error + + +class StrictListField(fields.List): + def _deserialize(self, value, attr, data): + try: + return super(StrictListField, self)._deserialize(value, attr, data) + except ValidationError as exc: + exc.data = [item for item in exc.data if item is not None] + raise exc + + +class AuthorSchema(StrictSchema): + name = fields.Str(required=True, validate=validate.Length(min=1, max=50)) + email = fields.Email(validate=validate.Length(min=1, max=50)) + maintainer = fields.Bool(default=False) + url = fields.Url(validate=validate.Length(min=1, max=255)) + + +class RepositorySchema(StrictSchema): + type = fields.Str( + required=True, + validate=validate.OneOf( + ["git", "hg", "svn"], + error="Invalid repository type, please use one of [git, hg, svn]", + ), + ) + url = fields.Str(required=True, validate=validate.Length(min=1, max=255)) + branch = fields.Str(validate=validate.Length(min=1, max=50)) + + +class ExportSchema(Schema): + include = StrictListField(fields.Str) + exclude = StrictListField(fields.Str) + + +class ExampleSchema(StrictSchema): + name = fields.Str( + required=True, + validate=[ + validate.Length(min=1, max=100), + validate.Regexp( + r"^[a-zA-Z\d\-\_/]+$", error="Only [a-zA-Z0-9-_/] chars are allowed" + ), + ], + ) + base = fields.Str(required=True) + files = StrictListField(fields.Str, required=True) + + +class ManifestSchema(Schema): + # Required fields + name = fields.Str(required=True, validate=validate.Length(min=1, max=100)) + version = fields.Str(required=True, validate=validate.Length(min=1, max=50)) + + # Optional fields + + authors = fields.Nested(AuthorSchema, many=True) + description = fields.Str(validate=validate.Length(min=1, max=1000)) + homepage = fields.Url(validate=validate.Length(min=1, max=255)) + license = fields.Str(validate=validate.Length(min=1, max=255)) + repository = fields.Nested(RepositorySchema) + export = fields.Nested(ExportSchema) + examples = fields.Nested(ExampleSchema, many=True) + + keywords = StrictListField( + fields.Str( + validate=[ + validate.Length(min=1, max=50), + validate.Regexp( + r"^[a-z\d\-\+\. ]+$", error="Only [a-z0-9-+. ] chars are allowed" + ), + ] + ) + ) + + platforms = StrictListField( + fields.Str( + validate=[ + validate.Length(min=1, max=50), + validate.Regexp( + r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed" + ), + ] + ) + ) + frameworks = StrictListField( + fields.Str( + validate=[ + validate.Length(min=1, max=50), + validate.Regexp( + r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed" + ), + ] + ) + ) + + # platform.json specific + title = fields.Str(validate=validate.Length(min=1, max=100)) + + # package.json specific + system = StrictListField( + fields.Str( + validate=[ + validate.Length(min=1, max=50), + validate.Regexp( + r"^[a-z\d\-_]+$", error="Only [a-z0-9-_] chars are allowed" + ), + ] + ) + ) + + def handle_error(self, error, data): + if self.strict: + raise ManifestValidationError(error, data) + + @validates("version") + def validate_version(self, value): # pylint: disable=no-self-use + try: + value = str(value) + assert "." in value + semantic_version.Version.coerce(value) + except (AssertionError, ValueError): + raise ValidationError( + "Invalid semantic versioning format, see https://semver.org/" + ) + + @validates("license") + def validate_license(self, value): + try: + spdx = self.load_spdx_licenses() + except requests.exceptions.RequestException: + raise ValidationError("Could not load SPDX licenses for validation") + for item in spdx.get("licenses", []): + if item.get("licenseId") == value: + return + raise ValidationError( + "Invalid SPDX license identifier. See valid identifiers at " + "https://spdx.org/licenses/" + ) + + @staticmethod + @memoized(expire="1h") + def load_spdx_licenses(): + r = requests.get( + "https://raw.githubusercontent.com/spdx/license-list-data" + "/v3.6/json/licenses.json" + ) + r.raise_for_status() + return r.json() diff --git a/setup.py b/setup.py index 60b0a115..64a83fbf 100644 --- a/setup.py +++ b/setup.py @@ -33,6 +33,7 @@ install_requires = [ "semantic_version>=2.8.1,<3", "tabulate>=0.8.3,<1", "pyelftools>=0.25,<1", + "marshmallow>=2.20.5,<3" ] setup( diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index db531b93..95a9af43 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -12,11 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import jsondiff import pytest -from platformio import datamodel from platformio.compat import WINDOWS -from platformio.package.manifest import model, parser +from platformio.package.manifest import parser +from platformio.package.manifest.schema import ManifestSchema, ManifestValidationError def test_library_json_parser(): @@ -31,14 +32,15 @@ def test_library_json_parser(): } """ mp = parser.LibraryJsonManifestParser(contents) - assert sorted(mp.as_dict().items()) == sorted( + assert not jsondiff.diff( + mp.as_dict(), { "name": "TestPackage", "platforms": ["atmelavr", "espressif8266"], "export": {"exclude": [".gitignore", "tests"], "include": ["mylib"]}, "keywords": ["kw1", "kw2", "kw3"], "homepage": "http://old.url.format", - }.items() + }, ) contents = """ @@ -52,13 +54,14 @@ def test_library_json_parser(): } """ mp = parser.LibraryJsonManifestParser(contents) - assert sorted(mp.as_dict().items()) == sorted( + assert not jsondiff.diff( + mp.as_dict(), { "keywords": ["sound", "audio", "music", "sd", "card", "playback"], "frameworks": ["arduino"], "export": {"exclude": ["audio_samples"]}, "platforms": ["atmelavr"], - }.items() + }, ) @@ -87,7 +90,8 @@ def test_module_json_parser(): } """ mp = parser.ModuleJsonManifestParser(contents) - assert sorted(mp.as_dict().items()) == sorted( + assert not jsondiff.diff( + mp.as_dict(), { "name": "YottaLibrary", "description": "This is Yotta library", @@ -97,15 +101,9 @@ def test_module_json_parser(): "platforms": ["*"], "frameworks": ["mbed"], "export": {"exclude": ["tests", "test", "*.doxyfile", "*.pdf"]}, - "authors": [ - { - "maintainer": False, - "email": "name@surname.com", - "name": "Name Surname", - } - ], + "authors": [{"email": "name@surname.com", "name": "Name Surname"}], "version": "1.2.3", - }.items() + }, ) @@ -118,24 +116,20 @@ author=SomeAuthor sentence=This is Arduino library """ mp = parser.LibraryPropertiesManifestParser(contents) - assert sorted(mp.as_dict().items()) == sorted( + assert not jsondiff.diff( + mp.as_dict(), { "name": "TestPackage", "version": "1.2.3", "description": "This is Arduino library", - "repository": None, "platforms": ["*"], "frameworks": ["arduino"], "export": { - "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"], - "include": None, + "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"] }, - "authors": [ - {"maintainer": False, "email": "info@author.com", "name": "SomeAuthor"} - ], + "authors": [{"email": "info@author.com", "name": "SomeAuthor"}], "keywords": ["uncategorized"], - "homepage": None, - }.items() + }, ) # Platforms ALL @@ -170,7 +164,6 @@ sentence=This is Arduino library data = parser.LibraryPropertiesManifestParser( "url=https://github.com/username/reponame.git\n" + contents ).as_dict() - assert data["homepage"] is None assert data["repository"] == { "type": "git", "url": "https://github.com/username/reponame.git", @@ -216,15 +209,20 @@ def test_library_json_model(): ] } """ - data = parser.ManifestParserFactory.new( + raw_data = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.LIBRARY_JSON ).as_dict() - m = model.StrictManifestModel(**data) - assert m.repository.url == "https://github.com/bblanchon/ArduinoJson.git" - assert m.examples[1].base == "examples/JsonHttpClient" - assert m.examples[1].files == ["JsonHttpClient.ino"] - assert m == model.StrictManifestModel( - **{ + + data, errors = ManifestSchema(strict=True).load(raw_data) + assert not errors + + assert data["repository"]["url"] == "https://github.com/bblanchon/ArduinoJson.git" + assert data["examples"][1]["base"] == "examples/JsonHttpClient" + assert data["examples"][1]["files"] == ["JsonHttpClient.ino"] + + assert not jsondiff.diff( + data, + { "name": "ArduinoJson", "keywords": ["json", "rest", "http", "web"], "description": "An elegant and efficient JSON library for embedded systems", @@ -232,21 +230,12 @@ def test_library_json_model(): "repository": { "url": "https://github.com/bblanchon/ArduinoJson.git", "type": "git", - "branch": None, }, "version": "6.12.0", "authors": [ - { - "name": "Benoit Blanchon", - "url": "https://blog.benoitblanchon.fr", - "maintainer": False, - "email": None, - } + {"name": "Benoit Blanchon", "url": "https://blog.benoitblanchon.fr"} ], - "export": { - "exclude": ["fuzzing", "scripts", "test", "third-party"], - "include": None, - }, + "export": {"exclude": ["fuzzing", "scripts", "test", "third-party"]}, "frameworks": ["arduino"], "platforms": ["*"], "license": "MIT", @@ -262,7 +251,7 @@ def test_library_json_model(): "files": ["JsonHttpClient.ino"], }, ], - } + }, ) @@ -278,42 +267,33 @@ category=Display url=https://github.com/olikraus/u8glib architectures=avr,sam """ - data = parser.ManifestParserFactory.new( + raw_data = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.LIBRARY_PROPERTIES ).as_dict() - m = model.StrictManifestModel(**data) - assert not m.get_exceptions() - assert m == model.StrictManifestModel( - **{ - "license": None, + + data, errors = ManifestSchema(strict=True).load(raw_data) + assert not errors + + assert not jsondiff.diff( + data, + { "description": ( "A library for monochrome TFTs and OLEDs. Supported display " "controller: SSD1306, SSD1309, SSD1322, SSD1325" ), - "repository": { - "url": "https://github.com/olikraus/u8glib", - "type": "git", - "branch": None, - }, + "repository": {"url": "https://github.com/olikraus/u8glib", "type": "git"}, "frameworks": ["arduino"], "platforms": ["atmelavr", "atmelsam"], "version": "1.19.1", "export": { - "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"], - "include": None, + "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"] }, "authors": [ - { - "url": None, - "maintainer": True, - "email": "olikraus@gmail.com", - "name": "oliver", - } + {"maintainer": True, "email": "olikraus@gmail.com", "name": "oliver"} ], "keywords": ["display"], - "homepage": None, "name": "U8glib", - } + }, ) # Broken fields @@ -330,7 +310,7 @@ architectures=* dot_a_linkage=false includes=MozziGuts.h """ - data = parser.ManifestParserFactory.new( + raw_data = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.LIBRARY_PROPERTIES, remote_url=( @@ -338,10 +318,13 @@ includes=MozziGuts.h "master/library.properties" ), ).as_dict() - m = model.ManifestModel(**data) - assert m.get_exceptions() - assert m == model.ManifestModel( - **{ + + data, errors = ManifestSchema(strict=False).load(raw_data) + assert errors["authors"] + + assert not jsondiff.diff( + data, + { "name": "Mozzi", "version": "1.0.3", "description": ( @@ -353,8 +336,7 @@ includes=MozziGuts.h "platforms": ["*"], "frameworks": ["arduino"], "export": { - "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"], - "include": None, + "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"] }, "authors": [ { @@ -365,7 +347,7 @@ includes=MozziGuts.h ], "keywords": ["signal", "input", "output"], "homepage": "https://sensorium.github.io/Mozzi/", - } + }, ) @@ -419,14 +401,16 @@ def test_platform_json_model(): } } """ - data = parser.ManifestParserFactory.new( + raw_data = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.PLATFORM_JSON ).as_dict() - data["frameworks"] = sorted(data["frameworks"]) - m = model.ManifestModel(**data) - assert m.frameworks == ["arduino", "simba"] - assert m == model.ManifestModel( - **{ + + data, errors = ManifestSchema(strict=False).load(raw_data) + assert not errors + + assert not jsondiff.diff( + data, + { "name": "atmelavr", "title": "Atmel AVR", "description": ( @@ -441,11 +425,10 @@ def test_platform_json_model(): "repository": { "url": "https://github.com/platformio/platform-atmelavr.git", "type": "git", - "branch": None, }, "frameworks": ["arduino", "simba"], "version": "1.15.0", - } + }, ) @@ -458,19 +441,21 @@ def test_package_json_model(): "version": "3.30101.0" } """ - data = parser.ManifestParserFactory.new( + raw_data = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.PACKAGE_JSON ).as_dict() - m = model.ManifestModel(**data) - assert m.system is None - assert m.homepage == "http://www.scons.org" - assert m == model.ManifestModel( - **{ + + data, errors = ManifestSchema(strict=False).load(raw_data) + assert not errors + + assert not jsondiff.diff( + data, + { "name": "tool-scons", "description": "SCons software construction tool", "homepage": "http://www.scons.org", "version": "3.30101.0", - } + }, ) mp = parser.ManifestParserFactory.new( @@ -543,20 +528,23 @@ def test_examples_from_dir(tmpdir_factory): # Do testing - data = parser.ManifestParserFactory.new_from_dir(str(package_dir)).as_dict() - assert isinstance(data["examples"], list) - assert len(data["examples"]) == 6 + raw_data = parser.ManifestParserFactory.new_from_dir(str(package_dir)).as_dict() + assert isinstance(raw_data["examples"], list) + assert len(raw_data["examples"]) == 6 def _sort_examples(items): for i, item in enumerate(items): items[i]["files"] = sorted(item["files"]) return sorted(items, key=lambda item: item["name"]) - data["examples"] = _sort_examples(data["examples"]) - m = model.ManifestModel(**data) - assert m.examples[3].name == "PlatformIO/hello" - assert m == model.ManifestModel( - **{ + raw_data["examples"] = _sort_examples(raw_data["examples"]) + + data, errors = ManifestSchema(strict=True).load(raw_data) + assert not errors + + assert not jsondiff.diff( + data, + { "version": "1.0.0", "name": "pkg", "examples": _sort_examples( @@ -599,84 +587,44 @@ def test_examples_from_dir(tmpdir_factory): }, ] ), - } + }, ) -def test_dict_of_type(): - class TestModel(datamodel.DataModel): - examples = datamodel.DataField(type=datamodel.DictOfType(model.ExampleModel)) - - class StrictTestModel(TestModel, datamodel.StrictDataModel): - pass - - # valid - m = TestModel( - examples={ - "valid": dict(name="Valid", base="valid", files=["valid.h"]), - "invalid": "test", - } - ) - assert list(m.examples.keys()) == ["valid"] - - # invalid - with pytest.raises(datamodel.DataFieldException): - StrictTestModel(examples=[dict(name="Valid", base="valid", files=["valid.h"])]) - - with pytest.raises(datamodel.DataFieldException): - StrictTestModel( - examples={ - "valid": dict(name="Valid", base="valid", files=["valid.h"]), - "invalid": "test", - } - ) - - def test_broken_models(): # non-strict mode - assert len(model.ManifestModel(name="MyPackage").get_exceptions()) == 4 - assert ( - model.ManifestModel(name="MyPackage", version="broken_version").version is None - ) + data, errors = ManifestSchema(strict=False).load(dict(name="MyPackage")) + assert set(errors.keys()) == set(["version"]) + assert data.get("version") is None # invalid keywords - m = model.ManifestModel(keywords=["kw1", "*^[]"]) - assert any( - "Value `*^[]` does not match RegExp" in str(e) for e in m.get_exceptions() - ) - assert m.keywords == ["kw1"] + data, errors = ManifestSchema(strict=False).load(dict(keywords=["kw1", "*^[]"])) + assert errors + assert data["keywords"] == ["kw1"] # strict mode - with pytest.raises(datamodel.DataFieldException) as excinfo: - assert model.StrictManifestModel(name="MyPackage") - assert excinfo.match(r"Missed value for `StrictManifestModel.[a-z]+` field") + with pytest.raises( + ManifestValidationError, match="Missing data for required field" + ): + ManifestSchema(strict=True).load(dict(name="MyPackage")) # broken SemVer with pytest.raises( - datamodel.DataFieldException, - match=( - "Invalid semantic versioning format for " - "`StrictManifestModel.version` field" - ), + ManifestValidationError, match=("Invalid semantic versioning format") ): - assert model.StrictManifestModel( - name="MyPackage", - description="MyDescription", - keywords=["a", "b"], - authors=[{"name": "Author"}], - version="broken_version", + ManifestSchema(strict=True).load( + dict(name="MyPackage", version="broken_version") ) - # broken value for DataModel - with pytest.raises( - datamodel.DataFieldException, - match=("Value `should be dict here` should be type of dictionary"), - ): - assert model.StrictManifestModel( - name="MyPackage", - description="MyDescription", - keywords=["a", "b"], - authors=["should be dict here"], - version="1.2.3", + # broken value for Nested + with pytest.raises(ManifestValidationError, match=r"authors.*Invalid input type"): + ManifestSchema(strict=True).load( + dict( + name="MyPackage", + description="MyDescription", + keywords=["a", "b"], + authors=["should be dict here"], + version="1.2.3", + ) ) diff --git a/tox.ini b/tox.ini index 3512bfb6..29384ca2 100644 --- a/tox.ini +++ b/tox.ini @@ -24,6 +24,7 @@ deps = pylint pytest pytest-xdist + jsondiff commands = {envpython} --version pylint --rcfile=./.pylintrc ./platformio From 83f25cbc16e7a2377336f80915af359fb8bbd2db Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 17 Oct 2019 12:38:35 +0300 Subject: [PATCH 100/221] Fix tests --- tests/test_pkgmanifest.py | 16 ++++++++-------- tox.ini | 2 -- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 95a9af43..ccafa35f 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -170,7 +170,7 @@ sentence=This is Arduino library } -def test_library_json_model(): +def test_library_json_schema(): contents = """ { "name": "ArduinoJson", @@ -255,7 +255,7 @@ def test_library_json_model(): ) -def test_library_properties_model(): +def test_library_properties_schema(): contents = """ name=U8glib version=1.19.1 @@ -351,7 +351,7 @@ includes=MozziGuts.h ) -def test_platform_json_model(): +def test_platform_json_schema(): contents = """ { "name": "atmelavr", @@ -404,7 +404,7 @@ def test_platform_json_model(): raw_data = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.PLATFORM_JSON ).as_dict() - + raw_data["frameworks"] = sorted(raw_data["frameworks"]) data, errors = ManifestSchema(strict=False).load(raw_data) assert not errors @@ -426,13 +426,13 @@ def test_platform_json_model(): "url": "https://github.com/platformio/platform-atmelavr.git", "type": "git", }, - "frameworks": ["arduino", "simba"], + "frameworks": sorted(["arduino", "simba"]), "version": "1.15.0", - }, + } ) -def test_package_json_model(): +def test_package_json_schema(): contents = """ { "name": "tool-scons", @@ -591,7 +591,7 @@ def test_examples_from_dir(tmpdir_factory): ) -def test_broken_models(): +def test_broken_schemas(): # non-strict mode data, errors = ManifestSchema(strict=False).load(dict(name="MyPackage")) assert set(errors.keys()) == set(["version"]) diff --git a/tox.ini b/tox.ini index 29384ca2..fc31f45e 100644 --- a/tox.ini +++ b/tox.ini @@ -50,8 +50,6 @@ commands = sphinx-build -W -b linkcheck docs docs/_build/html [testenv:skipexamples] -deps = - pytest commands = py.test -v --basetemp="{envtmpdir}" tests --ignore tests/test_examples.py From 31d4a5c72e25b10e6d35c81c1500cbcda9bfd93f Mon Sep 17 00:00:00 2001 From: valeros Date: Thu, 17 Oct 2019 13:42:00 +0300 Subject: [PATCH 101/221] Add collective stats info about project components to check report --- platformio/commands/check.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/platformio/commands/check.py b/platformio/commands/check.py index 0bf33012..c041d043 100644 --- a/platformio/commands/check.py +++ b/platformio/commands/check.py @@ -147,17 +147,19 @@ def cli( click.echo("No defects found") print_processing_footer(result) + component_stats = collect_component_stats(results) if json_output: - click.echo(dump_json_to_unicode(results_to_json(results))) + click.echo(dump_json_to_unicode( + results_to_json(results, component_stats))) elif not silent: - print_check_summary(results) + print_check_summary(results, component_stats) command_failed = any(r.get("succeeded") is False for r in results) if command_failed: raise exception.ReturnErrorCode(1) -def results_to_json(raw): +def results_to_json(raw, components): results = [] for item in raw: item.update( @@ -165,6 +167,7 @@ def results_to_json(raw): "ignored": item.get("succeeded") is None, "succeeded": bool(item.get("succeeded")), "defects": [d.to_json() for d in item.get("defects", [])], + "stats": [{k: v} for k, v in components.items()] } ) results.append(item) @@ -197,7 +200,7 @@ def print_processing_footer(result): ) -def print_defects_stats(results): +def collect_component_stats(results): components = dict() def _append_defect(component, defect): @@ -215,13 +218,17 @@ def print_defects_stats(results): component = dirname(component) _append_defect(component, defect) - if not components: + return dict(components) + + +def print_defects_stats(component_stats): + if not component_stats: return severity_labels = list(DefectItem.SEVERITY_LABELS.values()) severity_labels.reverse() tabular_data = list() - for k, v in components.items(): + for k, v in component_stats.items(): tool_defect = [v.get(s, 0) for s in severity_labels] tabular_data.append([k] + tool_defect) @@ -237,7 +244,7 @@ def print_defects_stats(results): click.echo() -def print_check_summary(results): +def print_check_summary(results, component_stats): click.echo() tabular_data = [] @@ -245,7 +252,7 @@ def print_check_summary(results): failed_nums = 0 duration = 0 - print_defects_stats(results) + print_defects_stats(component_stats) for result in results: duration += result.get("duration", 0) From 89843c0d652c7d2a295d7db23a1cd0cde03b61bf Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 17 Oct 2019 15:48:18 +0300 Subject: [PATCH 102/221] Fix issue with parsing library.properties when export field is used --- platformio/package/manifest/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index aa0e1c7e..dc7265b3 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -505,7 +505,7 @@ class LibraryPropertiesManifestParser(BaseManifestParser): or None ) if include: - result["include"] = include + result["include"] = [include] return result From 7bcfea13fb5b2daf07f4a4c6f2c9d1cb0412a30b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 17 Oct 2019 16:52:18 +0300 Subject: [PATCH 103/221] Fixed an issue with linking process when ``$LDSCRIPT`` contains a space in path --- HISTORY.rst | 1 + platformio/builder/tools/platformio.py | 2 +- platformio/package/manifest/schema.py | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 62d41eaa..61c48891 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -25,6 +25,7 @@ PlatformIO Core 4.0 * Fixed default PIO Unified Debugger configuration for `J-Link probe `__ * Fixed an issue when configuration file options partly ignored when using custom ``--project-conf`` (`issue #3034 `_) * Fixed an issue when installing a package using custom Git tag and submodules were not updated correctly (`issue #3060 `_) +* Fixed an issue with linking process when ``$LDSCRIPT`` contains a space in path 4.0.3 (2019-08-30) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index f032be94..c4e5dc8c 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -130,7 +130,7 @@ def BuildProgram(env): # append into the beginning a main LD script if env.get("LDSCRIPT_PATH") and not any("-Wl,-T" in f for f in env["LINKFLAGS"]): - env.Prepend(LINKFLAGS=["-T", "$LDSCRIPT_PATH"]) + env.Prepend(LINKFLAGS=["-T", env["LDSCRIPT_PATH"]]) # enable "cyclic reference" for linker if env.get("LIBS") and env.GetCompilerType() == "gcc": diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 75471323..f1d68e08 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -37,7 +37,8 @@ class StrictListField(fields.List): try: return super(StrictListField, self)._deserialize(value, attr, data) except ValidationError as exc: - exc.data = [item for item in exc.data if item is not None] + if exc.data: + exc.data = [item for item in exc.data if item is not None] raise exc From 6218b773fd014e8fdbca612c9d44b89bc6992f66 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 17 Oct 2019 18:48:59 +0300 Subject: [PATCH 104/221] Better support for file contents writing // Issue #2796 --- platformio/builder/tools/piolib.py | 33 ++++++++---------- platformio/builder/tools/piomaxlen.py | 4 +-- platformio/builder/tools/piomisc.py | 12 +++---- platformio/commands/check.py | 5 ++- platformio/commands/init.py | 45 ++++++++++++------------- platformio/commands/remote.py | 3 +- platformio/compat.py | 12 ------- platformio/debug/helpers.py | 10 +++--- platformio/fs.py | 21 +++++++++++- platformio/home/rpc/handlers/project.py | 3 +- platformio/ide/projectgenerator.py | 7 ++-- platformio/run/helpers.py | 9 ++--- platformio/test/processor.py | 5 ++- tests/test_pkgmanifest.py | 2 +- 14 files changed, 80 insertions(+), 91 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 5ca4b2a2..b27fbf9b 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -41,12 +41,7 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error from platformio import exception, fs, util from platformio.builder.tools import platformio as piotool -from platformio.compat import ( - WINDOWS, - get_file_contents, - hashlib_encode_data, - string_types, -) +from platformio.compat import WINDOWS, hashlib_encode_data, string_types from platformio.managers.lib import LibraryManager @@ -92,7 +87,7 @@ class LibBuilderFactory(object): fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT ): continue - content = get_file_contents(join(root, fname)) + content = fs.get_file_contents(join(root, fname)) if not content: continue if "Arduino.h" in content and include_re.search(content): @@ -1047,22 +1042,22 @@ def ConfigureProjectLibBuilder(env): title += " %s" % lb.version if vcs_info and vcs_info.get("version"): title += " #%s" % vcs_info.get("version") - sys.stdout.write("%s|-- %s" % (margin, title)) + click.echo("%s|-- %s" % (margin, title), nl=False) if int(ARGUMENTS.get("PIOVERBOSE", 0)): if vcs_info: - sys.stdout.write(" [%s]" % vcs_info.get("url")) - sys.stdout.write(" (") - sys.stdout.write(lb.path) - sys.stdout.write(")") - sys.stdout.write("\n") + click.echo(" [%s]" % vcs_info.get("url"), nl=False) + click.echo(" (", nl=False) + click.echo(lb.path, nl=False) + click.echo(")", nl=False) + click.echo("") if lb.depbuilders: _print_deps_tree(lb, level + 1) project = ProjectAsLibBuilder(env, "$PROJECT_DIR") ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project) - print ("LDF: Library Dependency Finder -> http://bit.ly/configure-pio-ldf") - print ( + click.echo("LDF: Library Dependency Finder -> http://bit.ly/configure-pio-ldf") + click.echo( "LDF Modes: Finder ~ %s, Compatibility ~ %s" % (ldf_mode, project.lib_compat_mode) ) @@ -1070,19 +1065,19 @@ def ConfigureProjectLibBuilder(env): project.install_dependencies() lib_builders = env.GetLibBuilders() - print ("Found %d compatible libraries" % len(lib_builders)) + click.echo("Found %d compatible libraries" % len(lib_builders)) - print ("Scanning dependencies...") + click.echo("Scanning dependencies...") project.search_deps_recursive() if ldf_mode.startswith("chain") and project.depbuilders: _correct_found_libs(lib_builders) if project.depbuilders: - print ("Dependency Graph") + click.echo("Dependency Graph") _print_deps_tree(project) else: - print ("No dependencies") + click.echo("No dependencies") return project diff --git a/platformio/builder/tools/piomaxlen.py b/platformio/builder/tools/piomaxlen.py index 04f1304f..3bf258db 100644 --- a/platformio/builder/tools/piomaxlen.py +++ b/platformio/builder/tools/piomaxlen.py @@ -18,6 +18,7 @@ from hashlib import md5 from os import makedirs from os.path import isdir, isfile, join +from platformio import fs from platformio.compat import WINDOWS, hashlib_encode_data # Windows CLI has limit with command length to 8192 @@ -66,8 +67,7 @@ def _file_long_data(env, data): ) if isfile(tmp_file): return tmp_file - with open(tmp_file, "w") as fp: - fp.write(data) + fs.write_file_contents(tmp_file, data) return tmp_file diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index dd7cffd6..20768671 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -25,7 +25,7 @@ from SCons.Action import Action # pylint: disable=import-error from SCons.Script import ARGUMENTS # pylint: disable=import-error from platformio import fs, util -from platformio.compat import get_file_contents, glob_escape +from platformio.compat import glob_escape from platformio.managers.core import get_core_package_dir from platformio.proc import exec_command @@ -62,7 +62,7 @@ class InoToCPPConverter(object): assert nodes lines = [] for node in nodes: - contents = get_file_contents(node.get_path()) + contents = fs.get_file_contents(node.get_path()) _lines = ['# 1 "%s"' % node.get_path().replace("\\", "/"), contents] if self.is_main_node(contents): lines = _lines + lines @@ -78,16 +78,14 @@ class InoToCPPConverter(object): def process(self, contents): out_file = self._main_ino + ".cpp" assert self._gcc_preprocess(contents, out_file) - contents = get_file_contents(out_file) + contents = fs.get_file_contents(out_file) contents = self._join_multiline_strings(contents) - with open(out_file, "w") as fp: - fp.write(self.append_prototypes(contents)) + fs.write_file_contents(out_file, self.append_prototypes(contents)) return out_file def _gcc_preprocess(self, contents, out_file): tmp_path = mkstemp()[1] - with open(tmp_path, "w") as fp: - fp.write(contents) + fs.write_file_contents(tmp_path, contents) self.env.Execute( self.env.VerboseAction( '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format( diff --git a/platformio/commands/check.py b/platformio/commands/check.py index c041d043..614f60d1 100644 --- a/platformio/commands/check.py +++ b/platformio/commands/check.py @@ -149,8 +149,7 @@ def cli( component_stats = collect_component_stats(results) if json_output: - click.echo(dump_json_to_unicode( - results_to_json(results, component_stats))) + click.echo(dump_json_to_unicode(results_to_json(results, component_stats))) elif not silent: print_check_summary(results, component_stats) @@ -167,7 +166,7 @@ def results_to_json(raw, components): "ignored": item.get("succeeded") is None, "succeeded": bool(item.get("succeeded")), "defects": [d.to_json() for d in item.get("defects", [])], - "stats": [{k: v} for k, v in components.items()] + "stats": [{k: v} for k, v in components.items()], } ) results.append(item) diff --git a/platformio/commands/init.py b/platformio/commands/init.py index ebf8a3f6..e37871c9 100644 --- a/platformio/commands/init.py +++ b/platformio/commands/init.py @@ -145,9 +145,9 @@ def init_base_project(project_dir): def init_include_readme(include_dir): - with open(join(include_dir, "README"), "w") as f: - f.write( - """ + fs.write_file_contents( + join(include_dir, "README"), + """ This directory is intended for project header files. A header file is a file containing C declarations and macro definitions @@ -186,15 +186,15 @@ Read more about using header files in official GCC documentation: * Computed Includes https://gcc.gnu.org/onlinedocs/cpp/Header-Files.html -""" - ) +""", + ) def init_lib_readme(lib_dir): - with open(join(lib_dir, "README"), "w") as f: - # pylint: disable=line-too-long - f.write( - """ + # pylint: disable=line-too-long + fs.write_file_contents( + join(lib_dir, "README"), + """ This directory is intended for project specific (private) libraries. PlatformIO will compile them to static libraries and link into executable file. @@ -240,14 +240,14 @@ libraries scanning project source files. More information about PlatformIO Library Dependency Finder - https://docs.platformio.org/page/librarymanager/ldf.html -""" - ) +""", + ) def init_test_readme(test_dir): - with open(join(test_dir, "README"), "w") as f: - f.write( - """ + fs.write_file_contents( + join(test_dir, "README"), + """ This directory is intended for PIO Unit Testing and project tests. Unit Testing is a software testing method by which individual units of @@ -258,17 +258,17 @@ in the development cycle. More information about PIO Unit Testing: - https://docs.platformio.org/page/plus/unit-testing.html -""" - ) +""", + ) def init_ci_conf(project_dir): conf_path = join(project_dir, ".travis.yml") if isfile(conf_path): return - with open(conf_path, "w") as f: - f.write( - """# Continuous Integration (CI) is the practice, in software + fs.write_file_contents( + conf_path, + """# Continuous Integration (CI) is the practice, in software # engineering, of merging all developer working copies with a shared mainline # several times a day < https://docs.platformio.org/page/ci/index.html > # @@ -335,16 +335,15 @@ def init_ci_conf(project_dir): # # script: # - platformio ci --lib="." --board=ID_1 --board=ID_2 --board=ID_N -""" - ) +""", + ) def init_cvs_ignore(project_dir): conf_path = join(project_dir, ".gitignore") if isfile(conf_path): return - with open(conf_path, "w") as fp: - fp.write(".pio\n") + fs.write_file_contents(conf_path, ".pio\n") def fill_project_envs( diff --git a/platformio/commands/remote.py b/platformio/commands/remote.py index e15bbf7a..a0707c1f 100644 --- a/platformio/commands/remote.py +++ b/platformio/commands/remote.py @@ -23,7 +23,6 @@ import click from platformio import exception, fs from platformio.commands.device import device_monitor as cmd_device_monitor -from platformio.compat import get_file_contents from platformio.managers.core import pioplus_call # pylint: disable=unused-argument @@ -201,7 +200,7 @@ def device_monitor(ctx, **kwargs): sleep(0.1) if not t.is_alive(): return - kwargs["port"] = get_file_contents(sock_file) + kwargs["port"] = fs.get_file_contents(sock_file) ctx.invoke(cmd_device_monitor, **kwargs) t.join(2) finally: diff --git a/platformio/compat.py b/platformio/compat.py index 15942556..37c63f2a 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -52,10 +52,6 @@ if PY2: return path return path.decode(get_filesystem_encoding()).encode("utf-8") - def get_file_contents(path): - with open(path) as f: - return f.read() - def hashlib_encode_data(data): if is_bytes(data): return data @@ -104,14 +100,6 @@ else: def path_to_unicode(path): return path - def get_file_contents(path): - try: - with open(path) as f: - return f.read() - except UnicodeDecodeError: - with open(path, encoding="latin-1") as f: - return f.read() - def hashlib_encode_data(data): if is_bytes(data): return data diff --git a/platformio/debug/helpers.py b/platformio/debug/helpers.py index d88307e8..db396649 100644 --- a/platformio/debug/helpers.py +++ b/platformio/debug/helpers.py @@ -220,14 +220,12 @@ def is_prog_obsolete(prog_path): break shasum.update(data) new_digest = shasum.hexdigest() - old_digest = None - if isfile(prog_hash_path): - with open(prog_hash_path, "r") as fp: - old_digest = fp.read() + old_digest = ( + fs.get_file_contents(prog_hash_path) if isfile(prog_hash_path) else None + ) if new_digest == old_digest: return False - with open(prog_hash_path, "w") as fp: - fp.write(new_digest) + fs.write_file_contents(prog_hash_path, new_digest) return True diff --git a/platformio/fs.py b/platformio/fs.py index 3e708484..080f1502 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import io import json import os import re @@ -23,7 +24,7 @@ from glob import glob import click from platformio import exception -from platformio.compat import WINDOWS, get_file_contents, glob_escape +from platformio.compat import WINDOWS, glob_escape class cd(object): @@ -48,6 +49,24 @@ def get_source_dir(): return os.path.dirname(curpath) +def get_file_contents(path): + try: + with open(path) as fp: + return fp.read() + except UnicodeDecodeError: + with io.open(path, encoding="latin-1") as fp: + return fp.read() + + +def write_file_contents(path, contents): + try: + with open(path, "w") as fp: + return fp.write(contents) + except UnicodeDecodeError: + with io.open(path, "w", encoding="latin-1") as fp: + return fp.write(contents) + + def load_json(file_path): try: with open(file_path, "r") as f: diff --git a/platformio/home/rpc/handlers/project.py b/platformio/home/rpc/handlers/project.py index f7a9b8f8..bb320ab9 100644 --- a/platformio/home/rpc/handlers/project.py +++ b/platformio/home/rpc/handlers/project.py @@ -208,8 +208,7 @@ class ProjectRPC(object): return project_dir if not isdir(src_dir): os.makedirs(src_dir) - with open(main_path, "w") as f: - f.write(main_content.strip()) + fs.write_file_contents(main_path, main_content.strip()) return project_dir def import_arduino(self, board, use_arduino_libs, arduino_project_dir): diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index defeaafa..b297e37b 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import codecs +import io import os import sys from os.path import abspath, basename, expanduser, isdir, isfile, join, relpath @@ -20,7 +20,6 @@ from os.path import abspath, basename, expanduser, isdir, isfile, join, relpath import bottle from platformio import fs, util -from platformio.compat import get_file_contents from platformio.proc import where_is_program from platformio.project.config import ProjectConfig from platformio.project.helpers import load_project_ide_data @@ -137,11 +136,11 @@ class ProjectGenerator(object): @staticmethod def _render_tpl(tpl_path, tpl_vars): - return bottle.template(get_file_contents(tpl_path), **tpl_vars) + return bottle.template(fs.get_file_contents(tpl_path), **tpl_vars) @staticmethod def _merge_contents(dst_path, contents): if basename(dst_path) == ".gitignore" and isfile(dst_path): return - with codecs.open(dst_path, "w", encoding="utf8") as fp: + with io.open(dst_path, "w", encoding="utf8") as fp: fp.write(contents) diff --git a/platformio/run/helpers.py b/platformio/run/helpers.py index a4c79e83..0d41a569 100644 --- a/platformio/run/helpers.py +++ b/platformio/run/helpers.py @@ -53,12 +53,9 @@ def clean_build_dir(build_dir, config): if isdir(build_dir): # check project structure - if isfile(checksum_file): - with open(checksum_file) as f: - if f.read() == checksum: - return + if isfile(checksum_file) and fs.get_file_contents(checksum_file) == checksum: + return fs.rmtree(build_dir) makedirs(build_dir) - with open(checksum_file, "w") as f: - f.write(checksum) + fs.write_file_contents(checksum_file, checksum) diff --git a/platformio/test/processor.py b/platformio/test/processor.py index e228a8c0..35500855 100644 --- a/platformio/test/processor.py +++ b/platformio/test/processor.py @@ -19,7 +19,7 @@ from string import Template import click -from platformio import exception +from platformio import exception, fs TRANSPORT_OPTIONS = { "arduino": { @@ -193,7 +193,6 @@ class TestProcessorBase(object): data = Template(tpl).substitute(baudrate=self.get_baudrate()) tmp_file = join(test_dir, "output_export.cpp") - with open(tmp_file, "w") as f: - f.write(data) + fs.write_file_contents(tmp_file, data) atexit.register(delete_tmptest_file, tmp_file) diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index ccafa35f..8096d9ba 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -428,7 +428,7 @@ def test_platform_json_schema(): }, "frameworks": sorted(["arduino", "simba"]), "version": "1.15.0", - } + }, ) From 5cfa2b7fdd2a4843587874576a538449f8f6faf3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 17 Oct 2019 19:28:57 +0300 Subject: [PATCH 105/221] Fix typo with UnicodeEncodeError // Issue #2796 --- platformio/fs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/fs.py b/platformio/fs.py index 080f1502..c81a50db 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -62,7 +62,7 @@ def write_file_contents(path, contents): try: with open(path, "w") as fp: return fp.write(contents) - except UnicodeDecodeError: + except UnicodeEncodeError: with io.open(path, "w", encoding="latin-1") as fp: return fp.write(contents) From be9aaf8902659587542047bf51b240efa5a65b23 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 17 Oct 2019 20:57:40 +0300 Subject: [PATCH 106/221] Be compatible with Python 3.8, on Windows skip HOME and check for USERPROFILE --- platformio/builder/tools/piolib.py | 14 +++----------- platformio/commands/ci.py | 4 ++-- platformio/fs.py | 9 +++++++++ platformio/home/rpc/handlers/app.py | 6 +++--- platformio/home/rpc/handlers/os.py | 6 +++--- platformio/home/rpc/handlers/project.py | 4 ++-- platformio/ide/projectgenerator.py | 4 ++-- platformio/project/config.py | 6 +++--- platformio/project/helpers.py | 6 +++--- platformio/project/options.py | 4 +++- 10 files changed, 33 insertions(+), 30 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index b27fbf9b..7f5484ce 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -22,16 +22,7 @@ import hashlib import os import re import sys -from os.path import ( - basename, - commonprefix, - expanduser, - isdir, - isfile, - join, - realpath, - sep, -) +from os.path import basename, commonprefix, isdir, isfile, join, realpath, sep import click import SCons.Scanner # pylint: disable=import-error @@ -943,7 +934,8 @@ def GetLibSourceDirs(env): items = env.GetProjectOption("lib_extra_dirs", []) items.extend(env["LIBSOURCE_DIRS"]) return [ - env.subst(expanduser(item) if item.startswith("~") else item) for item in items + env.subst(fs.expanduser(item) if item.startswith("~") else item) + for item in items ] diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index 6b98aacd..7228d001 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -14,7 +14,7 @@ from glob import glob from os import getenv, makedirs, remove -from os.path import abspath, basename, expanduser, isdir, isfile, join +from os.path import abspath, basename, isdir, isfile, join from shutil import copyfile, copytree from tempfile import mkdtemp @@ -34,7 +34,7 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument value = list(value) for i, p in enumerate(value): if p.startswith("~"): - value[i] = expanduser(p) + value[i] = fs.expanduser(p) value[i] = abspath(value[i]) if not glob(value[i]): invalid_path = p diff --git a/platformio/fs.py b/platformio/fs.py index c81a50db..23edb634 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -171,6 +171,15 @@ def to_unix_path(path): return re.sub(r"[\\]+", "/", path) +def expanduser(path): + """ + Be compatible with Python 3.8, on Windows skip HOME and check for USERPROFILE + """ + if not WINDOWS or not path.startswith("~") or "USERPROFILE" not in os.environ: + return os.path.expanduser(path) + return os.environ["USERPROFILE"] + path[1:] + + def rmtree(path): def _onerror(func, path, __): try: diff --git a/platformio/home/rpc/handlers/app.py b/platformio/home/rpc/handlers/app.py index cc3be81f..1fd49e22 100644 --- a/platformio/home/rpc/handlers/app.py +++ b/platformio/home/rpc/handlers/app.py @@ -14,9 +14,9 @@ from __future__ import absolute_import -from os.path import expanduser, join +from os.path import join -from platformio import __version__, app, util +from platformio import __version__, app, fs, util from platformio.project.helpers import get_project_core_dir, is_platformio_project @@ -54,7 +54,7 @@ class AppRPC(object): for name, data in app.DEFAULT_SETTINGS.items() } - storage["homeDir"] = expanduser("~") + storage["homeDir"] = fs.expanduser("~") storage["projectsDir"] = storage["coreSettings"]["projects_dir"]["value"] # skip non-existing recent projects diff --git a/platformio/home/rpc/handlers/os.py b/platformio/home/rpc/handlers/os.py index 7f5f5ece..92f4a008 100644 --- a/platformio/home/rpc/handlers/os.py +++ b/platformio/home/rpc/handlers/os.py @@ -19,12 +19,12 @@ import glob import os import shutil from functools import cmp_to_key -from os.path import expanduser, isdir, isfile, join +from os.path import isdir, isfile, join import click from twisted.internet import defer # pylint: disable=import-error -from platformio import app, util +from platformio import app, fs, util from platformio.compat import PY2, get_filesystem_encoding from platformio.home import helpers @@ -126,7 +126,7 @@ class OSRPC(object): items = [] if path.startswith("~"): - path = expanduser(path) + path = fs.expanduser(path) if not isdir(path): return items for item in os.listdir(path): diff --git a/platformio/home/rpc/handlers/project.py b/platformio/home/rpc/handlers/project.py index bb320ab9..aa60058e 100644 --- a/platformio/home/rpc/handlers/project.py +++ b/platformio/home/rpc/handlers/project.py @@ -17,7 +17,7 @@ from __future__ import absolute_import import os import shutil import time -from os.path import basename, expanduser, getmtime, isdir, isfile, join, realpath, sep +from os.path import basename, getmtime, isdir, isfile, join, realpath, sep import jsonrpc # pylint: disable=import-error @@ -56,7 +56,7 @@ class ProjectRPC(object): # skip non existing folders and resolve full path for key in ("envLibdepsDirs", "libExtraDirs"): data[key] = [ - expanduser(d) if d.startswith("~") else realpath(d) + fs.expanduser(d) if d.startswith("~") else realpath(d) for d in data[key] if isdir(d) ] diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index b297e37b..d4e13e5a 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -15,7 +15,7 @@ import io import os import sys -from os.path import abspath, basename, expanduser, isdir, isfile, join, relpath +from os.path import abspath, basename, isdir, isfile, join, relpath import bottle @@ -64,7 +64,7 @@ class ProjectGenerator(object): "project_name": basename(self.project_dir), "project_dir": self.project_dir, "env_name": self.env_name, - "user_home_dir": abspath(expanduser("~")), + "user_home_dir": abspath(fs.expanduser("~")), "platformio_path": sys.argv[0] if isfile(sys.argv[0]) else where_is_program("platformio"), diff --git a/platformio/project/config.py b/platformio/project/config.py index 988b53c2..4d22ea57 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -20,7 +20,7 @@ from hashlib import sha1 import click -from platformio import exception +from platformio import exception, fs from platformio.compat import WINDOWS, hashlib_encode_data from platformio.project.options import ProjectOptions @@ -106,7 +106,7 @@ class ProjectConfigBase(object): # load extra configs for pattern in self.get("platformio", "extra_configs", []): if pattern.startswith("~"): - pattern = os.path.expanduser(pattern) + pattern = fs.expanduser(pattern) for item in glob.glob(pattern): self.read(item) @@ -380,7 +380,7 @@ class ProjectConfigDirsMixin(object): ) if result.startswith("~"): - result = os.path.expanduser(result) + result = fs.expanduser(result) result = os.path.realpath(result) diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 5b921e9d..fbf86199 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -16,11 +16,11 @@ import json import os from hashlib import sha1 from os import walk -from os.path import dirname, expanduser, isdir, isfile, join +from os.path import dirname, isdir, isfile, join from click.testing import CliRunner -from platformio import __version__, exception +from platformio import __version__, exception, fs from platformio.compat import WINDOWS, hashlib_encode_data from platformio.project.config import ProjectConfig @@ -90,7 +90,7 @@ def get_project_libdeps_dir(): def get_default_projects_dir(): - docs_dir = join(expanduser("~"), "Documents") + docs_dir = join(fs.expanduser("~"), "Documents") try: assert WINDOWS import ctypes.wintypes # pylint: disable=import-outside-toplevel diff --git a/platformio/project/options.py b/platformio/project/options.py index f7294c40..02762783 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -19,6 +19,8 @@ from collections import OrderedDict, namedtuple import click +from platformio import fs + ConfigOptionClass = namedtuple( "ConfigOption", [ @@ -77,7 +79,7 @@ ProjectOptions = OrderedDict( name="core_dir", oldnames=["home_dir"], sysenvvar="PLATFORMIO_CORE_DIR", - default=os.path.join(os.path.expanduser("~"), ".platformio"), + default=os.path.join(fs.expanduser("~"), ".platformio"), ), ConfigPlatformioOption( name="globallib_dir", From 19a8326f0f756fb1205ee0ef775aa2d8f90b1568 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 17 Oct 2019 21:19:04 +0300 Subject: [PATCH 107/221] Fix test for package manifest --- Makefile | 2 +- tests/test_pkgmanifest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ff246abd..088bae11 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ black: black --target-version py27 ./tests test: - py.test --verbose --capture=no --exitfirst -n 3 --dist=loadscope tests --ignore tests/test_examples.py --ignore tests/test_pkgmanifest.py + py.test --verbose --capture=no --exitfirst -n 3 --dist=loadscope tests --ignore tests/test_examples.py before-commit: isort black lint test diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 8096d9ba..cbd632ed 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -153,7 +153,7 @@ sentence=This is Arduino library ).as_dict() assert data["export"] == { "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"], - "include": "libraries/TestPackage", + "include": ["libraries/TestPackage"], } assert data["repository"] == { "url": "https://github.com/username/reponame", From d44c60614d59069f295b98b19bbbf66efeb85fb6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 17 Oct 2019 23:40:30 +0300 Subject: [PATCH 108/221] Use direct LDSCRIPT_PATH only if script resolves --- platformio/builder/tools/platformio.py | 26 ++++++++++++++++---------- tests/test_pkgmanifest.py | 20 +++++++++++++------- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index c4e5dc8c..5c967d19 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -16,7 +16,6 @@ from __future__ import absolute_import import os import sys -from os.path import basename, dirname, isdir, join, realpath from SCons import Builder, Util # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error @@ -130,7 +129,14 @@ def BuildProgram(env): # append into the beginning a main LD script if env.get("LDSCRIPT_PATH") and not any("-Wl,-T" in f for f in env["LINKFLAGS"]): - env.Prepend(LINKFLAGS=["-T", env["LDSCRIPT_PATH"]]) + env.Prepend( + LINKFLAGS=[ + "-T", + env["LDSCRIPT_PATH"] + if os.path.isfile(env["LDSCRIPT_PATH"]) + else "$LDSCRIPT_PATH", + ] + ) # enable "cyclic reference" for linker if env.get("LIBS") and env.GetCompilerType() == "gcc": @@ -138,7 +144,7 @@ def BuildProgram(env): env.Append(_LIBFLAGS=" -Wl,--end-group") program = env.Program( - join("$BUILD_DIR", env.subst("$PROGNAME")), env["PIOBUILDFILES"] + os.path.join("$BUILD_DIR", env.subst("$PROGNAME")), env["PIOBUILDFILES"] ) env.Replace(PIOMAINPROG=program) @@ -181,13 +187,13 @@ def ParseFlagsExtended(env, flags): # pylint: disable=too-many-branches # fix relative CPPPATH & LIBPATH for k in ("CPPPATH", "LIBPATH"): for i, p in enumerate(result.get(k, [])): - if isdir(p): - result[k][i] = realpath(p) + if os.path.isdir(p): + result[k][i] = os.path.realpath(p) # fix relative path for "-include" for i, f in enumerate(result.get("CCFLAGS", [])): if isinstance(f, tuple) and f[0] == "-include": - result["CCFLAGS"][i] = (f[0], env.File(realpath(f[1].get_path()))) + result["CCFLAGS"][i] = (f[0], env.File(os.path.realpath(f[1].get_path()))) return result @@ -254,16 +260,16 @@ def CollectBuildFiles(env, variant_dir, src_dir, src_filter=None, duplicate=Fals src_dir = src_dir[:-1] for item in env.MatchSourceFiles(src_dir, src_filter): - _reldir = dirname(item) - _src_dir = join(src_dir, _reldir) if _reldir else src_dir - _var_dir = join(variant_dir, _reldir) if _reldir else variant_dir + _reldir = os.path.dirname(item) + _src_dir = os.path.join(src_dir, _reldir) if _reldir else src_dir + _var_dir = os.path.join(variant_dir, _reldir) if _reldir else variant_dir if _var_dir not in variants: variants.append(_var_dir) env.VariantDir(_var_dir, _src_dir, duplicate) if fs.path_endswith_ext(item, SRC_BUILD_EXT): - sources.append(env.File(join(_var_dir, basename(item)))) + sources.append(env.File(os.path.join(_var_dir, os.path.basename(item)))) return sources diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index cbd632ed..64b67047 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os + import jsondiff import pytest @@ -551,22 +553,26 @@ def test_examples_from_dir(tmpdir_factory): [ { "name": "PlatformIO/hello", - "base": "examples/PlatformIO/hello", - "files": ["platformio.ini", "include/main.h", "src/main.cpp"], + "base": os.path.join("examples", "PlatformIO", "hello"), + "files": [ + "platformio.ini", + os.path.join("include", "main.h"), + os.path.join("src", "main.cpp"), + ], }, { "name": "1_General/SomeSketchIno", - "base": "examples/1. General/SomeSketchIno", + "base": os.path.join("examples", "1. General", "SomeSketchIno"), "files": ["SomeSketchIno.ino"], }, { "name": "1_General/SomeSketchPde", - "base": "examples/1. General/SomeSketchPde", + "base": os.path.join("examples", "1. General", "SomeSketchPde"), "files": ["SomeSketchPde.pde"], }, { "name": "demo", - "base": "examples/demo", + "base": os.path.join("examples", "demo"), "files": ["demo.h", "util.h", "demo.cpp"], }, { @@ -574,8 +580,8 @@ def test_examples_from_dir(tmpdir_factory): "base": "examples/world", "files": [ "platformio.ini", - "include/world.h", - "src/world.c", + os.path.join("include", "world.h"), + os.path.join("src", "world.c"), "README", "extra.py", ], From 92d86192aa21130f719c028a101237626c00f9da Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 18 Oct 2019 15:05:11 +0300 Subject: [PATCH 109/221] Substitute LDSCRIPT with real value --- platformio/builder/tools/platformio.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 5c967d19..ae4625e7 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -129,14 +129,7 @@ def BuildProgram(env): # append into the beginning a main LD script if env.get("LDSCRIPT_PATH") and not any("-Wl,-T" in f for f in env["LINKFLAGS"]): - env.Prepend( - LINKFLAGS=[ - "-T", - env["LDSCRIPT_PATH"] - if os.path.isfile(env["LDSCRIPT_PATH"]) - else "$LDSCRIPT_PATH", - ] - ) + env.Prepend(LINKFLAGS=["-T", env.subst("$LDSCRIPT_PATH")]) # enable "cyclic reference" for linker if env.get("LIBS") and env.GetCompilerType() == "gcc": From f5807364e84a2c76c9b8222804c05211b0eb6f2a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 18 Oct 2019 15:20:52 +0300 Subject: [PATCH 110/221] Force to "backslashreplace" when UnicodeEncodeError arises when writing file // Issue #2796 --- docs | 2 +- platformio/fs.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index e8b93616..8d04b2e1 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit e8b9361615d00180d9cb01af0b401885da091df8 +Subproject commit 8d04b2e1f37808a1a0787942d3a208af389878cc diff --git a/platformio/fs.py b/platformio/fs.py index 23edb634..d75b7837 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -63,7 +63,7 @@ def write_file_contents(path, contents): with open(path, "w") as fp: return fp.write(contents) except UnicodeEncodeError: - with io.open(path, "w", encoding="latin-1") as fp: + with io.open(path, "w", encoding="latin-1", errors="backslashreplace") as fp: return fp.write(contents) From 065607b68c4c900341c2a6ba4ae028b4b4e7fe92 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 18 Oct 2019 15:41:52 +0300 Subject: [PATCH 111/221] Disable PyLint's "import-outside-toplevel" --- platformio/app.py | 7 ++----- platformio/commands/__init__.py | 6 ++---- platformio/commands/debug.py | 2 ++ platformio/commands/home.py | 4 ++-- platformio/fs.py | 2 +- platformio/managers/core.py | 2 +- platformio/managers/lib.py | 1 + platformio/managers/platform.py | 8 ++------ platformio/project/config.py | 2 +- platformio/project/helpers.py | 1 + platformio/test/processor.py | 1 + platformio/util.py | 8 +++++--- 12 files changed, 21 insertions(+), 23 deletions(-) diff --git a/platformio/app.py b/platformio/app.py index 42dac9c0..c0e7e48a 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -25,11 +25,8 @@ import requests from platformio import exception, fs, lockfile from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data from platformio.proc import is_ci -from platformio.project.helpers import ( - get_default_projects_dir, - get_project_cache_dir, - get_project_core_dir, -) +from platformio.project.helpers import (get_default_projects_dir, get_project_cache_dir, + get_project_core_dir) def projects_dir_validate(projects_dir): diff --git a/platformio/commands/__init__.py b/platformio/commands/__init__.py index 5dd6d34b..961f4932 100644 --- a/platformio/commands/__init__.py +++ b/platformio/commands/__init__.py @@ -58,14 +58,12 @@ class PlatformioCLI(click.MultiCommand): try: mod = __import__("platformio.commands." + cmd_name, None, None, ["cli"]) except ImportError: - try: - return self._handle_obsolate_command(cmd_name) - except AttributeError: - raise click.UsageError('No such command "%s"' % cmd_name, ctx) + raise click.UsageError('No such command "%s"' % cmd_name, ctx) return mod.cli @staticmethod def _handle_obsolate_command(name): + # pylint: disable=import-outside-toplevel if name == "platforms": from platformio.commands import platform diff --git a/platformio/commands/debug.py b/platformio/commands/debug.py index 21b0bfd8..1fb1b8e7 100644 --- a/platformio/commands/debug.py +++ b/platformio/commands/debug.py @@ -137,6 +137,8 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro # run debugging client inject_contrib_pysite() + + # pylint: disable=import-outside-toplevel from platformio.debug.client import GDBClient, reactor client = GDBClient(project_dir, __unprocessed, debug_options, env_options) diff --git a/platformio/commands/home.py b/platformio/commands/home.py index 38b371bb..89bd9d01 100644 --- a/platformio/commands/home.py +++ b/platformio/commands/home.py @@ -36,14 +36,14 @@ from platformio.managers.core import get_core_package_dir, inject_contrib_pysite ) @click.option("--no-open", is_flag=True) def cli(port, host, no_open): + # pylint: disable=import-error, import-outside-toplevel + # import contrib modules inject_contrib_pysite() - # pylint: disable=import-error from autobahn.twisted.resource import WebSocketResource from twisted.internet import reactor from twisted.web import server - # pylint: enable=import-error from platformio.home.rpc.handlers.app import AppRPC from platformio.home.rpc.handlers.ide import IDERPC from platformio.home.rpc.handlers.misc import MiscRPC diff --git a/platformio/fs.py b/platformio/fs.py index d75b7837..017b653a 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -93,7 +93,7 @@ def format_filesize(filesize): def ensure_udev_rules(): - from platformio.util import get_systype + from platformio.util import get_systype # pylint: disable=import-outside-toplevel def _rules_to_set(rules_path): return set( diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 4ff9e5da..4609dbad 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -103,7 +103,7 @@ def update_core_packages(only_check=False, silent=False): def inject_contrib_pysite(): - from site import addsitedir + from site import addsitedir # pylint: disable=import-outside-toplevel contrib_pysite_dir = get_core_package_dir("contrib-pysite") if contrib_pysite_dir in sys.path: diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index 85cf4b29..8a120c3a 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -264,6 +264,7 @@ class LibraryManager(BasePkgManager): fg="yellow", err=True, ) + # pylint: disable=import-outside-toplevel from platformio.commands.lib import print_lib_item for item in result["items"]: diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 2536386a..d9797516 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -27,12 +27,8 @@ from platformio import __version__, app, exception, fs, util from platformio.compat import PY2, hashlib_encode_data, is_bytes, load_python_module from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager -from platformio.proc import ( - BuildAsyncPipe, - copy_pythonpath_to_osenv, - exec_command, - get_pythonexe_path, -) +from platformio.proc import (BuildAsyncPipe, copy_pythonpath_to_osenv, exec_command, + get_pythonexe_path) from platformio.project.config import ProjectConfig try: diff --git a/platformio/project/config.py b/platformio/project/config.py index 4d22ea57..fdc00124 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -72,7 +72,7 @@ class ProjectConfigBase(object): @staticmethod def get_default_path(): - from platformio import app + from platformio import app # pylint: disable=import-outside-toplevel return app.get_session_var("custom_project_conf") or os.path.join( os.getcwd(), "platformio.ini" diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index fbf86199..deeeb6c2 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -136,6 +136,7 @@ def compute_project_checksum(config): def load_project_ide_data(project_dir, env_or_envs): + # pylint: disable=import-outside-toplevel from platformio.commands.run import cli as cmd_run assert env_or_envs diff --git a/platformio/test/processor.py b/platformio/test/processor.py index 35500855..6c900934 100644 --- a/platformio/test/processor.py +++ b/platformio/test/processor.py @@ -112,6 +112,7 @@ class TestProcessorBase(object): self.cmd_ctx.meta[CTX_META_TEST_RUNNING_NAME] = self.test_name try: + # pylint: disable=import-outside-toplevel from platformio.commands.run import cli as cmd_run return self.cmd_ctx.invoke( diff --git a/platformio/util.py b/platformio/util.py index 48f679c2..26529dbb 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -126,6 +126,7 @@ def change_filemtime(path, mtime): def get_serial_ports(filter_hwid=False): try: + # pylint: disable=import-outside-toplevel from serial.tools.list_ports import comports except ImportError: raise exception.GetSerialPortsError(os.name) @@ -190,8 +191,9 @@ def get_logical_devices(): def get_mdns_services(): + # pylint: disable=import-outside-toplevel try: - import zeroconf # pylint: disable=import-outside-toplevel + import zeroconf except ImportError: from site import addsitedir from platformio.managers.core import get_core_package_dir @@ -282,7 +284,7 @@ def _api_request_session(): def _get_api_result( url, params=None, data=None, auth=None # pylint: disable=too-many-branches ): - from platformio.app import get_setting + from platformio.app import get_setting # pylint: disable=import-outside-toplevel result = {} r = None @@ -326,7 +328,7 @@ def _get_api_result( def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): - from platformio.app import ContentCache + from platformio.app import ContentCache # pylint: disable=import-outside-toplevel total = 0 max_retries = 5 From 4d84d03a636c177c93cf6dc8a962fdd14f91c859 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 18 Oct 2019 15:56:41 +0300 Subject: [PATCH 112/221] Black --- platformio/app.py | 7 +++++-- platformio/managers/platform.py | 8 ++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/platformio/app.py b/platformio/app.py index c0e7e48a..42dac9c0 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -25,8 +25,11 @@ import requests from platformio import exception, fs, lockfile from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data from platformio.proc import is_ci -from platformio.project.helpers import (get_default_projects_dir, get_project_cache_dir, - get_project_core_dir) +from platformio.project.helpers import ( + get_default_projects_dir, + get_project_cache_dir, + get_project_core_dir, +) def projects_dir_validate(projects_dir): diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index d9797516..2536386a 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -27,8 +27,12 @@ from platformio import __version__, app, exception, fs, util from platformio.compat import PY2, hashlib_encode_data, is_bytes, load_python_module from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager -from platformio.proc import (BuildAsyncPipe, copy_pythonpath_to_osenv, exec_command, - get_pythonexe_path) +from platformio.proc import ( + BuildAsyncPipe, + copy_pythonpath_to_osenv, + exec_command, + get_pythonexe_path, +) from platformio.project.config import ProjectConfig try: From 77f8414c639338ea7179067c4f3a70b6d1b4fba5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 18 Oct 2019 15:56:50 +0300 Subject: [PATCH 113/221] Better explanation about encoding error // Resolve #2796 --- platformio/builder/tools/piomisc.py | 6 ++++-- platformio/fs.py | 11 +++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 20768671..9c7bf548 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -80,12 +80,14 @@ class InoToCPPConverter(object): assert self._gcc_preprocess(contents, out_file) contents = fs.get_file_contents(out_file) contents = self._join_multiline_strings(contents) - fs.write_file_contents(out_file, self.append_prototypes(contents)) + fs.write_file_contents( + out_file, self.append_prototypes(contents), errors="backslashreplace" + ) return out_file def _gcc_preprocess(self, contents, out_file): tmp_path = mkstemp()[1] - fs.write_file_contents(tmp_path, contents) + fs.write_file_contents(tmp_path, contents, errors="backslashreplace") self.env.Execute( self.env.VerboseAction( '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format( diff --git a/platformio/fs.py b/platformio/fs.py index 017b653a..16dcb3e8 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -58,12 +58,19 @@ def get_file_contents(path): return fp.read() -def write_file_contents(path, contents): +def write_file_contents(path, contents, errors=None): try: with open(path, "w") as fp: return fp.write(contents) except UnicodeEncodeError: - with io.open(path, "w", encoding="latin-1", errors="backslashreplace") as fp: + if errors: + click.secho( + "Warning! There is a problem with contents encoding, please remove " + "invalid characters (non-ASCII or non-UT8) in %s" % path, + fg="yellow", + err=True, + ) + with io.open(path, "w", encoding="latin-1", errors=errors) as fp: return fp.write(contents) From f78a1a7b15a8d180c700b6864b1145c51240e431 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 18 Oct 2019 22:00:28 +0300 Subject: [PATCH 114/221] Show encoding error when can't read a file // Issue #2796 --- platformio/fs.py | 13 ++++++------- tests/test_pkgmanifest.py | 7 ++++++- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/platformio/fs.py b/platformio/fs.py index 16dcb3e8..18ad38b2 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -54,6 +54,12 @@ def get_file_contents(path): with open(path) as fp: return fp.read() except UnicodeDecodeError: + click.secho( + "Unicode decode error has occurred, please remove invalid " + "(non-ASCII or non-UTF8) characters from %s file" % path, + fg="yellow", + err=True, + ) with io.open(path, encoding="latin-1") as fp: return fp.read() @@ -63,13 +69,6 @@ def write_file_contents(path, contents, errors=None): with open(path, "w") as fp: return fp.write(contents) except UnicodeEncodeError: - if errors: - click.secho( - "Warning! There is a problem with contents encoding, please remove " - "invalid characters (non-ASCII or non-UT8) in %s" % path, - fg="yellow", - err=True, - ) with io.open(path, "w", encoding="latin-1", errors=errors) as fp: return fp.write(contents) diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 64b67047..212793a1 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -13,6 +13,7 @@ # limitations under the License. import os +import re import jsondiff import pytest @@ -534,9 +535,13 @@ def test_examples_from_dir(tmpdir_factory): assert isinstance(raw_data["examples"], list) assert len(raw_data["examples"]) == 6 + def _to_unix_path(path): + return re.sub(r"[\\/]+", "/", path) + def _sort_examples(items): for i, item in enumerate(items): - items[i]["files"] = sorted(item["files"]) + items[i]["base"] = _to_unix_path(items[i]["base"]) + items[i]["files"] = [_to_unix_path(f) for f in sorted(items[i]["files"])] return sorted(items, key=lambda item: item["name"]) raw_data["examples"] = _sort_examples(raw_data["examples"]) From d771816b0230292177ed96039d1a6544b764d214 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 19 Oct 2019 12:42:43 +0300 Subject: [PATCH 115/221] Automatically change dir to project for RPC "config_call"; add "envs" and "descrption" for project entities --- platformio/home/rpc/handlers/project.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/platformio/home/rpc/handlers/project.py b/platformio/home/rpc/handlers/project.py index aa60058e..02c44e7e 100644 --- a/platformio/home/rpc/handlers/project.py +++ b/platformio/home/rpc/handlers/project.py @@ -34,6 +34,9 @@ from platformio.project.helpers import is_platformio_project class ProjectRPC(object): @staticmethod def config_call(path, method, *args): + if isfile(path): + with fs.cd(os.path.dirname(path)): + return getattr(ProjectConfig(path), method)(*args) return getattr(ProjectConfig(path), method)(*args) @staticmethod @@ -41,10 +44,11 @@ class ProjectRPC(object): def _get_project_data(): data = {"boards": [], "envLibdepsDirs": [], "libExtraDirs": []} config = ProjectConfig() - libdeps_dir = config.get_optional_dir("libdeps") - + data["envs"] = config.envs() + data["description"] = config.get("platformio", "description") data["libExtraDirs"].extend(config.get("platformio", "lib_extra_dirs", [])) + libdeps_dir = config.get_optional_dir("libdeps") for section in config.sections(): if not section.startswith("env:"): continue @@ -94,6 +98,8 @@ class ProjectRPC(object): "name": _path_to_name(project_dir), "modified": int(getmtime(project_dir)), "boards": boards, + "description": data.get("description"), + "envs": data.get("envs", []), "envLibStorages": [ {"name": basename(d), "path": d} for d in data.get("envLibdepsDirs", []) From 6bf8bec22d4d3fde8eb7f37ae0f1f9b942d91c13 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 19 Oct 2019 12:43:43 +0300 Subject: [PATCH 116/221] Bump version to 4.1.0b5 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 28d0a201..0a46fb32 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, "0b4") +VERSION = (4, 1, "0b5") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 971cd2ca0fa92c115bdc4b9a08632d151511f167 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 21 Oct 2019 00:12:04 +0300 Subject: [PATCH 117/221] Export device info in pair with sizedata --- platformio/builder/tools/piosize.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/platformio/builder/tools/piosize.py b/platformio/builder/tools/piosize.py index db8f2b5e..36f3bb3c 100644 --- a/platformio/builder/tools/piosize.py +++ b/platformio/builder/tools/piosize.py @@ -136,7 +136,17 @@ def _calculate_firmware_size(sections): def DumpSizeData(_, target, source, env): # pylint: disable=unused-argument - data = {"memory": {}, "version": 1} + data = {"device": {}, "memory": {}, "version": 1} + + board = env.BoardConfig() + if board: + data["device"] = { + "mcu": board.get("build.mcu", ""), + "cpu": board.get("build.cpu", ""), + "frequency": board.get("build.f_cpu"), + "flash": int(board.get("upload.maximum_size", 0)), + "ram": int(board.get("upload.maximum_ram_size", 0)), + } elf_path = env.subst("$PIOMAINPROG") From 9fef7f0ba901275b2c26605c56829f6c17d12a86 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 21 Oct 2019 15:53:25 +0300 Subject: [PATCH 118/221] Docs: Sync TI MSP430 dev/platfom --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 8d04b2e1..b456c59a 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 8d04b2e1f37808a1a0787942d3a208af389878cc +Subproject commit b456c59a5a27cf57024932ae8d94f1337f434de5 From f34745bef93ce16d1cb6515d66377744809fedea Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 21 Oct 2019 15:57:34 +0300 Subject: [PATCH 119/221] Parse device frequency in int format for size data --- platformio/builder/tools/piosize.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/platformio/builder/tools/piosize.py b/platformio/builder/tools/piosize.py index 36f3bb3c..bf44a826 100644 --- a/platformio/builder/tools/piosize.py +++ b/platformio/builder/tools/piosize.py @@ -147,6 +147,8 @@ def DumpSizeData(_, target, source, env): # pylint: disable=unused-argument "flash": int(board.get("upload.maximum_size", 0)), "ram": int(board.get("upload.maximum_ram_size", 0)), } + if data["device"]["frequency"] and data["device"]["frequency"].endswith("L"): + data["device"]["frequency"] = int(data["device"]["frequency"][0:-1]) elf_path = env.subst("$PIOMAINPROG") From 941c0f4297e647c9ad8a2cb66875f621e7828c68 Mon Sep 17 00:00:00 2001 From: valeros Date: Mon, 21 Oct 2019 23:26:28 +0300 Subject: [PATCH 120/221] Improve the speed of memory use report generation --- platformio/builder/tools/piosize.py | 65 ++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 16 deletions(-) diff --git a/platformio/builder/tools/piosize.py b/platformio/builder/tools/piosize.py index bf44a826..6338726d 100644 --- a/platformio/builder/tools/piosize.py +++ b/platformio/builder/tools/piosize.py @@ -17,8 +17,8 @@ from __future__ import absolute_import import sys -from os import environ -from os.path import join +from os import environ, makedirs, remove +from os.path import join, isdir from elftools.elf.descriptions import describe_sh_flags from elftools.elf.elffile import ELFFile @@ -27,10 +27,42 @@ from platformio.compat import dump_json_to_unicode from platformio.proc import exec_command -def _get_file_location(env, elf_path, addr, sysenv): - cmd = [env.subst("$CC").replace("-gcc", "-addr2line"), "-e", elf_path, hex(addr)] +def _run_tool(cmd, env, tool_args): + sysenv = environ.copy() + sysenv["PATH"] = str(env["ENV"]["PATH"]) + + build_dir = env.subst("$BUILD_DIR") + if not isdir(build_dir): + makedirs(build_dir) + tmp_file = join(build_dir, "size-data-longcmd.txt") + + with open(tmp_file, "w") as fp: + fp.write("\n".join(tool_args)) + + cmd.append("@" + tmp_file) result = exec_command(cmd, env=sysenv) - return result["out"].strip().replace("\\", "/") + remove(tmp_file) + + return result + + +def _get_symbol_locations(env, elf_path, addrs): + cmd = [env.subst("$CC").replace("-gcc", "-addr2line"), "-e", elf_path] + result = _run_tool(cmd, env, addrs) + locations = [line for line in result["out"].split("\n") if line] + assert(len(addrs) == len(locations)) + + return dict(zip(addrs, [l.strip().replace("\\", "/") for l in locations])) + + +def _get_demangled_names(env, mangled_names): + result = _run_tool( + [env.subst("$CC").replace("-gcc", "-c++filt")], env, mangled_names) + demangled_names = [line for line in result["out"].split("\n") if line] + assert(len(mangled_names) == len(demangled_names)) + + return dict(zip(mangled_names, [dn.strip().replace( + "::__FUNCTION__", "") for dn in demangled_names])) def _determine_section(sections, symbol_addr): @@ -42,15 +74,6 @@ def _determine_section(sections, symbol_addr): return "unknown" -def _demangle_cpp_name(env, symbol_name, sysenv): - cmd = [env.subst("$CC").replace("-gcc", "-c++filt"), symbol_name] - result = exec_command(cmd, env=sysenv) - demangled_name = result["out"].strip() - if "(" in demangled_name: - demangled_name = demangled_name[0 : demangled_name.find("(")] - return demangled_name - - def _is_ram_section(section): return ( section.get("type", "") in ("SHT_NOBITS", "SHT_PROGBITS") @@ -97,6 +120,8 @@ def _collect_symbols_info(env, elffile, elf_path, sections): sysenv = environ.copy() sysenv["PATH"] = str(env["ENV"]["PATH"]) + symbol_addrs = [] + mangled_names = [] for s in symbol_section.iter_symbols(): symbol_info = s.entry["st_info"] symbol_addr = s["st_value"] @@ -109,7 +134,6 @@ def _collect_symbols_info(env, elffile, elf_path, sections): symbol = { "addr": symbol_addr, "bind": symbol_info["bind"], - "location": _get_file_location(env, elf_path, symbol_addr, sysenv), "name": s.name, "type": symbol_type, "size": symbol_size, @@ -117,10 +141,19 @@ def _collect_symbols_info(env, elffile, elf_path, sections): } if s.name.startswith("_Z"): - symbol["demangled_name"] = _demangle_cpp_name(env, s.name, sysenv) + mangled_names.append(s.name) + symbol_addrs.append(hex(symbol_addr)) symbols.append(symbol) + symbol_locations = _get_symbol_locations( + env, elf_path, symbol_addrs) + demangled_names = _get_demangled_names(env, mangled_names) + for symbol in symbols: + if symbol["name"].startswith("_Z"): + symbol["demangled_name"] = demangled_names.get(symbol["name"]) + symbol["location"] = symbol_locations.get(hex(symbol["addr"])) + return symbols From 56ac577b0a278b3e2835a80a33dbe2f4bbd8b53d Mon Sep 17 00:00:00 2001 From: valeros Date: Tue, 22 Oct 2019 12:10:48 +0300 Subject: [PATCH 121/221] Fix case with empty arguments when generating sizedata report --- platformio/builder/tools/piosize.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/platformio/builder/tools/piosize.py b/platformio/builder/tools/piosize.py index 6338726d..c1f6c4fc 100644 --- a/platformio/builder/tools/piosize.py +++ b/platformio/builder/tools/piosize.py @@ -47,6 +47,8 @@ def _run_tool(cmd, env, tool_args): def _get_symbol_locations(env, elf_path, addrs): + if not addrs: + return {} cmd = [env.subst("$CC").replace("-gcc", "-addr2line"), "-e", elf_path] result = _run_tool(cmd, env, addrs) locations = [line for line in result["out"].split("\n") if line] @@ -56,6 +58,8 @@ def _get_symbol_locations(env, elf_path, addrs): def _get_demangled_names(env, mangled_names): + if not mangled_names: + return {} result = _run_tool( [env.subst("$CC").replace("-gcc", "-c++filt")], env, mangled_names) demangled_names = [line for line in result["out"].split("\n") if line] From 374379ba0300aeaff7ada2eccbccc5ff27604d74 Mon Sep 17 00:00:00 2001 From: valeros Date: Tue, 22 Oct 2019 21:52:55 +0300 Subject: [PATCH 122/221] Skip .debug sections when generating memory use report --- platformio/builder/tools/piosize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/builder/tools/piosize.py b/platformio/builder/tools/piosize.py index c1f6c4fc..75fe944b 100644 --- a/platformio/builder/tools/piosize.py +++ b/platformio/builder/tools/piosize.py @@ -96,7 +96,7 @@ def _is_valid_symbol(symbol_name, symbol_type, symbol_address): def _collect_sections_info(elffile): sections = {} for section in elffile.iter_sections(): - if section.is_null(): + if section.is_null() or section.name.startswith(".debug"): continue section_type = section["sh_type"] From 9c7cc87c5f7c7990085d73bfbd9318b89f003457 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 23 Oct 2019 16:05:27 +0300 Subject: [PATCH 123/221] Move command related modules to "commands" package --- platformio/builder/tools/piosize.py | 20 ++++++----- platformio/commands/__init__.py | 36 ++++++++----------- platformio/{ => commands}/check/__init__.py | 0 .../commands/{check.py => check/command.py} | 4 +-- platformio/{ => commands}/check/defect.py | 0 .../{ => commands}/check/tools/__init__.py | 4 +-- platformio/{ => commands}/check/tools/base.py | 2 +- .../{ => commands}/check/tools/clangtidy.py | 4 +-- .../{ => commands}/check/tools/cppcheck.py | 4 +-- platformio/commands/ci.py | 2 +- platformio/{ => commands}/debug/__init__.py | 0 platformio/{ => commands}/debug/client.py | 6 ++-- .../commands/{debug.py => debug/command.py} | 4 +-- platformio/{ => commands}/debug/helpers.py | 2 +- platformio/{ => commands}/debug/initcfgs.py | 0 platformio/{ => commands}/debug/process.py | 0 platformio/{ => commands}/debug/server.py | 2 +- platformio/{ => commands}/home/__init__.py | 0 .../commands/{home.py => home/command.py} | 16 ++++----- platformio/{ => commands}/home/helpers.py | 0 .../{ => commands}/home/rpc/__init__.py | 0 .../home/rpc/handlers/__init__.py | 0 .../{ => commands}/home/rpc/handlers/app.py | 0 .../{ => commands}/home/rpc/handlers/ide.py | 0 .../{ => commands}/home/rpc/handlers/misc.py | 2 +- .../{ => commands}/home/rpc/handlers/os.py | 2 +- .../home/rpc/handlers/piocore.py | 2 +- .../home/rpc/handlers/project.py | 4 +-- platformio/{ => commands}/home/rpc/server.py | 0 platformio/{ => commands}/home/web.py | 0 platformio/{ => commands}/run/__init__.py | 0 .../commands/{run.py => run/command.py} | 6 ++-- platformio/{ => commands}/run/helpers.py | 0 platformio/{ => commands}/run/processor.py | 2 +- platformio/{ => commands}/test/__init__.py | 0 .../commands/{test.py => test/command.py} | 4 +-- platformio/{ => commands}/test/embedded.py | 2 +- platformio/{ => commands}/test/native.py | 2 +- platformio/{ => commands}/test/processor.py | 2 +- platformio/project/helpers.py | 2 +- tests/commands/test_check.py | 2 +- tests/test_builder.py | 2 +- 42 files changed, 68 insertions(+), 72 deletions(-) rename platformio/{ => commands}/check/__init__.py (100%) rename platformio/commands/{check.py => check/command.py} (98%) rename platformio/{ => commands}/check/defect.py (100%) rename platformio/{ => commands}/check/tools/__init__.py (88%) rename platformio/{ => commands}/check/tools/base.py (98%) rename platformio/{ => commands}/check/tools/clangtidy.py (94%) rename platformio/{ => commands}/check/tools/cppcheck.py (97%) rename platformio/{ => commands}/debug/__init__.py (100%) rename platformio/{ => commands}/debug/client.py (98%) rename platformio/commands/{debug.py => debug/command.py} (97%) rename platformio/{ => commands}/debug/helpers.py (99%) rename platformio/{ => commands}/debug/initcfgs.py (100%) rename platformio/{ => commands}/debug/process.py (100%) rename platformio/{ => commands}/debug/server.py (98%) rename platformio/{ => commands}/home/__init__.py (100%) rename platformio/commands/{home.py => home/command.py} (86%) rename platformio/{ => commands}/home/helpers.py (100%) rename platformio/{ => commands}/home/rpc/__init__.py (100%) rename platformio/{ => commands}/home/rpc/handlers/__init__.py (100%) rename platformio/{ => commands}/home/rpc/handlers/app.py (100%) rename platformio/{ => commands}/home/rpc/handlers/ide.py (100%) rename platformio/{ => commands}/home/rpc/handlers/misc.py (97%) rename platformio/{ => commands}/home/rpc/handlers/os.py (99%) rename platformio/{ => commands}/home/rpc/handlers/piocore.py (99%) rename platformio/{ => commands}/home/rpc/handlers/project.py (98%) rename platformio/{ => commands}/home/rpc/server.py (100%) rename platformio/{ => commands}/home/web.py (100%) rename platformio/{ => commands}/run/__init__.py (100%) rename platformio/commands/{run.py => run/command.py} (97%) rename platformio/{ => commands}/run/helpers.py (100%) rename platformio/{ => commands}/run/processor.py (97%) rename platformio/{ => commands}/test/__init__.py (100%) rename platformio/commands/{test.py => test/command.py} (98%) rename platformio/{ => commands}/test/embedded.py (98%) rename platformio/{ => commands}/test/native.py (95%) rename platformio/{ => commands}/test/processor.py (98%) diff --git a/platformio/builder/tools/piosize.py b/platformio/builder/tools/piosize.py index 75fe944b..593375f1 100644 --- a/platformio/builder/tools/piosize.py +++ b/platformio/builder/tools/piosize.py @@ -18,7 +18,7 @@ from __future__ import absolute_import import sys from os import environ, makedirs, remove -from os.path import join, isdir +from os.path import isdir, join from elftools.elf.descriptions import describe_sh_flags from elftools.elf.elffile import ELFFile @@ -52,7 +52,7 @@ def _get_symbol_locations(env, elf_path, addrs): cmd = [env.subst("$CC").replace("-gcc", "-addr2line"), "-e", elf_path] result = _run_tool(cmd, env, addrs) locations = [line for line in result["out"].split("\n") if line] - assert(len(addrs) == len(locations)) + assert len(addrs) == len(locations) return dict(zip(addrs, [l.strip().replace("\\", "/") for l in locations])) @@ -61,12 +61,17 @@ def _get_demangled_names(env, mangled_names): if not mangled_names: return {} result = _run_tool( - [env.subst("$CC").replace("-gcc", "-c++filt")], env, mangled_names) + [env.subst("$CC").replace("-gcc", "-c++filt")], env, mangled_names + ) demangled_names = [line for line in result["out"].split("\n") if line] - assert(len(mangled_names) == len(demangled_names)) + assert len(mangled_names) == len(demangled_names) - return dict(zip(mangled_names, [dn.strip().replace( - "::__FUNCTION__", "") for dn in demangled_names])) + return dict( + zip( + mangled_names, + [dn.strip().replace("::__FUNCTION__", "") for dn in demangled_names], + ) + ) def _determine_section(sections, symbol_addr): @@ -150,8 +155,7 @@ def _collect_symbols_info(env, elffile, elf_path, sections): symbol_addrs.append(hex(symbol_addr)) symbols.append(symbol) - symbol_locations = _get_symbol_locations( - env, elf_path, symbol_addrs) + symbol_locations = _get_symbol_locations(env, elf_path, symbol_addrs) demangled_names = _get_demangled_names(env, mangled_names) for symbol in symbols: if symbol["name"].startswith("_Z"): diff --git a/platformio/commands/__init__.py b/platformio/commands/__init__.py index 961f4932..bc018f8c 100644 --- a/platformio/commands/__init__.py +++ b/platformio/commands/__init__.py @@ -13,7 +13,6 @@ # limitations under the License. import os -from os.path import dirname, isfile, join import click @@ -22,6 +21,10 @@ class PlatformioCLI(click.MultiCommand): leftover_args = [] + def __init__(self, *args, **kwargs): + super(PlatformioCLI, self).__init__(*args, **kwargs) + self._pio_cmds_dir = os.path.dirname(__file__) + @staticmethod def in_silence(): args = PlatformioCLI.leftover_args @@ -42,34 +45,23 @@ class PlatformioCLI(click.MultiCommand): def list_commands(self, ctx): cmds = [] - cmds_dir = dirname(__file__) - for name in os.listdir(cmds_dir): - if name.startswith("__init__"): + for cmd_name in os.listdir(self._pio_cmds_dir): + if cmd_name.startswith("__init__"): continue - if isfile(join(cmds_dir, name, "command.py")): - cmds.append(name) - elif name.endswith(".py"): - cmds.append(name[:-3]) + if os.path.isfile(os.path.join(self._pio_cmds_dir, cmd_name, "command.py")): + cmds.append(cmd_name) + elif cmd_name.endswith(".py"): + cmds.append(cmd_name[:-3]) cmds.sort() return cmds def get_command(self, ctx, cmd_name): mod = None try: - mod = __import__("platformio.commands." + cmd_name, None, None, ["cli"]) + mod_path = "platformio.commands." + cmd_name + if os.path.isfile(os.path.join(self._pio_cmds_dir, cmd_name, "command.py")): + mod_path = "platformio.commands.%s.command" % cmd_name + mod = __import__(mod_path, None, None, ["cli"]) except ImportError: raise click.UsageError('No such command "%s"' % cmd_name, ctx) return mod.cli - - @staticmethod - def _handle_obsolate_command(name): - # pylint: disable=import-outside-toplevel - if name == "platforms": - from platformio.commands import platform - - return platform.cli - if name == "serialports": - from platformio.commands import device - - return device.cli - raise AttributeError() diff --git a/platformio/check/__init__.py b/platformio/commands/check/__init__.py similarity index 100% rename from platformio/check/__init__.py rename to platformio/commands/check/__init__.py diff --git a/platformio/commands/check.py b/platformio/commands/check/command.py similarity index 98% rename from platformio/commands/check.py rename to platformio/commands/check/command.py index 614f60d1..d40058bc 100644 --- a/platformio/commands/check.py +++ b/platformio/commands/check/command.py @@ -24,8 +24,8 @@ import click from tabulate import tabulate from platformio import app, exception, fs, util -from platformio.check.defect import DefectItem -from platformio.check.tools import CheckToolFactory +from platformio.commands.check.defect import DefectItem +from platformio.commands.check.tools import CheckToolFactory from platformio.compat import dump_json_to_unicode from platformio.project.config import ProjectConfig from platformio.project.helpers import find_project_dir_above, get_project_dir diff --git a/platformio/check/defect.py b/platformio/commands/check/defect.py similarity index 100% rename from platformio/check/defect.py rename to platformio/commands/check/defect.py diff --git a/platformio/check/tools/__init__.py b/platformio/commands/check/tools/__init__.py similarity index 88% rename from platformio/check/tools/__init__.py rename to platformio/commands/check/tools/__init__.py index 0649e81d..4c8a5e72 100644 --- a/platformio/check/tools/__init__.py +++ b/platformio/commands/check/tools/__init__.py @@ -13,8 +13,8 @@ # limitations under the License. from platformio import exception -from platformio.check.tools.clangtidy import ClangtidyCheckTool -from platformio.check.tools.cppcheck import CppcheckCheckTool +from platformio.commands.check.tools.clangtidy import ClangtidyCheckTool +from platformio.commands.check.tools.cppcheck import CppcheckCheckTool class CheckToolFactory(object): diff --git a/platformio/check/tools/base.py b/platformio/commands/check/tools/base.py similarity index 98% rename from platformio/check/tools/base.py rename to platformio/commands/check/tools/base.py index 3306dd3c..78eac5dd 100644 --- a/platformio/check/tools/base.py +++ b/platformio/commands/check/tools/base.py @@ -15,7 +15,7 @@ import click from platformio import fs, proc -from platformio.check.defect import DefectItem +from platformio.commands.check.defect import DefectItem from platformio.project.helpers import get_project_dir, load_project_ide_data diff --git a/platformio/check/tools/clangtidy.py b/platformio/commands/check/tools/clangtidy.py similarity index 94% rename from platformio/check/tools/clangtidy.py rename to platformio/commands/check/tools/clangtidy.py index 36c00c26..20b4a5de 100644 --- a/platformio/check/tools/clangtidy.py +++ b/platformio/commands/check/tools/clangtidy.py @@ -15,8 +15,8 @@ import re from os.path import join -from platformio.check.defect import DefectItem -from platformio.check.tools.base import CheckToolBase +from platformio.commands.check.defect import DefectItem +from platformio.commands.check.tools.base import CheckToolBase from platformio.managers.core import get_core_package_dir diff --git a/platformio/check/tools/cppcheck.py b/platformio/commands/check/tools/cppcheck.py similarity index 97% rename from platformio/check/tools/cppcheck.py rename to platformio/commands/check/tools/cppcheck.py index aacb24f8..9eef560f 100644 --- a/platformio/check/tools/cppcheck.py +++ b/platformio/commands/check/tools/cppcheck.py @@ -16,8 +16,8 @@ from os import remove from os.path import isfile, join from tempfile import NamedTemporaryFile -from platformio.check.defect import DefectItem -from platformio.check.tools.base import CheckToolBase +from platformio.commands.check.defect import DefectItem +from platformio.commands.check.tools.base import CheckToolBase from platformio.managers.core import get_core_package_dir diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index 7228d001..4cdf227c 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -23,7 +23,7 @@ import click from platformio import app, fs from platformio.commands.init import cli as cmd_init from platformio.commands.init import validate_boards -from platformio.commands.run import cli as cmd_run +from platformio.commands.run.command import cli as cmd_run from platformio.compat import glob_escape from platformio.exception import CIBuildEnvsEmpty from platformio.project.config import ProjectConfig diff --git a/platformio/debug/__init__.py b/platformio/commands/debug/__init__.py similarity index 100% rename from platformio/debug/__init__.py rename to platformio/commands/debug/__init__.py diff --git a/platformio/debug/client.py b/platformio/commands/debug/client.py similarity index 98% rename from platformio/debug/client.py rename to platformio/commands/debug/client.py index eca409e5..990340e8 100644 --- a/platformio/debug/client.py +++ b/platformio/commands/debug/client.py @@ -27,10 +27,10 @@ from twisted.internet import stdio # pylint: disable=import-error from twisted.internet import task # pylint: disable=import-error from platformio import app, exception, fs, proc, util +from platformio.commands.debug import helpers, initcfgs +from platformio.commands.debug.process import BaseProcess +from platformio.commands.debug.server import DebugServer from platformio.compat import hashlib_encode_data -from platformio.debug import helpers, initcfgs -from platformio.debug.process import BaseProcess -from platformio.debug.server import DebugServer from platformio.project.helpers import get_project_cache_dir from platformio.telemetry import MeasurementProtocol diff --git a/platformio/commands/debug.py b/platformio/commands/debug/command.py similarity index 97% rename from platformio/commands/debug.py rename to platformio/commands/debug/command.py index 1fb1b8e7..c99cc548 100644 --- a/platformio/commands/debug.py +++ b/platformio/commands/debug/command.py @@ -22,7 +22,7 @@ from os.path import isfile import click from platformio import app, exception, fs, proc, util -from platformio.debug import helpers +from platformio.commands.debug import helpers from platformio.managers.core import inject_contrib_pysite from platformio.project.config import ProjectConfig from platformio.project.helpers import is_platformio_project, load_project_ide_data @@ -139,7 +139,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro inject_contrib_pysite() # pylint: disable=import-outside-toplevel - from platformio.debug.client import GDBClient, reactor + from platformio.commands.debug.client import GDBClient, reactor client = GDBClient(project_dir, __unprocessed, debug_options, env_options) client.spawn(configuration["gdb_path"], configuration["prog_path"]) diff --git a/platformio/debug/helpers.py b/platformio/commands/debug/helpers.py similarity index 99% rename from platformio/debug/helpers.py rename to platformio/commands/debug/helpers.py index db396649..67255c46 100644 --- a/platformio/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -21,7 +21,7 @@ from os.path import isfile from platformio import exception, fs, util from platformio.commands.platform import platform_install as cmd_platform_install -from platformio.commands.run import cli as cmd_run +from platformio.commands.run.command import cli as cmd_run from platformio.managers.platform import PlatformFactory from platformio.project.config import ProjectConfig diff --git a/platformio/debug/initcfgs.py b/platformio/commands/debug/initcfgs.py similarity index 100% rename from platformio/debug/initcfgs.py rename to platformio/commands/debug/initcfgs.py diff --git a/platformio/debug/process.py b/platformio/commands/debug/process.py similarity index 100% rename from platformio/debug/process.py rename to platformio/commands/debug/process.py diff --git a/platformio/debug/server.py b/platformio/commands/debug/server.py similarity index 98% rename from platformio/debug/server.py rename to platformio/commands/debug/server.py index 49d730a0..18b39e41 100644 --- a/platformio/debug/server.py +++ b/platformio/commands/debug/server.py @@ -19,7 +19,7 @@ from twisted.internet import error # pylint: disable=import-error from twisted.internet import reactor # pylint: disable=import-error from platformio import exception, fs, util -from platformio.debug.process import BaseProcess +from platformio.commands.debug.process import BaseProcess from platformio.proc import where_is_program diff --git a/platformio/home/__init__.py b/platformio/commands/home/__init__.py similarity index 100% rename from platformio/home/__init__.py rename to platformio/commands/home/__init__.py diff --git a/platformio/commands/home.py b/platformio/commands/home/command.py similarity index 86% rename from platformio/commands/home.py rename to platformio/commands/home/command.py index 89bd9d01..a6340b5d 100644 --- a/platformio/commands/home.py +++ b/platformio/commands/home/command.py @@ -44,14 +44,14 @@ def cli(port, host, no_open): from twisted.internet import reactor from twisted.web import server - from platformio.home.rpc.handlers.app import AppRPC - from platformio.home.rpc.handlers.ide import IDERPC - from platformio.home.rpc.handlers.misc import MiscRPC - from platformio.home.rpc.handlers.os import OSRPC - from platformio.home.rpc.handlers.piocore import PIOCoreRPC - from platformio.home.rpc.handlers.project import ProjectRPC - from platformio.home.rpc.server import JSONRPCServerFactory - from platformio.home.web import WebRoot + from platformio.commands.home.rpc.handlers.app import AppRPC + from platformio.commands.home.rpc.handlers.ide import IDERPC + from platformio.commands.home.rpc.handlers.misc import MiscRPC + from platformio.commands.home.rpc.handlers.os import OSRPC + from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC + from platformio.commands.home.rpc.handlers.project import ProjectRPC + from platformio.commands.home.rpc.server import JSONRPCServerFactory + from platformio.commands.home.web import WebRoot factory = JSONRPCServerFactory() factory.addHandler(AppRPC(), namespace="app") diff --git a/platformio/home/helpers.py b/platformio/commands/home/helpers.py similarity index 100% rename from platformio/home/helpers.py rename to platformio/commands/home/helpers.py diff --git a/platformio/home/rpc/__init__.py b/platformio/commands/home/rpc/__init__.py similarity index 100% rename from platformio/home/rpc/__init__.py rename to platformio/commands/home/rpc/__init__.py diff --git a/platformio/home/rpc/handlers/__init__.py b/platformio/commands/home/rpc/handlers/__init__.py similarity index 100% rename from platformio/home/rpc/handlers/__init__.py rename to platformio/commands/home/rpc/handlers/__init__.py diff --git a/platformio/home/rpc/handlers/app.py b/platformio/commands/home/rpc/handlers/app.py similarity index 100% rename from platformio/home/rpc/handlers/app.py rename to platformio/commands/home/rpc/handlers/app.py diff --git a/platformio/home/rpc/handlers/ide.py b/platformio/commands/home/rpc/handlers/ide.py similarity index 100% rename from platformio/home/rpc/handlers/ide.py rename to platformio/commands/home/rpc/handlers/ide.py diff --git a/platformio/home/rpc/handlers/misc.py b/platformio/commands/home/rpc/handlers/misc.py similarity index 97% rename from platformio/home/rpc/handlers/misc.py rename to platformio/commands/home/rpc/handlers/misc.py index 5503cf8c..004c14a3 100644 --- a/platformio/home/rpc/handlers/misc.py +++ b/platformio/commands/home/rpc/handlers/misc.py @@ -18,7 +18,7 @@ import time from twisted.internet import defer, reactor # pylint: disable=import-error from platformio import app -from platformio.home.rpc.handlers.os import OSRPC +from platformio.commands.home.rpc.handlers.os import OSRPC class MiscRPC(object): diff --git a/platformio/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py similarity index 99% rename from platformio/home/rpc/handlers/os.py rename to platformio/commands/home/rpc/handlers/os.py index 92f4a008..1be28fc8 100644 --- a/platformio/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -25,8 +25,8 @@ import click from twisted.internet import defer # pylint: disable=import-error from platformio import app, fs, util +from platformio.commands.home import helpers from platformio.compat import PY2, get_filesystem_encoding -from platformio.home import helpers class OSRPC(object): diff --git a/platformio/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py similarity index 99% rename from platformio/home/rpc/handlers/piocore.py rename to platformio/commands/home/rpc/handlers/piocore.py index f2a692d0..9ef39a03 100644 --- a/platformio/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -26,8 +26,8 @@ from twisted.internet import threads # pylint: disable=import-error from twisted.internet import utils # pylint: disable=import-error from platformio import __main__, __version__, fs +from platformio.commands.home import helpers from platformio.compat import PY2, get_filesystem_encoding, is_bytes, string_types -from platformio.home import helpers try: from thread import get_ident as thread_get_ident diff --git a/platformio/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py similarity index 98% rename from platformio/home/rpc/handlers/project.py rename to platformio/commands/home/rpc/handlers/project.py index 02c44e7e..b9e3948f 100644 --- a/platformio/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -22,9 +22,9 @@ from os.path import basename, getmtime, isdir, isfile, join, realpath, sep import jsonrpc # pylint: disable=import-error from platformio import exception, fs +from platformio.commands.home.rpc.handlers.app import AppRPC +from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC from platformio.compat import PY2, get_filesystem_encoding -from platformio.home.rpc.handlers.app import AppRPC -from platformio.home.rpc.handlers.piocore import PIOCoreRPC from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager from platformio.project.config import ProjectConfig diff --git a/platformio/home/rpc/server.py b/platformio/commands/home/rpc/server.py similarity index 100% rename from platformio/home/rpc/server.py rename to platformio/commands/home/rpc/server.py diff --git a/platformio/home/web.py b/platformio/commands/home/web.py similarity index 100% rename from platformio/home/web.py rename to platformio/commands/home/web.py diff --git a/platformio/run/__init__.py b/platformio/commands/run/__init__.py similarity index 100% rename from platformio/run/__init__.py rename to platformio/commands/run/__init__.py diff --git a/platformio/commands/run.py b/platformio/commands/run/command.py similarity index 97% rename from platformio/commands/run.py rename to platformio/commands/run/command.py index e378c528..5058c159 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run/command.py @@ -22,11 +22,11 @@ from tabulate import tabulate from platformio import app, exception, fs, util from platformio.commands.device import device_monitor as cmd_device_monitor +from platformio.commands.run.helpers import clean_build_dir, handle_legacy_libdeps +from platformio.commands.run.processor import EnvironmentProcessor +from platformio.commands.test.processor import CTX_META_TEST_IS_RUNNING from platformio.project.config import ProjectConfig from platformio.project.helpers import find_project_dir_above -from platformio.run.helpers import clean_build_dir, handle_legacy_libdeps -from platformio.run.processor import EnvironmentProcessor -from platformio.test.processor import CTX_META_TEST_IS_RUNNING # pylint: disable=too-many-arguments,too-many-locals,too-many-branches diff --git a/platformio/run/helpers.py b/platformio/commands/run/helpers.py similarity index 100% rename from platformio/run/helpers.py rename to platformio/commands/run/helpers.py diff --git a/platformio/run/processor.py b/platformio/commands/run/processor.py similarity index 97% rename from platformio/run/processor.py rename to platformio/commands/run/processor.py index 6b7a9e20..3366c1e1 100644 --- a/platformio/run/processor.py +++ b/platformio/commands/run/processor.py @@ -14,8 +14,8 @@ from platformio import exception, telemetry from platformio.commands.platform import platform_install as cmd_platform_install +from platformio.commands.test.processor import CTX_META_TEST_RUNNING_NAME from platformio.managers.platform import PlatformFactory -from platformio.test.processor import CTX_META_TEST_RUNNING_NAME # pylint: disable=too-many-instance-attributes diff --git a/platformio/test/__init__.py b/platformio/commands/test/__init__.py similarity index 100% rename from platformio/test/__init__.py rename to platformio/commands/test/__init__.py diff --git a/platformio/commands/test.py b/platformio/commands/test/command.py similarity index 98% rename from platformio/commands/test.py rename to platformio/commands/test/command.py index 49b188df..cb1c8117 100644 --- a/platformio/commands/test.py +++ b/platformio/commands/test/command.py @@ -23,9 +23,9 @@ import click from tabulate import tabulate from platformio import app, exception, fs, util +from platformio.commands.test.embedded import EmbeddedTestProcessor +from platformio.commands.test.native import NativeTestProcessor from platformio.project.config import ProjectConfig -from platformio.test.embedded import EmbeddedTestProcessor -from platformio.test.native import NativeTestProcessor @click.command("test", short_help="Unit Testing") diff --git a/platformio/test/embedded.py b/platformio/commands/test/embedded.py similarity index 98% rename from platformio/test/embedded.py rename to platformio/commands/test/embedded.py index 6c6726e2..dc0d9ef0 100644 --- a/platformio/test/embedded.py +++ b/platformio/commands/test/embedded.py @@ -18,8 +18,8 @@ import click import serial from platformio import exception, util +from platformio.commands.test.processor import TestProcessorBase from platformio.managers.platform import PlatformFactory -from platformio.test.processor import TestProcessorBase class EmbeddedTestProcessor(TestProcessorBase): diff --git a/platformio/test/native.py b/platformio/commands/test/native.py similarity index 95% rename from platformio/test/native.py rename to platformio/commands/test/native.py index f028e4f3..a73c03fe 100644 --- a/platformio/test/native.py +++ b/platformio/commands/test/native.py @@ -15,8 +15,8 @@ from os.path import join from platformio import proc +from platformio.commands.test.processor import TestProcessorBase from platformio.proc import LineBufferedAsyncPipe -from platformio.test.processor import TestProcessorBase class NativeTestProcessor(TestProcessorBase): diff --git a/platformio/test/processor.py b/platformio/commands/test/processor.py similarity index 98% rename from platformio/test/processor.py rename to platformio/commands/test/processor.py index 6c900934..ed4f935c 100644 --- a/platformio/test/processor.py +++ b/platformio/commands/test/processor.py @@ -113,7 +113,7 @@ class TestProcessorBase(object): try: # pylint: disable=import-outside-toplevel - from platformio.commands.run import cli as cmd_run + from platformio.commands.run.command import cli as cmd_run return self.cmd_ctx.invoke( cmd_run, diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index deeeb6c2..642e1a7e 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -137,7 +137,7 @@ def compute_project_checksum(config): def load_project_ide_data(project_dir, env_or_envs): # pylint: disable=import-outside-toplevel - from platformio.commands.run import cli as cmd_run + from platformio.commands.run.command import cli as cmd_run assert env_or_envs envs = env_or_envs diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index e4bb6db9..fbb5b881 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -17,7 +17,7 @@ from os.path import isfile, join import pytest -from platformio.commands.check import cli as cmd_check +from platformio.commands.check.command import cli as cmd_check DEFAULT_CONFIG = """ [env:native] diff --git a/tests/test_builder.py b/tests/test_builder.py index d4f8012f..9dec2e0f 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from platformio.commands.run import cli as cmd_run +from platformio.commands.run.command import cli as cmd_run def test_build_flags(clirunner, validate_cliresult, tmpdir): From 95d1f4379961e6cdef8d71b526a5b959dc4ee752 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 23 Oct 2019 18:49:08 +0300 Subject: [PATCH 124/221] Sync docs with ST STM32 dev/platform --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index b456c59a..1f833f57 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit b456c59a5a27cf57024932ae8d94f1337f434de5 +Subproject commit 1f833f57255b09222abd052067612cdd941be4db From b533d7a1dde6377c65ec9c6cb0f18f9de0a28ce7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 23 Oct 2019 22:31:26 +0300 Subject: [PATCH 125/221] LDF: Check global CPPPATH when looking for dependencies --- platformio/builder/tools/piolib.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 7f5484ce..2858eaa3 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -304,7 +304,10 @@ class LibBuilderBase(object): ): # all include directories if not LibBuilderBase._INCLUDE_DIRS_CACHE: - LibBuilderBase._INCLUDE_DIRS_CACHE = [] + LibBuilderBase._INCLUDE_DIRS_CACHE = [ + self.env.Dir(os.path.realpath(d) if os.path.isdir(d) else d) + for d in self.envorigin.get("CPPPATH", []) + ] for lb in self.env.GetLibBuilders(): LibBuilderBase._INCLUDE_DIRS_CACHE.extend( [self.env.Dir(d) for d in lb.get_include_dirs()] @@ -346,7 +349,7 @@ class LibBuilderBase(object): self.env.File(path), self.env, tuple(include_dirs) ) - # print(path, map(lambda n: n.get_abspath(), candidates)) + # print(path, [c.get_abspath() for c in candidates]) for item in candidates: if item not in result: result.append(item) From 6684ac5a571fc0a40cfaf46f9df5119c05569521 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 23 Oct 2019 22:55:02 +0300 Subject: [PATCH 126/221] LDF: Check project include dirs before looking for dependencies --- platformio/builder/tools/piolib.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 2858eaa3..3c34748e 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -305,8 +305,10 @@ class LibBuilderBase(object): # all include directories if not LibBuilderBase._INCLUDE_DIRS_CACHE: LibBuilderBase._INCLUDE_DIRS_CACHE = [ - self.env.Dir(os.path.realpath(d) if os.path.isdir(d) else d) - for d in self.envorigin.get("CPPPATH", []) + self.env.Dir(d) + for d in ProjectAsLibBuilder( + self.envorigin, "$PROJECT_DIR" + ).get_include_dirs() ] for lb in self.env.GetLibBuilders(): LibBuilderBase._INCLUDE_DIRS_CACHE.extend( From dd1da95a40c49ccc2b191bc2f1a3e787bd6d18cf Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 24 Oct 2019 00:28:03 +0300 Subject: [PATCH 127/221] Fix issue when wrong library was picked up by LDF when framework is not declared --- platformio/builder/tools/piolib.py | 1 - 1 file changed, 1 deletion(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 3c34748e..234a621a 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -956,7 +956,6 @@ def IsCompatibleLibBuilder(env, lb, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))) return False if ( compat_mode in ("soft", "strict") - and "PIOFRAMEWORK" in env and not lb.is_frameworks_compatible(env.get("PIOFRAMEWORK", [])) ): if verbose: From 334d50c36797e1a1c0c36590e4693e7a02836d7b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 24 Oct 2019 13:42:46 +0300 Subject: [PATCH 128/221] Use package parser for package manager and LDF --- platformio/builder/tools/piolib.py | 66 +++++++------------------- platformio/managers/lib.py | 68 --------------------------- platformio/managers/package.py | 20 ++++---- platformio/package/manifest/parser.py | 55 ++++++++++++---------- tests/test_pkgmanifest.py | 17 ++++++- 5 files changed, 70 insertions(+), 156 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 234a621a..69d645c5 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -17,7 +17,6 @@ from __future__ import absolute_import -import codecs import hashlib import os import re @@ -34,6 +33,7 @@ from platformio import exception, fs, util from platformio.builder.tools import platformio as piotool from platformio.compat import WINDOWS, hashlib_encode_data, string_types from platformio.managers.lib import LibraryManager +from platformio.package.manifest.parser import ManifestParserFactory class LibBuilderFactory(object): @@ -456,17 +456,10 @@ class UnknownLibBuilder(LibBuilderBase): class ArduinoLibBuilder(LibBuilderBase): def load_manifest(self): - manifest = {} - if not isfile(join(self.path, "library.properties")): - return manifest manifest_path = join(self.path, "library.properties") - with codecs.open(manifest_path, encoding="utf-8") as fp: - for line in fp.readlines(): - if "=" not in line: - continue - key, value = line.split("=", 1) - manifest[key.strip()] = value.strip() - return manifest + if not isfile(manifest_path): + return {} + return ManifestParserFactory.new_from_file(manifest_path).as_dict() def get_include_dirs(self): include_dirs = LibBuilderBase.get_include_dirs(self) @@ -510,24 +503,7 @@ class ArduinoLibBuilder(LibBuilderBase): return util.items_in_list(frameworks, ["arduino", "energia"]) def is_platforms_compatible(self, platforms): - platforms_map = { - "avr": ["atmelavr"], - "sam": ["atmelsam"], - "samd": ["atmelsam"], - "esp8266": ["espressif8266"], - "esp32": ["espressif32"], - "arc32": ["intel_arc32"], - "stm32": ["ststm32"], - "nrf5": ["nordicnrf51", "nordicnrf52"], - } - items = [] - for arch in self._manifest.get("architectures", "").split(","): - arch = arch.strip().lower() - if arch == "*": - items = "*" - break - if arch in platforms_map: - items.extend(platforms_map[arch]) + items = self._manifest.get("platforms", []) if not items: return LibBuilderBase.is_platforms_compatible(self, platforms) return util.items_in_list(platforms, items) @@ -535,9 +511,10 @@ class ArduinoLibBuilder(LibBuilderBase): class MbedLibBuilder(LibBuilderBase): def load_manifest(self): - if not isfile(join(self.path, "module.json")): + manifest_path = join(self.path, "module.json") + if not isfile(manifest_path): return {} - return fs.load_json(join(self.path, "module.json")) + return ManifestParserFactory.new_from_file(manifest_path).as_dict() @property def include_dir(self): @@ -682,20 +659,12 @@ class MbedLibBuilder(LibBuilderBase): class PlatformIOLibBuilder(LibBuilderBase): def load_manifest(self): - assert isfile(join(self.path, "library.json")) - manifest = fs.load_json(join(self.path, "library.json")) - assert "name" in manifest + manifest_path = join(self.path, "library.json") + if not isfile(manifest_path): + return {} + return ManifestParserFactory.new_from_file(manifest_path).as_dict() - # replace "espressif" old name dev/platform with ESP8266 - if "platforms" in manifest: - manifest["platforms"] = [ - "espressif8266" if p == "espressif" else p - for p in util.items_to_list(manifest["platforms"]) - ] - - return manifest - - def _is_arduino_manifest(self): + def _has_arduino_manifest(self): return isfile(join(self.path, "library.properties")) @property @@ -718,7 +687,7 @@ class PlatformIOLibBuilder(LibBuilderBase): return self._manifest.get("build").get("srcFilter") if self.env["SRC_FILTER"]: return self.env["SRC_FILTER"] - if self._is_arduino_manifest(): + if self._has_arduino_manifest(): return ArduinoLibBuilder.src_filter.fget(self) return LibBuilderBase.src_filter.fget(self) @@ -789,7 +758,7 @@ class PlatformIOLibBuilder(LibBuilderBase): # backwards compatibility with PlatformIO 2.0 if ( "build" not in self._manifest - and self._is_arduino_manifest() + and self._has_arduino_manifest() and not isdir(join(self.path, "src")) and isdir(join(self.path, "utility")) ): @@ -954,9 +923,8 @@ def IsCompatibleLibBuilder(env, lb, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))) if verbose: sys.stderr.write("Platform incompatible library %s\n" % lb.path) return False - if ( - compat_mode in ("soft", "strict") - and not lb.is_frameworks_compatible(env.get("PIOFRAMEWORK", [])) + if compat_mode in ("soft", "strict") and not lb.is_frameworks_compatible( + env.get("PIOFRAMEWORK", []) ): if verbose: sys.stderr.write("Framework incompatible library %s\n" % lb.path) diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index 8a120c3a..e85a1225 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -16,7 +16,6 @@ # pylint: disable=too-many-return-statements import json -import re from glob import glob from os.path import isdir, join @@ -62,73 +61,6 @@ class LibraryManager(BasePkgManager): return None - def load_manifest(self, pkg_dir): - manifest = BasePkgManager.load_manifest(self, pkg_dir) - if not manifest: - return manifest - - # if Arduino library.properties - if "sentence" in manifest: - manifest["frameworks"] = ["arduino"] - manifest["description"] = manifest["sentence"] - del manifest["sentence"] - - if "author" in manifest: - if isinstance(manifest["author"], dict): - manifest["authors"] = [manifest["author"]] - else: - manifest["authors"] = [{"name": manifest["author"]}] - del manifest["author"] - - if "authors" in manifest and not isinstance(manifest["authors"], list): - manifest["authors"] = [manifest["authors"]] - - if "keywords" not in manifest: - keywords = [] - for keyword in re.split( - r"[\s/]+", manifest.get("category", "Uncategorized") - ): - keyword = keyword.strip() - if not keyword: - continue - keywords.append(keyword.lower()) - manifest["keywords"] = keywords - if "category" in manifest: - del manifest["category"] - - # don't replace VCS URL - if "url" in manifest and "description" in manifest: - manifest["homepage"] = manifest["url"] - del manifest["url"] - - if "architectures" in manifest: - platforms = [] - platforms_map = { - "avr": "atmelavr", - "sam": "atmelsam", - "samd": "atmelsam", - "esp8266": "espressif8266", - "esp32": "espressif32", - "arc32": "intel_arc32", - } - for arch in manifest["architectures"].split(","): - arch = arch.strip() - if arch == "*": - platforms = "*" - break - if arch in platforms_map: - platforms.append(platforms_map[arch]) - manifest["platforms"] = platforms - del manifest["architectures"] - - # convert listed items via comma to array - for key in ("keywords", "frameworks", "platforms"): - if key not in manifest or not isinstance(manifest[key], string_types): - continue - manifest[key] = [i.strip() for i in manifest[key].split(",") if i.strip()] - - return manifest - @staticmethod def normalize_dependencies(dependencies): if not dependencies: diff --git a/platformio/managers/package.py b/platformio/managers/package.py index f940767b..ba7fb494 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import codecs import hashlib import json import os @@ -29,6 +28,10 @@ from platformio import __version__, app, exception, fs, telemetry, util from platformio.compat import hashlib_encode_data from platformio.downloader import FileDownloader from platformio.lockfile import LockFile +from platformio.package.manifest.parser import ( + ManifestParserError, + ManifestParserFactory, +) from platformio.unpacker import FileUnpacker from platformio.vcsclient import VCSClientFactory @@ -326,7 +329,7 @@ class PkgInstallerMixin(object): def manifest_exists(self, pkg_dir): return self.get_manifest_path(pkg_dir) or self.get_src_manifest_path(pkg_dir) - def load_manifest(self, pkg_dir): + def load_manifest(self, pkg_dir): # pylint: disable=too-many-branches cache_key = "load_manifest-%s" % pkg_dir result = self.cache_get(cache_key) if result: @@ -342,15 +345,10 @@ class PkgInstallerMixin(object): if not manifest_path and not src_manifest_path: return None - if manifest_path and manifest_path.endswith(".json"): - manifest = fs.load_json(manifest_path) - elif manifest_path and manifest_path.endswith(".properties"): - with codecs.open(manifest_path, encoding="utf-8") as fp: - for line in fp.readlines(): - if "=" not in line: - continue - key, value = line.split("=", 1) - manifest[key.strip()] = value.strip() + try: + manifest = ManifestParserFactory.new_from_file(manifest_path).as_dict() + except ManifestParserError: + pass if src_manifest: if "version" in src_manifest: diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index dc7265b3..fb6f06db 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -46,6 +46,8 @@ class ManifestFileType(object): return ManifestFileType.MODULE_JSON if uri.endswith("package.json"): return ManifestFileType.PACKAGE_JSON + if uri.endswith("library.json"): + return ManifestFileType.LIBRARY_JSON return None @@ -331,18 +333,16 @@ class LibraryJsonManifestParser(BaseManifestParser): class ModuleJsonManifestParser(BaseManifestParser): def parse(self, contents): data = json.loads(contents) - return dict( - name=data["name"], - version=data["version"], - keywords=data.get("keywords"), - description=data["description"], - frameworks=["mbed"], - platforms=["*"], - homepage=data.get("homepage"), - export={"exclude": ["tests", "test", "*.doxyfile", "*.pdf"]}, - authors=self._parse_authors(data.get("author")), - license=self._parse_license(data.get("licenses")), - ) + data["frameworks"] = ["mbed"] + data["platforms"] = ["*"] + data["export"] = {"exclude": ["tests", "test", "*.doxyfile", "*.pdf"]} + if "author" in data: + data["authors"] = self._parse_authors(data.get("author")) + del data["author"] + if "licenses" in data: + data["license"] = self._parse_license(data.get("licenses")) + del data["licenses"] + return data def _parse_authors(self, raw): if not raw: @@ -364,23 +364,26 @@ class ModuleJsonManifestParser(BaseManifestParser): class LibraryPropertiesManifestParser(BaseManifestParser): def parse(self, contents): - properties = self._parse_properties(contents) - repository = self._parse_repository(properties) - homepage = properties.get("url") + data = self._parse_properties(contents) + repository = self._parse_repository(data) + homepage = data.get("url") if repository and repository["url"] == homepage: homepage = None - return dict( - frameworks=["arduino"], - homepage=homepage, - repository=repository or None, - name=properties.get("name"), - version=properties.get("version"), - description=self._parse_description(properties), - platforms=self._parse_platforms(properties) or ["*"], - keywords=self._parse_keywords(properties), - authors=self._parse_authors(properties) or None, - export=self._parse_export(), + data.update( + dict( + frameworks=["arduino"], + homepage=homepage, + repository=repository or None, + description=self._parse_description(data), + platforms=self._parse_platforms(data) or ["*"], + keywords=self._parse_keywords(data), + export=self._parse_export(), + ) ) + if "author" in data: + data["authors"] = self._parse_authors(data) + del data["author"] + return data @staticmethod def _parse_properties(contents): diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 212793a1..f779b488 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -31,7 +31,11 @@ def test_library_json_parser(): "platforms": ["atmelavr", "espressif"], "url": "http://old.url.format", "exclude": [".gitignore", "tests"], - "include": "mylib" + "include": "mylib", + "build": { + "flags": ["-DHELLO"] + }, + "customField": "Custom Value" } """ mp = parser.LibraryJsonManifestParser(contents) @@ -43,6 +47,8 @@ def test_library_json_parser(): "export": {"exclude": [".gitignore", "tests"], "include": ["mylib"]}, "keywords": ["kw1", "kw2", "kw3"], "homepage": "http://old.url.format", + "build": {"flags": ["-DHELLO"]}, + "customField": "Custom Value", }, ) @@ -89,9 +95,11 @@ def test_module_json_parser(): "type": "git", "url": "git@github.com:username/repo.git" }, - "version": "1.2.3" + "version": "1.2.3", + "customField": "Custom Value" } """ + mp = parser.ModuleJsonManifestParser(contents) assert not jsondiff.diff( mp.as_dict(), @@ -106,6 +114,8 @@ def test_module_json_parser(): "export": {"exclude": ["tests", "test", "*.doxyfile", "*.pdf"]}, "authors": [{"email": "name@surname.com", "name": "Name Surname"}], "version": "1.2.3", + "repository": {"type": "git", "url": "git@github.com:username/repo.git"}, + "customField": "Custom Value", }, ) @@ -117,6 +127,7 @@ name=TestPackage version=1.2.3 author=SomeAuthor sentence=This is Arduino library +customField=Custom Value """ mp = parser.LibraryPropertiesManifestParser(contents) assert not jsondiff.diff( @@ -125,6 +136,7 @@ sentence=This is Arduino library "name": "TestPackage", "version": "1.2.3", "description": "This is Arduino library", + "sentence": "This is Arduino library", "platforms": ["*"], "frameworks": ["arduino"], "export": { @@ -132,6 +144,7 @@ sentence=This is Arduino library }, "authors": [{"email": "info@author.com", "name": "SomeAuthor"}], "keywords": ["uncategorized"], + "customField": "Custom Value", }, ) From 798b12ce7b097fdcb5056d84054fe157aa12a9ff Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 24 Oct 2019 14:55:45 +0300 Subject: [PATCH 129/221] Fixed security issue when extracting items from TAR archive // Resolve #2995 --- HISTORY.rst | 1 + platformio/unpacker.py | 52 +++++++++++++++++++++++++++++++++--------- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 61c48891..43e0ec6e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -26,6 +26,7 @@ PlatformIO Core 4.0 * Fixed an issue when configuration file options partly ignored when using custom ``--project-conf`` (`issue #3034 `_) * Fixed an issue when installing a package using custom Git tag and submodules were not updated correctly (`issue #3060 `_) * Fixed an issue with linking process when ``$LDSCRIPT`` contains a space in path +* Fixed security issue when extracting items from TAR archive (`issue #2995 `_) 4.0.3 (2019-08-30) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/unpacker.py b/platformio/unpacker.py index 41b455a2..eabb03d4 100644 --- a/platformio/unpacker.py +++ b/platformio/unpacker.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os import chmod -from os.path import exists, join +import os from tarfile import open as tarfile_open from time import mktime from zipfile import ZipFile @@ -33,6 +32,9 @@ class ArchiveBase(object): def get_item_filename(self, item): raise NotImplementedError() + def is_link(self, item): + raise NotImplementedError() + def extract_item(self, item, dest_dir): self._afo.extract(item, dest_dir) self.after_extract(item, dest_dir) @@ -55,9 +57,36 @@ class TARArchive(ArchiveBase): return item.name @staticmethod - def islink(item): + def is_link(item): return item.islnk() or item.issym() + @staticmethod + def resolve_path(path): + return os.path.realpath(os.path.abspath(path)) + + def is_bad_path(self, path, base): + return not self.resolve_path(os.path.join(base, path)).startswith(base) + + def is_bad_link(self, tarinfo, base): + return self.is_bad_path( + tarinfo.linkname, + base=self.resolve_path(os.path.join(base, os.path.dirname(tarinfo.name))), + ) + + def extract_item(self, item, dest_dir): + bad_conds = [ + self.is_link(item) and self.is_bad_link(item, dest_dir), + not self.is_link(item) and self.is_bad_path(item.name, dest_dir), + ] + if not any(bad_conds): + super(TARArchive, self).extract_item(item, dest_dir) + else: + click.secho( + "Blocked insecure item `%s` from archive" % item.name, + fg="red", + err=True, + ) + class ZIPArchive(ArchiveBase): def __init__(self, archpath): @@ -67,24 +96,25 @@ class ZIPArchive(ArchiveBase): def preserve_permissions(item, dest_dir): attrs = item.external_attr >> 16 if attrs: - chmod(join(dest_dir, item.filename), attrs) + os.chmod(os.path.join(dest_dir, item.filename), attrs) @staticmethod def preserve_mtime(item, dest_dir): util.change_filemtime( - join(dest_dir, item.filename), + os.path.join(dest_dir, item.filename), mktime(tuple(item.date_time) + tuple([0, 0, 0])), ) + @staticmethod + def is_link(_): + return False + def get_items(self): return self._afo.infolist() def get_item_filename(self, item): return item.filename - def islink(self, item): - raise NotImplementedError() - def after_extract(self, item, dest_dir): self.preserve_permissions(item, dest_dir) self.preserve_mtime(item, dest_dir) @@ -96,7 +126,7 @@ class FileUnpacker(object): self._unpacker = None def __enter__(self): - if self.archpath.lower().endswith((".gz", ".bz2")): + if self.archpath.lower().endswith((".gz", ".bz2", ".tar")): self._unpacker = TARArchive(self.archpath) elif self.archpath.lower().endswith(".zip"): self._unpacker = ZIPArchive(self.archpath) @@ -126,9 +156,9 @@ class FileUnpacker(object): # check on disk for item in self._unpacker.get_items(): filename = self._unpacker.get_item_filename(item) - item_path = join(dest_dir, filename) + item_path = os.path.join(dest_dir, filename) try: - if not self._unpacker.islink(item) and not exists(item_path): + if not self._unpacker.is_link(item) and not os.path.exists(item_path): raise exception.ExtractArchiveItemError(filename, dest_dir) except NotImplementedError: pass From 0b500dba5468fd4688a46b6501dce291d8f5530c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 24 Oct 2019 15:10:11 +0300 Subject: [PATCH 130/221] Handle legacy "system": "all" for package manifest --- platformio/managers/package.py | 2 +- platformio/package/manifest/parser.py | 2 +- tests/test_pkgmanifest.py | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/platformio/managers/package.py b/platformio/managers/package.py index ba7fb494..de87904b 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -79,7 +79,7 @@ class PkgRepoMixin(object): @staticmethod def is_system_compatible(valid_systems): - if valid_systems in (None, "all", "*"): + if not valid_systems or "*" in valid_systems: return True if not isinstance(valid_systems, list): valid_systems = list([valid_systems]) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index fb6f06db..22c21b7b 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -537,7 +537,7 @@ class PackageJsonManifestParser(BaseManifestParser): def _parse_system(data): if "system" not in data: return data - if data["system"] in ("*", ["*"]): + if data["system"] in ("*", ["*"], "all"): del data["system"] return data if not isinstance(data["system"], list): diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index f779b488..18078a7f 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -479,6 +479,11 @@ def test_package_json_schema(): ) assert "system" not in mp.as_dict() + mp = parser.ManifestParserFactory.new( + '{"system": "all"}', parser.ManifestFileType.PACKAGE_JSON + ) + assert "system" not in mp.as_dict() + mp = parser.ManifestParserFactory.new( '{"system": "darwin_x86_64"}', parser.ManifestFileType.PACKAGE_JSON ) From 69d9438c716452030d6bd74076e1df255dbfb887 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 24 Oct 2019 15:39:41 +0300 Subject: [PATCH 131/221] Temporary disable security checking for Tar items --- platformio/unpacker.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/platformio/unpacker.py b/platformio/unpacker.py index eabb03d4..f00a6c84 100644 --- a/platformio/unpacker.py +++ b/platformio/unpacker.py @@ -73,19 +73,19 @@ class TARArchive(ArchiveBase): base=self.resolve_path(os.path.join(base, os.path.dirname(tarinfo.name))), ) - def extract_item(self, item, dest_dir): - bad_conds = [ - self.is_link(item) and self.is_bad_link(item, dest_dir), - not self.is_link(item) and self.is_bad_path(item.name, dest_dir), - ] - if not any(bad_conds): - super(TARArchive, self).extract_item(item, dest_dir) - else: - click.secho( - "Blocked insecure item `%s` from archive" % item.name, - fg="red", - err=True, - ) + # def extract_item(self, item, dest_dir): + # bad_conds = [ + # self.is_link(item) and self.is_bad_link(item, dest_dir), + # not self.is_link(item) and self.is_bad_path(item.name, dest_dir), + # ] + # if not any(bad_conds): + # super(TARArchive, self).extract_item(item, dest_dir) + # else: + # click.secho( + # "Blocked insecure item `%s` from archive" % item.name, + # fg="red", + # err=True, + # ) class ZIPArchive(ArchiveBase): From 2388b2a62be5c220736dbf205bb33d413fa04cea Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 24 Oct 2019 16:24:53 +0300 Subject: [PATCH 132/221] Fixed security issue when extracting items from TAR archive // Issue #2995 --- platformio/unpacker.py | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/platformio/unpacker.py b/platformio/unpacker.py index f00a6c84..980b43db 100644 --- a/platformio/unpacker.py +++ b/platformio/unpacker.py @@ -67,25 +67,24 @@ class TARArchive(ArchiveBase): def is_bad_path(self, path, base): return not self.resolve_path(os.path.join(base, path)).startswith(base) - def is_bad_link(self, tarinfo, base): - return self.is_bad_path( - tarinfo.linkname, - base=self.resolve_path(os.path.join(base, os.path.dirname(tarinfo.name))), - ) + def is_bad_link(self, item, base): + return not self.resolve_path( + os.path.join(os.path.join(base, os.path.dirname(item.name)), item.linkname) + ).startswith(base) - # def extract_item(self, item, dest_dir): - # bad_conds = [ - # self.is_link(item) and self.is_bad_link(item, dest_dir), - # not self.is_link(item) and self.is_bad_path(item.name, dest_dir), - # ] - # if not any(bad_conds): - # super(TARArchive, self).extract_item(item, dest_dir) - # else: - # click.secho( - # "Blocked insecure item `%s` from archive" % item.name, - # fg="red", - # err=True, - # ) + def extract_item(self, item, dest_dir): + bad_conds = [ + self.is_bad_path(item.name, dest_dir), + self.is_link(item) and self.is_bad_link(item, dest_dir), + ] + if not any(bad_conds): + super(TARArchive, self).extract_item(item, dest_dir) + else: + click.secho( + "Blocked insecure item `%s` from TAR archive" % item.name, + fg="red", + err=True, + ) class ZIPArchive(ArchiveBase): From 234585dc97c24c05f1c24b5ff590c301b9cb726f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 24 Oct 2019 16:39:11 +0300 Subject: [PATCH 133/221] Fixed an issue with project generator when ``src_build_flags`` were not respected // Resolve #3137 --- HISTORY.rst | 1 + platformio/builder/main.py | 2 +- platformio/builder/tools/pioide.py | 10 +++++----- platformio/builder/tools/platformio.py | 6 +++--- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 43e0ec6e..df2da41f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -27,6 +27,7 @@ PlatformIO Core 4.0 * Fixed an issue when installing a package using custom Git tag and submodules were not updated correctly (`issue #3060 `_) * Fixed an issue with linking process when ``$LDSCRIPT`` contains a space in path * Fixed security issue when extracting items from TAR archive (`issue #2995 `_) +* Fixed an issue with project generator when ``src_build_flags`` were not respected (`issue #3137 `_) 4.0.3 (2019-08-30) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/main.py b/platformio/builder/main.py index be5a4e07..81aefa09 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -189,7 +189,7 @@ if "idedata" in COMMAND_LINE_TARGETS: print ( "\n%s\n" % dump_json_to_unicode( - env.DumpIDEData(projenv) # pylint: disable=undefined-variable + projenv.DumpIDEData() # pylint: disable=undefined-variable ) ) env.Exit(0) diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 606a3bb0..26544cfe 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -25,11 +25,11 @@ from platformio.managers.core import get_core_package_dir from platformio.proc import exec_command, where_is_program -def _dump_includes(env, projenv): +def _dump_includes(env): includes = [] - for item in projenv.get("CPPPATH", []): - includes.append(projenv.subst(item)) + for item in env.get("CPPPATH", []): + includes.append(env.subst(item)) # installed libs for lb in env.GetLibBuilders(): @@ -138,7 +138,7 @@ def _get_svd_path(env): return None -def DumpIDEData(env, projenv): +def DumpIDEData(env): LINTCCOM = "$CFLAGS $CCFLAGS $CPPFLAGS" LINTCXXCOM = "$CXXFLAGS $CCFLAGS $CPPFLAGS" @@ -146,7 +146,7 @@ def DumpIDEData(env, projenv): "env_name": env["PIOENV"], "libsource_dirs": [env.subst(l) for l in env.GetLibSourceDirs()], "defines": _dump_defines(env), - "includes": _dump_includes(env, projenv), + "includes": _dump_includes(env), "cc_flags": env.subst(LINTCCOM), "cxx_flags": env.subst(LINTCXXCOM), "cc_path": where_is_program(env.subst("$CC"), env.subst("${ENV['PATH']}")), diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index ae4625e7..5d7fff09 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -124,9 +124,6 @@ def BuildProgram(env): if "__test" in COMMAND_LINE_TARGETS: env.ConfigureTestTarget() - # build project with dependencies - _build_project_deps(env) - # append into the beginning a main LD script if env.get("LDSCRIPT_PATH") and not any("-Wl,-T" in f for f in env["LINKFLAGS"]): env.Prepend(LINKFLAGS=["-T", env.subst("$LDSCRIPT_PATH")]) @@ -136,6 +133,9 @@ def BuildProgram(env): env.Prepend(_LIBFLAGS="-Wl,--start-group ") env.Append(_LIBFLAGS=" -Wl,--end-group") + # build project with dependencies + _build_project_deps(env) + program = env.Program( os.path.join("$BUILD_DIR", env.subst("$PROGNAME")), env["PIOBUILDFILES"] ) From 601989c5ffe04630963b98c16ff5ea8c7ff39523 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 24 Oct 2019 16:56:28 +0300 Subject: [PATCH 134/221] Escape "\" char in GDB console output --- platformio/commands/debug/helpers.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index 67255c46..9175d070 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import re import sys import time from fnmatch import fnmatch @@ -30,12 +31,16 @@ class GDBBytesIO(BytesIO): # pylint: disable=too-few-public-methods STDOUT = sys.stdout + @staticmethod + def escape(text): + return re.sub(r"\\+", "\\\\", text) + def write(self, text): if "\n" in text: for line in text.strip().split("\n"): - self.STDOUT.write('~"%s\\n"\n' % line) + self.STDOUT.write('~"%s\\n"\n' % self.escape(line)) else: - self.STDOUT.write('~"%s"' % text) + self.STDOUT.write('~"%s"' % self.escape(text)) self.STDOUT.flush() From ed6c9a08cebae5a59197ea23539a548296ead819 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 24 Oct 2019 17:21:02 +0300 Subject: [PATCH 135/221] Add custom "PLATFORMIO_BUILD_DEBUG" target for CLion --- platformio/ide/tpls/clion/CMakeLists.txt.tpl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/platformio/ide/tpls/clion/CMakeLists.txt.tpl b/platformio/ide/tpls/clion/CMakeLists.txt.tpl index c3ff24f9..9fb475f0 100644 --- a/platformio/ide/tpls/clion/CMakeLists.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeLists.txt.tpl @@ -62,6 +62,12 @@ add_custom_target( WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) +add_custom_target( + PLATFORMIO_BUILD_DEBUG ALL + COMMAND ${PLATFORMIO_CMD} -f -c clion run --target debug "$<$>:-e${CMAKE_BUILD_TYPE}>" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) + add_custom_target( PLATFORMIO_UPDATE_ALL ALL COMMAND ${PLATFORMIO_CMD} -f -c clion update From 70b484a2c2fcd92ea30778840b520e3bc872c584 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 24 Oct 2019 17:34:49 +0300 Subject: [PATCH 136/221] Escape "\" char in GDB console output --- platformio/commands/debug/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index 9175d070..858be66b 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -33,7 +33,7 @@ class GDBBytesIO(BytesIO): # pylint: disable=too-few-public-methods @staticmethod def escape(text): - return re.sub(r"\\+", "\\\\", text) + return re.sub(r"\\+", "\\\\\\\\", text) def write(self, text): if "\n" in text: From 3177aaf591aaed7807b0a6f5e810668e7193ae10 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 24 Oct 2019 19:43:13 +0300 Subject: [PATCH 137/221] Fixed an issue when booleans in "platformio.ini" are not parsed properly // Resolve #3022 --- HISTORY.rst | 1 + docs | 2 +- platformio/builder/tools/pioplatform.py | 10 ++++++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index df2da41f..f635db39 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -28,6 +28,7 @@ PlatformIO Core 4.0 * Fixed an issue with linking process when ``$LDSCRIPT`` contains a space in path * Fixed security issue when extracting items from TAR archive (`issue #2995 `_) * Fixed an issue with project generator when ``src_build_flags`` were not respected (`issue #3137 `_) +* Fixed an issue when booleans in "platformio.ini" are not parsed properly (`issue #3022 `_) 4.0.3 (2019-08-30) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index 1f833f57..46b1dee7 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 1f833f57255b09222abd052067612cdd941be4db +Subproject commit 46b1dee7f1426eeedd9f16aaef7dcddffdd51b37 diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 9571ea75..a2b7679f 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -93,8 +93,14 @@ def LoadPioPlatform(env): # update board manifest with overridden data from INI config board_config = env.BoardConfig() for option, value in env.GetProjectOptions(): - if option.startswith("board_"): - board_config.update(option.lower()[6:], value) + if not option.startswith("board_"): + continue + option = option.lower()[6:] + if isinstance(board_config.get(option), bool): + value = str(value).lower() in ("1", "yes", "true") + elif isinstance(board_config.get(option), int): + value = int(value) + board_config.update(option, value) # load default variables from board config for option_meta in ProjectOptions.values(): From 8127fd99600ae367ea4845e75ad0295caa7d71d9 Mon Sep 17 00:00:00 2001 From: valeros Date: Thu, 24 Oct 2019 20:44:34 +0300 Subject: [PATCH 138/221] Export correct stats for each check tool --- platformio/commands/check/command.py | 48 ++++++++++++++++------------ 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py index d40058bc..2542ce17 100644 --- a/platformio/commands/check/command.py +++ b/platformio/commands/check/command.py @@ -137,6 +137,7 @@ def cli( result["succeeded"] = rc == 0 and not any( d.severity == DefectItem.SEVERITY_HIGH for d in result["defects"] ) + result["stats"] = collect_component_stats(result) results.append(result) if verbose: @@ -147,26 +148,24 @@ def cli( click.echo("No defects found") print_processing_footer(result) - component_stats = collect_component_stats(results) if json_output: - click.echo(dump_json_to_unicode(results_to_json(results, component_stats))) + click.echo(dump_json_to_unicode(results_to_json(results))) elif not silent: - print_check_summary(results, component_stats) + print_check_summary(results) command_failed = any(r.get("succeeded") is False for r in results) if command_failed: raise exception.ReturnErrorCode(1) -def results_to_json(raw, components): +def results_to_json(raw): results = [] for item in raw: item.update( { "ignored": item.get("succeeded") is None, "succeeded": bool(item.get("succeeded")), - "defects": [d.to_json() for d in item.get("defects", [])], - "stats": [{k: v} for k, v in components.items()], + "defects": [d.to_json() for d in item.get("defects", [])] } ) results.append(item) @@ -199,7 +198,7 @@ def print_processing_footer(result): ) -def collect_component_stats(results): +def collect_component_stats(result): components = dict() def _append_defect(component, defect): @@ -207,20 +206,29 @@ def collect_component_stats(results): components[component] = Counter() components[component].update({DefectItem.SEVERITY_LABELS[defect.severity]: 1}) - for result in results: - for defect in result.get("defects", []): - component = dirname(defect.file) or defect.file - _append_defect(component, defect) + for defect in result.get("defects", []): + component = dirname(defect.file) or defect.file + _append_defect(component, defect) - if component.startswith(get_project_dir()): - while os.sep in component: - component = dirname(component) - _append_defect(component, defect) + if component.startswith(get_project_dir()): + while os.sep in component: + component = dirname(component) + _append_defect(component, defect) - return dict(components) + return components -def print_defects_stats(component_stats): +def print_defects_stats(results): + if not results: + return + + component_stats = {} + for r in results: + for k, v in r.get("stats", {}).items(): + if not component_stats.get(k): + component_stats[k] = Counter() + component_stats[k].update(r["stats"][k]) + if not component_stats: return @@ -233,7 +241,7 @@ def print_defects_stats(component_stats): total = ["Total"] + [sum(d) for d in list(zip(*tabular_data))[1:]] tabular_data.sort() - tabular_data.append([]) # Empty line as delimeter + tabular_data.append([]) # Empty line as delimiter tabular_data.append(total) headers = ["Component"] @@ -243,7 +251,7 @@ def print_defects_stats(component_stats): click.echo() -def print_check_summary(results, component_stats): +def print_check_summary(results): click.echo() tabular_data = [] @@ -251,7 +259,7 @@ def print_check_summary(results, component_stats): failed_nums = 0 duration = 0 - print_defects_stats(component_stats) + print_defects_stats(results) for result in results: duration += result.get("duration", 0) From 5345dd267479f05c88ee1386ba876d69d140ae4d Mon Sep 17 00:00:00 2001 From: valeros Date: Thu, 24 Oct 2019 21:35:04 +0300 Subject: [PATCH 139/221] Give a proper name to method that converts defect item to dict --- platformio/commands/check/command.py | 2 +- platformio/commands/check/defect.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py index 2542ce17..03e5a018 100644 --- a/platformio/commands/check/command.py +++ b/platformio/commands/check/command.py @@ -165,7 +165,7 @@ def results_to_json(raw): { "ignored": item.get("succeeded") is None, "succeeded": bool(item.get("succeeded")), - "defects": [d.to_json() for d in item.get("defects", [])] + "defects": [d.as_dict() for d in item.get("defects", [])] } ) results.append(item) diff --git a/platformio/commands/check/defect.py b/platformio/commands/check/defect.py index adef3666..c28e60c1 100644 --- a/platformio/commands/check/defect.py +++ b/platformio/commands/check/defect.py @@ -81,7 +81,7 @@ class DefectItem(object): return key raise Exception("Unknown severity label -> %s" % label) - def to_json(self): + def as_dict(self): return { "severity": self.SEVERITY_LABELS[self.severity], "category": self.category, From 784a5cd349f56bcf8a66a669ae506e4992b3de81 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 25 Oct 2019 00:33:04 +0300 Subject: [PATCH 140/221] Add support for "Build Middlewares" --- HISTORY.rst | 1 + docs | 2 +- platformio/builder/tools/platformio.py | 28 ++++++++++++++++++++++++-- platformio/commands/check/command.py | 2 +- 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index f635db39..e3ce94bd 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,6 +18,7 @@ PlatformIO Core 4.0 - Unused variables or functions - Out of scope memory usage. +* Added support for `Build Middlewares `__: configure custom build flags per specific file, skip any build nodes from a framework, replace build file with another on-the-fly, etc. * Extend project environment configuration in "platformio.ini" with other sections using a new `extends `__ option (`issue #2953 `_) * Generate ``.ccls`` LSP file for `Emacs `__ cross references, hierarchies, completion and semantic highlighting * Added ``--no-ansi`` flag for `PIO Core `__ to disable ANSI control characters diff --git a/docs b/docs index 46b1dee7..733ab67e 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 46b1dee7f1426eeedd9f16aaef7dcddffdd51b37 +Subproject commit 733ab67e51bf7b567e73ca9b1545410b0397f167 diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 5d7fff09..64638450 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -14,10 +14,12 @@ from __future__ import absolute_import +import fnmatch import os import sys from SCons import Builder, Util # pylint: disable=import-error +from SCons.Node import FS # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from SCons.Script import AlwaysBuild # pylint: disable=import-error from SCons.Script import DefaultEnvironment # pylint: disable=import-error @@ -244,7 +246,9 @@ def MatchSourceFiles(env, src_dir, src_filter=None): ) -def CollectBuildFiles(env, variant_dir, src_dir, src_filter=None, duplicate=False): +def CollectBuildFiles( + env, variant_dir, src_dir, src_filter=None, duplicate=False +): # pylint: disable=too-many-locals sources = [] variants = [] @@ -264,9 +268,24 @@ def CollectBuildFiles(env, variant_dir, src_dir, src_filter=None, duplicate=Fals if fs.path_endswith_ext(item, SRC_BUILD_EXT): sources.append(env.File(os.path.join(_var_dir, os.path.basename(item)))) + for callback, pattern in env.get("__PIO_BUILD_MIDDLEWARES", []): + tmp = [] + for node in sources: + if pattern and not fnmatch.fnmatch(node.get_path(), pattern): + tmp.append(node) + continue + n = callback(node) + if n: + tmp.append(n) + sources = tmp + return sources +def AddBuildMiddleware(env, callback, pattern=None): + env.Append(__PIO_BUILD_MIDDLEWARES=[(callback, pattern)]) + + def BuildFrameworks(env, frameworks): if not frameworks: return @@ -309,7 +328,11 @@ def BuildLibrary(env, variant_dir, src_dir, src_filter=None): def BuildSources(env, variant_dir, src_dir, src_filter=None): nodes = env.CollectBuildFiles(variant_dir, src_dir, src_filter) - DefaultEnvironment().Append(PIOBUILDFILES=[env.Object(node) for node in nodes]) + DefaultEnvironment().Append( + PIOBUILDFILES=[ + env.Object(node) if isinstance(node, FS.File) else node for node in nodes + ] + ) def exists(_): @@ -323,6 +346,7 @@ def generate(env): env.AddMethod(ProcessUnFlags) env.AddMethod(MatchSourceFiles) env.AddMethod(CollectBuildFiles) + env.AddMethod(AddBuildMiddleware) env.AddMethod(BuildFrameworks) env.AddMethod(BuildLibrary) env.AddMethod(BuildSources) diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py index 03e5a018..e7ba650a 100644 --- a/platformio/commands/check/command.py +++ b/platformio/commands/check/command.py @@ -165,7 +165,7 @@ def results_to_json(raw): { "ignored": item.get("succeeded") is None, "succeeded": bool(item.get("succeeded")), - "defects": [d.as_dict() for d in item.get("defects", [])] + "defects": [d.as_dict() for d in item.get("defects", [])], } ) results.append(item) From 5e681ec03cb03790439f47996dd8c3c3c5fd7712 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 25 Oct 2019 14:01:46 +0300 Subject: [PATCH 141/221] ProjectRPC.config_call accepts first argument as dict/kwargs for Config.init --- platformio/commands/home/rpc/handlers/project.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index b9e3948f..dbd41a55 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -28,16 +28,19 @@ from platformio.compat import PY2, get_filesystem_encoding from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager from platformio.project.config import ProjectConfig -from platformio.project.helpers import is_platformio_project +from platformio.project.helpers import get_project_dir, is_platformio_project class ProjectRPC(object): @staticmethod - def config_call(path, method, *args): - if isfile(path): - with fs.cd(os.path.dirname(path)): - return getattr(ProjectConfig(path), method)(*args) - return getattr(ProjectConfig(path), method)(*args) + def config_call(init_kwargs, method, *args): + assert isinstance(init_kwargs, dict) + assert "path" in init_kwargs + project_dir = get_project_dir() + if isfile(init_kwargs["path"]): + project_dir = os.path.dirname(init_kwargs["path"]) + with fs.cd(project_dir): + return getattr(ProjectConfig(**init_kwargs), method)(*args) @staticmethod def _get_projects(project_dirs=None): From 087a8f6dd023b37eb45a7423f9a6b5a045627a2f Mon Sep 17 00:00:00 2001 From: valeros Date: Fri, 25 Oct 2019 14:27:47 +0300 Subject: [PATCH 142/221] Fix Visual Studio template files encoding // Resolve #3183 --- platformio/ide/tpls/visualstudio/platformio.vcxproj.filters.tpl | 2 +- platformio/ide/tpls/visualstudio/platformio.vcxproj.tpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/ide/tpls/visualstudio/platformio.vcxproj.filters.tpl b/platformio/ide/tpls/visualstudio/platformio.vcxproj.filters.tpl index b29cb31f..cce99afe 100644 --- a/platformio/ide/tpls/visualstudio/platformio.vcxproj.filters.tpl +++ b/platformio/ide/tpls/visualstudio/platformio.vcxproj.filters.tpl @@ -1,4 +1,4 @@ - + diff --git a/platformio/ide/tpls/visualstudio/platformio.vcxproj.tpl b/platformio/ide/tpls/visualstudio/platformio.vcxproj.tpl index 82a011ad..fa10ad19 100644 --- a/platformio/ide/tpls/visualstudio/platformio.vcxproj.tpl +++ b/platformio/ide/tpls/visualstudio/platformio.vcxproj.tpl @@ -1,4 +1,4 @@ - + {{env_path}} From 8a6892bf3ce65e3f2b3f7134737272c4bd0e974a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 25 Oct 2019 14:33:22 +0300 Subject: [PATCH 143/221] Fixed an issue with invalid encoding when generating project for Visual Studio // Issue #3183 --- HISTORY.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/HISTORY.rst b/HISTORY.rst index e3ce94bd..0d3e7e8b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -30,6 +30,7 @@ PlatformIO Core 4.0 * Fixed security issue when extracting items from TAR archive (`issue #2995 `_) * Fixed an issue with project generator when ``src_build_flags`` were not respected (`issue #3137 `_) * Fixed an issue when booleans in "platformio.ini" are not parsed properly (`issue #3022 `_) +* Fixed an issue with invalid encoding when generating project for Visual Studio (`issue #3183 `_) 4.0.3 (2019-08-30) ~~~~~~~~~~~~~~~~~~ From 4486a85d4c3e6ed3776dedd17004c5ab1bc777ee Mon Sep 17 00:00:00 2001 From: valeros Date: Fri, 25 Oct 2019 15:40:50 +0300 Subject: [PATCH 144/221] Introduce new flag --fail-on-defect to pio check --- platformio/commands/check/command.py | 15 +++++++++++---- tests/commands/test_check.py | 26 ++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py index e7ba650a..b6fc2bec 100644 --- a/platformio/commands/check/command.py +++ b/platformio/commands/check/command.py @@ -56,6 +56,7 @@ from platformio.project.helpers import find_project_dir_above, get_project_dir @click.option("-s", "--silent", is_flag=True) @click.option("-v", "--verbose", is_flag=True) @click.option("--json-output", is_flag=True) +@click.option("--fail-on-defect", is_flag=True) def cli( environment, project_dir, @@ -66,6 +67,7 @@ def cli( silent, verbose, json_output, + fail_on_defect ): app.set_session_var("custom_project_conf", project_conf) @@ -134,9 +136,11 @@ def cli( result["defects"] = ct.get_defects() result["duration"] = time() - result["duration"] - result["succeeded"] = rc == 0 and not any( - d.severity == DefectItem.SEVERITY_HIGH for d in result["defects"] - ) + + result["succeeded"] = rc == 0 + if fail_on_defect: + result["succeeded"] = rc == 0 and not any( + d.severity == DefectItem.SEVERITY_HIGH for d in result["defects"]) result["stats"] = collect_component_stats(result) results.append(result) @@ -144,7 +148,10 @@ def cli( click.echo("\n".join(repr(d) for d in result["defects"])) if not json_output and not silent: - if not result["defects"]: + if rc != 0: + click.echo("Error: %s failed to perform check! Please " + "examine tool output in verbose mode." % tool) + elif not result["defects"]: click.echo("No defects found") print_processing_footer(result) diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index fbb5b881..028798ef 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -280,3 +280,29 @@ R21.4 text. assert result.exit_code != 0 assert "R21.3 Found MISRA defect" in result.output assert not isfile(join(str(check_dir), "src", "main.cpp.dump")) + + +def test_check_fails_on_defects_only_with_flag(clirunner, tmpdir): + config = DEFAULT_CONFIG + "\ncheck_tool = cppcheck, clangtidy" + tmpdir.join("platformio.ini").write(config) + tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) + + default_result = clirunner.invoke( + cmd_check, ["--project-dir", str(tmpdir)]) + + result_with_flag = clirunner.invoke( + cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect"]) + + assert default_result.exit_code == 0 + assert result_with_flag.exit_code != 0 + + +def test_check_bad_tool_flag_fails_check(clirunner, tmpdir): + config = DEFAULT_CONFIG + "\ncheck_tool = cppcheck, clangtidy" + config += "\ncheck_flags = --unknown-flag" + tmpdir.join("platformio.ini").write(config) + tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) + + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) + + assert result.exit_code != 0 From f3d8c30f954425a84ba97204cf343e75c037e89e Mon Sep 17 00:00:00 2001 From: valeros Date: Fri, 25 Oct 2019 15:50:19 +0300 Subject: [PATCH 145/221] Skip ignored environments when exporting check report in JSON format --- platformio/commands/check/command.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py index b6fc2bec..d1f01f5c 100644 --- a/platformio/commands/check/command.py +++ b/platformio/commands/check/command.py @@ -168,9 +168,10 @@ def cli( def results_to_json(raw): results = [] for item in raw: + if item.get("succeeded") is None: + continue item.update( { - "ignored": item.get("succeeded") is None, "succeeded": bool(item.get("succeeded")), "defects": [d.as_dict() for d in item.get("defects", [])], } From 49acf4bdb918a12781cec78a9a7e8f84de05c0be Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 25 Oct 2019 17:27:51 +0300 Subject: [PATCH 146/221] Minimum supported version of PIO Plus Core is 2.5.8 --- platformio/managers/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 4609dbad..856edecb 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -25,8 +25,8 @@ from platformio.project.config import ProjectConfig CORE_PACKAGES = { "contrib-piohome": "^2.3.2", - "contrib-pysite": "~2.%d%d.190418" % (sys.version_info[0], sys.version_info[1]), - "tool-pioplus": "^2.5.6", + "contrib-pysite": "~2.%d%d.0" % (sys.version_info[0], sys.version_info[1]), + "tool-pioplus": "^2.5.8", "tool-unity": "~1.20403.0", "tool-scons": "~2.20501.7" if PY2 else "~3.30101.0", "tool-cppcheck": "~1.189.0", From 4e43e7d3c3804b7efc9672233d6e0a3e8879058f Mon Sep 17 00:00:00 2001 From: valeros Date: Fri, 25 Oct 2019 17:43:52 +0300 Subject: [PATCH 147/221] Fix code formatting --- platformio/commands/check/command.py | 12 ++++++++---- tests/commands/test_check.py | 6 +++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py index d1f01f5c..3d430b02 100644 --- a/platformio/commands/check/command.py +++ b/platformio/commands/check/command.py @@ -67,7 +67,7 @@ def cli( silent, verbose, json_output, - fail_on_defect + fail_on_defect, ): app.set_session_var("custom_project_conf", project_conf) @@ -140,7 +140,9 @@ def cli( result["succeeded"] = rc == 0 if fail_on_defect: result["succeeded"] = rc == 0 and not any( - d.severity == DefectItem.SEVERITY_HIGH for d in result["defects"]) + d.severity == DefectItem.SEVERITY_HIGH + for d in result["defects"] + ) result["stats"] = collect_component_stats(result) results.append(result) @@ -149,8 +151,10 @@ def cli( if not json_output and not silent: if rc != 0: - click.echo("Error: %s failed to perform check! Please " - "examine tool output in verbose mode." % tool) + click.echo( + "Error: %s failed to perform check! Please " + "examine tool output in verbose mode." % tool + ) elif not result["defects"]: click.echo("No defects found") print_processing_footer(result) diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index 028798ef..b079aaad 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -287,11 +287,11 @@ def test_check_fails_on_defects_only_with_flag(clirunner, tmpdir): tmpdir.join("platformio.ini").write(config) tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) - default_result = clirunner.invoke( - cmd_check, ["--project-dir", str(tmpdir)]) + default_result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) result_with_flag = clirunner.invoke( - cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect"]) + cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect"] + ) assert default_result.exit_code == 0 assert result_with_flag.exit_code != 0 From 2e5dabb91374656afcfd3f97f14f3d46c3feadc2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 25 Oct 2019 19:33:22 +0300 Subject: [PATCH 148/221] Fix issue with custom board_ options --- platformio/builder/tools/pioplatform.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index a2b7679f..459b6a0b 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -96,10 +96,13 @@ def LoadPioPlatform(env): if not option.startswith("board_"): continue option = option.lower()[6:] - if isinstance(board_config.get(option), bool): - value = str(value).lower() in ("1", "yes", "true") - elif isinstance(board_config.get(option), int): - value = int(value) + try: + if isinstance(board_config.get(option), bool): + value = str(value).lower() in ("1", "yes", "true") + elif isinstance(board_config.get(option), int): + value = int(value) + except KeyError: + pass board_config.update(option, value) # load default variables from board config From a2451a716db5ba633260c63f5437be4d0c472ba2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 25 Oct 2019 20:01:31 +0300 Subject: [PATCH 149/221] PIO Home 3.0 with Project Inspect --- HISTORY.rst | 7 +++++++ docs | 2 +- platformio/managers/core.py | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 0d3e7e8b..14258008 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,6 +18,13 @@ PlatformIO Core 4.0 - Unused variables or functions - Out of scope memory usage. +* `PlatformIO Home 3.0 `__ + + - Static Code Analysis + - Firmware File Explorer + - Firmware Memory Inspection + - Firmware Sections & Symbols Viewer. + * Added support for `Build Middlewares `__: configure custom build flags per specific file, skip any build nodes from a framework, replace build file with another on-the-fly, etc. * Extend project environment configuration in "platformio.ini" with other sections using a new `extends `__ option (`issue #2953 `_) * Generate ``.ccls`` LSP file for `Emacs `__ cross references, hierarchies, completion and semantic highlighting diff --git a/docs b/docs index 733ab67e..3f5ecce6 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 733ab67e51bf7b567e73ca9b1545410b0397f167 +Subproject commit 3f5ecce6556575ebc8942369891050d7a0ac4e08 diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 856edecb..f9ad5c0b 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -24,7 +24,7 @@ from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path from platformio.project.config import ProjectConfig CORE_PACKAGES = { - "contrib-piohome": "^2.3.2", + "contrib-piohome": ">=3.0.0-beta.1,<4", "contrib-pysite": "~2.%d%d.0" % (sys.version_info[0], sys.version_info[1]), "tool-pioplus": "^2.5.8", "tool-unity": "~1.20403.0", From f3b8ae42240720dcaa1f9f77ebd592cf4bb615fb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 25 Oct 2019 20:02:01 +0300 Subject: [PATCH 150/221] Bump version to 4.1.0rc1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 0a46fb32..0f62b997 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, "0b5") +VERSION = (4, 1, "0rc1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 0e7a2b314157a38c256eab29db7496b5ee8eaaa6 Mon Sep 17 00:00:00 2001 From: valeros Date: Fri, 25 Oct 2019 20:08:04 +0300 Subject: [PATCH 151/221] Automatically detect source files language when invoking cppcheck --- platformio/commands/check/tools/base.py | 11 +++++++++++ platformio/commands/check/tools/cppcheck.py | 3 +++ 2 files changed, 14 insertions(+) diff --git a/platformio/commands/check/tools/base.py b/platformio/commands/check/tools/base.py index 78eac5dd..d199f131 100644 --- a/platformio/commands/check/tools/base.py +++ b/platformio/commands/check/tools/base.py @@ -13,6 +13,7 @@ # limitations under the License. import click +import os from platformio import fs, proc from platformio.commands.check.defect import DefectItem @@ -126,6 +127,16 @@ class CheckToolBase(object): # pylint: disable=too-many-instance-attributes get_project_dir(), self.options.get("filter"), file_extensions ) + def get_source_language(self): + with fs.cd(get_project_dir()): + for _, __, files in os.walk(self.config.get_optional_dir("src")): + for name in files: + if "." not in name: + continue + if os.path.splitext(name)[1].lower() in (".cpp", ".cxx", ".ino"): + return "c++" + return "c" + def check(self, on_defect_callback=None): self._on_defect_callback = on_defect_callback cmd = self.configure_command() diff --git a/platformio/commands/check/tools/cppcheck.py b/platformio/commands/check/tools/cppcheck.py index 9eef560f..b9a8578b 100644 --- a/platformio/commands/check/tools/cppcheck.py +++ b/platformio/commands/check/tools/cppcheck.py @@ -90,6 +90,9 @@ class CppcheckCheckTool(CheckToolBase): % "<&PIO&>".join(["{0}={{{0}}}".format(f) for f in self.defect_fields]) ) + if self.get_source_language() == "c++": + cmd.append("--language=c++") + flags = self.get_flags("cppcheck") if not self.is_flag_set("--platform", flags): cmd.append("--platform=unspecified") From 48651286b64d448cc26715a68c55853d86c12bbe Mon Sep 17 00:00:00 2001 From: valeros Date: Fri, 25 Oct 2019 20:59:36 +0300 Subject: [PATCH 152/221] Automatically detect C++ standard version when invoking cppcheck --- platformio/commands/check/tools/base.py | 5 ++++- platformio/commands/check/tools/cppcheck.py | 13 ++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/platformio/commands/check/tools/base.py b/platformio/commands/check/tools/base.py index d199f131..ae4bf7b5 100644 --- a/platformio/commands/check/tools/base.py +++ b/platformio/commands/check/tools/base.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import click import os +import click + from platformio import fs, proc from platformio.commands.check.defect import DefectItem from platformio.project.helpers import get_project_dir, load_project_ide_data @@ -26,6 +27,7 @@ class CheckToolBase(object): # pylint: disable=too-many-instance-attributes self.envname = envname self.options = options self.cpp_defines = [] + self.cpp_flags = [] self.cpp_includes = [] self._defects = [] @@ -50,6 +52,7 @@ class CheckToolBase(object): # pylint: disable=too-many-instance-attributes data = load_project_ide_data(project_dir, envname) if not data: return + self.cpp_flags = data.get("cxx_flags", "").split(" ") self.cpp_includes = data.get("includes", []) self.cpp_defines = data.get("defines", []) self.cpp_defines.extend(self._get_toolchain_defines(data.get("cc_path"))) diff --git a/platformio/commands/check/tools/cppcheck.py b/platformio/commands/check/tools/cppcheck.py index b9a8578b..17d1f041 100644 --- a/platformio/commands/check/tools/cppcheck.py +++ b/platformio/commands/check/tools/cppcheck.py @@ -90,9 +90,6 @@ class CppcheckCheckTool(CheckToolBase): % "<&PIO&>".join(["{0}={{{0}}}".format(f) for f in self.defect_fields]) ) - if self.get_source_language() == "c++": - cmd.append("--language=c++") - flags = self.get_flags("cppcheck") if not self.is_flag_set("--platform", flags): cmd.append("--platform=unspecified") @@ -106,6 +103,16 @@ class CppcheckCheckTool(CheckToolBase): ] cmd.append("--enable=%s" % ",".join(enabled_checks)) + if not self.is_flag_set("--language", flags): + if self.get_source_language() == "c++": + cmd.append("--language=c++") + + if not self.is_flag_set("--std", flags): + for f in self.cpp_flags: + if "-std" in f: + # Standards with GNU extensions are not allowed + cmd.append("-" + f.replace("gnu", "c")) + cmd.extend(["-D%s" % d for d in self.cpp_defines]) cmd.extend(flags) From d6205792470206b7f093f56c7d16e9b0d9d9d74d Mon Sep 17 00:00:00 2001 From: valeros Date: Fri, 25 Oct 2019 21:04:30 +0300 Subject: [PATCH 153/221] Fix tests for check command according to updated exit codes --- tests/commands/test_check.py | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index b079aaad..c5a8a7ff 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -90,7 +90,7 @@ def test_check_cli_output(clirunner, check_dir): errors, warnings, style = count_defects(result.output) - assert result.exit_code != 0 + assert result.exit_code == 0 assert errors + warnings + style == EXPECTED_DEFECTS @@ -119,7 +119,7 @@ def test_check_severity_threshold(clirunner, check_dir): errors, warnings, style = count_defects(result.output) - assert result.exit_code != 0 + assert result.exit_code == 0 assert errors == EXPECTED_ERRORS assert warnings == 0 assert style == 0 @@ -143,7 +143,7 @@ def test_check_silent_mode(clirunner, check_dir): errors, warnings, style = count_defects(result.output) - assert result.exit_code != 0 + assert result.exit_code == 0 assert errors == EXPECTED_ERRORS assert warnings == 0 assert style == 0 @@ -158,7 +158,7 @@ def test_check_filter_sources(clirunner, check_dir): errors, warnings, style = count_defects(result.output) - assert result.exit_code != 0 + assert result.exit_code == 0 assert errors == EXPECTED_ERRORS assert warnings == EXPECTED_WARNINGS assert style == EXPECTED_STYLE @@ -277,7 +277,7 @@ R21.4 text. cmd_check, ["--project-dir", str(check_dir), "--flags=--addon=misra.json"] ) - assert result.exit_code != 0 + assert result.exit_code == 0 assert "R21.3 Found MISRA defect" in result.output assert not isfile(join(str(check_dir), "src", "main.cpp.dump")) @@ -295,14 +295,3 @@ def test_check_fails_on_defects_only_with_flag(clirunner, tmpdir): assert default_result.exit_code == 0 assert result_with_flag.exit_code != 0 - - -def test_check_bad_tool_flag_fails_check(clirunner, tmpdir): - config = DEFAULT_CONFIG + "\ncheck_tool = cppcheck, clangtidy" - config += "\ncheck_flags = --unknown-flag" - tmpdir.join("platformio.ini").write(config) - tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) - - result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) - - assert result.exit_code != 0 From 5573c3871cc4d16d945478ae12e3f12e1dd578fa Mon Sep 17 00:00:00 2001 From: valeros Date: Mon, 28 Oct 2019 13:38:46 +0200 Subject: [PATCH 154/221] Allow cppcheck suppress individual defects by default --- platformio/commands/check/tools/cppcheck.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/platformio/commands/check/tools/cppcheck.py b/platformio/commands/check/tools/cppcheck.py index 17d1f041..f5a95ed5 100644 --- a/platformio/commands/check/tools/cppcheck.py +++ b/platformio/commands/check/tools/cppcheck.py @@ -91,6 +91,10 @@ class CppcheckCheckTool(CheckToolBase): ) flags = self.get_flags("cppcheck") + if not flags: + # by default user can suppress reporting individual defects + # directly in code // cppcheck-suppress warningID + cmd.append("--inline-suppr") if not self.is_flag_set("--platform", flags): cmd.append("--platform=unspecified") if not self.is_flag_set("--enable", flags): From 4ff7c868ef861137676855bf2b63af6d41ac6530 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 28 Oct 2019 16:34:26 +0200 Subject: [PATCH 155/221] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 3f5ecce6..a4227316 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 3f5ecce6556575ebc8942369891050d7a0ac4e08 +Subproject commit a4227316bebed4001cdece12826951302418d45f From 39a7062503d45d19b8ca67263fadb87bf9d9ec15 Mon Sep 17 00:00:00 2001 From: valeros Date: Mon, 28 Oct 2019 18:12:39 +0200 Subject: [PATCH 156/221] Fix types of defect fields column and line --- platformio/commands/check/defect.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/commands/check/defect.py b/platformio/commands/check/defect.py index c28e60c1..03e59c3c 100644 --- a/platformio/commands/check/defect.py +++ b/platformio/commands/check/defect.py @@ -45,8 +45,8 @@ class DefectItem(object): self.severity = severity self.category = category self.message = message - self.line = line - self.column = column + self.line = int(line) + self.column = int(column) self.callstack = callstack self.cwe = cwe self.id = id From 187e30d0556be3ad02edf3ceea343eedd5ccfdff Mon Sep 17 00:00:00 2001 From: valeros Date: Mon, 28 Oct 2019 18:30:22 +0200 Subject: [PATCH 157/221] Export full path to file with defect --- platformio/commands/check/command.py | 8 ++++---- platformio/commands/check/defect.py | 8 ++++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py index 3d430b02..be270b3d 100644 --- a/platformio/commands/check/command.py +++ b/platformio/commands/check/command.py @@ -159,10 +159,10 @@ def cli( click.echo("No defects found") print_processing_footer(result) - if json_output: - click.echo(dump_json_to_unicode(results_to_json(results))) - elif not silent: - print_check_summary(results) + if json_output: + click.echo(dump_json_to_unicode(results_to_json(results))) + elif not silent: + print_check_summary(results) command_failed = any(r.get("succeeded") is False for r in results) if command_failed: diff --git a/platformio/commands/check/defect.py b/platformio/commands/check/defect.py index 03e59c3c..ca5d0e03 100644 --- a/platformio/commands/check/defect.py +++ b/platformio/commands/check/defect.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os.path import relpath +from os.path import isabs, isfile, join, relpath import click @@ -82,11 +82,15 @@ class DefectItem(object): raise Exception("Unknown severity label -> %s" % label) def as_dict(self): + filepath = self.file + if not isabs(filepath) and isfile(join(get_project_dir(), filepath)): + filepath = join(get_project_dir(), filepath) + return { "severity": self.SEVERITY_LABELS[self.severity], "category": self.category, "message": self.message, - "file": self.file, + "file": filepath, "line": self.line, "column": self.column, "callstack": self.callstack, From 0194e094106bba97eed0cbe15b676abe5b5472fb Mon Sep 17 00:00:00 2001 From: valeros Date: Mon, 28 Oct 2019 18:37:14 +0200 Subject: [PATCH 158/221] Use simple abspath to get absolute path to file with defect --- platformio/commands/check/defect.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/platformio/commands/check/defect.py b/platformio/commands/check/defect.py index ca5d0e03..e864356e 100644 --- a/platformio/commands/check/defect.py +++ b/platformio/commands/check/defect.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os.path import isabs, isfile, join, relpath +from os.path import abspath, relpath import click @@ -82,15 +82,11 @@ class DefectItem(object): raise Exception("Unknown severity label -> %s" % label) def as_dict(self): - filepath = self.file - if not isabs(filepath) and isfile(join(get_project_dir(), filepath)): - filepath = join(get_project_dir(), filepath) - return { "severity": self.SEVERITY_LABELS[self.severity], "category": self.category, "message": self.message, - "file": filepath, + "file": abspath(self.file), "line": self.line, "column": self.column, "callstack": self.callstack, From 3236fb6b3db779587557a349877927d6c0131312 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 29 Oct 2019 17:01:20 +0200 Subject: [PATCH 159/221] Return file+line for sizedata instead of "location" --- platformio/builder/tools/piosize.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/platformio/builder/tools/piosize.py b/platformio/builder/tools/piosize.py index 593375f1..84d3bf12 100644 --- a/platformio/builder/tools/piosize.py +++ b/platformio/builder/tools/piosize.py @@ -160,8 +160,16 @@ def _collect_symbols_info(env, elffile, elf_path, sections): for symbol in symbols: if symbol["name"].startswith("_Z"): symbol["demangled_name"] = demangled_names.get(symbol["name"]) - symbol["location"] = symbol_locations.get(hex(symbol["addr"])) - + location = symbol_locations.get(hex(symbol["addr"])) + if not location or "?" in location: + continue + symbol["file"] = location + symbol["line"] = 0 + if ":" in location: + file_, line = location.rsplit(":", 1) + if line.isdigit(): + symbol["file"] = file_ + symbol["line"] = int(line) return symbols @@ -210,10 +218,7 @@ def DumpSizeData(_, target, source, env): # pylint: disable=unused-argument files = dict() for symbol in _collect_symbols_info(env, elffile, elf_path, sections): - file_path, _ = symbol.get("location").rsplit(":", 1) - if not file_path or file_path.startswith("?"): - file_path = "unknown" - + file_path = symbol.get("file") or "unknown" if not files.get(file_path, {}): files[file_path] = {"symbols": [], "ram_size": 0, "flash_size": 0} From 7cad06ea18e7dc4a1385617c5a5268de5c797f40 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 29 Oct 2019 17:12:18 +0200 Subject: [PATCH 160/221] Minor test fixes --- tests/commands/test_init.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/commands/test_init.py b/tests/commands/test_init.py index 309b47c3..4310fc41 100644 --- a/tests/commands/test_init.py +++ b/tests/commands/test_init.py @@ -72,7 +72,7 @@ def test_init_ide_atom(clirunner, validate_cliresult, tmpdir): assert all( [tmpdir.join(f).check() for f in (".clang_complete", ".gcc-flags.json")] ) - assert "arduinoavr" in tmpdir.join(".clang_complete").read() + assert "framework-arduino" in tmpdir.join(".clang_complete").read() # switch to NodeMCU result = clirunner.invoke(cmd_init, ["--ide", "atom", "-b", "nodemcuv2"]) @@ -84,7 +84,7 @@ def test_init_ide_atom(clirunner, validate_cliresult, tmpdir): result = clirunner.invoke(cmd_init, ["--ide", "atom"]) validate_cliresult(result) validate_pioproject(str(tmpdir)) - assert "arduinoavr" in tmpdir.join(".clang_complete").read() + assert "framework-arduino" in tmpdir.join(".clang_complete").read() def test_init_ide_eclipse(clirunner, validate_cliresult): From 7dbeab11a50cb7f0c23b3d216c2d876cf64ad875 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 29 Oct 2019 17:36:36 +0200 Subject: [PATCH 161/221] Add new OS.RPC "open_file" method for PIO Home --- platformio/commands/home/rpc/handlers/os.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index 1be28fc8..c019fbc7 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -82,6 +82,10 @@ class OSRPC(object): path.encode(get_filesystem_encoding()) if PY2 else path, locate=True ) + @staticmethod + def open_file(path): + return click.launch(path.encode(get_filesystem_encoding()) if PY2 else path) + @staticmethod def is_file(path): return isfile(path) From 8d0584aa590a8cce0fc4828e4e5ca8862f95ccf2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 29 Oct 2019 17:37:09 +0200 Subject: [PATCH 162/221] Add new IDE RPC "open_text_document" method for PIO Home --- platformio/commands/home/rpc/handlers/ide.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/platformio/commands/home/rpc/handlers/ide.py b/platformio/commands/home/rpc/handlers/ide.py index af5b474e..e3ad75f3 100644 --- a/platformio/commands/home/rpc/handlers/ide.py +++ b/platformio/commands/home/rpc/handlers/ide.py @@ -22,7 +22,7 @@ class IDERPC(object): def __init__(self): self._queue = {} - def send_command(self, command, params, sid=0): + def send_command(self, sid, command, params): if not self._queue.get(sid): raise jsonrpc.exceptions.JSONRPCDispatchException( code=4005, message="PIO Home IDE agent is not started" @@ -38,5 +38,10 @@ class IDERPC(object): self._queue[sid].append(defer.Deferred()) return self._queue[sid][-1] - def open_project(self, project_dir, sid=0): - return self.send_command("open_project", project_dir, sid) + def open_project(self, sid, project_dir): + return self.send_command(sid, "open_project", project_dir) + + def open_text_document(self, sid, path, line=None, column=None): + return self.send_command( + sid, "open_text_document", dict(path=path, line=line, column=column) + ) From 0222c56c4da19402c7931882d62d9156f4d4af38 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 29 Oct 2019 17:43:48 +0200 Subject: [PATCH 163/221] Use file system encoding when decoding subprocess output // Resolve #2890 --- platformio/proc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/proc.py b/platformio/proc.py index b17170dc..85336be9 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -19,7 +19,7 @@ from os.path import isdir, isfile, join, normpath from threading import Thread from platformio import exception -from platformio.compat import WINDOWS, string_types +from platformio.compat import WINDOWS, get_filesystem_encoding, string_types class AsyncPipeBase(object): @@ -122,7 +122,7 @@ def exec_command(*args, **kwargs): for k, v in result.items(): if isinstance(result[k], bytes): try: - result[k] = result[k].decode(sys.getdefaultencoding()) + result[k] = result[k].decode(get_filesystem_encoding()) except UnicodeDecodeError: result[k] = result[k].decode("latin-1") if v and isinstance(v, string_types): From 737c29b510fb062a81c6971e8117f34caaf0d00e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 29 Oct 2019 18:02:32 +0200 Subject: [PATCH 164/221] Update PIO Home to 3.0.0-beta.2 --- platformio/managers/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/managers/core.py b/platformio/managers/core.py index f9ad5c0b..c3fcc757 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -24,7 +24,7 @@ from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path from platformio.project.config import ProjectConfig CORE_PACKAGES = { - "contrib-piohome": ">=3.0.0-beta.1,<4", + "contrib-piohome": ">=3.0.0-beta.2,<4", "contrib-pysite": "~2.%d%d.0" % (sys.version_info[0], sys.version_info[1]), "tool-pioplus": "^2.5.8", "tool-unity": "~1.20403.0", From 6e03aa3a3d860c60fbde3fd57612d9e4b7bb527c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 29 Oct 2019 18:03:07 +0200 Subject: [PATCH 165/221] Bump version to 4.1.0rc2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 0f62b997..6cce0f67 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, "0rc1") +VERSION = (4, 1, "0rc2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 94f565db8418022c9ce92ac69d1b84e8b524b601 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 29 Oct 2019 18:11:09 +0200 Subject: [PATCH 166/221] Show warning about restart IDE to affect PIO Home changes --- platformio/commands/upgrade.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index a933ab7f..947b7a0b 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -19,7 +19,7 @@ from zipfile import ZipFile import click import requests -from platformio import VERSION, __version__, exception, util +from platformio import VERSION, __version__, app, exception, util from platformio.compat import WINDOWS from platformio.proc import exec_command, get_pythonexe_path from platformio.project.helpers import get_project_cache_dir @@ -64,6 +64,10 @@ def cli(dev): ) click.echo("Release notes: ", nl=False) click.secho("https://docs.platformio.org/en/latest/history.html", fg="cyan") + if app.get_session_var("caller_id"): + click.secho( + "Warning! Please restart IDE to affect PIO Home changes", fg="yellow" + ) except Exception as e: # pylint: disable=broad-except if not r: raise exception.UpgradeError("\n".join([str(cmd), str(e)])) From c4e76745858127df1e8243c5f7e631d1e1704d41 Mon Sep 17 00:00:00 2001 From: valeros Date: Wed, 30 Oct 2019 12:23:33 +0200 Subject: [PATCH 167/221] Don't pass header files to Cppcheck --- platformio/commands/check/tools/cppcheck.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/platformio/commands/check/tools/cppcheck.py b/platformio/commands/check/tools/cppcheck.py index f5a95ed5..20a28ec5 100644 --- a/platformio/commands/check/tools/cppcheck.py +++ b/platformio/commands/check/tools/cppcheck.py @@ -136,7 +136,10 @@ class CppcheckCheckTool(CheckToolBase): return fp.name def _generate_src_file(self): - return self._create_tmp_file("\n".join(self.get_project_src_files())) + src_files = [ + f for f in self.get_project_src_files() if not f.endswith((".h", ".hpp")) + ] + return self._create_tmp_file("\n".join(src_files)) def _generate_inc_file(self): return self._create_tmp_file("\n".join(self.cpp_includes)) From d815daed2977a389125d779619ff20074540a9e7 Mon Sep 17 00:00:00 2001 From: valeros Date: Wed, 30 Oct 2019 13:38:46 +0200 Subject: [PATCH 168/221] Allow specifying defect level that will cause failure --- platformio/commands/check/command.py | 8 ++++-- tests/commands/test_check.py | 43 ++++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py index be270b3d..cb155e8a 100644 --- a/platformio/commands/check/command.py +++ b/platformio/commands/check/command.py @@ -56,7 +56,11 @@ from platformio.project.helpers import find_project_dir_above, get_project_dir @click.option("-s", "--silent", is_flag=True) @click.option("-v", "--verbose", is_flag=True) @click.option("--json-output", is_flag=True) -@click.option("--fail-on-defect", is_flag=True) +@click.option( + "--fail-on-defect", + multiple=True, + type=click.Choice(DefectItem.SEVERITY_LABELS.values()), +) def cli( environment, project_dir, @@ -140,7 +144,7 @@ def cli( result["succeeded"] = rc == 0 if fail_on_defect: result["succeeded"] = rc == 0 and not any( - d.severity == DefectItem.SEVERITY_HIGH + DefectItem.SEVERITY_LABELS[d.severity] in fail_on_defect for d in result["defects"] ) result["stats"] = collect_component_stats(result) diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index c5a8a7ff..59bfff0f 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -32,25 +32,26 @@ void run_defects() { int* doubleFreePi = (int*)malloc(sizeof(int)); *doubleFreePi=2; free(doubleFreePi); - free(doubleFreePi); + free(doubleFreePi); /* High */ /* Reading uninitialized memory */ int* uninitializedPi = (int*)malloc(sizeof(int)); - *uninitializedPi++; + *uninitializedPi++; /* High + Medium*/ free(uninitializedPi); /* Delete instead of delete [] */ int* wrongDeletePi = new int[10]; wrongDeletePi++; - delete wrongDeletePi; + delete wrongDeletePi; /* High */ /* Index out of bounds */ int arr[10]; for(int i=0; i < 11; i++) { - arr[i] = 0; + arr[i] = 0; /* High */ } } +/* Low */ void unusedFuntion(){ } @@ -197,7 +198,7 @@ def test_check_success_if_no_errors(clirunner, tmpdir): """ #include -void unused_functin(){ +void unused_function(){ int unusedVar = 0; int* iP = &unusedVar; *iP++; @@ -290,8 +291,38 @@ def test_check_fails_on_defects_only_with_flag(clirunner, tmpdir): default_result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) result_with_flag = clirunner.invoke( - cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect"] + cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect=high"] ) assert default_result.exit_code == 0 assert result_with_flag.exit_code != 0 + + +def test_check_fails_on_defects_only_on_specified_level(clirunner, tmpdir): + config = DEFAULT_CONFIG + "\ncheck_tool = cppcheck, clangtidy" + tmpdir.join("platformio.ini").write(config) + tmpdir.mkdir("src").join("main.c").write( + """ +#include + +void unused_function(){ + int unusedVar = 0; + int* iP = &unusedVar; + *iP++; +} + +int main() { +} +""" + ) + + high_result = clirunner.invoke( + cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect=high"] + ) + + low_result = clirunner.invoke( + cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect=low"] + ) + + assert high_result.exit_code == 0 + assert low_result.exit_code != 0 From 2d4722477e7740f5c90a724d95dc5f2201316ea4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 30 Oct 2019 18:58:49 +0200 Subject: [PATCH 169/221] Automatically shutdown PIO Home server when no clients for 1 hour --- platformio/commands/home/rpc/server.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/platformio/commands/home/rpc/server.py b/platformio/commands/home/rpc/server.py index d7b73bb4..4eb2aceb 100644 --- a/platformio/commands/home/rpc/server.py +++ b/platformio/commands/home/rpc/server.py @@ -18,12 +18,23 @@ import click import jsonrpc from autobahn.twisted.websocket import WebSocketServerFactory, WebSocketServerProtocol from jsonrpc.exceptions import JSONRPCDispatchException -from twisted.internet import defer +from twisted.internet import defer, reactor from platformio.compat import PY2, dump_json_to_unicode, is_bytes class JSONRPCServerProtocol(WebSocketServerProtocol): + def onOpen(self): + self.factory.connection_nums += 1 + if self.factory.shutdown_timer: + self.factory.shutdown_timer.cancel() + self.factory.shutdown_timer = None + + def onClose(self, wasClean, code, reason): # pylint: disable=unused-argument + self.factory.connection_nums -= 1 + if self.factory.connection_nums == 0: + self.factory.shutdownByTimeout() + def onMessage(self, payload, isBinary): # pylint: disable=unused-argument # click.echo("> %s" % payload) response = jsonrpc.JSONRPCResponseManager.handle( @@ -65,11 +76,24 @@ class JSONRPCServerProtocol(WebSocketServerProtocol): class JSONRPCServerFactory(WebSocketServerFactory): + SHUTDOWN_TIMEOUT = 3600 # in seconds + protocol = JSONRPCServerProtocol + connection_nums = 0 + shutdown_timer = 0 def __init__(self): super(JSONRPCServerFactory, self).__init__() self.dispatcher = jsonrpc.Dispatcher() + def shutdownByTimeout(self): + def _auto_shutdown_server(): + click.echo("Automatically shutdown server on timeout") + reactor.stop() + + self.shutdown_timer = reactor.callLater( + self.SHUTDOWN_TIMEOUT, _auto_shutdown_server + ) + def addHandler(self, handler, namespace): self.dispatcher.build_method_map(handler, prefix="%s." % namespace) From 3146ab5d12274512f7d5bbe9a216dc469271dc6a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 30 Oct 2019 19:09:32 +0200 Subject: [PATCH 170/221] Allow export project config data as Tuple --- platformio/project/config.py | 3 ++ tests/test_projectconf.py | 93 ++++++++++++++++++++++++++++++++++-- 2 files changed, 91 insertions(+), 5 deletions(-) diff --git a/platformio/project/config.py b/platformio/project/config.py index fdc00124..170ec766 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -412,6 +412,9 @@ class ProjectConfig(ProjectConfigBase, ProjectConfigDirsMixin): def as_dict(self): return {s: self.items(s, as_dict=True) for s in self.sections()} + def as_tuple(self): + return [(s, self.items(s)) for s in self.sections()] + def to_json(self): return json.dumps(self.as_dict()) diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index 2ec24521..8b4d3445 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -242,27 +242,27 @@ def test_items(config): ("lib_flags", "-lc -lm"), ("extra_flags", None), ("lib_ignore", "LibIgnoreCustom"), - ] # yapf: disable + ] assert config.items(env="base") == [ ("build_flags", ["-D DEBUG=1"]), ("targets", []), ("monitor_speed", "115200"), ("lib_deps", ["Lib1", "Lib2"]), ("lib_ignore", ["LibIgnoreCustom"]), - ] # yapf: disable + ] assert config.items(env="extra_1") == [ ("build_flags", ["-lc -lm -D DEBUG=1"]), ("lib_deps", ["574"]), ("monitor_speed", "115200"), ("lib_ignore", ["LibIgnoreCustom"]), - ] # yapf: disable + ] assert config.items(env="extra_2") == [ ("build_flags", ["-Og"]), ("lib_ignore", ["LibIgnoreCustom", "Lib3"]), ("upload_port", "/dev/extra_2/port"), ("monitor_speed", "115200"), ("lib_deps", ["Lib1", "Lib2"]), - ] # yapf: disable + ] assert config.items(env="test_extends") == [ ("extends", ["strict_settings"]), ("build_flags", ["-D RELEASE"]), @@ -271,4 +271,87 @@ def test_items(config): ("monitor_speed", "9600"), ("lib_deps", ["Lib1", "Lib2"]), ("lib_ignore", ["LibIgnoreCustom"]), - ] # yapf: disable + ] + + +def test_as_tuple(config): + assert config.as_tuple() == [ + ( + "platformio", + [ + ("extra_configs", ["extra_envs.ini", "extra_debug.ini"]), + ("default_envs", ["base", "extra_2"]), + ("workspace_dir", "/tmp/pio-workspaces/$PROJECT_HASH"), + ], + ), + ( + "env", + [ + ("monitor_speed", "115200"), + ("lib_deps", ["Lib1", "Lib2"]), + ("lib_ignore", ["LibIgnoreCustom"]), + ], + ), + ("strict_ldf", [("lib_ldf_mode", "chain+"), ("lib_compat_mode", "strict")]), + ("monitor_custom", [("monitor_speed", "9600")]), + ( + "strict_settings", + [ + ("extends", "strict_ldf, monitor_custom"), + ("build_flags", "-D RELEASE"), + ("lib_ldf_mode", "chain+"), + ("lib_compat_mode", "strict"), + ("monitor_speed", "9600"), + ], + ), + ( + "custom", + [ + ("debug_flags", "-D DEBUG=1"), + ("lib_flags", "-lc -lm"), + ("extra_flags", None), + ("lib_ignore", "LibIgnoreCustom"), + ], + ), + ( + "env:base", + [ + ("build_flags", ["-D DEBUG=1"]), + ("targets", []), + ("monitor_speed", "115200"), + ("lib_deps", ["Lib1", "Lib2"]), + ("lib_ignore", ["LibIgnoreCustom"]), + ], + ), + ( + "env:test_extends", + [ + ("extends", ["strict_settings"]), + ("build_flags", ["-D RELEASE"]), + ("lib_ldf_mode", "chain+"), + ("lib_compat_mode", "strict"), + ("monitor_speed", "9600"), + ("lib_deps", ["Lib1", "Lib2"]), + ("lib_ignore", ["LibIgnoreCustom"]), + ], + ), + ( + "env:extra_1", + [ + ("build_flags", ["-lc -lm -D DEBUG=1"]), + ("lib_deps", ["574"]), + ("monitor_speed", "115200"), + ("lib_ignore", ["LibIgnoreCustom"]), + ], + ), + ( + "env:extra_2", + [ + ("build_flags", ["-Og"]), + ("lib_ignore", ["LibIgnoreCustom", "Lib3"]), + ("upload_port", "/dev/extra_2/port"), + ("monitor_speed", "115200"), + ("lib_deps", ["Lib1", "Lib2"]), + ], + ), + ] From 257a8c63d20879e930cef7330cb3e3bf074718ba Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 30 Oct 2019 20:28:38 +0200 Subject: [PATCH 171/221] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index a4227316..865c2c05 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit a4227316bebed4001cdece12826951302418d45f +Subproject commit 865c2c05a8cca6b40f2687ade48c531160e0f973 From 7b6bab7f4e656d5b5c998cb680123b7bacca72a8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 30 Oct 2019 20:40:26 +0200 Subject: [PATCH 172/221] Update memory usage banner --- platformio/builder/tools/pioupload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py index a77c6726..c1449f80 100644 --- a/platformio/builder/tools/pioupload.py +++ b/platformio/builder/tools/pioupload.py @@ -249,7 +249,7 @@ def CheckUploadSize(_, target, source, env): program_size = _calculate_size(output, env.get("SIZEPROGREGEXP")) data_size = _calculate_size(output, env.get("SIZEDATAREGEXP")) - print ("Memory Usage -> http://bit.ly/pio-memory-usage") + print ('Advanced Memory Usage is available via "PlatformIO Home > Project Inspect"') if data_max_size and data_size > -1: print ("DATA: %s" % _format_availale_bytes(data_size, data_max_size)) if program_size > -1: From 6d69c25a2feaa2285b85da5f8a4f65f0df5fe795 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 30 Oct 2019 20:43:37 +0200 Subject: [PATCH 173/221] Use locale encoding to decode subprocess output // Resolve #2890 --- platformio/compat.py | 5 +++++ platformio/proc.py | 11 +++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/platformio/compat.py b/platformio/compat.py index 37c63f2a..ac21016e 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -17,6 +17,7 @@ import inspect import json +import locale import os import re import sys @@ -30,6 +31,10 @@ def get_filesystem_encoding(): return sys.getfilesystemencoding() or sys.getdefaultencoding() +def get_locale_encoding(): + return locale.getdefaultlocale()[1] + + def get_class_attributes(cls): attributes = inspect.getmembers(cls, lambda a: not (inspect.isroutine(a))) return { diff --git a/platformio/proc.py b/platformio/proc.py index 85336be9..80e50201 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -19,7 +19,12 @@ from os.path import isdir, isfile, join, normpath from threading import Thread from platformio import exception -from platformio.compat import WINDOWS, get_filesystem_encoding, string_types +from platformio.compat import ( + WINDOWS, + get_filesystem_encoding, + get_locale_encoding, + string_types, +) class AsyncPipeBase(object): @@ -122,7 +127,9 @@ def exec_command(*args, **kwargs): for k, v in result.items(): if isinstance(result[k], bytes): try: - result[k] = result[k].decode(get_filesystem_encoding()) + result[k] = result[k].decode( + get_locale_encoding() or get_filesystem_encoding() + ) except UnicodeDecodeError: result[k] = result[k].decode("latin-1") if v and isinstance(v, string_types): From cd6137bdb02a72a148b2b54d8fd851ce81429972 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 31 Oct 2019 00:43:44 +0200 Subject: [PATCH 174/221] Bump version to 4.1.0rc3 --- platformio/__init__.py | 2 +- platformio/managers/core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 6cce0f67..17ea3358 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, "0rc2") +VERSION = (4, 1, "0rc3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/managers/core.py b/platformio/managers/core.py index c3fcc757..4a857e29 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -24,7 +24,7 @@ from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path from platformio.project.config import ProjectConfig CORE_PACKAGES = { - "contrib-piohome": ">=3.0.0-beta.2,<4", + "contrib-piohome": ">=3.0.0-beta.3,<4", "contrib-pysite": "~2.%d%d.0" % (sys.version_info[0], sys.version_info[1]), "tool-pioplus": "^2.5.8", "tool-unity": "~1.20403.0", From 8ccf9d2e5304d223a84d9344cd7558144ff2317a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 31 Oct 2019 15:26:34 +0200 Subject: [PATCH 175/221] Implement project config "update" with "clear" option --- platformio/project/config.py | 15 ++++ tests/test_projectconf.py | 141 +++++++++++++++-------------------- 2 files changed, 77 insertions(+), 79 deletions(-) diff --git a/platformio/project/config.py b/platformio/project/config.py index 170ec766..73d5e087 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -225,6 +225,10 @@ class ProjectConfigBase(object): value = "\n".join(value) if value: value = "\n" + value # start from a new line + elif isinstance(value, bool): + value = "yes" if value else "no" + elif isinstance(value, (int, float)): + value = str(value) self._parser.set(section, option, value) def getraw(self, section, option): @@ -418,6 +422,16 @@ class ProjectConfig(ProjectConfigBase, ProjectConfigDirsMixin): def to_json(self): return json.dumps(self.as_dict()) + def update(self, data, clear=False): + assert isinstance(data, list) + if clear: + self._parser = ConfigParser.ConfigParser() + for section, options in data: + if not self._parser.has_section(section): + self._parser.add_section(section) + for option, value in options: + self.set(section, option, value) + def save(self, path=None): path = path or self.path if path in self._instances: @@ -425,3 +439,4 @@ class ProjectConfig(ProjectConfigBase, ProjectConfigDirsMixin): with open(path or self.path, "w") as fp: fp.write(CONFIG_HEADER) self._parser.write(fp) + return True diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index 8b4d3445..c95584e9 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -274,84 +274,67 @@ def test_items(config): ] -def test_as_tuple(config): +def test_update_and_save(tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("project") + tmpdir.join("platformio.ini").write( + """ +[platformio] +extra_configs = a.ini, b.ini + +[env:myenv] +board = myboard + """ + ) + config = ProjectConfig(tmpdir.join("platformio.ini").strpath) + assert config.envs() == ["myenv"] + assert config.as_tuple()[0][1][0][1] == ["a.ini", "b.ini"] + + config.update( + [ + ["platformio", [("extra_configs", ["extra.ini"])]], + ["env:myenv", [("framework", ["espidf", "arduino"])]], + ["check_types", [("float_option", 13.99), ("bool_option", True)]], + ] + ) + config.get("platformio", "extra_configs") == "extra.ini" + config.remove_section("platformio") assert config.as_tuple() == [ - ( - "platformio", - [ - ("extra_configs", ["extra_envs.ini", "extra_debug.ini"]), - ("default_envs", ["base", "extra_2"]), - ("workspace_dir", "/tmp/pio-workspaces/$PROJECT_HASH"), - ], - ), - ( - "env", - [ - ("monitor_speed", "115200"), - ("lib_deps", ["Lib1", "Lib2"]), - ("lib_ignore", ["LibIgnoreCustom"]), - ], - ), - ("strict_ldf", [("lib_ldf_mode", "chain+"), ("lib_compat_mode", "strict")]), - ("monitor_custom", [("monitor_speed", "9600")]), - ( - "strict_settings", - [ - ("extends", "strict_ldf, monitor_custom"), - ("build_flags", "-D RELEASE"), - ("lib_ldf_mode", "chain+"), - ("lib_compat_mode", "strict"), - ("monitor_speed", "9600"), - ], - ), - ( - "custom", - [ - ("debug_flags", "-D DEBUG=1"), - ("lib_flags", "-lc -lm"), - ("extra_flags", None), - ("lib_ignore", "LibIgnoreCustom"), - ], - ), - ( - "env:base", - [ - ("build_flags", ["-D DEBUG=1"]), - ("targets", []), - ("monitor_speed", "115200"), - ("lib_deps", ["Lib1", "Lib2"]), - ("lib_ignore", ["LibIgnoreCustom"]), - ], - ), - ( - "env:test_extends", - [ - ("extends", ["strict_settings"]), - ("build_flags", ["-D RELEASE"]), - ("lib_ldf_mode", "chain+"), - ("lib_compat_mode", "strict"), - ("monitor_speed", "9600"), - ("lib_deps", ["Lib1", "Lib2"]), - ("lib_ignore", ["LibIgnoreCustom"]), - ], - ), - ( - "env:extra_1", - [ - ("build_flags", ["-lc -lm -D DEBUG=1"]), - ("lib_deps", ["574"]), - ("monitor_speed", "115200"), - ("lib_ignore", ["LibIgnoreCustom"]), - ], - ), - ( - "env:extra_2", - [ - ("build_flags", ["-Og"]), - ("lib_ignore", ["LibIgnoreCustom", "Lib3"]), - ("upload_port", "/dev/extra_2/port"), - ("monitor_speed", "115200"), - ("lib_deps", ["Lib1", "Lib2"]), - ], - ), + ("env:myenv", [("board", "myboard"), ("framework", ["espidf", "arduino"])]), + ("check_types", [("float_option", "13.99"), ("bool_option", "yes")]), + ] + + config.save() + lines = [ + line.strip() + for line in tmpdir.join("platformio.ini").readlines() + if line.strip() and not line.startswith((";", "#")) + ] + assert lines == [ + "[env:myenv]", + "board = myboard", + "framework =", + "espidf", + "arduino", + "[check_types]", + "float_option = 13.99", + "bool_option = yes", + ] + + +def test_update_and_clear(tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("project") + tmpdir.join("platformio.ini").write( + """ +[platformio] +extra_configs = a.ini, b.ini + +[env:myenv] +board = myboard + """ + ) + config = ProjectConfig(tmpdir.join("platformio.ini").strpath) + assert config.sections() == ["platformio", "env:myenv"] + config.update([["mysection", [("opt1", "value1"), ("opt2", "value2")]]], clear=True) + assert config.as_tuple() == [ + ("mysection", [("opt1", "value1"), ("opt2", "value2")]) ] From 9b65a091dac91c53a7e5ab37353d4609871f752b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 31 Oct 2019 15:27:34 +0200 Subject: [PATCH 176/221] Export config dump/load and schema to PIO Home Project.RPC --- .../commands/home/rpc/handlers/project.py | 15 ++ platformio/project/options.py | 235 +++++++++++++----- 2 files changed, 181 insertions(+), 69 deletions(-) diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index dbd41a55..d00370c5 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -29,6 +29,7 @@ from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager from platformio.project.config import ProjectConfig from platformio.project.helpers import get_project_dir, is_platformio_project +from platformio.project.options import get_config_options_schema class ProjectRPC(object): @@ -42,6 +43,20 @@ class ProjectRPC(object): with fs.cd(project_dir): return getattr(ProjectConfig(**init_kwargs), method)(*args) + @staticmethod + def config_load(path): + return ProjectConfig(path).as_tuple() + + @staticmethod + def config_dump(path, data): + config = ProjectConfig(path) + config.update(data, clear=True) + return config.save() + + @staticmethod + def get_config_schema(): + return get_config_options_schema() + @staticmethod def _get_projects(project_dirs=None): def _get_project_data(): diff --git a/platformio/project/options.py b/platformio/project/options.py index 02762783..d4e2e962 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -15,40 +15,58 @@ # pylint: disable=redefined-builtin, too-many-arguments import os -from collections import OrderedDict, namedtuple +from collections import OrderedDict import click from platformio import fs -ConfigOptionClass = namedtuple( - "ConfigOption", - [ - "scope", - "name", - "type", - "multiple", - "sysenvvar", - "buildenvvar", - "oldnames", - "default", - ], -) +class ConfigOption(object): # pylint: disable=too-many-instance-attributes + def __init__( + self, + scope, + group, + name, + type=str, + multiple=False, + sysenvvar=None, + buildenvvar=None, + oldnames=None, + default=None, + description=None, + ): + self.scope = scope + self.group = group + self.name = name + self.type = type + self.multiple = multiple + self.sysenvvar = sysenvvar + self.buildenvvar = buildenvvar + self.oldnames = oldnames + self.default = default + self.description = description -def ConfigOption( - scope, - name, - type=str, - multiple=False, - sysenvvar=None, - buildenvvar=None, - oldnames=None, - default=None, -): - return ConfigOptionClass( - scope, name, type, multiple, sysenvvar, buildenvvar, oldnames, default - ) + def as_dict(self): + result = dict( + scope=self.scope, + group=self.group, + name=self.name, + type="string", + multiple=self.multiple, + default=self.default, + description=self.description, + ) + if isinstance(self.type, click.ParamType): + result["type"] = self.type.name + + if isinstance(self.type, (click.IntRange, click.FloatRange)): + result["min"] = self.type.min + result["max"] = self.type.max + if isinstance(self.type, click.Choice): + result["choices"] = self.type.choices + + return result def ConfigPlatformioOption(*args, **kwargs): @@ -66,90 +84,110 @@ ProjectOptions = OrderedDict( # # [platformio] # - ConfigPlatformioOption(name="description"), + ConfigPlatformioOption(group="generic", name="description"), ConfigPlatformioOption( + group="generic", name="default_envs", oldnames=["env_default"], multiple=True, sysenvvar="PLATFORMIO_DEFAULT_ENVS", ), - ConfigPlatformioOption(name="extra_configs", multiple=True), + ConfigPlatformioOption( + group="generic", name="extra_configs", multiple=True + ), # Dirs ConfigPlatformioOption( + group="directory", name="core_dir", oldnames=["home_dir"], sysenvvar="PLATFORMIO_CORE_DIR", default=os.path.join(fs.expanduser("~"), ".platformio"), ), ConfigPlatformioOption( + group="directory", name="globallib_dir", sysenvvar="PLATFORMIO_GLOBALLIB_DIR", default=os.path.join("$PROJECT_CORE_DIR", "lib"), ), ConfigPlatformioOption( + group="directory", name="platforms_dir", sysenvvar="PLATFORMIO_PLATFORMS_DIR", default=os.path.join("$PROJECT_CORE_DIR", "platforms"), ), ConfigPlatformioOption( + group="directory", name="packages_dir", sysenvvar="PLATFORMIO_PACKAGES_DIR", default=os.path.join("$PROJECT_CORE_DIR", "packages"), ), ConfigPlatformioOption( + group="directory", name="cache_dir", sysenvvar="PLATFORMIO_CACHE_DIR", default=os.path.join("$PROJECT_CORE_DIR", ".cache"), ), ConfigPlatformioOption( - name="build_cache_dir", sysenvvar="PLATFORMIO_BUILD_CACHE_DIR" + group="directory", + name="build_cache_dir", + sysenvvar="PLATFORMIO_BUILD_CACHE_DIR", ), ConfigPlatformioOption( + group="directory", name="workspace_dir", sysenvvar="PLATFORMIO_WORKSPACE_DIR", default=os.path.join("$PROJECT_DIR", ".pio"), ), ConfigPlatformioOption( + group="directory", name="build_dir", sysenvvar="PLATFORMIO_BUILD_DIR", default=os.path.join("$PROJECT_WORKSPACE_DIR", "build"), ), ConfigPlatformioOption( + group="directory", name="libdeps_dir", sysenvvar="PLATFORMIO_LIBDEPS_DIR", default=os.path.join("$PROJECT_WORKSPACE_DIR", "libdeps"), ), ConfigPlatformioOption( + group="directory", name="lib_dir", sysenvvar="PLATFORMIO_LIB_DIR", default=os.path.join("$PROJECT_DIR", "lib"), ), ConfigPlatformioOption( + group="directory", name="include_dir", sysenvvar="PLATFORMIO_INCLUDE_DIR", default=os.path.join("$PROJECT_DIR", "include"), ), ConfigPlatformioOption( + group="directory", name="src_dir", sysenvvar="PLATFORMIO_SRC_DIR", default=os.path.join("$PROJECT_DIR", "src"), ), ConfigPlatformioOption( + group="directory", name="test_dir", sysenvvar="PLATFORMIO_TEST_DIR", default=os.path.join("$PROJECT_DIR", "test"), ), ConfigPlatformioOption( + group="directory", name="boards_dir", sysenvvar="PLATFORMIO_BOARDS_DIR", default=os.path.join("$PROJECT_DIR", "boards"), ), ConfigPlatformioOption( + group="directory", name="data_dir", sysenvvar="PLATFORMIO_DATA_DIR", default=os.path.join("$PROJECT_DIR", "data"), ), ConfigPlatformioOption( + group="directory", name="shared_dir", sysenvvar="PLATFORMIO_SHARED_DIR", default=os.path.join("$PROJECT_DIR", "shared"), @@ -157,147 +195,206 @@ ProjectOptions = OrderedDict( # # [env] # - ConfigEnvOption(name="extends", multiple=True), - # Generic - ConfigEnvOption(name="platform", buildenvvar="PIOPLATFORM"), - ConfigEnvOption(name="platform_packages", multiple=True), + # Platform ConfigEnvOption( - name="framework", multiple=True, buildenvvar="PIOFRAMEWORK" + group="platform", name="platform", buildenvvar="PIOPLATFORM" + ), + ConfigEnvOption(group="platform", name="platform_packages", multiple=True), + ConfigEnvOption( + group="platform", + name="framework", + multiple=True, + buildenvvar="PIOFRAMEWORK", ), # Board - ConfigEnvOption(name="board", buildenvvar="BOARD"), + ConfigEnvOption(group="board", name="board", buildenvvar="BOARD"), ConfigEnvOption( - name="board_build.mcu", oldnames=["board_mcu"], buildenvvar="BOARD_MCU" + group="board", + name="board_build.mcu", + oldnames=["board_mcu"], + buildenvvar="BOARD_MCU", ), ConfigEnvOption( + group="board", name="board_build.f_cpu", oldnames=["board_f_cpu"], buildenvvar="BOARD_F_CPU", ), ConfigEnvOption( + group="board", name="board_build.f_flash", oldnames=["board_f_flash"], buildenvvar="BOARD_F_FLASH", ), ConfigEnvOption( + group="board", name="board_build.flash_mode", oldnames=["board_flash_mode"], buildenvvar="BOARD_FLASH_MODE", ), # Build - ConfigEnvOption(name="build_type", type=click.Choice(["release", "debug"])), ConfigEnvOption( + group="build", + name="build_type", + type=click.Choice(["release", "debug"]), + ), + ConfigEnvOption( + group="build", name="build_flags", multiple=True, sysenvvar="PLATFORMIO_BUILD_FLAGS", buildenvvar="BUILD_FLAGS", ), ConfigEnvOption( + group="build", name="src_build_flags", multiple=True, sysenvvar="PLATFORMIO_SRC_BUILD_FLAGS", buildenvvar="SRC_BUILD_FLAGS", ), ConfigEnvOption( + group="build", name="build_unflags", multiple=True, sysenvvar="PLATFORMIO_BUILD_UNFLAGS", buildenvvar="BUILD_UNFLAGS", ), ConfigEnvOption( + group="build", name="src_filter", multiple=True, sysenvvar="PLATFORMIO_SRC_FILTER", buildenvvar="SRC_FILTER", ), - ConfigEnvOption(name="targets", multiple=True), + ConfigEnvOption(group="build", name="targets", multiple=True), # Upload ConfigEnvOption( + group="upload", name="upload_port", sysenvvar="PLATFORMIO_UPLOAD_PORT", buildenvvar="UPLOAD_PORT", ), - ConfigEnvOption(name="upload_protocol", buildenvvar="UPLOAD_PROTOCOL"), ConfigEnvOption( - name="upload_speed", type=click.INT, buildenvvar="UPLOAD_SPEED" + group="upload", name="upload_protocol", buildenvvar="UPLOAD_PROTOCOL" ), ConfigEnvOption( + group="upload", + name="upload_speed", + type=click.INT, + buildenvvar="UPLOAD_SPEED", + ), + ConfigEnvOption( + group="upload", name="upload_flags", multiple=True, sysenvvar="PLATFORMIO_UPLOAD_FLAGS", buildenvvar="UPLOAD_FLAGS", ), ConfigEnvOption( - name="upload_resetmethod", buildenvvar="UPLOAD_RESETMETHOD" + group="upload", + name="upload_resetmethod", + buildenvvar="UPLOAD_RESETMETHOD", + ), + ConfigEnvOption( + group="upload", name="upload_command", buildenvvar="UPLOADCMD" ), - ConfigEnvOption(name="upload_command", buildenvvar="UPLOADCMD"), # Monitor - ConfigEnvOption(name="monitor_port"), - ConfigEnvOption(name="monitor_speed", oldnames=["monitor_baud"]), - ConfigEnvOption(name="monitor_rts", type=click.IntRange(0, 1)), - ConfigEnvOption(name="monitor_dtr", type=click.IntRange(0, 1)), - ConfigEnvOption(name="monitor_flags", multiple=True), + ConfigEnvOption(group="monitor", name="monitor_port"), + ConfigEnvOption( + group="monitor", name="monitor_speed", oldnames=["monitor_baud"] + ), + ConfigEnvOption( + group="monitor", name="monitor_rts", type=click.IntRange(0, 1) + ), + ConfigEnvOption( + group="monitor", name="monitor_dtr", type=click.IntRange(0, 1) + ), + ConfigEnvOption(group="monitor", name="monitor_flags", multiple=True), # Library ConfigEnvOption( + group="lib", name="lib_deps", oldnames=["lib_use", "lib_force", "lib_install"], multiple=True, ), - ConfigEnvOption(name="lib_ignore", multiple=True), + ConfigEnvOption(group="lib", name="lib_ignore", multiple=True), ConfigEnvOption( + group="lib", name="lib_extra_dirs", multiple=True, sysenvvar="PLATFORMIO_LIB_EXTRA_DIRS", ), ConfigEnvOption( + group="lib", name="lib_ldf_mode", type=click.Choice(["off", "chain", "deep", "chain+", "deep+"]), ), ConfigEnvOption( - name="lib_compat_mode", type=click.Choice(["off", "soft", "strict"]) + group="lib", + name="lib_compat_mode", + type=click.Choice(["off", "soft", "strict"]), + ), + ConfigEnvOption( + group="lib", name="lib_archive", type=click.BOOL, default=True ), - ConfigEnvOption(name="lib_archive", type=click.BOOL), # Test - ConfigEnvOption(name="test_filter", multiple=True), - ConfigEnvOption(name="test_ignore", multiple=True), - ConfigEnvOption(name="test_port"), - ConfigEnvOption(name="test_speed", type=click.INT), - ConfigEnvOption(name="test_transport"), - ConfigEnvOption(name="test_build_project_src", type=click.BOOL), - # Debug - ConfigEnvOption(name="debug_tool"), - ConfigEnvOption(name="debug_init_break"), - ConfigEnvOption(name="debug_init_cmds", multiple=True), - ConfigEnvOption(name="debug_extra_cmds", multiple=True), + ConfigEnvOption(group="test", name="test_filter", multiple=True), + ConfigEnvOption(group="test", name="test_ignore", multiple=True), + ConfigEnvOption(group="test", name="test_port"), + ConfigEnvOption(group="test", name="test_speed", type=click.INT), + ConfigEnvOption(group="test", name="test_transport"), ConfigEnvOption( - name="debug_load_cmds", oldnames=["debug_load_cmd"], multiple=True + group="test", + name="test_build_project_src", + type=click.BOOL, + default=False, + ), + # Debug + ConfigEnvOption(group="debug", name="debug_tool"), + ConfigEnvOption(group="debug", name="debug_init_break"), + ConfigEnvOption(group="debug", name="debug_init_cmds", multiple=True), + ConfigEnvOption(group="debug", name="debug_extra_cmds", multiple=True), + ConfigEnvOption( + group="debug", + name="debug_load_cmds", + oldnames=["debug_load_cmd"], + multiple=True, ), ConfigEnvOption( + group="debug", name="debug_load_mode", type=click.Choice(["always", "modified", "manual"]), ), - ConfigEnvOption(name="debug_server", multiple=True), - ConfigEnvOption(name="debug_port"), + ConfigEnvOption(group="debug", name="debug_server", multiple=True), + ConfigEnvOption(group="debug", name="debug_port"), ConfigEnvOption( + group="debug", name="debug_svd_path", type=click.Path(exists=True, file_okay=True, dir_okay=False), ), # Check - ConfigEnvOption(name="check_tool", multiple=True), - ConfigEnvOption(name="check_filter", multiple=True), - ConfigEnvOption(name="check_flags", multiple=True), + ConfigEnvOption(group="check", name="check_tool", multiple=True), + ConfigEnvOption(group="check", name="check_filter", multiple=True), + ConfigEnvOption(group="check", name="check_flags", multiple=True), ConfigEnvOption( + group="check", name="check_severity", multiple=True, type=click.Choice(["low", "medium", "high"]), ), - # Other + # Advanced ConfigEnvOption( + group="advanced", name="extra_scripts", oldnames=["extra_script"], multiple=True, sysenvvar="PLATFORMIO_EXTRA_SCRIPTS", ), + ConfigEnvOption(group="advanced", name="extends", multiple=True), ] ] ) + + +def get_config_options_schema(): + return [option.as_dict() for option in ProjectOptions.values()] From a18f8b2a4c3d34db177933f5e95ceb8e4dfed95a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 31 Oct 2019 15:28:02 +0200 Subject: [PATCH 177/221] Use default values from project options --- platformio/builder/tools/piolib.py | 7 ++++--- platformio/builder/tools/platformio.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 69d645c5..8c63934e 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -211,7 +211,7 @@ class LibBuilderBase(object): @property def lib_archive(self): - return self.env.GetProjectOption("lib_archive", True) + return self.env.GetProjectOption("lib_archive") @property def lib_ldf_mode(self): @@ -711,8 +711,9 @@ class PlatformIOLibBuilder(LibBuilderBase): @property def lib_archive(self): - global_value = self.env.GetProjectOption("lib_archive") - if global_value is not None: + unique_value = "_not_declared_%s" % id(self) + global_value = self.env.GetProjectOption("lib_archive", unique_value) + if global_value != unique_value: return global_value return self._manifest.get("build", {}).get( "libArchive", LibBuilderBase.lib_archive.fget(self) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 64638450..123e7d43 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -70,7 +70,7 @@ def _build_project_deps(env): projenv.BuildSources( "$BUILD_TEST_DIR", "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER" ) - if not is_test or env.GetProjectOption("test_build_project_src", False): + if not is_test or env.GetProjectOption("test_build_project_src"): projenv.BuildSources( "$BUILD_SRC_DIR", "$PROJECT_SRC_DIR", env.get("SRC_FILTER") ) From a7f8838d9a516ffe8485d9ce77ec80c6d317166c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 31 Oct 2019 18:52:13 +0200 Subject: [PATCH 178/221] Format code --- platformio/builder/main.py | 6 +++--- platformio/builder/tools/piomisc.py | 6 +++--- platformio/builder/tools/pioplatform.py | 2 +- platformio/builder/tools/pioupload.py | 22 +++++++++++----------- platformio/project/options.py | 2 +- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 81aefa09..f9b86cfd 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -132,7 +132,7 @@ if env.GetOption("clean"): env.PioClean(env.subst("$BUILD_DIR")) env.Exit(0) elif not int(ARGUMENTS.get("PIOVERBOSE", 0)): - print ("Verbose mode can be enabled via `-v, --verbose` option") + print("Verbose mode can be enabled via `-v, --verbose` option") env.LoadProjectOptions() env.LoadPioPlatform() @@ -181,12 +181,12 @@ AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS)) ############################################################################## if "envdump" in COMMAND_LINE_TARGETS: - print (env.Dump()) + print(env.Dump()) env.Exit(0) if "idedata" in COMMAND_LINE_TARGETS: Import("projenv") - print ( + print( "\n%s\n" % dump_json_to_unicode( projenv.DumpIDEData() # pylint: disable=undefined-variable diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 9c7bf548..5ef48d20 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -294,17 +294,17 @@ def VerboseAction(_, act, actstr): def PioClean(env, clean_dir): if not isdir(clean_dir): - print ("Build environment is clean") + print("Build environment is clean") env.Exit(0) clean_rel_path = relpath(clean_dir) for root, _, files in walk(clean_dir): for f in files: dst = join(root, f) remove(dst) - print ( + print( "Removed %s" % (dst if clean_rel_path.startswith(".") else relpath(dst)) ) - print ("Done cleaning") + print("Done cleaning") fs.rmtree(clean_dir) env.Exit(0) diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 459b6a0b..f910aaed 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -223,7 +223,7 @@ def PrintConfiguration(env): # pylint: disable=too-many-statements _get_packages_data(), ): if data and len(data) > 1: - print (" ".join(data)) + print(" ".join(data)) def exists(_): diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py index c1449f80..15356434 100644 --- a/platformio/builder/tools/pioupload.py +++ b/platformio/builder/tools/pioupload.py @@ -45,7 +45,7 @@ def FlushSerialBuffer(env, port): def TouchSerialPort(env, port, baudrate): port = env.subst(port) - print ("Forcing reset using %dbps open/close on port %s" % (baudrate, port)) + print("Forcing reset using %dbps open/close on port %s" % (baudrate, port)) try: s = Serial(port=port, baudrate=baudrate) s.setDTR(False) @@ -56,7 +56,7 @@ def TouchSerialPort(env, port, baudrate): def WaitForNewSerialPort(env, before): - print ("Waiting for the new upload port...") + print("Waiting for the new upload port...") prev_port = env.subst("$UPLOAD_PORT") new_port = None elapsed = 0 @@ -145,7 +145,7 @@ def AutodetectUploadPort(*args, **kwargs): return port if "UPLOAD_PORT" in env and not _get_pattern(): - print (env.subst("Use manually specified: $UPLOAD_PORT")) + print(env.subst("Use manually specified: $UPLOAD_PORT")) return if env.subst("$UPLOAD_PROTOCOL") == "mbed" or ( @@ -160,7 +160,7 @@ def AutodetectUploadPort(*args, **kwargs): env.Replace(UPLOAD_PORT=_look_for_serial_port()) if env.subst("$UPLOAD_PORT"): - print (env.subst("Auto-detected: $UPLOAD_PORT")) + print(env.subst("Auto-detected: $UPLOAD_PORT")) else: sys.stderr.write( "Error: Please specify `upload_port` for environment or use " @@ -179,7 +179,7 @@ def UploadToDisk(_, target, source, env): if not isfile(fpath): continue copyfile(fpath, join(env.subst("$UPLOAD_PORT"), "%s.%s" % (progname, ext))) - print ( + print( "Firmware has been successfully uploaded.\n" "(Some boards may require manual hard reset)" ) @@ -249,13 +249,13 @@ def CheckUploadSize(_, target, source, env): program_size = _calculate_size(output, env.get("SIZEPROGREGEXP")) data_size = _calculate_size(output, env.get("SIZEDATAREGEXP")) - print ('Advanced Memory Usage is available via "PlatformIO Home > Project Inspect"') + print('Advanced Memory Usage is available via "PlatformIO Home > Project Inspect"') if data_max_size and data_size > -1: - print ("DATA: %s" % _format_availale_bytes(data_size, data_max_size)) + print("DATA: %s" % _format_availale_bytes(data_size, data_max_size)) if program_size > -1: - print ("PROGRAM: %s" % _format_availale_bytes(program_size, program_max_size)) + print("PROGRAM: %s" % _format_availale_bytes(program_size, program_max_size)) if int(ARGUMENTS.get("PIOVERBOSE", 0)): - print (output) + print(output) # raise error # if data_max_size and data_size > data_max_size: @@ -277,9 +277,9 @@ def PrintUploadInfo(env): if "BOARD" in env: available.extend(env.BoardConfig().get("upload", {}).get("protocols", [])) if available: - print ("AVAILABLE: %s" % ", ".join(sorted(set(available)))) + print("AVAILABLE: %s" % ", ".join(sorted(set(available)))) if configured: - print ("CURRENT: upload_protocol = %s" % configured) + print("CURRENT: upload_protocol = %s" % configured) def exists(_): diff --git a/platformio/project/options.py b/platformio/project/options.py index d4e2e962..91c61b73 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -397,4 +397,4 @@ ProjectOptions = OrderedDict( def get_config_options_schema(): - return [option.as_dict() for option in ProjectOptions.values()] + return [opt.as_dict() for opt in ProjectOptions.values()] From 2c2b4196858e97b4d096906fab756a595348f498 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 31 Oct 2019 18:52:26 +0200 Subject: [PATCH 179/221] Docs: Sync nRF52 dev/platform --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 865c2c05..2c0c2222 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 865c2c05a8cca6b40f2687ade48c531160e0f973 +Subproject commit 2c0c2222a55284686f82022025370d856c6d8700 From 1d5d09feab2640d372e42744e12356605d84e697 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 31 Oct 2019 22:04:57 +0200 Subject: [PATCH 180/221] Fixed an issue when Project Config Parser does not remove in-line comments when Python 3 is used // Remove #3213 --- HISTORY.rst | 1 + platformio/project/config.py | 8 ++++++-- tests/test_projectconf.py | 4 ++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 14258008..0b7f21ca 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -38,6 +38,7 @@ PlatformIO Core 4.0 * Fixed an issue with project generator when ``src_build_flags`` were not respected (`issue #3137 `_) * Fixed an issue when booleans in "platformio.ini" are not parsed properly (`issue #3022 `_) * Fixed an issue with invalid encoding when generating project for Visual Studio (`issue #3183 `_) +* Fixed an issue when Project Config Parser does not remove in-line comments when Python 3 is used (`issue #3213 `_) 4.0.3 (2019-08-30) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/project/config.py b/platformio/project/config.py index 73d5e087..763ff66f 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -21,7 +21,7 @@ from hashlib import sha1 import click from platformio import exception, fs -from platformio.compat import WINDOWS, hashlib_encode_data +from platformio.compat import PY2, WINDOWS, hashlib_encode_data from platformio.project.options import ProjectOptions try: @@ -84,7 +84,11 @@ class ProjectConfigBase(object): self.expand_interpolations = expand_interpolations self.warnings = [] self._parsed = [] - self._parser = ConfigParser.ConfigParser() + self._parser = ( + ConfigParser.ConfigParser() + if PY2 + else ConfigParser.ConfigParser(inline_comment_prefixes=("#", ";")) + ) if path and os.path.isfile(path): self.read(path, parse_extra) diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index c95584e9..b118a2ef 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -28,9 +28,9 @@ extra_configs = # global options per [env:*] [env] -monitor_speed = 115200 +monitor_speed = 115200 ; inline comment lib_deps = - Lib1 + Lib1 ; inline comment in multi-line value Lib2 lib_ignore = ${custom.lib_ignore} From 3de2d84e2bc35789af6031c10f33277e1bbf35ff Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 31 Oct 2019 22:42:22 +0200 Subject: [PATCH 181/221] Fixed an issue with a GCC Linter for PlatformIO IDE for Atom // Resolve #3218 --- HISTORY.rst | 3 ++- platformio/ide/tpls/atom/.gcc-flags.json.tpl | 2 +- platformio/ide/tpls/vim/.gcc-flags.json.tpl | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 0b7f21ca..dfe12b1a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,7 +18,7 @@ PlatformIO Core 4.0 - Unused variables or functions - Out of scope memory usage. -* `PlatformIO Home 3.0 `__ +* `PlatformIO Home 3.0 `__ and Project Inspection - Static Code Analysis - Firmware File Explorer @@ -39,6 +39,7 @@ PlatformIO Core 4.0 * Fixed an issue when booleans in "platformio.ini" are not parsed properly (`issue #3022 `_) * Fixed an issue with invalid encoding when generating project for Visual Studio (`issue #3183 `_) * Fixed an issue when Project Config Parser does not remove in-line comments when Python 3 is used (`issue #3213 `_) +* Fixed an issue with a GCC Linter for PlatformIO IDE for Atom (`issue #3218 `_) 4.0.3 (2019-08-30) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/ide/tpls/atom/.gcc-flags.json.tpl b/platformio/ide/tpls/atom/.gcc-flags.json.tpl index cbb47418..d2ddcf78 100644 --- a/platformio/ide/tpls/atom/.gcc-flags.json.tpl +++ b/platformio/ide/tpls/atom/.gcc-flags.json.tpl @@ -1,4 +1,4 @@ -% _defines = " ".join(["-D%s" % d for d in defines]) +% _defines = " ".join(["-D%s" % d.replace(" ", "\\\\ ") for d in defines]) { "execPath": "{{ cxx_path }}", "gccDefaultCFlags": "-fsyntax-only {{! cc_flags.replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}", diff --git a/platformio/ide/tpls/vim/.gcc-flags.json.tpl b/platformio/ide/tpls/vim/.gcc-flags.json.tpl index b9b29fdb..d2ddcf78 100644 --- a/platformio/ide/tpls/vim/.gcc-flags.json.tpl +++ b/platformio/ide/tpls/vim/.gcc-flags.json.tpl @@ -1,9 +1,9 @@ -% _defines = " ".join(["-D%s" % d for d in defines]) +% _defines = " ".join(["-D%s" % d.replace(" ", "\\\\ ") for d in defines]) { "execPath": "{{ cxx_path }}", "gccDefaultCFlags": "-fsyntax-only {{! cc_flags.replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}", "gccDefaultCppFlags": "-fsyntax-only {{! cxx_flags.replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}", "gccErrorLimit": 15, - "gccIncludePaths": "{{! ','.join("'{}'".format(inc) for inc in includes)}}", + "gccIncludePaths": "{{ ','.join(includes) }}", "gccSuppressWarnings": false } From cd3d6383372657aba51db91abc59eb51fdf0a7a1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 1 Nov 2019 12:05:13 +0200 Subject: [PATCH 182/221] Disable parsing of extra configs for PIO Home Project RPC load/dump methods --- platformio/commands/home/rpc/handlers/project.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index d00370c5..b7981f56 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -45,11 +45,11 @@ class ProjectRPC(object): @staticmethod def config_load(path): - return ProjectConfig(path).as_tuple() + return ProjectConfig(path, parse_extra=False).as_tuple() @staticmethod def config_dump(path, data): - config = ProjectConfig(path) + config = ProjectConfig(path, parse_extra=False) config.update(data, clear=True) return config.save() From da928efb43d7b7566558b0c4cfb76c2a5c249749 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 1 Nov 2019 14:40:03 +0200 Subject: [PATCH 183/221] Added "--shutdown-timeout" option to PIO Home Server --- HISTORY.rst | 1 + docs | 2 +- platformio/commands/home/command.py | 13 +++++++++++-- platformio/commands/home/rpc/server.py | 10 ++++++---- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index dfe12b1a..7cba3999 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -29,6 +29,7 @@ PlatformIO Core 4.0 * Extend project environment configuration in "platformio.ini" with other sections using a new `extends `__ option (`issue #2953 `_) * Generate ``.ccls`` LSP file for `Emacs `__ cross references, hierarchies, completion and semantic highlighting * Added ``--no-ansi`` flag for `PIO Core `__ to disable ANSI control characters +* Added ``--shutdown-timeout`` option to `PIO Home Server `__ * Fixed an issue with project generator for `CLion IDE `__ when 2 environments were used (`issue #2824 `_) * Fixed default PIO Unified Debugger configuration for `J-Link probe `__ * Fixed an issue when configuration file options partly ignored when using custom ``--project-conf`` (`issue #3034 `_) diff --git a/docs b/docs index 2c0c2222..22ac1a3e 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 2c0c2222a55284686f82022025370d856c6d8700 +Subproject commit 22ac1a3ea0f7d895822e391e8441a91bcd201d39 diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py index a6340b5d..56298ea1 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home/command.py @@ -35,7 +35,16 @@ from platformio.managers.core import get_core_package_dir, inject_contrib_pysite ), ) @click.option("--no-open", is_flag=True) -def cli(port, host, no_open): +@click.option( + "--shutdown-timeout", + default=0, + type=int, + help=( + "Automatically shutdown server on timeout (in seconds) when no clients " + "are connected. Default is 0 which means never auto shutdown" + ), +) +def cli(port, host, no_open, shutdown_timeout): # pylint: disable=import-error, import-outside-toplevel # import contrib modules @@ -53,7 +62,7 @@ def cli(port, host, no_open): from platformio.commands.home.rpc.server import JSONRPCServerFactory from platformio.commands.home.web import WebRoot - factory = JSONRPCServerFactory() + factory = JSONRPCServerFactory(shutdown_timeout) factory.addHandler(AppRPC(), namespace="app") factory.addHandler(IDERPC(), namespace="ide") factory.addHandler(MiscRPC(), namespace="misc") diff --git a/platformio/commands/home/rpc/server.py b/platformio/commands/home/rpc/server.py index 4eb2aceb..1924754f 100644 --- a/platformio/commands/home/rpc/server.py +++ b/platformio/commands/home/rpc/server.py @@ -76,23 +76,25 @@ class JSONRPCServerProtocol(WebSocketServerProtocol): class JSONRPCServerFactory(WebSocketServerFactory): - SHUTDOWN_TIMEOUT = 3600 # in seconds - protocol = JSONRPCServerProtocol connection_nums = 0 shutdown_timer = 0 - def __init__(self): + def __init__(self, shutdown_timeout=0): super(JSONRPCServerFactory, self).__init__() + self.shutdown_timeout = shutdown_timeout self.dispatcher = jsonrpc.Dispatcher() def shutdownByTimeout(self): + if self.shutdown_timeout < 1: + return + def _auto_shutdown_server(): click.echo("Automatically shutdown server on timeout") reactor.stop() self.shutdown_timer = reactor.callLater( - self.SHUTDOWN_TIMEOUT, _auto_shutdown_server + self.shutdown_timeout, _auto_shutdown_server ) def addHandler(self, handler, namespace): From 88db253515b9666e69de85fdba3530c54e221127 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 1 Nov 2019 18:28:20 +0200 Subject: [PATCH 184/221] Ignore duplicate library storages --- platformio/managers/platform.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 2536386a..fd670895 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -674,7 +674,7 @@ class PlatformBase(PlatformPackagesMixin, PlatformRunMixin): self.packages[name]["optional"] = True def get_lib_storages(self): - storages = [] + storages = {} for opts in (self.frameworks or {}).values(): if "package" not in opts: continue @@ -682,7 +682,7 @@ class PlatformBase(PlatformPackagesMixin, PlatformRunMixin): if not pkg_dir or not isdir(join(pkg_dir, "libraries")): continue libs_dir = join(pkg_dir, "libraries") - storages.append({"name": opts["package"], "path": libs_dir}) + storages[libs_dir] = opts["package"] libcores_dir = join(libs_dir, "__cores__") if not isdir(libcores_dir): continue @@ -690,14 +690,9 @@ class PlatformBase(PlatformPackagesMixin, PlatformRunMixin): libcore_dir = join(libcores_dir, item) if not isdir(libcore_dir): continue - storages.append( - { - "name": "%s-core-%s" % (opts["package"], item), - "path": libcore_dir, - } - ) + storages[libcore_dir] = "%s-core-%s" % (opts["package"], item) - return storages + return [dict(name=name, path=path) for path, name in storages.items()] class PlatformBoardConfig(object): From 53c561e8956c52c2150816eddd2df235944ed5d6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 1 Nov 2019 18:33:21 +0200 Subject: [PATCH 185/221] Bump version to 4.1.0rc4 --- platformio/__init__.py | 2 +- platformio/managers/core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 17ea3358..c964118b 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, "0rc3") +VERSION = (4, 1, "0rc4") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 4a857e29..f81764b2 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -24,7 +24,7 @@ from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path from platformio.project.config import ProjectConfig CORE_PACKAGES = { - "contrib-piohome": ">=3.0.0-beta.3,<4", + "contrib-piohome": ">=3.0.0-beta.4,<3.1.0", "contrib-pysite": "~2.%d%d.0" % (sys.version_info[0], sys.version_info[1]), "tool-pioplus": "^2.5.8", "tool-unity": "~1.20403.0", From 3630084a64dc7d8f07b159855233b6eb118292f5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 1 Nov 2019 18:53:00 +0200 Subject: [PATCH 186/221] Docs: Sync Kendryte K210 dev/platform --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 22ac1a3e..00a49426 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 22ac1a3ea0f7d895822e391e8441a91bcd201d39 +Subproject commit 00a49426f39578532e80121414e8a3f5f0bff087 From 0a4bc1d4e3b68109055f33ac0dd1ecaad2c76501 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 2 Nov 2019 19:41:39 +0200 Subject: [PATCH 187/221] Add "description" for project config options, configure "default" values --- docs | 2 +- platformio/project/config.py | 9 +- platformio/project/options.py | 399 +++++++++++++++++++++++++++++----- tests/test_projectconf.py | 15 +- 4 files changed, 355 insertions(+), 70 deletions(-) diff --git a/docs b/docs index 00a49426..29115fe8 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 00a49426f39578532e80121414e8a3f5f0bff087 +Subproject commit 29115fe80b25f47b2837231e09f0f6ff169d0391 diff --git a/platformio/project/config.py b/platformio/project/config.py index 763ff66f..0ca94704 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -290,7 +290,9 @@ class ProjectConfigBase(object): value = envvar_value # option is not specified by user - if value is None: + if value is None or ( + option_meta.multiple and value == [] and option_meta.default + ): return default if default is not None else option_meta.default try: @@ -417,14 +419,11 @@ class ProjectConfig(ProjectConfigBase, ProjectConfigDirsMixin): def __repr__(self): return "" % (self.path or "in-memory") - def as_dict(self): - return {s: self.items(s, as_dict=True) for s in self.sections()} - def as_tuple(self): return [(s, self.items(s)) for s in self.sections()] def to_json(self): - return json.dumps(self.as_dict()) + return json.dumps(self.as_tuple()) def update(self, data, clear=False): assert isinstance(data, list) diff --git a/platformio/project/options.py b/platformio/project/options.py index 91c61b73..02cea3a3 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -28,34 +28,35 @@ class ConfigOption(object): # pylint: disable=too-many-instance-attributes scope, group, name, + description, type=str, multiple=False, sysenvvar=None, buildenvvar=None, oldnames=None, default=None, - description=None, ): self.scope = scope self.group = group self.name = name + self.description = description self.type = type self.multiple = multiple self.sysenvvar = sysenvvar self.buildenvvar = buildenvvar self.oldnames = oldnames self.default = default - self.description = description def as_dict(self): result = dict( scope=self.scope, group=self.group, name=self.name, + description=self.description, type="string", multiple=self.multiple, + sysenvvar=self.sysenvvar, default=self.default, - description=self.description, ) if isinstance(self.type, click.ParamType): result["type"] = self.type.name @@ -84,21 +85,39 @@ ProjectOptions = OrderedDict( # # [platformio] # - ConfigPlatformioOption(group="generic", name="description"), + ConfigPlatformioOption( + group="generic", + name="description", + description="Describe a project with a short information", + ), ConfigPlatformioOption( group="generic", name="default_envs", + description=( + "Configure a list with environments which PlatformIO should " + "process by default" + ), oldnames=["env_default"], multiple=True, sysenvvar="PLATFORMIO_DEFAULT_ENVS", ), ConfigPlatformioOption( - group="generic", name="extra_configs", multiple=True + group="generic", + name="extra_configs", + description=( + "Extend main configuration with the extra configuration files" + ), + multiple=True, ), # Dirs ConfigPlatformioOption( group="directory", name="core_dir", + description=( + "PlatformIO Core location where it keeps installed development " + "platforms, packages, global libraries, " + "and other internal information" + ), oldnames=["home_dir"], sysenvvar="PLATFORMIO_CORE_DIR", default=os.path.join(fs.expanduser("~"), ".platformio"), @@ -106,89 +125,146 @@ ProjectOptions = OrderedDict( ConfigPlatformioOption( group="directory", name="globallib_dir", + description=( + "A library folder/storage where PlatformIO Library Dependency " + "Finder (LDF) looks for global libraries" + ), sysenvvar="PLATFORMIO_GLOBALLIB_DIR", default=os.path.join("$PROJECT_CORE_DIR", "lib"), ), ConfigPlatformioOption( group="directory", name="platforms_dir", + description=( + "A location where PlatformIO Core keeps installed development " + "platforms" + ), sysenvvar="PLATFORMIO_PLATFORMS_DIR", default=os.path.join("$PROJECT_CORE_DIR", "platforms"), ), ConfigPlatformioOption( group="directory", name="packages_dir", + description=( + "A location where PlatformIO Core keeps installed packages" + ), sysenvvar="PLATFORMIO_PACKAGES_DIR", default=os.path.join("$PROJECT_CORE_DIR", "packages"), ), ConfigPlatformioOption( group="directory", name="cache_dir", + description=( + "A location where PlatformIO Core stores caching information " + "(requests to PlatformIO Registry, downloaded packages and " + "other service information)" + ), sysenvvar="PLATFORMIO_CACHE_DIR", default=os.path.join("$PROJECT_CORE_DIR", ".cache"), ), ConfigPlatformioOption( group="directory", name="build_cache_dir", + description=( + "A location where PlatformIO Core keeps derived files from a " + "build system (objects, firmwares, ELFs) and caches them between " + "build environments" + ), sysenvvar="PLATFORMIO_BUILD_CACHE_DIR", ), ConfigPlatformioOption( group="directory", name="workspace_dir", + description=( + "A path to a project workspace directory where PlatformIO keeps " + "by default compiled objects, static libraries, firmwares, and " + "external library dependencies" + ), sysenvvar="PLATFORMIO_WORKSPACE_DIR", default=os.path.join("$PROJECT_DIR", ".pio"), ), ConfigPlatformioOption( group="directory", name="build_dir", + description=( + "PlatformIO Build System uses this folder for project environments" + " to store compiled object files, static libraries, firmwares, " + "and other cached information" + ), sysenvvar="PLATFORMIO_BUILD_DIR", default=os.path.join("$PROJECT_WORKSPACE_DIR", "build"), ), ConfigPlatformioOption( group="directory", name="libdeps_dir", + description=( + "Internal storage where Library Manager will install project " + "dependencies declared via `lib_deps` option" + ), sysenvvar="PLATFORMIO_LIBDEPS_DIR", default=os.path.join("$PROJECT_WORKSPACE_DIR", "libdeps"), ), - ConfigPlatformioOption( - group="directory", - name="lib_dir", - sysenvvar="PLATFORMIO_LIB_DIR", - default=os.path.join("$PROJECT_DIR", "lib"), - ), ConfigPlatformioOption( group="directory", name="include_dir", + description=( + "A default location for project header files. PlatformIO Build " + "System automatically adds this path to CPPPATH scope" + ), sysenvvar="PLATFORMIO_INCLUDE_DIR", default=os.path.join("$PROJECT_DIR", "include"), ), ConfigPlatformioOption( group="directory", name="src_dir", + description=( + "A default location where PlatformIO Build System looks for the " + "project C/C++ source files" + ), sysenvvar="PLATFORMIO_SRC_DIR", default=os.path.join("$PROJECT_DIR", "src"), ), + ConfigPlatformioOption( + group="directory", + name="lib_dir", + description="A storage for the custom/private project libraries", + sysenvvar="PLATFORMIO_LIB_DIR", + default=os.path.join("$PROJECT_DIR", "lib"), + ), + ConfigPlatformioOption( + group="directory", + name="data_dir", + description=( + "A data directory to store contents which can be uploaded to " + "file system (SPIFFS, etc.)" + ), + sysenvvar="PLATFORMIO_DATA_DIR", + default=os.path.join("$PROJECT_DIR", "data"), + ), ConfigPlatformioOption( group="directory", name="test_dir", + description=( + "A location where PIO Unit Testing engine looks for " + "test source files" + ), sysenvvar="PLATFORMIO_TEST_DIR", default=os.path.join("$PROJECT_DIR", "test"), ), ConfigPlatformioOption( group="directory", name="boards_dir", + description="A global storage for custom board manifests", sysenvvar="PLATFORMIO_BOARDS_DIR", default=os.path.join("$PROJECT_DIR", "boards"), ), - ConfigPlatformioOption( - group="directory", - name="data_dir", - sysenvvar="PLATFORMIO_DATA_DIR", - default=os.path.join("$PROJECT_DIR", "data"), - ), ConfigPlatformioOption( group="directory", name="shared_dir", + description=( + "A location which PIO Remote uses to synchronize extra files " + "between remote machine." + ), sysenvvar="PLATFORMIO_SHARED_DIR", default=os.path.join("$PROJECT_DIR", "shared"), ), @@ -197,38 +273,56 @@ ProjectOptions = OrderedDict( # # Platform ConfigEnvOption( - group="platform", name="platform", buildenvvar="PIOPLATFORM" + group="platform", + name="platform", + description="A name or specification of development platform", + buildenvvar="PIOPLATFORM", + ), + ConfigEnvOption( + group="platform", + name="platform_packages", + description="Custom packages and specifications", + multiple=True, ), - ConfigEnvOption(group="platform", name="platform_packages", multiple=True), ConfigEnvOption( group="platform", name="framework", + description="A list of project dependent frameworks", multiple=True, buildenvvar="PIOFRAMEWORK", ), # Board - ConfigEnvOption(group="board", name="board", buildenvvar="BOARD"), + ConfigEnvOption( + group="board", + name="board", + description="A board ID", + buildenvvar="BOARD", + ), ConfigEnvOption( group="board", name="board_build.mcu", + description="A custom board MCU", oldnames=["board_mcu"], buildenvvar="BOARD_MCU", ), ConfigEnvOption( group="board", name="board_build.f_cpu", + description="A custom MCU frequency", oldnames=["board_f_cpu"], buildenvvar="BOARD_F_CPU", ), ConfigEnvOption( group="board", name="board_build.f_flash", + description="A custom flash frequency", oldnames=["board_f_flash"], buildenvvar="BOARD_F_FLASH", ), ConfigEnvOption( group="board", name="board_build.flash_mode", + description="A custom flash mode", oldnames=["board_flash_mode"], buildenvvar="BOARD_FLASH_MODE", ), @@ -236,11 +330,17 @@ ProjectOptions = OrderedDict( ConfigEnvOption( group="build", name="build_type", + description="Project build configuration", type=click.Choice(["release", "debug"]), + default="release", ), ConfigEnvOption( group="build", name="build_flags", + description=( + "Custom build flags/options for preprocessing, compilation, " + "assembly, and linking processes" + ), multiple=True, sysenvvar="PLATFORMIO_BUILD_FLAGS", buildenvvar="BUILD_FLAGS", @@ -248,6 +348,10 @@ ProjectOptions = OrderedDict( ConfigEnvOption( group="build", name="src_build_flags", + description=( + "The same as `build_flags` but configures flags the only for " + "project source files (`src` folder)" + ), multiple=True, sysenvvar="PLATFORMIO_SRC_BUILD_FLAGS", buildenvvar="SRC_BUILD_FLAGS", @@ -255,6 +359,7 @@ ProjectOptions = OrderedDict( ConfigEnvOption( group="build", name="build_unflags", + description="A list with flags/option which should be removed", multiple=True, sysenvvar="PLATFORMIO_BUILD_UNFLAGS", buildenvvar="BUILD_UNFLAGS", @@ -262,30 +367,51 @@ ProjectOptions = OrderedDict( ConfigEnvOption( group="build", name="src_filter", + description=( + "Control which source files should be included/excluded from a " + "build process" + ), multiple=True, sysenvvar="PLATFORMIO_SRC_FILTER", buildenvvar="SRC_FILTER", + default="+<*> -<.git/> -<.svn/>", + ), + ConfigEnvOption( + group="build", + name="targets", + description="A custom list of targets for PlatformIO Build System", + multiple=True, ), - ConfigEnvOption(group="build", name="targets", multiple=True), # Upload ConfigEnvOption( group="upload", name="upload_port", + description=( + "An upload port which `uploader` tool uses for a firmware flashing" + ), sysenvvar="PLATFORMIO_UPLOAD_PORT", buildenvvar="UPLOAD_PORT", ), ConfigEnvOption( - group="upload", name="upload_protocol", buildenvvar="UPLOAD_PROTOCOL" + group="upload", + name="upload_protocol", + description="A protocol that `uploader` tool uses to talk to a board", + buildenvvar="UPLOAD_PROTOCOL", ), ConfigEnvOption( group="upload", name="upload_speed", + description=( + "A connection speed (baud rate) which `uploader` tool uses when " + "sending firmware to a board" + ), type=click.INT, buildenvvar="UPLOAD_SPEED", ), ConfigEnvOption( group="upload", name="upload_flags", + description="An extra flags for `uploader` tool", multiple=True, sysenvvar="PLATFORMIO_UPLOAD_FLAGS", buildenvvar="UPLOAD_FLAGS", @@ -293,104 +419,265 @@ ProjectOptions = OrderedDict( ConfigEnvOption( group="upload", name="upload_resetmethod", + description="A custom reset method", buildenvvar="UPLOAD_RESETMETHOD", ), ConfigEnvOption( - group="upload", name="upload_command", buildenvvar="UPLOADCMD" + group="upload", + name="upload_command", + description=( + "A custom upload command which overwrites a default from " + "development platform" + ), + buildenvvar="UPLOADCMD", ), # Monitor - ConfigEnvOption(group="monitor", name="monitor_port"), ConfigEnvOption( - group="monitor", name="monitor_speed", oldnames=["monitor_baud"] + group="monitor", + name="monitor_port", + description="A port, a number or a device name", ), ConfigEnvOption( - group="monitor", name="monitor_rts", type=click.IntRange(0, 1) + group="monitor", + name="monitor_speed", + description="A monitor speed (baud rate)", + type=click.INT, + oldnames=["monitor_baud"], + default=9600, ), ConfigEnvOption( - group="monitor", name="monitor_dtr", type=click.IntRange(0, 1) + group="monitor", + name="monitor_rts", + description="A monitor initial RTS line state", + type=click.IntRange(0, 1), + ), + ConfigEnvOption( + group="monitor", + name="monitor_dtr", + description="A monitor initial DTR line state", + type=click.IntRange(0, 1), + ), + ConfigEnvOption( + group="monitor", + name="monitor_flags", + description=( + "The extra flags and options for `platformio device monitor` " + "command" + ), + multiple=True, ), - ConfigEnvOption(group="monitor", name="monitor_flags", multiple=True), # Library ConfigEnvOption( - group="lib", + group="library", name="lib_deps", + description=( + "A list of project library dependencies which should be installed " + "automatically before a build process" + ), oldnames=["lib_use", "lib_force", "lib_install"], multiple=True, ), - ConfigEnvOption(group="lib", name="lib_ignore", multiple=True), ConfigEnvOption( - group="lib", + group="library", + name="lib_ignore", + description=( + "A list of library names which should be ignored by " + "Library Dependency Finder (LDF)" + ), + multiple=True, + ), + ConfigEnvOption( + group="library", name="lib_extra_dirs", + description=( + "A list of extra directories/storages where Library Dependency " + "Finder (LDF) will look for dependencies" + ), multiple=True, sysenvvar="PLATFORMIO_LIB_EXTRA_DIRS", ), ConfigEnvOption( - group="lib", + group="library", name="lib_ldf_mode", + description=( + "Control how Library Dependency Finder (LDF) should analyze " + "dependencies (`#include` directives)" + ), type=click.Choice(["off", "chain", "deep", "chain+", "deep+"]), + default="chain", ), ConfigEnvOption( - group="lib", + group="library", name="lib_compat_mode", + description=( + "Configure a strictness (compatibility mode by frameworks, " + "development platforms) of Library Dependency Finder (LDF)" + ), type=click.Choice(["off", "soft", "strict"]), + default="soft", ), ConfigEnvOption( - group="lib", name="lib_archive", type=click.BOOL, default=True + group="library", + name="lib_archive", + description=( + "Create an archive (`*.a`, static library) from the object files " + "and link it into a firmware (program)" + ), + type=click.BOOL, + default=True, + ), + # Check + ConfigEnvOption( + group="check", + name="check_tool", + description="A list of check tools used for analysis", + type=click.Choice(["cppcheck", "clangtidy"]), + multiple=True, + default=["cppcheck"], + ), + ConfigEnvOption( + group="check", + name="check_filter", + description=( + "Configure a list of source files which should be " + "included/excluded from a check process" + ), + multiple=True, + ), + ConfigEnvOption( + group="check", + name="check_flags", + description="An extra flags to be passed to a check tool", + multiple=True, + ), + ConfigEnvOption( + group="check", + name="check_severity", + description="List of defect severity types for analysis", + multiple=True, + type=click.Choice(["low", "medium", "high"]), + default=["low", "medium", "high"], ), # Test - ConfigEnvOption(group="test", name="test_filter", multiple=True), - ConfigEnvOption(group="test", name="test_ignore", multiple=True), - ConfigEnvOption(group="test", name="test_port"), - ConfigEnvOption(group="test", name="test_speed", type=click.INT), - ConfigEnvOption(group="test", name="test_transport"), + ConfigEnvOption( + group="test", + name="test_filter", + description="Process tests where the name matches specified patterns", + multiple=True, + ), + ConfigEnvOption( + group="test", + name="test_ignore", + description="Ignore tests where the name matches specified patterns", + multiple=True, + ), + ConfigEnvOption( + group="test", + name="test_port", + description="A serial port to communicate with a target device", + ), + ConfigEnvOption( + group="test", + name="test_speed", + description="A connection speed (baud rate) to communicate with a target device", + type=click.INT, + ), + ConfigEnvOption(group="test", name="test_transport", description="",), ConfigEnvOption( group="test", name="test_build_project_src", + description="", type=click.BOOL, default=False, ), # Debug - ConfigEnvOption(group="debug", name="debug_tool"), - ConfigEnvOption(group="debug", name="debug_init_break"), - ConfigEnvOption(group="debug", name="debug_init_cmds", multiple=True), - ConfigEnvOption(group="debug", name="debug_extra_cmds", multiple=True), + ConfigEnvOption( + group="debug", + name="debug_tool", + description="A name of debugging tool", + ), + ConfigEnvOption( + group="debug", + name="debug_init_break", + description=( + "An initial breakpoint that makes program stop whenever a " + "certain point in the program is reached" + ), + default="tbreak main", + ), + ConfigEnvOption( + group="debug", + name="debug_init_cmds", + description="Initial commands to be passed to a back-end debugger", + multiple=True, + ), + ConfigEnvOption( + group="debug", + name="debug_extra_cmds", + description="An extra commands to be passed to a back-end debugger", + multiple=True, + ), ConfigEnvOption( group="debug", name="debug_load_cmds", + description=( + "A list of commands to be used to load program/firmware " + "to a target device" + ), oldnames=["debug_load_cmd"], multiple=True, + default=["load"], ), ConfigEnvOption( group="debug", name="debug_load_mode", + description=( + "Allows one to control when PlatformIO should load debugging " + "firmware to the end target" + ), type=click.Choice(["always", "modified", "manual"]), + default="always", + ), + ConfigEnvOption( + group="debug", + name="debug_server", + description="Allows one to setup a custom debugging server", + multiple=True, + ), + ConfigEnvOption( + group="debug", + name="debug_port", + description=( + "A debugging port of a remote target (a serial device or " + "network address)" + ), ), - ConfigEnvOption(group="debug", name="debug_server", multiple=True), - ConfigEnvOption(group="debug", name="debug_port"), ConfigEnvOption( group="debug", name="debug_svd_path", + description=( + "A custom path to SVD file which contains information about " + "device peripherals" + ), type=click.Path(exists=True, file_okay=True, dir_okay=False), ), - # Check - ConfigEnvOption(group="check", name="check_tool", multiple=True), - ConfigEnvOption(group="check", name="check_filter", multiple=True), - ConfigEnvOption(group="check", name="check_flags", multiple=True), - ConfigEnvOption( - group="check", - name="check_severity", - multiple=True, - type=click.Choice(["low", "medium", "high"]), - ), # Advanced + ConfigEnvOption( + group="advanced", + name="extends", + description=( + "Inherit configuration from other sections or build environments" + ), + multiple=True, + ), ConfigEnvOption( group="advanced", name="extra_scripts", + description="A list of PRE and POST extra scripts", oldnames=["extra_script"], multiple=True, sysenvvar="PLATFORMIO_EXTRA_SCRIPTS", ), - ConfigEnvOption(group="advanced", name="extends", multiple=True), ] ] ) diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index b118a2ef..a10b33a3 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -116,9 +116,8 @@ def test_defaults(config): assert config.get_optional_dir("core") == os.path.join( os.path.expanduser("~"), ".platformio" ) - assert config.get_optional_dir("build_cache") == os.environ.get( - "PLATFORMIO_BUILD_CACHE_DIR" - ) + assert config.get("env:extra_2", "lib_compat_mode") == "soft" + assert config.get("env:extra_2", "build_type") == "release" def test_sections(config): @@ -232,7 +231,7 @@ def test_get_value(config): assert config.get("custom", "debug_flags") == "-D DEBUG=1" assert config.get("env:extra_1", "build_flags") == ["-lc -lm -D DEBUG=1"] assert config.get("env:extra_2", "build_flags") == ["-Og"] - assert config.get("env:extra_2", "monitor_speed") == "115200" + assert config.get("env:extra_2", "monitor_speed") == 115200 assert config.get("env:base", "build_flags") == ["-D DEBUG=1"] @@ -246,21 +245,21 @@ def test_items(config): assert config.items(env="base") == [ ("build_flags", ["-D DEBUG=1"]), ("targets", []), - ("monitor_speed", "115200"), + ("monitor_speed", 115200), ("lib_deps", ["Lib1", "Lib2"]), ("lib_ignore", ["LibIgnoreCustom"]), ] assert config.items(env="extra_1") == [ ("build_flags", ["-lc -lm -D DEBUG=1"]), ("lib_deps", ["574"]), - ("monitor_speed", "115200"), + ("monitor_speed", 115200), ("lib_ignore", ["LibIgnoreCustom"]), ] assert config.items(env="extra_2") == [ ("build_flags", ["-Og"]), ("lib_ignore", ["LibIgnoreCustom", "Lib3"]), ("upload_port", "/dev/extra_2/port"), - ("monitor_speed", "115200"), + ("monitor_speed", 115200), ("lib_deps", ["Lib1", "Lib2"]), ] assert config.items(env="test_extends") == [ @@ -268,7 +267,7 @@ def test_items(config): ("build_flags", ["-D RELEASE"]), ("lib_ldf_mode", "chain+"), ("lib_compat_mode", "strict"), - ("monitor_speed", "9600"), + ("monitor_speed", 9600), ("lib_deps", ["Lib1", "Lib2"]), ("lib_ignore", ["LibIgnoreCustom"]), ] From 7784743cb1863c16dddc479e2dcb4360c3f2e750 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 2 Nov 2019 19:44:28 +0200 Subject: [PATCH 188/221] Switch to default values from project configuration options --- platformio/builder/tools/piolib.py | 25 +++++++++++-------------- platformio/commands/check/command.py | 4 ++-- platformio/commands/debug/helpers.py | 10 +++------- 3 files changed, 16 insertions(+), 23 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 8c63934e..d5c58ff8 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -34,6 +34,7 @@ from platformio.builder.tools import platformio as piotool from platformio.compat import WINDOWS, hashlib_encode_data, string_types from platformio.managers.lib import LibraryManager from platformio.package.manifest.parser import ManifestParserFactory +from platformio.project.options import ProjectOptions class LibBuilderFactory(object): @@ -90,12 +91,6 @@ class LibBuilderFactory(object): class LibBuilderBase(object): - LDF_MODES = ["off", "chain", "deep", "chain+", "deep+"] - LDF_MODE_DEFAULT = "chain" - - COMPAT_MODES = ["off", "soft", "strict"] - COMPAT_MODE_DEFAULT = "soft" - CLASSIC_SCANNER = SCons.Scanner.C.CScanner() CCONDITIONAL_SCANNER = SCons.Scanner.C.CConditionalScanner() # Max depth of nested includes: @@ -215,35 +210,37 @@ class LibBuilderBase(object): @property def lib_ldf_mode(self): - return self.env.GetProjectOption("lib_ldf_mode", self.LDF_MODE_DEFAULT) + return self.env.GetProjectOption("lib_ldf_mode") @staticmethod def validate_ldf_mode(mode): + ldf_modes = ProjectOptions["env.lib_ldf_mode"].type.choices if isinstance(mode, string_types): mode = mode.strip().lower() - if mode in LibBuilderBase.LDF_MODES: + if mode in ldf_modes: return mode try: - return LibBuilderBase.LDF_MODES[int(mode)] + return ldf_modes[int(mode)] except (IndexError, ValueError): pass - return LibBuilderBase.LDF_MODE_DEFAULT + return ProjectOptions["env.lib_ldf_mode"].default @property def lib_compat_mode(self): - return self.env.GetProjectOption("lib_compat_mode", self.COMPAT_MODE_DEFAULT) + return self.env.GetProjectOption("lib_compat_mode") @staticmethod def validate_compat_mode(mode): + compat_modes = ProjectOptions["env.lib_compat_mode"].type.choices if isinstance(mode, string_types): mode = mode.strip().lower() - if mode in LibBuilderBase.COMPAT_MODES: + if mode in compat_modes: return mode try: - return LibBuilderBase.COMPAT_MODES[int(mode)] + return compat_modes[int(mode)] except (IndexError, ValueError): pass - return LibBuilderBase.COMPAT_MODE_DEFAULT + return ProjectOptions["env.lib_compat_mode"].default def is_platforms_compatible(self, platforms): return True diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py index cb155e8a..0d0b6f27 100644 --- a/platformio/commands/check/command.py +++ b/platformio/commands/check/command.py @@ -117,10 +117,10 @@ def cli( flags=flags or env_options.get("check_flags"), severity=[DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_HIGH]] if silent - else (severity or env_options.get("check_severity")), + else severity or config.get("env:" + envname, "check_severity"), ) - for tool in env_options.get("check_tool", ["cppcheck"]): + for tool in config.get("env:" + envname, "check_tool"): if skipenv: results.append({"env": envname, "tool": tool}) continue diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index 858be66b..a3058776 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -139,15 +139,11 @@ def validate_debug_options(cmd_ctx, env_options): load_cmds=_cleanup_cmds( env_options.get( "debug_load_cmds", - tool_settings.get("load_cmds", tool_settings.get("load_cmd", "load")), + tool_settings.get("load_cmds", tool_settings.get("load_cmd")), ) ), - load_mode=env_options.get( - "debug_load_mode", tool_settings.get("load_mode", "always") - ), - init_break=env_options.get( - "debug_init_break", tool_settings.get("init_break", "tbreak main") - ), + load_mode=env_options.get("debug_load_mode", tool_settings.get("load_mode")), + init_break=env_options.get("debug_init_break", tool_settings.get("init_break")), init_cmds=_cleanup_cmds( env_options.get("debug_init_cmds", tool_settings.get("init_cmds")) ), From 0488cc4086a2e69fe1e623b003b3328d703f28d3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 2 Nov 2019 19:48:41 +0200 Subject: [PATCH 189/221] Typo fix --- platformio/project/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/project/options.py b/platformio/project/options.py index 02cea3a3..58149df9 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -263,7 +263,7 @@ ProjectOptions = OrderedDict( name="shared_dir", description=( "A location which PIO Remote uses to synchronize extra files " - "between remote machine." + "between remote machines" ), sysenvvar="PLATFORMIO_SHARED_DIR", default=os.path.join("$PROJECT_DIR", "shared"), From 0dcc6f350da2a0c192ddbf60aaa10caea0680717 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 2 Nov 2019 19:49:34 +0200 Subject: [PATCH 190/221] Bump version to 4.1.0rc5 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index c964118b..116bcf70 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, "0rc4") +VERSION = (4, 1, "0rc5") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 73bcf1849878e394fc28fa1e3356ddb4046471af Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 2 Nov 2019 22:54:57 +0200 Subject: [PATCH 191/221] Fix broken debugger --- platformio/commands/debug/helpers.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index a3058776..3e53be8a 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -25,6 +25,7 @@ from platformio.commands.platform import platform_install as cmd_platform_instal from platformio.commands.run.command import cli as cmd_run from platformio.managers.platform import PlatformFactory from platformio.project.config import ProjectConfig +from platformio.project.options import ProjectOptions class GDBBytesIO(BytesIO): # pylint: disable=too-few-public-methods @@ -139,11 +140,26 @@ def validate_debug_options(cmd_ctx, env_options): load_cmds=_cleanup_cmds( env_options.get( "debug_load_cmds", - tool_settings.get("load_cmds", tool_settings.get("load_cmd")), + tool_settings.get( + "load_cmds", + tool_settings.get( + "load_cmd", ProjectOptions["env.debug_load_cmds"].default + ), + ), ) ), - load_mode=env_options.get("debug_load_mode", tool_settings.get("load_mode")), - init_break=env_options.get("debug_init_break", tool_settings.get("init_break")), + load_mode=env_options.get( + "debug_load_mode", + tool_settings.get( + "load_mode", ProjectOptions["env.debug_load_mode"].default + ), + ), + init_break=env_options.get( + "debug_init_break", + tool_settings.get( + "init_break", ProjectOptions["env.debug_init_break"].default + ), + ), init_cmds=_cleanup_cmds( env_options.get("debug_init_cmds", tool_settings.get("init_cmds")) ), From b45261c3dc66e7089e016ebad1121f72fe7ffb80 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 2 Nov 2019 22:56:11 +0200 Subject: [PATCH 192/221] Change initial debug configuration to: reset/halt, load, init break points --- platformio/commands/debug/initcfgs.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/platformio/commands/debug/initcfgs.py b/platformio/commands/debug/initcfgs.py index 9b70babe..e80b8604 100644 --- a/platformio/commands/debug/initcfgs.py +++ b/platformio/commands/debug/initcfgs.py @@ -22,11 +22,10 @@ define pio_reset_target end target extended-remote $DEBUG_PORT -$INIT_BREAK -pio_reset_halt_target -$LOAD_CMDS monitor init pio_reset_halt_target +$LOAD_CMDS +$INIT_BREAK """ GDB_STUTIL_INIT_CONFIG = """ @@ -57,9 +56,9 @@ define pio_reset_target end target extended-remote $DEBUG_PORT -$INIT_BREAK pio_reset_halt_target $LOAD_CMDS +$INIT_BREAK """ GDB_BLACKMAGIC_INIT_CONFIG = """ @@ -81,8 +80,8 @@ target extended-remote $DEBUG_PORT monitor swdp_scan attach 1 set mem inaccessible-by-default off -$INIT_BREAK $LOAD_CMDS +$INIT_BREAK set language c set *0xE000ED0C = 0x05FA0004 @@ -101,10 +100,10 @@ define pio_reset_target end target extended-remote $DEBUG_PORT -$INIT_BREAK monitor erase -$LOAD_CMDS pio_reset_halt_target +$LOAD_CMDS +$INIT_BREAK """ GDB_QEMU_INIT_CONFIG = """ @@ -117,7 +116,7 @@ define pio_reset_target end target extended-remote $DEBUG_PORT -$INIT_BREAK -$LOAD_CMDS pio_reset_halt_target +$LOAD_CMDS +$INIT_BREAK """ From 40109263f0efd73fb4fab78f03f9222045ae4165 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 2 Nov 2019 23:09:56 +0200 Subject: [PATCH 193/221] Fix initial debug configuration for J-Link --- platformio/commands/debug/initcfgs.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/platformio/commands/debug/initcfgs.py b/platformio/commands/debug/initcfgs.py index e80b8604..5596ffa3 100644 --- a/platformio/commands/debug/initcfgs.py +++ b/platformio/commands/debug/initcfgs.py @@ -39,16 +39,15 @@ define pio_reset_target end target extended-remote $DEBUG_PORT -$INIT_BREAK pio_reset_halt_target $LOAD_CMDS -pio_reset_halt_target +$INIT_BREAK """ GDB_JLINK_INIT_CONFIG = """ define pio_reset_halt_target - monitor halt monitor reset + monitor halt end define pio_reset_target @@ -56,6 +55,8 @@ define pio_reset_target end target extended-remote $DEBUG_PORT +monitor clrbp +monitor speed auto pio_reset_halt_target $LOAD_CMDS $INIT_BREAK From 484ea15959723b68c40b2027eddeb407b5789636 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 2 Nov 2019 23:14:16 +0200 Subject: [PATCH 194/221] Bump version to 4.1.0rc6 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 116bcf70..ff70e462 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, "0rc5") +VERSION = (4, 1, "0rc6") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From e57871cab7479c616eb74d3e6fd7a4e5655715c3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 3 Nov 2019 22:22:40 +0200 Subject: [PATCH 195/221] Print a building mode --- platformio/builder/main.py | 9 ++++++--- platformio/builder/tools/platformio.py | 2 ++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index f9b86cfd..6286bd57 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -132,7 +132,10 @@ if env.GetOption("clean"): env.PioClean(env.subst("$BUILD_DIR")) env.Exit(0) elif not int(ARGUMENTS.get("PIOVERBOSE", 0)): - print("Verbose mode can be enabled via `-v, --verbose` option") + click.echo("Verbose mode can be enabled via `-v, --verbose` option") + +if not isdir(env.subst("$BUILD_DIR")): + makedirs(env.subst("$BUILD_DIR")) env.LoadProjectOptions() env.LoadPioPlatform() @@ -181,12 +184,12 @@ AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS)) ############################################################################## if "envdump" in COMMAND_LINE_TARGETS: - print(env.Dump()) + click.echo(env.Dump()) env.Exit(0) if "idedata" in COMMAND_LINE_TARGETS: Import("projenv") - print( + click.echo( "\n%s\n" % dump_json_to_unicode( projenv.DumpIDEData() # pylint: disable=undefined-variable diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 123e7d43..5fac08eb 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -151,6 +151,8 @@ def BuildProgram(env): ) ) + print("Building in %s mode" % env.GetProjectOption("build_type")) + return program From b45abf67a5a03ee6fb4415c85e04991b1df51624 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 4 Nov 2019 15:36:23 +0200 Subject: [PATCH 196/221] Fix broken debug configuration --- examples | 2 +- platformio/commands/debug/initcfgs.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/examples b/examples index bf351dbc..9070288c 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit bf351dbce107e39850bbc95385a6986b3088c302 +Subproject commit 9070288cff31e353ee307b1b108662ff8c06a9b2 diff --git a/platformio/commands/debug/initcfgs.py b/platformio/commands/debug/initcfgs.py index 5596ffa3..fbd207f2 100644 --- a/platformio/commands/debug/initcfgs.py +++ b/platformio/commands/debug/initcfgs.py @@ -23,15 +23,15 @@ end target extended-remote $DEBUG_PORT monitor init -pio_reset_halt_target $LOAD_CMDS +pio_reset_halt_target $INIT_BREAK """ GDB_STUTIL_INIT_CONFIG = """ define pio_reset_halt_target - monitor halt monitor reset + monitor halt end define pio_reset_target @@ -39,8 +39,8 @@ define pio_reset_target end target extended-remote $DEBUG_PORT -pio_reset_halt_target $LOAD_CMDS +pio_reset_halt_target $INIT_BREAK """ @@ -57,8 +57,8 @@ end target extended-remote $DEBUG_PORT monitor clrbp monitor speed auto -pio_reset_halt_target $LOAD_CMDS +pio_reset_halt_target $INIT_BREAK """ @@ -102,8 +102,8 @@ end target extended-remote $DEBUG_PORT monitor erase -pio_reset_halt_target $LOAD_CMDS +pio_reset_halt_target $INIT_BREAK """ @@ -117,7 +117,7 @@ define pio_reset_target end target extended-remote $DEBUG_PORT -pio_reset_halt_target $LOAD_CMDS +pio_reset_halt_target $INIT_BREAK """ From 39639d45fea8753869feeed4f879533811a9222d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 4 Nov 2019 15:36:45 +0200 Subject: [PATCH 197/221] Bump version to 4.1.0rc7 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index ff70e462..e1e11af1 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, "0rc6") +VERSION = (4, 1, "0rc7") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 3289e84b210d6730b734100d6f861e942dd29e22 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 4 Nov 2019 18:22:28 +0200 Subject: [PATCH 198/221] Refactor PIO Check from "check_filters" to "check_patterns" --- platformio/commands/check/command.py | 18 ++++++--------- platformio/commands/check/tools/base.py | 24 ++++++++++++++++---- platformio/commands/check/tools/clangtidy.py | 2 +- platformio/commands/check/tools/cppcheck.py | 4 ++-- platformio/project/options.py | 6 ++--- 5 files changed, 32 insertions(+), 22 deletions(-) diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py index 0d0b6f27..24575436 100644 --- a/platformio/commands/check/command.py +++ b/platformio/commands/check/command.py @@ -17,7 +17,7 @@ import os from collections import Counter -from os.path import basename, dirname, isfile +from os.path import dirname, isfile from time import time import click @@ -48,7 +48,7 @@ from platformio.project.helpers import find_project_dir_above, get_project_dir exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True ), ) -@click.option("--filter", multiple=True, help="Pattern: + -") +@click.option("--pattern", multiple=True) @click.option("--flags", multiple=True) @click.option( "--severity", multiple=True, type=click.Choice(DefectItem.SEVERITY_LABELS.values()) @@ -65,7 +65,7 @@ def cli( environment, project_dir, project_conf, - filter, + pattern, flags, severity, silent, @@ -102,18 +102,14 @@ def cli( "%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v) ) - default_filter = [ - "+<%s/>" % basename(d) - for d in ( - config.get_optional_dir("src"), - config.get_optional_dir("include"), - ) + default_patterns = [ + config.get_optional_dir("src"), + config.get_optional_dir("include"), ] - tool_options = dict( verbose=verbose, silent=silent, - filter=filter or env_options.get("check_filter", default_filter), + patterns=pattern or env_options.get("check_patterns", default_patterns), flags=flags or env_options.get("check_flags"), severity=[DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_HIGH]] if silent diff --git a/platformio/commands/check/tools/base.py b/platformio/commands/check/tools/base.py index ae4bf7b5..59951288 100644 --- a/platformio/commands/check/tools/base.py +++ b/platformio/commands/check/tools/base.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import glob import os import click @@ -124,11 +125,24 @@ class CheckToolBase(object): # pylint: disable=too-many-instance-attributes def clean_up(self): pass - def get_project_src_files(self): - file_extensions = ["h", "hpp", "c", "cc", "cpp", "ino"] - return fs.match_src_files( - get_project_dir(), self.options.get("filter"), file_extensions - ) + def get_project_target_files(self): + allowed_extensions = (".h", ".hpp", ".c", ".cc", ".cpp", ".ino") + result = [] + + def _add_file(path): + if not path.endswith(allowed_extensions): + return + result.append(os.path.abspath(path)) + + for pattern in self.options["patterns"]: + for item in glob.glob(pattern): + if not os.path.isdir(item): + _add_file(item) + for root, _, files in os.walk(item, followlinks=True): + for f in files: + _add_file(os.path.join(root, f)) + + return result def get_source_language(self): with fs.cd(get_project_dir()): diff --git a/platformio/commands/check/tools/clangtidy.py b/platformio/commands/check/tools/clangtidy.py index 20b4a5de..b7845f8b 100644 --- a/platformio/commands/check/tools/clangtidy.py +++ b/platformio/commands/check/tools/clangtidy.py @@ -58,7 +58,7 @@ class ClangtidyCheckTool(CheckToolBase): cmd.append("--checks=*") cmd.extend(flags) - cmd.extend(self.get_project_src_files()) + cmd.extend(self.get_project_target_files()) cmd.append("--") cmd.extend(["-D%s" % d for d in self.cpp_defines]) diff --git a/platformio/commands/check/tools/cppcheck.py b/platformio/commands/check/tools/cppcheck.py index 20a28ec5..3d92bc56 100644 --- a/platformio/commands/check/tools/cppcheck.py +++ b/platformio/commands/check/tools/cppcheck.py @@ -137,7 +137,7 @@ class CppcheckCheckTool(CheckToolBase): def _generate_src_file(self): src_files = [ - f for f in self.get_project_src_files() if not f.endswith((".h", ".hpp")) + f for f in self.get_project_target_files() if not f.endswith((".h", ".hpp")) ] return self._create_tmp_file("\n".join(src_files)) @@ -152,7 +152,7 @@ class CppcheckCheckTool(CheckToolBase): # delete temporary dump files generated by addons if not self.is_flag_set("--addon", self.get_flags("cppcheck")): return - for f in self.get_project_src_files(): + for f in self.get_project_target_files(): dump_file = f + ".dump" if isfile(dump_file): remove(dump_file) diff --git a/platformio/project/options.py b/platformio/project/options.py index 58149df9..b4c1814f 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -537,10 +537,10 @@ ProjectOptions = OrderedDict( ), ConfigEnvOption( group="check", - name="check_filter", + name="check_patterns", description=( - "Configure a list of source files which should be " - "included/excluded from a check process" + "Configure a list of target files or directories for checking " + "(Unix shell-style wildcards)" ), multiple=True, ), From dfd853fa876b518f9c2206f3701939e1bea44203 Mon Sep 17 00:00:00 2001 From: valeros Date: Mon, 4 Nov 2019 21:34:39 +0200 Subject: [PATCH 199/221] Update tests for check command with a new flag "pattern" that supersedes "filter" --- tests/commands/test_check.py | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index 59bfff0f..920ef5fd 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -51,11 +51,8 @@ void run_defects() { } } -/* Low */ -void unusedFuntion(){ -} - int main() { + int uninitializedVar; /* Low */ run_defects(); } """ @@ -150,11 +147,15 @@ def test_check_silent_mode(clirunner, check_dir): assert style == 0 -def test_check_filter_sources(clirunner, check_dir): - check_dir.mkdir(join("src", "app")).join("additional.cpp").write(TEST_CODE) +def test_check_custom_pattern_absolute_path(clirunner, tmpdir_factory): + project_dir = tmpdir_factory.mktemp("project") + project_dir.join("platformio.ini").write(DEFAULT_CONFIG) + + check_dir = tmpdir_factory.mktemp("custom_src_dir") + check_dir.join("main.cpp").write(TEST_CODE) result = clirunner.invoke( - cmd_check, ["--project-dir", str(check_dir), "--filter=-<*> +"] + cmd_check, ["--project-dir", str(project_dir), "--pattern=" + str(check_dir)] ) errors, warnings, style = count_defects(result.output) @@ -165,6 +166,23 @@ def test_check_filter_sources(clirunner, check_dir): assert style == EXPECTED_STYLE +def test_check_custom_pattern_relative_path(clirunner, tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("project") + tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) + + tmpdir.mkdir("app").join("main.cpp").write(TEST_CODE) + tmpdir.mkdir("prj").join("test.cpp").write(TEST_CODE) + + result = clirunner.invoke( + cmd_check, ["--project-dir", str(tmpdir), "--pattern=app", "--pattern=prj"] + ) + + errors, warnings, style = count_defects(result.output) + + assert result.exit_code == 0 + assert errors + warnings + style == EXPECTED_DEFECTS * 2 + + def test_check_no_source_files(clirunner, tmpdir): tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) tmpdir.mkdir("src") @@ -326,3 +344,4 @@ int main() { assert high_result.exit_code == 0 assert low_result.exit_code != 0 + From 3f4aa320c28fec0efa22c6a59b72c7ae8a50071a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 4 Nov 2019 21:52:42 +0200 Subject: [PATCH 200/221] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 29115fe8..e6c05ca3 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 29115fe80b25f47b2837231e09f0f6ff169d0391 +Subproject commit e6c05ca3da57393515f4b87df5a79bdd07d91e29 From 7345d3ea19c400cdf51f403cb72fcb03c02f25ee Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 5 Nov 2019 00:17:39 +0200 Subject: [PATCH 201/221] Improve dump of config data --- .../commands/home/rpc/handlers/project.py | 6 +- platformio/project/config.py | 15 ++-- tests/commands/test_check.py | 1 - tests/test_projectconf.py | 84 +++++++++++++++++-- 4 files changed, 89 insertions(+), 17 deletions(-) diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index b7981f56..e21331d8 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -45,11 +45,13 @@ class ProjectRPC(object): @staticmethod def config_load(path): - return ProjectConfig(path, parse_extra=False).as_tuple() + return ProjectConfig( + path, parse_extra=False, expand_interpolations=False + ).as_tuple() @staticmethod def config_dump(path, data): - config = ProjectConfig(path, parse_extra=False) + config = ProjectConfig(path, parse_extra=False, expand_interpolations=False) config.update(data, clear=True) return config.save() diff --git a/platformio/project/config.py b/platformio/project/config.py index 0ca94704..524879e3 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -92,6 +92,8 @@ class ProjectConfigBase(object): if path and os.path.isfile(path): self.read(path, parse_extra) + self._maintain_renaimed_options() + def __getattr__(self, name): return getattr(self._parser, name) @@ -114,8 +116,6 @@ class ProjectConfigBase(object): for item in glob.glob(pattern): self.read(item) - self._maintain_renaimed_options() - def _maintain_renaimed_options(self): # legacy `lib_extra_dirs` in [platformio] if self._parser.has_section("platformio") and self._parser.has_option( @@ -195,6 +195,9 @@ class ProjectConfigBase(object): if not section: section = "env:" + env + if not self.expand_interpolations: + return self._parser.options(section) + for _, option in self.walk_options(section): if option not in result: result.append(option) @@ -260,7 +263,7 @@ class ProjectConfigBase(object): return os.getenv(option) return self.getraw(section, option) - def get(self, section, option, default=None): + def get(self, section, option, default=None): # pylint: disable=too-many-branches value = None try: value = self.getraw(section, option) @@ -296,12 +299,14 @@ class ProjectConfigBase(object): return default if default is not None else option_meta.default try: - return self._cast_to(value, option_meta.type) + return self.cast_to(value, option_meta.type) except click.BadParameter as e: + if not self.expand_interpolations: + return value raise exception.ProjectOptionValueError(e.format_message(), option, section) @staticmethod - def _cast_to(value, to_type): + def cast_to(value, to_type): items = value if not isinstance(value, (list, tuple)): items = [value] diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index 920ef5fd..64e4602e 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -344,4 +344,3 @@ int main() { assert high_result.exit_code == 0 assert low_result.exit_code != 0 - diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index a10b33a3..cfede890 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -28,7 +28,8 @@ extra_configs = # global options per [env:*] [env] -monitor_speed = 115200 ; inline comment +monitor_speed = 9600 ; inline comment +custom_monitor_speed = 115200 lib_deps = Lib1 ; inline comment in multi-line value Lib2 @@ -39,7 +40,7 @@ lib_ldf_mode = chain+ lib_compat_mode = strict [monitor_custom] -monitor_speed = 9600 +monitor_speed = ${env.custom_monitor_speed} [strict_settings] extends = strict_ldf, monitor_custom @@ -53,11 +54,13 @@ lib_ignore = LibIgnoreCustom [env:base] build_flags = ${custom.debug_flags} ${custom.extra_flags} +lib_compat_mode = ${strict_ldf.strict} targets = [env:test_extends] extends = strict_settings + """ EXTRA_ENVS_CONFIG = """ @@ -146,8 +149,10 @@ def test_envs(config): def test_options(config): assert config.options(env="base") == [ "build_flags", + "lib_compat_mode", "targets", "monitor_speed", + "custom_monitor_speed", "lib_deps", "lib_ignore", ] @@ -157,6 +162,7 @@ def test_options(config): "lib_ldf_mode", "lib_compat_mode", "monitor_speed", + "custom_monitor_speed", "lib_deps", "lib_ignore", ] @@ -192,6 +198,7 @@ def test_sysenv_options(config): "lib_ldf_mode", "lib_compat_mode", "monitor_speed", + "custom_monitor_speed", "lib_deps", "lib_ignore", "upload_port", @@ -223,15 +230,15 @@ def test_getraw_value(config): # extended assert config.getraw("env:test_extends", "lib_ldf_mode") == "chain+" - assert config.getraw("env", "monitor_speed") == "115200" - assert config.getraw("env:test_extends", "monitor_speed") == "9600" + assert config.getraw("env", "monitor_speed") == "9600" + assert config.getraw("env:test_extends", "monitor_speed") == "115200" def test_get_value(config): assert config.get("custom", "debug_flags") == "-D DEBUG=1" assert config.get("env:extra_1", "build_flags") == ["-lc -lm -D DEBUG=1"] assert config.get("env:extra_2", "build_flags") == ["-Og"] - assert config.get("env:extra_2", "monitor_speed") == 115200 + assert config.get("env:extra_2", "monitor_speed") == 9600 assert config.get("env:base", "build_flags") == ["-D DEBUG=1"] @@ -244,22 +251,26 @@ def test_items(config): ] assert config.items(env="base") == [ ("build_flags", ["-D DEBUG=1"]), + ("lib_compat_mode", "soft"), ("targets", []), - ("monitor_speed", 115200), + ("monitor_speed", 9600), + ("custom_monitor_speed", "115200"), ("lib_deps", ["Lib1", "Lib2"]), ("lib_ignore", ["LibIgnoreCustom"]), ] assert config.items(env="extra_1") == [ ("build_flags", ["-lc -lm -D DEBUG=1"]), ("lib_deps", ["574"]), - ("monitor_speed", 115200), + ("monitor_speed", 9600), + ("custom_monitor_speed", "115200"), ("lib_ignore", ["LibIgnoreCustom"]), ] assert config.items(env="extra_2") == [ ("build_flags", ["-Og"]), ("lib_ignore", ["LibIgnoreCustom", "Lib3"]), ("upload_port", "/dev/extra_2/port"), - ("monitor_speed", 115200), + ("monitor_speed", 9600), + ("custom_monitor_speed", "115200"), ("lib_deps", ["Lib1", "Lib2"]), ] assert config.items(env="test_extends") == [ @@ -267,7 +278,8 @@ def test_items(config): ("build_flags", ["-D RELEASE"]), ("lib_ldf_mode", "chain+"), ("lib_compat_mode", "strict"), - ("monitor_speed", 9600), + ("monitor_speed", 115200), + ("custom_monitor_speed", "115200"), ("lib_deps", ["Lib1", "Lib2"]), ("lib_ignore", ["LibIgnoreCustom"]), ] @@ -337,3 +349,57 @@ board = myboard assert config.as_tuple() == [ ("mysection", [("opt1", "value1"), ("opt2", "value2")]) ] + + +def test_dump(tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("project") + tmpdir.join("platformio.ini").write(BASE_CONFIG) + tmpdir.join("extra_envs.ini").write(EXTRA_ENVS_CONFIG) + tmpdir.join("extra_debug.ini").write(EXTRA_DEBUG_CONFIG) + config = ProjectConfig( + tmpdir.join("platformio.ini").strpath, + parse_extra=False, + expand_interpolations=False, + ) + assert config.as_tuple() == [ + ( + "platformio", + [ + ("extra_configs", ["extra_envs.ini", "extra_debug.ini"]), + ("default_envs", ["base", "extra_2"]), + ], + ), + ( + "env", + [ + ("monitor_speed", 9600), + ("custom_monitor_speed", "115200"), + ("lib_deps", ["Lib1", "Lib2"]), + ("lib_ignore", ["${custom.lib_ignore}"]), + ], + ), + ("strict_ldf", [("lib_ldf_mode", "chain+"), ("lib_compat_mode", "strict")]), + ("monitor_custom", [("monitor_speed", "${env.custom_monitor_speed}")]), + ( + "strict_settings", + [("extends", "strict_ldf, monitor_custom"), ("build_flags", "-D RELEASE")], + ), + ( + "custom", + [ + ("debug_flags", "-D RELEASE"), + ("lib_flags", "-lc -lm"), + ("extra_flags", "${sysenv.__PIO_TEST_CNF_EXTRA_FLAGS}"), + ("lib_ignore", "LibIgnoreCustom"), + ], + ), + ( + "env:base", + [ + ("build_flags", ["${custom.debug_flags} ${custom.extra_flags}"]), + ("lib_compat_mode", "${strict_ldf.strict}"), + ("targets", []), + ], + ), + ("env:test_extends", [("extends", ["strict_settings"])]), + ] From 182835fabfb31225b169a084b4f6d7219303aa1b Mon Sep 17 00:00:00 2001 From: valeros Date: Tue, 5 Nov 2019 11:36:20 +0200 Subject: [PATCH 202/221] Rename check_patterns option to check_pattern --- platformio/commands/check/command.py | 2 +- platformio/commands/check/tools/base.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py index 24575436..9761c863 100644 --- a/platformio/commands/check/command.py +++ b/platformio/commands/check/command.py @@ -109,7 +109,7 @@ def cli( tool_options = dict( verbose=verbose, silent=silent, - patterns=pattern or env_options.get("check_patterns", default_patterns), + pattern=pattern or env_options.get("check_pattern", default_patterns), flags=flags or env_options.get("check_flags"), severity=[DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_HIGH]] if silent diff --git a/platformio/commands/check/tools/base.py b/platformio/commands/check/tools/base.py index 59951288..6a0d10b2 100644 --- a/platformio/commands/check/tools/base.py +++ b/platformio/commands/check/tools/base.py @@ -134,7 +134,7 @@ class CheckToolBase(object): # pylint: disable=too-many-instance-attributes return result.append(os.path.abspath(path)) - for pattern in self.options["patterns"]: + for pattern in self.options["pattern"]: for item in glob.glob(pattern): if not os.path.isdir(item): _add_file(item) From 83110326f40e9803675b4cf42a28360548e95cee Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 5 Nov 2019 12:02:12 +0200 Subject: [PATCH 203/221] Rename "check_pattern" option --- docs | 2 +- platformio/project/options.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index e6c05ca3..8086fc8c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit e6c05ca3da57393515f4b87df5a79bdd07d91e29 +Subproject commit 8086fc8c6ac06666b9afb6407c2d21f60fccfe18 diff --git a/platformio/project/options.py b/platformio/project/options.py index b4c1814f..3294084d 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -537,7 +537,7 @@ ProjectOptions = OrderedDict( ), ConfigEnvOption( group="check", - name="check_patterns", + name="check_pattern", description=( "Configure a list of target files or directories for checking " "(Unix shell-style wildcards)" From 30bc691c95feb8dbfa17cd4f59c7307e7333ca77 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 5 Nov 2019 12:08:29 +0200 Subject: [PATCH 204/221] Fix test with a missed library --- tests/test_maintenance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_maintenance.py b/tests/test_maintenance.py index a1298211..c004f28f 100644 --- a/tests/test_maintenance.py +++ b/tests/test_maintenance.py @@ -53,7 +53,7 @@ def test_check_pio_upgrade(clirunner, isolated_pio_home, validate_cliresult): def test_check_lib_updates(clirunner, isolated_pio_home, validate_cliresult): # install obsolete library - result = clirunner.invoke(cli_pio, ["lib", "-g", "install", "ArduinoJson@<5.7"]) + result = clirunner.invoke(cli_pio, ["lib", "-g", "install", "ArduinoJson@<6.13"]) validate_cliresult(result) # reset check time @@ -88,7 +88,7 @@ def test_check_and_update_libraries(clirunner, isolated_pio_home, validate_clire validate_cliresult(result) assert "There are the new updates for libraries (ArduinoJson)" in result.output assert "Please wait while updating libraries" in result.output - assert re.search(r"Updating ArduinoJson\s+@ 5.6.7\s+\[[\d\.]+\]", result.output) + assert re.search(r"Updating ArduinoJson\s+@ 6.12.0\s+\[[\d\.]+\]", result.output) # check updated version result = clirunner.invoke(cli_pio, ["lib", "-g", "list", "--json-output"]) From 9786b3e1b9eb0721760c1bf0fbf59bf9b2de448d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 5 Nov 2019 15:16:35 +0200 Subject: [PATCH 205/221] Fix CLion integration when project name contains a space --- platformio/ide/tpls/clion/CMakeLists.txt.tpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/ide/tpls/clion/CMakeLists.txt.tpl b/platformio/ide/tpls/clion/CMakeLists.txt.tpl index 9fb475f0..0b758531 100644 --- a/platformio/ide/tpls/clion/CMakeLists.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeLists.txt.tpl @@ -6,7 +6,7 @@ # The `CMakeListsUser.txt` will not be overwritten by PlatformIO. cmake_minimum_required(VERSION 3.2) -project({{project_name}}) +project("{{project_name}}") include(CMakeListsPrivate.txt) @@ -86,4 +86,4 @@ add_custom_target( WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) -add_executable(${PROJECT_NAME} ${SRC_LIST}) +add_executable(pioprog ${SRC_LIST}) From 66cc557d2f295ed4263bd525d0dc4d4043743d8b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 5 Nov 2019 15:16:50 +0200 Subject: [PATCH 206/221] Export SVD Path to CLion --- platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl index 40f31e09..c7c7c163 100644 --- a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl @@ -31,6 +31,9 @@ set(CMAKE_CONFIGURATION_TYPES "{{ env_name }}" CACHE STRING "" FORCE) % end set(PLATFORMIO_CMD "{{ _normalize_path(platformio_path) }}") +% if svd_path: +set(SVD_PATH "{{ _normalize_path(svd_path) }}") +% end SET(CMAKE_C_COMPILER "{{ _normalize_path(cc_path) }}") SET(CMAKE_CXX_COMPILER "{{ _normalize_path(cxx_path) }}") From f3992f8e533f75e9c0e2d7aa5c3448a9c2bfeccd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 5 Nov 2019 15:59:06 +0200 Subject: [PATCH 207/221] Create dummy target for CLION --- platformio/ide/tpls/clion/CMakeLists.txt.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/ide/tpls/clion/CMakeLists.txt.tpl b/platformio/ide/tpls/clion/CMakeLists.txt.tpl index 0b758531..302fb6e1 100644 --- a/platformio/ide/tpls/clion/CMakeLists.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeLists.txt.tpl @@ -86,4 +86,4 @@ add_custom_target( WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) -add_executable(pioprog ${SRC_LIST}) +add_executable(Z_DUMMY_TARGET ${SRC_LIST}) From bcf09964ab18d579dd6fe18c398e135d5a04c25a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 6 Nov 2019 00:03:08 +0200 Subject: [PATCH 208/221] Better formatting for multi-line values in config option --- platformio/project/config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/platformio/project/config.py b/platformio/project/config.py index 524879e3..27b116c1 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -230,12 +230,13 @@ class ProjectConfigBase(object): def set(self, section, option, value): if isinstance(value, (list, tuple)): value = "\n".join(value) - if value: - value = "\n" + value # start from a new line elif isinstance(value, bool): value = "yes" if value else "no" elif isinstance(value, (int, float)): value = str(value) + # start multi-line value from a new line + if "\n" in value and not value.startswith("\n"): + value = "\n" + value self._parser.set(section, option, value) def getraw(self, section, option): From e5ec4de3a4ff6b524e60cc398b367cb2d0202c2e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 6 Nov 2019 00:05:17 +0200 Subject: [PATCH 209/221] Extend PIO Home RPC with "project.config_update_description(path, text)" method --- platformio/commands/home/rpc/handlers/project.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index e21331d8..dcb42ed7 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -55,6 +55,14 @@ class ProjectRPC(object): config.update(data, clear=True) return config.save() + @staticmethod + def config_update_description(path, text): + config = ProjectConfig(path, parse_extra=False, expand_interpolations=False) + if not config.has_section("platformio"): + config.add_section("platformio") + config.set("platformio", "description", text) + return config.save() + @staticmethod def get_config_schema(): return get_config_options_schema() From e0c174b9b6885b7576ba4be0b02f6469e0940890 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 6 Nov 2019 00:48:28 +0200 Subject: [PATCH 210/221] Improve PIO Home Misc RPC --- platformio/commands/home/rpc/handlers/misc.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/platformio/commands/home/rpc/handlers/misc.py b/platformio/commands/home/rpc/handlers/misc.py index 004c14a3..992eeebe 100644 --- a/platformio/commands/home/rpc/handlers/misc.py +++ b/platformio/commands/home/rpc/handlers/misc.py @@ -22,8 +22,8 @@ from platformio.commands.home.rpc.handlers.os import OSRPC class MiscRPC(object): - def load_latest_tweets(self, username): - cache_key = "piohome_latest_tweets_" + str(username) + def load_latest_tweets(self, data_url): + cache_key = data_url cache_valid = "7d" with app.ContentCache() as cc: cache_data = cc.get(cache_key) @@ -32,20 +32,17 @@ class MiscRPC(object): # automatically update cache in background every 12 hours if cache_data["time"] < (time.time() - (3600 * 12)): reactor.callLater( - 5, self._preload_latest_tweets, username, cache_key, cache_valid + 5, self._preload_latest_tweets, data_url, cache_key, cache_valid ) return cache_data["result"] - result = self._preload_latest_tweets(username, cache_key, cache_valid) + result = self._preload_latest_tweets(data_url, cache_key, cache_valid) return result @staticmethod @defer.inlineCallbacks - def _preload_latest_tweets(username, cache_key, cache_valid): - result = yield OSRPC.fetch_content( - "https://api.platformio.org/tweets/" + username - ) - result = json.loads(result) + def _preload_latest_tweets(data_url, cache_key, cache_valid): + result = json.loads((yield OSRPC.fetch_content(data_url))) with app.ContentCache() as cc: cc.set( cache_key, From c1f62f8eada8c643ba3d6a21443c9cacd6b0615c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 6 Nov 2019 01:01:07 +0200 Subject: [PATCH 211/221] Bump version to 4.1.0rc8 --- platformio/__init__.py | 2 +- platformio/managers/core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index e1e11af1..54d3766e 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, "0rc7") +VERSION = (4, 1, "0rc8") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/managers/core.py b/platformio/managers/core.py index f81764b2..a3a33868 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -24,7 +24,7 @@ from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path from platformio.project.config import ProjectConfig CORE_PACKAGES = { - "contrib-piohome": ">=3.0.0-beta.4,<3.1.0", + "contrib-piohome": ">=3.0.0-beta.5,<3.1.0", "contrib-pysite": "~2.%d%d.0" % (sys.version_info[0], sys.version_info[1]), "tool-pioplus": "^2.5.8", "tool-unity": "~1.20403.0", From 2cce47a13d721d66f0a622dfc92bc1e3cfe10f91 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 6 Nov 2019 12:20:42 +0200 Subject: [PATCH 212/221] Fix "pio_reset_run_target" for JLink debug probe --- platformio/commands/debug/client.py | 12 ++++++------ platformio/commands/debug/initcfgs.py | 16 +++++++++------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py index 990340e8..f7a81968 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/client.py @@ -133,18 +133,18 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes commands = self.debug_options["init_cmds"] commands.extend(self.debug_options["extra_cmds"]) - if not any("define pio_reset_target" in cmd for cmd in commands): + if not any("define pio_reset_run_target" in cmd for cmd in commands): commands = [ - "define pio_reset_target", - " echo Warning! Undefined pio_reset_target command\\n", - " mon reset", + "define pio_reset_run_target", + " echo Warning! Undefined pio_reset_run_target command\\n", + " monitor reset", "end", ] + commands if not any("define pio_reset_halt_target" in cmd for cmd in commands): commands = [ "define pio_reset_halt_target", " echo Warning! Undefined pio_reset_halt_target command\\n", - " mon reset halt", + " monitor reset halt", "end", ] + commands if not any("define pio_restart_target" in cmd for cmd in commands): @@ -193,7 +193,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes if b"-gdb-exit" in data or data.strip() in (b"q", b"quit"): # Allow terminating via SIGINT/CTRL+C signal.signal(signal.SIGINT, signal.default_int_handler) - self.transport.write(b"pio_reset_target\n") + self.transport.write(b"pio_reset_run_target\n") self.transport.write(data) def processEnded(self, reason): # pylint: disable=unused-argument diff --git a/platformio/commands/debug/initcfgs.py b/platformio/commands/debug/initcfgs.py index fbd207f2..b241efc6 100644 --- a/platformio/commands/debug/initcfgs.py +++ b/platformio/commands/debug/initcfgs.py @@ -17,7 +17,7 @@ define pio_reset_halt_target monitor reset halt end -define pio_reset_target +define pio_reset_run_target monitor reset end @@ -34,7 +34,7 @@ define pio_reset_halt_target monitor halt end -define pio_reset_target +define pio_reset_run_target monitor reset end @@ -50,8 +50,10 @@ define pio_reset_halt_target monitor halt end -define pio_reset_target +define pio_reset_run_target + monitor clrbp monitor reset + monitor go end target extended-remote $DEBUG_PORT @@ -73,7 +75,7 @@ define pio_reset_halt_target set language auto end -define pio_reset_target +define pio_reset_run_target pio_reset_halt_target end @@ -97,7 +99,7 @@ GDB_MSPDEBUG_INIT_CONFIG = """ define pio_reset_halt_target end -define pio_reset_target +define pio_reset_run_target end target extended-remote $DEBUG_PORT @@ -112,8 +114,8 @@ define pio_reset_halt_target monitor system_reset end -define pio_reset_target - pio_reset_halt_target +define pio_reset_run_target + monitor system_reset end target extended-remote $DEBUG_PORT From 17ff3250c95f0b036457463c673058d708dfcd94 Mon Sep 17 00:00:00 2001 From: valeros Date: Wed, 6 Nov 2019 12:34:43 +0200 Subject: [PATCH 213/221] Export uppercased driver name in sizedata report on Windows --- platformio/builder/tools/piosize.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/platformio/builder/tools/piosize.py b/platformio/builder/tools/piosize.py index 84d3bf12..83b3a3f5 100644 --- a/platformio/builder/tools/piosize.py +++ b/platformio/builder/tools/piosize.py @@ -18,13 +18,14 @@ from __future__ import absolute_import import sys from os import environ, makedirs, remove -from os.path import isdir, join +from os.path import isdir, join, splitdrive from elftools.elf.descriptions import describe_sh_flags from elftools.elf.elffile import ELFFile from platformio.compat import dump_json_to_unicode from platformio.proc import exec_command +from platformio.util import get_systype def _run_tool(cmd, env, tool_args): @@ -54,7 +55,7 @@ def _get_symbol_locations(env, elf_path, addrs): locations = [line for line in result["out"].split("\n") if line] assert len(addrs) == len(locations) - return dict(zip(addrs, [l.strip().replace("\\", "/") for l in locations])) + return dict(zip(addrs, [l.strip() for l in locations])) def _get_demangled_names(env, mangled_names): @@ -163,6 +164,9 @@ def _collect_symbols_info(env, elffile, elf_path, sections): location = symbol_locations.get(hex(symbol["addr"])) if not location or "?" in location: continue + if "windows" in get_systype(): + drive, tail = splitdrive(location) + location = join(drive.upper(), tail) symbol["file"] = location symbol["line"] = 0 if ":" in location: From 70f4fa26656d648af861ff771e9c6580e982dbce Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 6 Nov 2019 20:06:30 +0200 Subject: [PATCH 214/221] Remove platformio.description if none value is passed --- platformio/commands/home/rpc/handlers/project.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index dcb42ed7..80340a5a 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -60,7 +60,13 @@ class ProjectRPC(object): config = ProjectConfig(path, parse_extra=False, expand_interpolations=False) if not config.has_section("platformio"): config.add_section("platformio") - config.set("platformio", "description", text) + if text: + config.set("platformio", "description", text) + else: + if config.has_option("platformio", "description"): + config.remove_option("platformio", "description") + if not config.options("platformio"): + config.remove_section("platformio") return config.save() @staticmethod From 6b44a8ae75cbdc709c7538035287b58305e59fb0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 6 Nov 2019 22:25:14 +0200 Subject: [PATCH 215/221] Use BuildAsyncPipe only if TTY stream --- platformio/managers/platform.py | 41 +++++++++++++++++---------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index fd670895..ec7bd2a2 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -23,16 +23,10 @@ from os.path import basename, dirname, isdir, isfile, join import click import semantic_version -from platformio import __version__, app, exception, fs, util +from platformio import __version__, app, exception, fs, proc, util from platformio.compat import PY2, hashlib_encode_data, is_bytes, load_python_module from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager -from platformio.proc import ( - BuildAsyncPipe, - copy_pythonpath_to_osenv, - exec_command, - get_pythonexe_path, -) from platformio.project.config import ProjectConfig try: @@ -409,7 +403,7 @@ class PlatformRunMixin(object): def _run_scons(self, variables, targets, jobs): args = [ - get_pythonexe_path(), + proc.get_pythonexe_path(), join(get_core_package_dir("tool-scons"), "script", "scons"), "-Q", "--warn=no-no-parallel-support", @@ -434,18 +428,25 @@ class PlatformRunMixin(object): except IOError: pass - copy_pythonpath_to_osenv() - result = exec_command( - args, - stdout=BuildAsyncPipe( - line_callback=self._on_stdout_line, - data_callback=lambda data: _write_and_flush(sys.stdout, data), - ), - stderr=BuildAsyncPipe( - line_callback=self._on_stderr_line, - data_callback=lambda data: _write_and_flush(sys.stderr, data), - ), - ) + proc.copy_pythonpath_to_osenv() + if click._compat.isatty(sys.stdout): + result = proc.exec_command( + args, + stdout=proc.BuildAsyncPipe( + line_callback=self._on_stdout_line, + data_callback=lambda data: _write_and_flush(sys.stdout, data), + ), + stderr=proc.BuildAsyncPipe( + line_callback=self._on_stderr_line, + data_callback=lambda data: _write_and_flush(sys.stderr, data), + ), + ) + else: + result = proc.exec_command( + args, + stdout=proc.LineBufferedAsyncPipe(line_callback=self._on_stdout_line), + stderr=proc.LineBufferedAsyncPipe(line_callback=self._on_stderr_line), + ) return result def _on_stdout_line(self, line): From 67aea4db3f36faf3df80bc86e671c45f4d3f2be9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 6 Nov 2019 22:30:58 +0200 Subject: [PATCH 216/221] Fix issue with GDB/MI Stream Records for PIO Debugger --- platformio/commands/debug/client.py | 15 +++++------ platformio/commands/debug/command.py | 25 +++++++++++------- platformio/commands/debug/helpers.py | 38 +++++++++++++++++++--------- platformio/commands/debug/server.py | 6 +++++ 4 files changed, 55 insertions(+), 29 deletions(-) diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py index f7a81968..72a12a84 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/client.py @@ -30,7 +30,7 @@ from platformio import app, exception, fs, proc, util from platformio.commands.debug import helpers, initcfgs from platformio.commands.debug.process import BaseProcess from platformio.commands.debug.server import DebugServer -from platformio.compat import hashlib_encode_data +from platformio.compat import hashlib_encode_data, is_bytes from platformio.project.helpers import get_project_cache_dir from platformio.telemetry import MeasurementProtocol @@ -223,10 +223,9 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes self._handle_error(data) def console_log(self, msg): - if helpers.is_mi_mode(self.args): - self.outReceived(('~"%s\\n"\n' % msg).encode()) - else: - self.outReceived(("%s\n" % msg).encode()) + if helpers.is_gdbmi_mode(): + msg = helpers.escape_gdbmi_stream("~", msg) + self.outReceived(msg if is_bytes(msg) else msg.encode()) def _auto_exec_continue(self): auto_exec_delay = 0.5 # in seconds @@ -239,14 +238,14 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes if not self.debug_options["init_break"] or self._target_is_run: return self.console_log( - "PlatformIO: Resume the execution to `debug_init_break = %s`" + "PlatformIO: Resume the execution to `debug_init_break = %s`\n" % self.debug_options["init_break"] ) self.console_log( - "PlatformIO: More configuration options -> http://bit.ly/pio-debug" + "PlatformIO: More configuration options -> http://bit.ly/pio-debug\n" ) self.transport.write( - b"0-exec-continue\n" if helpers.is_mi_mode(self.args) else b"continue\n" + b"0-exec-continue\n" if helpers.is_gdbmi_mode() else b"continue\n" ) self._target_is_run = True diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index c99cc548..ab273063 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -90,10 +90,12 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro try: fs.ensure_udev_rules() except exception.InvalidUdevRules as e: - for line in str(e).split("\n") + [""]: - click.echo( - ('~"%s\\n"' if helpers.is_mi_mode(__unprocessed) else "%s") % line - ) + click.echo( + helpers.escape_gdbmi_stream("~", str(e) + "\n") + if helpers.is_gdbmi_mode() + else str(e) + "\n", + nl=False, + ) debug_options["load_cmds"] = helpers.configure_esp32_load_cmds( debug_options, configuration @@ -118,12 +120,17 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro debug_options["load_cmds"] = [] if rebuild_prog: - if helpers.is_mi_mode(__unprocessed): - click.echo('~"Preparing firmware for debugging...\\n"') - output = helpers.GDBBytesIO() - with util.capture_std_streams(output): + if helpers.is_gdbmi_mode(): + click.echo( + helpers.escape_gdbmi_stream( + "~", "Preparing firmware for debugging...\n" + ), + nl=False, + ) + stream = helpers.GDBMIConsoleStream() + with util.capture_std_streams(stream): helpers.predebug_project(ctx, project_dir, env_name, preload, verbose) - output.close() + stream.close() else: click.echo("Preparing firmware for debugging...") helpers.predebug_project(ctx, project_dir, env_name, preload, verbose) diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index 3e53be8a..e8e6c525 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -21,32 +21,46 @@ from io import BytesIO from os.path import isfile from platformio import exception, fs, util +from platformio.commands import PlatformioCLI from platformio.commands.platform import platform_install as cmd_platform_install from platformio.commands.run.command import cli as cmd_run +from platformio.compat import is_bytes from platformio.managers.platform import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.options import ProjectOptions -class GDBBytesIO(BytesIO): # pylint: disable=too-few-public-methods +class GDBMIConsoleStream(BytesIO): # pylint: disable=too-few-public-methods STDOUT = sys.stdout - @staticmethod - def escape(text): - return re.sub(r"\\+", "\\\\\\\\", text) - def write(self, text): - if "\n" in text: - for line in text.strip().split("\n"): - self.STDOUT.write('~"%s\\n"\n' % self.escape(line)) - else: - self.STDOUT.write('~"%s"' % self.escape(text)) + self.STDOUT.write(escape_gdbmi_stream("~", text)) self.STDOUT.flush() -def is_mi_mode(args): - return "--interpreter" in " ".join(args) +def is_gdbmi_mode(): + return "--interpreter" in " ".join(PlatformioCLI.leftover_args) + + +def escape_gdbmi_stream(prefix, stream): + bytes_stream = False + if is_bytes(stream): + bytes_stream = True + stream = stream.decode() + + if not stream: + return b"" if bytes_stream else "" + + ends_nl = stream.endswith("\n") + stream = re.sub(r"\\+", "\\\\\\\\", stream) + stream = stream.replace('"', '\\"') + stream = stream.replace("\n", "\\n") + stream = '%s"%s"' % (prefix, stream) + if ends_nl: + stream += "\n" + + return stream.encode() if bytes_stream else stream def get_default_debug_env(config): diff --git a/platformio/commands/debug/server.py b/platformio/commands/debug/server.py index 18b39e41..cdd7fa32 100644 --- a/platformio/commands/debug/server.py +++ b/platformio/commands/debug/server.py @@ -19,6 +19,7 @@ from twisted.internet import error # pylint: disable=import-error from twisted.internet import reactor # pylint: disable=import-error from platformio import exception, fs, util +from platformio.commands.debug.helpers import escape_gdbmi_stream, is_gdbmi_mode from platformio.commands.debug.process import BaseProcess from platformio.proc import where_is_program @@ -119,6 +120,11 @@ class DebugServer(BaseProcess): def get_debug_port(self): return self._debug_port + def outReceived(self, data): + super(DebugServer, self).outReceived( + escape_gdbmi_stream("@", data) if is_gdbmi_mode() else data + ) + def processEnded(self, reason): self._process_ended = True super(DebugServer, self).processEnded(reason) From 4b3f593df99293afa57730265a963c6f53b47345 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 6 Nov 2019 23:24:49 +0200 Subject: [PATCH 217/221] Bump version to 4.1.0rc9 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 54d3766e..6e834f90 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, "0rc8") +VERSION = (4, 1, "0rc9") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 2408c0a4c71f10919923f38b746d43dfc36b2054 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 6 Nov 2019 23:53:38 +0200 Subject: [PATCH 218/221] Fix incorrect info about build_type --- platformio/builder/tools/platformio.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 5fac08eb..818c07ff 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -114,10 +114,11 @@ def BuildProgram(env): # process framework scripts env.BuildFrameworks(env.get("PIOFRAMEWORK")) - if ( + is_build_type_debug = ( set(["debug", "sizedata"]) & set(COMMAND_LINE_TARGETS) or env.GetProjectOption("build_type") == "debug" - ): + ) + if is_build_type_debug: env.ConfigureDebugFlags() # remove specified flags @@ -151,7 +152,7 @@ def BuildProgram(env): ) ) - print("Building in %s mode" % env.GetProjectOption("build_type")) + print("Building in %s mode" % ("debug" if is_build_type_debug else "release")) return program From 95c1b0214c4ca3c18b28ce6d9774c078ce6dd666 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 7 Nov 2019 15:24:47 +0200 Subject: [PATCH 219/221] Rename "check_pattern" to "check_patterns" --- docs | 2 +- platformio/commands/check/command.py | 2 +- platformio/commands/check/tools/base.py | 2 +- platformio/project/options.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs b/docs index 8086fc8c..35356aec 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 8086fc8c6ac06666b9afb6407c2d21f60fccfe18 +Subproject commit 35356aeca0786029398c9adf6c7aad209279cad7 diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py index 9761c863..24575436 100644 --- a/platformio/commands/check/command.py +++ b/platformio/commands/check/command.py @@ -109,7 +109,7 @@ def cli( tool_options = dict( verbose=verbose, silent=silent, - pattern=pattern or env_options.get("check_pattern", default_patterns), + patterns=pattern or env_options.get("check_patterns", default_patterns), flags=flags or env_options.get("check_flags"), severity=[DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_HIGH]] if silent diff --git a/platformio/commands/check/tools/base.py b/platformio/commands/check/tools/base.py index 6a0d10b2..59951288 100644 --- a/platformio/commands/check/tools/base.py +++ b/platformio/commands/check/tools/base.py @@ -134,7 +134,7 @@ class CheckToolBase(object): # pylint: disable=too-many-instance-attributes return result.append(os.path.abspath(path)) - for pattern in self.options["pattern"]: + for pattern in self.options["patterns"]: for item in glob.glob(pattern): if not os.path.isdir(item): _add_file(item) diff --git a/platformio/project/options.py b/platformio/project/options.py index 3294084d..b4c1814f 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -537,7 +537,7 @@ ProjectOptions = OrderedDict( ), ConfigEnvOption( group="check", - name="check_pattern", + name="check_patterns", description=( "Configure a list of target files or directories for checking " "(Unix shell-style wildcards)" From 71168b1a5f3a6d8a9999b0f041d3fa20d5c0b920 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 7 Nov 2019 16:49:34 +0200 Subject: [PATCH 220/221] Replace IoT with Embedded --- README.rst | 14 ++++++++------ docs | 2 +- platformio/__init__.py | 10 ++++++---- platformio/maintenance.py | 2 +- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index abb3f5f6..c7be1767 100644 --- a/README.rst +++ b/README.rst @@ -34,16 +34,19 @@ PlatformIO .. image:: https://raw.githubusercontent.com/platformio/platformio-web/develop/app/images/platformio-ide-laptop.png :target: https://platformio.org?utm_source=github&utm_medium=core -`PlatformIO `_ is an open source ecosystem for IoT -development. Cross-platform IDE and unified debugger. Remote unit testing and -firmware updates. +`PlatformIO `_ an open source ecosystem for embedded development + +* **Cross-platform IDE** and **Unified Debugger** +* **Static Code Analyzer** and **Remote Unit Testing** +* **Multi-platform** and **Multi-architecture Build System** +* **Firmware File Explorer** and **Memory Inspection**. Get Started ----------- * `What is PlatformIO? `_ -Open Source +Instruments ----------- * `PlatformIO IDE `_ @@ -57,11 +60,10 @@ Open Source PIO Plus -------- +* `PIO Check `_ * `PIO Remote `_ * `PIO Unified Debugger `_ * `PIO Unit Testing `_ -* `Cloud IDEs Integration `_ -* `Integration Services `_ Registry -------- diff --git a/docs b/docs index 35356aec..5ee19204 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 35356aeca0786029398c9adf6c7aad209279cad7 +Subproject commit 5ee192047c6d58aa7087f24ad9b5f4d1ffce68e1 diff --git a/platformio/__init__.py b/platformio/__init__.py index 6e834f90..74f1af2b 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -17,11 +17,13 @@ __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" __description__ = ( - "An open source ecosystem for IoT development. " - "Cross-platform IDE and unified debugger. " - "Remote unit testing and firmware updates. " + "An open source ecosystem for embedded development. " + "Cross-platform IDE and Unified Debugger. " + "Static Code Analyzer and Remote Unit Testing. " + "Multi-platform and Multi-architecture Build System. " + "Firmware File Explorer and Memory Inspection. " "Arduino, ARM mbed, Espressif (ESP8266/ESP32), STM32, PIC32, nRF51/nRF52, " - "FPGA, CMSIS, SPL, AVR, Samsung ARTIK, libOpenCM3" + "RISC-V, FPGA, CMSIS, SPL, AVR, Samsung ARTIK, libOpenCM3" ) __url__ = "https://platformio.org" diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 046555a2..94a9140a 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -180,7 +180,7 @@ def after_upgrade(ctx): ) if not getenv("PLATFORMIO_IDE"): click.echo( - "- %s PlatformIO IDE for IoT development > %s" + "- %s PlatformIO IDE for embedded development > %s" % ( click.style("try", fg="cyan"), click.style("https://platformio.org/platformio-ide", fg="cyan"), From d043412e0f2f00e393ac46d961dc611d68415cd1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 7 Nov 2019 16:54:12 +0200 Subject: [PATCH 221/221] Bump version to 4.1.0 --- HISTORY.rst | 2 +- docs | 2 +- platformio/__init__.py | 2 +- platformio/managers/core.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 7cba3999..13a15253 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,7 +6,7 @@ Release Notes PlatformIO Core 4.0 ------------------- -4.1.0 (2019-??-??) +4.1.0 (2019-11-07) ~~~~~~~~~~~~~~~~~~ * `PIO Check `__ – automated code analysis without hassle: diff --git a/docs b/docs index 5ee19204..28f91efb 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 5ee192047c6d58aa7087f24ad9b5f4d1ffce68e1 +Subproject commit 28f91efb24b70301c7357956b2aa88dae0ad6cdd diff --git a/platformio/__init__.py b/platformio/__init__.py index 74f1af2b..893d83db 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, "0rc9") +VERSION = (4, 1, 0) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/managers/core.py b/platformio/managers/core.py index a3a33868..8e66aa19 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -24,7 +24,7 @@ from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path from platformio.project.config import ProjectConfig CORE_PACKAGES = { - "contrib-piohome": ">=3.0.0-beta.5,<3.1.0", + "contrib-piohome": "~3.0.0", "contrib-pysite": "~2.%d%d.0" % (sys.version_info[0], sys.version_info[1]), "tool-pioplus": "^2.5.8", "tool-unity": "~1.20403.0",