Merge branch 'release/v4.1.0'

.appveyor.yml

@@ -6,14 +6,15 @@ platform:
 environment:
   matrix:
     - TOXENV: "py27"
-      PLATFORMIO_BUILD_CACHE_DIR: C:/Temp/PIO_Build_Cache_P2_{build}
+      PLATFORMIO_BUILD_CACHE_DIR: C:\Temp\PIO_Build_Cache_P2_{build}
 
     - TOXENV: "py36"
-      PLATFORMIO_BUILD_CACHE_DIR: C:/Temp/PIO_Build_Cache_P3_{build}
+      PLATFORMIO_BUILD_CACHE_DIR: C:\Temp\PIO_Build_Cache_P3_{build}
 
 install:
   - cmd: git submodule update --init --recursive
   - cmd: SET PATH=C:\MinGW\bin;%PATH%
+  - cmd: SET PLATFORMIO_CORE_DIR=C:\.pio
   - cmd: pip install --force-reinstall tox
 
 test_script:

.isort.cfg

@@ -1,3 +1,3 @@
 [settings]
-line_length=79
-known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial,twisted,autobahn,jsonrpc,tabulate
+line_length=88
+known_third_party=SCons, twisted, autobahn, jsonrpc

.pylintrc

@@ -1,5 +1,7 @@
 [MESSAGES CONTROL]
 disable=
+    bad-continuation,
+    bad-whitespace,
     missing-docstring,
     ungrouped-imports,
     invalid-name,
@@ -9,4 +11,5 @@ disable=
     too-few-public-methods,
     useless-object-inheritance,
     useless-import-alias,
-    fixme
+    fixme,
+    bad-option-value

.readthedocs.yml

@@ -0,0 +1,12 @@
+# See https://docs.readthedocs.io/en/stable/config-file/index.html
+
+version: 2
+
+sphinx:
+  configuration: docs/conf.py
+
+formats:
+  - pdf
+
+submodules:
+  include: all

HISTORY.rst

@@ -3,8 +3,44 @@ Release Notes
 
 .. _release_notes_4_0:
 
-PlatformIO 4.0
---------------
+PlatformIO Core 4.0
+-------------------
+
+4.1.0 (2019-11-07)
+~~~~~~~~~~~~~~~~~~
+
+* `PIO Check <http://docs.platformio.org/page/plus/pio-check.html>`__ – automated code analysis without hassle:
+
+  - Potential NULL pointer dereferences
+  - Possible indexing beyond array bounds
+  - Suspicious assignments
+  - Reads of potentially uninitialized objects
+  - Unused variables or functions
+  - Out of scope memory usage.
+
+* `PlatformIO Home 3.0 <http://docs.platformio.org/page/home/index.html>`__ and Project Inspection
+
+  - Static Code Analysis
+  - Firmware File Explorer
+  - Firmware Memory Inspection
+  - Firmware Sections & Symbols Viewer.
+
+* Added support for `Build Middlewares <http://docs.platformio.org/page/projectconf/advanced_scripting.html#build-middlewares>`__: configure custom build flags per specific file, skip any build nodes from a framework, replace build file with another on-the-fly, etc.
+* Extend project environment configuration in "platformio.ini" with other sections using a new `extends <http://docs.platformio.org/page/projectconf/section_env_advanced.html#extends>`__ option (`issue #2953 <https://github.com/platformio/platformio-core/issues/2953>`_)
+* Generate ``.ccls`` LSP file for `Emacs <https://docs.platformio.org/page/ide/emacs.html>`__ cross references, hierarchies, completion and semantic highlighting
+* Added ``--no-ansi`` flag for `PIO Core <http://docs.platformio.org/page/userguide/index.html>`__ to disable ANSI control characters
+* Added ``--shutdown-timeout`` option to `PIO Home Server <http://docs.platformio.org/page/userguide/cmd_home.html>`__
+* Fixed an issue with project generator for `CLion IDE <http://docs.platformio.org/page/ide/clion.html>`__ when 2 environments were used (`issue #2824 <https://github.com/platformio/platformio-core/issues/2824>`_)
+* Fixed default PIO Unified Debugger configuration for `J-Link probe <http://docs.platformio.org/page/plus/debug-tools/jlink.html>`__
+* Fixed an issue when configuration file options partly ignored when using custom ``--project-conf`` (`issue #3034 <https://github.com/platformio/platformio-core/issues/3034>`_)
+* Fixed an issue when installing a package using custom Git tag and submodules were not updated correctly (`issue #3060 <https://github.com/platformio/platformio-core/issues/3060>`_)
+* Fixed an issue with linking process when ``$LDSCRIPT`` contains a space in path
+* Fixed security issue when extracting items from TAR archive (`issue #2995 <https://github.com/platformio/platformio-core/issues/2995>`_)
+* Fixed an issue with project generator when ``src_build_flags`` were not respected (`issue #3137 <https://github.com/platformio/platformio-core/issues/3137>`_)
+* Fixed an issue when booleans in "platformio.ini" are not parsed properly (`issue #3022 <https://github.com/platformio/platformio-core/issues/3022>`_)
+* Fixed an issue with invalid encoding when generating project for Visual Studio (`issue #3183 <https://github.com/platformio/platformio-core/issues/3183>`_)
+* Fixed an issue when Project Config Parser does not remove in-line comments when Python 3 is used (`issue #3213 <https://github.com/platformio/platformio-core/issues/3213>`_)
+* Fixed an issue with a GCC Linter for PlatformIO IDE for Atom (`issue #3218 <https://github.com/platformio/platformio-core/issues/3218>`_)
 
 4.0.3 (2019-08-30)
 ~~~~~~~~~~~~~~~~~~
@@ -104,8 +140,8 @@ PlatformIO 4.0
   - Fixed "systemd-udevd" warnings in `99-platformio-udev.rules <http://docs.platformio.org/page/faq.html#platformio-udev-rules>`__ (`issue #2442 <https://github.com/platformio/platformio-core/issues/2442>`_)
   - Fixed an issue when package cache (Library Manager) expires too fast (`issue #2559 <https://github.com/platformio/platformio-core/issues/2559>`_)
 
-PlatformIO 3.0
---------------
+PlatformIO Core 3.0
+-------------------
 
 3.6.7 (2019-04-23)
 ~~~~~~~~~~~~~~~~~~
@@ -705,8 +741,8 @@ PlatformIO 3.0
   (`issue #742 <https://github.com/platformio/platformio-core/issues/742>`_)
 * Stopped supporting Python 2.6
 
-PlatformIO 2.0
---------------
+PlatformIO Core 2.0
+--------------------
 
 2.11.2 (2016-08-02)
 ~~~~~~~~~~~~~~~~~~~
@@ -1491,8 +1527,8 @@ PlatformIO 2.0
 * Fixed bug with creating copies of source files
   (`issue #177 <https://github.com/platformio/platformio-core/issues/177>`_)
 
-PlatformIO 1.0
---------------
+PlatformIO Core 1.0
+-------------------
 
 1.5.0 (2015-05-15)
 ~~~~~~~~~~~~~~~~~~
@@ -1682,8 +1718,8 @@ PlatformIO 1.0
   error (`issue #81 <https://github.com/platformio/platformio-core/issues/81>`_)
 * Several bug fixes, increased stability and performance improvements
 
-PlatformIO 0.0
---------------
+PlatformIO Core 0.0
+-------------------
 
 0.10.2 (2015-01-06)
 ~~~~~~~~~~~~~~~~~~~

Makefile

@@ -5,13 +5,14 @@ isort:
 	isort -rc ./platformio
 	isort -rc ./tests
 
-yapf:
-	yapf --recursive --in-place platformio/
+black:
+	black --target-version py27 ./platformio
+	black --target-version py27 ./tests
 
 test:
-	py.test --verbose --capture=no --exitfirst -n 3 --dist=loadscope tests --ignore tests/test_examples.py --ignore tests/test_pkgmanifest.py
+	py.test --verbose --capture=no --exitfirst -n 3 --dist=loadscope tests --ignore tests/test_examples.py
 
-before-commit: isort yapf lint test
+before-commit: isort black lint test
 
 clean-docs:
 	rm -rf docs/_build

README.rst

@@ -34,16 +34,19 @@ PlatformIO
 .. image:: https://raw.githubusercontent.com/platformio/platformio-web/develop/app/images/platformio-ide-laptop.png
     :target: https://platformio.org?utm_source=github&utm_medium=core
 
-`PlatformIO <https://platformio.org?utm_source=github&utm_medium=core>`_ is an open source ecosystem for IoT
-development. Cross-platform IDE and unified debugger. Remote unit testing and
-firmware updates.
+`PlatformIO <https://platformio.org?utm_source=github&utm_medium=core>`_ an open source ecosystem for embedded development
+
+* **Cross-platform IDE** and **Unified Debugger**
+* **Static Code Analyzer** and **Remote Unit Testing**
+* **Multi-platform** and **Multi-architecture Build System**
+* **Firmware File Explorer** and **Memory Inspection**.
 
 Get Started
 -----------
 
 * `What is PlatformIO? <https://docs.platformio.org/en/latest/what-is-platformio.html?utm_source=github&utm_medium=core>`_
 
-Open Source
+Instruments
 -----------
 
 * `PlatformIO IDE <https://platformio.org/platformio-ide?utm_source=github&utm_medium=core>`_
@@ -57,11 +60,10 @@ Open Source
 PIO Plus
 --------
 
+* `PIO Check <https://docs.platformio.org/page/plus/pio-check.html?utm_source=github&utm_medium=core>`_
 * `PIO Remote <https://docs.platformio.org/page/plus/pio-remote.html?utm_source=github&utm_medium=core>`_
 * `PIO Unified Debugger <https://docs.platformio.org/page/plus/debugging.html?utm_source=github&utm_medium=core>`_
 * `PIO Unit Testing <https://docs.platformio.org/en/latest/plus/unit-testing.html?utm_source=github&utm_medium=core>`_
-* `Cloud IDEs Integration <https://docs.platformio.org/en/latest/ide.html?utm_source=github&utm_medium=core#solution-pio-delivery>`_
-* `Integration Services <https://platformio.org/pricing?utm_source=github&utm_medium=core#enterprise-features>`_
 
 Registry
 --------
@@ -93,6 +95,7 @@ Development Platforms
 * `RISC-V <https://platformio.org/platforms/riscv?utm_source=github&utm_medium=core>`_
 * `RISC-V GAP <https://platformio.org/platforms/riscv_gap?utm_source=github&utm_medium=core>`_
 * `Samsung ARTIK <https://platformio.org/platforms/samsung_artik?utm_source=github&utm_medium=core>`_
+* `Shakti <https://platformio.org/platforms/shakti?utm_source=github&utm_medium=core>`_
 * `Silicon Labs EFM32 <https://platformio.org/platforms/siliconlabsefm32?utm_source=github&utm_medium=core>`_
 * `ST STM32 <https://platformio.org/platforms/ststm32?utm_source=github&utm_medium=core>`_
 * `ST STM8 <https://platformio.org/platforms/ststm8?utm_source=github&utm_medium=core>`_
@@ -117,6 +120,7 @@ Frameworks
 * `mbed <https://platformio.org/frameworks/mbed?utm_source=github&utm_medium=core>`_
 * `PULP OS <https://platformio.org/frameworks/pulp-os?utm_source=github&utm_medium=core>`_
 * `Pumbaa <https://platformio.org/frameworks/pumbaa?utm_source=github&utm_medium=core>`_
+* `Shakti <https://platformio.org/frameworks/shakti?utm_source=github&utm_medium=core>`_
 * `Simba <https://platformio.org/frameworks/simba?utm_source=github&utm_medium=core>`_
 * `SPL <https://platformio.org/frameworks/spl?utm_source=github&utm_medium=core>`_
 * `STM32Cube <https://platformio.org/frameworks/stm32cube?utm_source=github&utm_medium=core>`_

platformio/__init__.py

@@ -12,16 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-VERSION = (4, 0, 3)
+VERSION = (4, 1, 0)
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"
 __description__ = (
-    "An open source ecosystem for IoT development. "
-    "Cross-platform IDE and unified debugger. "
-    "Remote unit testing and firmware updates. "
+    "An open source ecosystem for embedded development. "
+    "Cross-platform IDE and Unified Debugger. "
+    "Static Code Analyzer and Remote Unit Testing. "
+    "Multi-platform and Multi-architecture Build System. "
+    "Firmware File Explorer and Memory Inspection. "
     "Arduino, ARM mbed, Espressif (ESP8266/ESP32), STM32, PIC32, nRF51/nRF52, "
-    "FPGA, CMSIS, SPL, AVR, Samsung ARTIK, libOpenCM3")
+    "RISC-V, FPGA, CMSIS, SPL, AVR, Samsung ARTIK, libOpenCM3"
+)
 __url__ = "https://platformio.org"
 
 __author__ = "PlatformIO"

platformio/__main__.py

@@ -23,22 +23,42 @@ from platformio.commands import PlatformioCLI
 from platformio.compat import CYGWIN
 
 
-@click.command(cls=PlatformioCLI,
-               context_settings=dict(help_option_names=["-h", "--help"]))
+@click.command(
+    cls=PlatformioCLI, context_settings=dict(help_option_names=["-h", "--help"])
+)
 @click.version_option(__version__, prog_name="PlatformIO")
-@click.option("--force",
-              "-f",
-              is_flag=True,
-              help="Force to accept any confirmation prompts.")
-@click.option("--caller", "-c", help="Caller ID (service).")
+@click.option("--force", "-f", is_flag=True, help="DEPRECATE")
+@click.option("--caller", "-c", help="Caller ID (service)")
+@click.option("--no-ansi", is_flag=True, help="Do not print ANSI control characters")
 @click.pass_context
-def cli(ctx, force, caller):
+def cli(ctx, force, caller, no_ansi):
+    try:
+        if (
+            no_ansi
+            or str(
+                os.getenv("PLATFORMIO_NO_ANSI", os.getenv("PLATFORMIO_DISABLE_COLOR"))
+            ).lower()
+            == "true"
+        ):
+            # pylint: disable=protected-access
+            click._compat.isatty = lambda stream: False
+        elif (
+            str(
+                os.getenv("PLATFORMIO_FORCE_ANSI", os.getenv("PLATFORMIO_FORCE_COLOR"))
+            ).lower()
+            == "true"
+        ):
+            # pylint: disable=protected-access
+            click._compat.isatty = lambda stream: True
+    except:  # pylint: disable=bare-except
+        pass
+
     maintenance.on_platformio_start(ctx, force, caller)
 
 
 @cli.resultcallback()
 @click.pass_context
-def process_result(ctx, result, force, caller):  # pylint: disable=W0613
+def process_result(ctx, result, *_, **__):
     maintenance.on_platformio_end(ctx, result)
 
 
@@ -50,21 +70,12 @@ def configure():
     # https://urllib3.readthedocs.org
     # /en/latest/security.html#insecureplatformwarning
     try:
-        import urllib3
+        import urllib3  # pylint: disable=import-outside-toplevel
+
         urllib3.disable_warnings()
     except (AttributeError, ImportError):
         pass
 
-    try:
-        if str(os.getenv("PLATFORMIO_DISABLE_COLOR", "")).lower() == "true":
-            # pylint: disable=protected-access
-            click._compat.isatty = lambda stream: False
-        elif str(os.getenv("PLATFORMIO_FORCE_COLOR", "")).lower() == "true":
-            # pylint: disable=protected-access
-            click._compat.isatty = lambda stream: True
-    except:  # pylint: disable=bare-except
-        pass
-
     # Handle IOError issue with VSCode's Terminal (Windows)
     click_echo_origin = [click.echo, click.secho]
 
@@ -73,7 +84,8 @@ def configure():
             click_echo_origin[origin](*args, **kwargs)
         except IOError:
             (sys.stderr.write if kwargs.get("err") else sys.stdout.write)(
-                "%s\n" % (args[0] if args else ""))
+                "%s\n" % (args[0] if args else "")
+            )
 
     click.echo = lambda *args, **kwargs: _safe_echo(0, *args, **kwargs)
     click.secho = lambda *args, **kwargs: _safe_echo(1, *args, **kwargs)
@@ -87,7 +99,7 @@ def main(argv=None):
         sys.argv = argv
     try:
         configure()
-        cli(None, None, None)
+        cli()  # pylint: disable=no-value-for-parameter
     except SystemExit:
         pass
     except Exception as e:  # pylint: disable=broad-except

platformio/app.py

@@ -17,30 +17,19 @@ import hashlib
 import os
 import uuid
 from os import environ, getenv, listdir, remove
-from os.path import abspath, dirname, expanduser, isdir, isfile, join
+from os.path import abspath, dirname, isdir, isfile, join
 from time import time
 
 import requests
 
 from platformio import exception, fs, lockfile
-from platformio.compat import (WINDOWS, dump_json_to_unicode,
-                               hashlib_encode_data)
+from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data
 from platformio.proc import is_ci
-from platformio.project.helpers import (get_project_cache_dir,
-                                        get_project_core_dir)
-
-
-def get_default_projects_dir():
-    docs_dir = join(expanduser("~"), "Documents")
-    try:
-        assert WINDOWS
-        import ctypes.wintypes
-        buf = ctypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH)
-        ctypes.windll.shell32.SHGetFolderPathW(None, 5, None, 0, buf)
-        docs_dir = buf.value
-    except:  # pylint: disable=bare-except
-        pass
-    return join(docs_dir, "PlatformIO", "Projects")
+from platformio.project.helpers import (
+    get_default_projects_dir,
+    get_project_cache_dir,
+    get_project_core_dir,
+)
 
 
 def projects_dir_validate(projects_dir):
@@ -51,53 +40,53 @@ def projects_dir_validate(projects_dir):
 DEFAULT_SETTINGS = {
     "auto_update_libraries": {
         "description": "Automatically update libraries (Yes/No)",
-        "value": False
+        "value": False,
     },
     "auto_update_platforms": {
         "description": "Automatically update platforms (Yes/No)",
-        "value": False
+        "value": False,
     },
     "check_libraries_interval": {
         "description": "Check for the library updates interval (days)",
-        "value": 7
+        "value": 7,
     },
     "check_platformio_interval": {
         "description": "Check for the new PlatformIO interval (days)",
-        "value": 3
+        "value": 3,
     },
     "check_platforms_interval": {
         "description": "Check for the platform updates interval (days)",
-        "value": 7
+        "value": 7,
     },
     "enable_cache": {
         "description": "Enable caching for API requests and Library Manager",
-        "value": True
-    },
-    "strict_ssl": {
-        "description": "Strict SSL for PlatformIO Services",
-        "value": False
+        "value": True,
     },
+    "strict_ssl": {"description": "Strict SSL for PlatformIO Services", "value": False},
     "enable_telemetry": {
-        "description":
-        ("Telemetry service <http://bit.ly/pio-telemetry> (Yes/No)"),
-        "value": True
+        "description": ("Telemetry service <http://bit.ly/pio-telemetry> (Yes/No)"),
+        "value": True,
     },
     "force_verbose": {
         "description": "Force verbose output when processing environments",
-        "value": False
+        "value": False,
     },
     "projects_dir": {
         "description": "Default location for PlatformIO projects (PIO Home)",
         "value": get_default_projects_dir(),
-        "validator": projects_dir_validate
+        "validator": projects_dir_validate,
     },
 }
 
-SESSION_VARS = {"command_ctx": None, "force_option": False, "caller_id": None}
+SESSION_VARS = {
+    "command_ctx": None,
+    "force_option": False,
+    "caller_id": None,
+    "custom_project_conf": None,
+}
 
 
 class State(object):
-
     def __init__(self, path=None, lock=False):
         self.path = path
         self.lock = lock
@@ -113,8 +102,12 @@ class State(object):
             if isfile(self.path):
                 self._storage = fs.load_json(self.path)
             assert isinstance(self._storage, dict)
-        except (AssertionError, ValueError, UnicodeDecodeError,
-                exception.InvalidJSONFile):
+        except (
+            AssertionError,
+            ValueError,
+            UnicodeDecodeError,
+            exception.InvalidJSONFile,
+        ):
             self._storage = {}
         return self
 
@@ -174,7 +167,6 @@ class State(object):
 
 
 class ContentCache(object):
-
     def __init__(self, cache_dir=None):
         self.cache_dir = None
         self._db_path = None
@@ -277,8 +269,11 @@ class ContentCache(object):
                     continue
                 expire, path = line.split("=")
                 try:
-                    if time() < int(expire) and isfile(path) and \
-                            path not in paths_for_delete:
+                    if (
+                        time() < int(expire)
+                        and isfile(path)
+                        and path not in paths_for_delete
+                    ):
                         newlines.append(line)
                         continue
                 except ValueError:
@@ -317,11 +312,11 @@ def sanitize_setting(name, value):
     defdata = DEFAULT_SETTINGS[name]
     try:
         if "validator" in defdata:
-            value = defdata['validator'](value)
-        elif isinstance(defdata['value'], bool):
+            value = defdata["validator"](value)
+        elif isinstance(defdata["value"], bool):
             if not isinstance(value, bool):
                 value = str(value).lower() in ("true", "yes", "y", "1")
-        elif isinstance(defdata['value'], int):
+        elif isinstance(defdata["value"], int):
             value = int(value)
     except Exception:
         raise exception.InvalidSettingValue(value, name)
@@ -351,24 +346,24 @@ def get_setting(name):
         return sanitize_setting(name, getenv(_env_name))
 
     with State() as state:
-        if "settings" in state and name in state['settings']:
-            return state['settings'][name]
+        if "settings" in state and name in state["settings"]:
+            return state["settings"][name]
 
-    return DEFAULT_SETTINGS[name]['value']
+    return DEFAULT_SETTINGS[name]["value"]
 
 
 def set_setting(name, value):
     with State(lock=True) as state:
         if "settings" not in state:
-            state['settings'] = {}
-        state['settings'][name] = sanitize_setting(name, value)
+            state["settings"] = {}
+        state["settings"][name] = sanitize_setting(name, value)
         state.modified = True
 
 
 def reset_settings():
     with State(lock=True) as state:
         if "settings" in state:
-            del state['settings']
+            del state["settings"]
 
 
 def get_session_var(name, default=None):
@@ -381,11 +376,13 @@ def set_session_var(name, value):
 
 
 def is_disabled_progressbar():
-    return any([
-        get_session_var("force_option"),
-        is_ci(),
-        getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true"
-    ])
+    return any(
+        [
+            get_session_var("force_option"),
+            is_ci(),
+            getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true",
+        ]
+    )
 
 
 def get_cid():
@@ -397,15 +394,22 @@ def get_cid():
         uid = getenv("C9_UID")
     elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")):
         try:
-            uid = requests.get("{api}/user?token={token}".format(
-                api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")),
-                token=getenv("USER_TOKEN"))).json().get("id")
+            uid = (
+                requests.get(
+                    "{api}/user?token={token}".format(
+                        api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")),
+                        token=getenv("USER_TOKEN"),
+                    )
+                )
+                .json()
+                .get("id")
+            )
         except:  # pylint: disable=bare-except
             pass
     if not uid:
         uid = uuid.getnode()
     cid = uuid.UUID(bytes=hashlib.md5(hashlib_encode_data(uid)).digest())
     cid = str(cid)
-    if WINDOWS or os.getuid() > 0:  # yapf: disable pylint: disable=no-member
+    if WINDOWS or os.getuid() > 0:  # pylint: disable=no-member
         set_state_item("cid", cid)
     return cid

platformio/builder/main.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import sys
 from os import environ, makedirs
 from os.path import isdir, join
 from time import time
@@ -28,10 +29,10 @@ from SCons.Script import Import  # pylint: disable=import-error
 from SCons.Script import Variables  # pylint: disable=import-error
 
 from platformio import fs
-from platformio.compat import PY2, dump_json_to_unicode
+from platformio.compat import dump_json_to_unicode
 from platformio.managers.platform import PlatformBase
 from platformio.proc import get_pythonexe_path
-from platformio.project import helpers as project_helpers
+from platformio.project.helpers import get_project_dir
 
 AllowSubstExceptions(NameError)
 
@@ -43,48 +44,44 @@ clivars.AddVariables(
     ("PROJECT_CONFIG",),
     ("PIOENV",),
     ("PIOTEST_RUNNING_NAME",),
-    ("UPLOAD_PORT",)
-)  # yapf: disable
+    ("UPLOAD_PORT",),
+)
 
 DEFAULT_ENV_OPTIONS = dict(
     tools=[
-        "ar", "gas", "gcc", "g++", "gnulink", "platformio", "pioplatform",
-        "pioproject", "piowinhooks", "piolib", "pioupload", "piomisc", "pioide"
+        "ar",
+        "gas",
+        "gcc",
+        "g++",
+        "gnulink",
+        "platformio",
+        "pioplatform",
+        "pioproject",
+        "piomaxlen",
+        "piolib",
+        "pioupload",
+        "piomisc",
+        "pioide",
+        "piosize",
     ],
     toolpath=[join(fs.get_source_dir(), "builder", "tools")],
     variables=clivars,
-
     # Propagating External Environment
     ENV=environ,
     UNIX_TIME=int(time()),
-    PROJECT_DIR=project_helpers.get_project_dir(),
-    PROJECTCORE_DIR=project_helpers.get_project_core_dir(),
-    PROJECTPACKAGES_DIR=project_helpers.get_project_packages_dir(),
-    PROJECTWORKSPACE_DIR=project_helpers.get_project_workspace_dir(),
-    PROJECTLIBDEPS_DIR=project_helpers.get_project_libdeps_dir(),
-    PROJECTINCLUDE_DIR=project_helpers.get_project_include_dir(),
-    PROJECTSRC_DIR=project_helpers.get_project_src_dir(),
-    PROJECTTEST_DIR=project_helpers.get_project_test_dir(),
-    PROJECTDATA_DIR=project_helpers.get_project_data_dir(),
-    PROJECTBUILD_DIR=project_helpers.get_project_build_dir(),
-    BUILDCACHE_DIR=project_helpers.get_project_optional_dir("build_cache_dir"),
-    BUILD_DIR=join("$PROJECTBUILD_DIR", "$PIOENV"),
-    BUILDSRC_DIR=join("$BUILD_DIR", "src"),
-    BUILDTEST_DIR=join("$BUILD_DIR", "test"),
+    BUILD_DIR=join("$PROJECT_BUILD_DIR", "$PIOENV"),
+    BUILD_SRC_DIR=join("$BUILD_DIR", "src"),
+    BUILD_TEST_DIR=join("$BUILD_DIR", "test"),
     LIBPATH=["$BUILD_DIR"],
-    LIBSOURCE_DIRS=[
-        project_helpers.get_project_lib_dir(),
-        join("$PROJECTLIBDEPS_DIR", "$PIOENV"),
-        project_helpers.get_project_global_lib_dir()
-    ],
     PROGNAME="program",
     PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
-    PYTHONEXE=get_pythonexe_path())
+    PYTHONEXE=get_pythonexe_path(),
+)
 
 if not int(ARGUMENTS.get("PIOVERBOSE", 0)):
-    DEFAULT_ENV_OPTIONS['ARCOMSTR'] = "Archiving $TARGET"
-    DEFAULT_ENV_OPTIONS['LINKCOMSTR'] = "Linking $TARGET"
-    DEFAULT_ENV_OPTIONS['RANLIBCOMSTR'] = "Indexing $TARGET"
+    DEFAULT_ENV_OPTIONS["ARCOMSTR"] = "Archiving $TARGET"
+    DEFAULT_ENV_OPTIONS["LINKCOMSTR"] = "Linking $TARGET"
+    DEFAULT_ENV_OPTIONS["RANLIBCOMSTR"] = "Indexing $TARGET"
     for k in ("ASCOMSTR", "ASPPCOMSTR", "CCCOMSTR", "CXXCOMSTR"):
         DEFAULT_ENV_OPTIONS[k] = "Compiling $TARGET"
 
@@ -94,31 +91,59 @@ env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS)
 env.Replace(
     **{
         key: PlatformBase.decode_scons_arg(env[key])
-        for key in list(clivars.keys()) if key in env
-    })
+        for key in list(clivars.keys())
+        if key in env
+    }
+)
 
-if env.subst("$BUILDCACHE_DIR"):
-    if not isdir(env.subst("$BUILDCACHE_DIR")):
-        makedirs(env.subst("$BUILDCACHE_DIR"))
-    env.CacheDir("$BUILDCACHE_DIR")
+# Setup project optional directories
+config = env.GetProjectConfig()
+env.Replace(
+    PROJECT_DIR=get_project_dir(),
+    PROJECT_CORE_DIR=config.get_optional_dir("core"),
+    PROJECT_PACKAGES_DIR=config.get_optional_dir("packages"),
+    PROJECT_WORKSPACE_DIR=config.get_optional_dir("workspace"),
+    PROJECT_LIBDEPS_DIR=config.get_optional_dir("libdeps"),
+    PROJECT_INCLUDE_DIR=config.get_optional_dir("include"),
+    PROJECT_SRC_DIR=config.get_optional_dir("src"),
+    PROJECTSRC_DIR=config.get_optional_dir("src"),  # legacy for dev/platform
+    PROJECT_TEST_DIR=config.get_optional_dir("test"),
+    PROJECT_DATA_DIR=config.get_optional_dir("data"),
+    PROJECTDATA_DIR=config.get_optional_dir("data"),  # legacy for dev/platform
+    PROJECT_BUILD_DIR=config.get_optional_dir("build"),
+    BUILD_CACHE_DIR=config.get_optional_dir("build_cache"),
+    LIBSOURCE_DIRS=[
+        config.get_optional_dir("lib"),
+        join("$PROJECT_LIBDEPS_DIR", "$PIOENV"),
+        config.get_optional_dir("globallib"),
+    ],
+)
+
+if env.subst("$BUILD_CACHE_DIR"):
+    if not isdir(env.subst("$BUILD_CACHE_DIR")):
+        makedirs(env.subst("$BUILD_CACHE_DIR"))
+    env.CacheDir("$BUILD_CACHE_DIR")
 
 if int(ARGUMENTS.get("ISATTY", 0)):
     # pylint: disable=protected-access
     click._compat.isatty = lambda stream: True
 
-if env.GetOption('clean'):
+if env.GetOption("clean"):
     env.PioClean(env.subst("$BUILD_DIR"))
     env.Exit(0)
 elif not int(ARGUMENTS.get("PIOVERBOSE", 0)):
-    print("Verbose mode can be enabled via `-v, --verbose` option")
+    click.echo("Verbose mode can be enabled via `-v, --verbose` option")
+
+if not isdir(env.subst("$BUILD_DIR")):
+    makedirs(env.subst("$BUILD_DIR"))
 
 env.LoadProjectOptions()
 env.LoadPioPlatform()
 
 env.SConscriptChdir(0)
 env.SConsignFile(
-    join("$PROJECTBUILD_DIR",
-         ".sconsign.dblite" if PY2 else ".sconsign3.dblite"))
+    join("$BUILD_DIR", ".sconsign.py%d%d" % (sys.version_info[0], sys.version_info[1]))
+)
 
 for item in env.GetExtraScripts("pre"):
     env.SConscript(item, exports="env")
@@ -145,10 +170,13 @@ if env.get("SIZETOOL") and "nobuild" not in COMMAND_LINE_TARGETS:
     Default("checkprogsize")
 
 # Print configured protocols
-env.AddPreAction(["upload", "program"],
-                 env.VerboseAction(
-                     lambda source, target, env: env.PrintUploadInfo(),
-                     "Configuring upload protocol..."))
+env.AddPreAction(
+    ["upload", "program"],
+    env.VerboseAction(
+        lambda source, target, env: env.PrintUploadInfo(),
+        "Configuring upload protocol...",
+    ),
+)
 
 AlwaysBuild(env.Alias("debug", DEFAULT_TARGETS))
 AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS))
@@ -156,12 +184,26 @@ AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS))
 ##############################################################################
 
 if "envdump" in COMMAND_LINE_TARGETS:
-    print(env.Dump())
+    click.echo(env.Dump())
     env.Exit(0)
 
 if "idedata" in COMMAND_LINE_TARGETS:
     Import("projenv")
-    print("\n%s\n" % dump_json_to_unicode(
-        env.DumpIDEData(projenv)  # pylint: disable=undefined-variable
-    ))
+    click.echo(
+        "\n%s\n"
+        % dump_json_to_unicode(
+            projenv.DumpIDEData()  # pylint: disable=undefined-variable
+        )
+    )
     env.Exit(0)
+
+if "sizedata" in COMMAND_LINE_TARGETS:
+    AlwaysBuild(
+        env.Alias(
+            "sizedata",
+            DEFAULT_TARGETS,
+            env.VerboseAction(env.DumpSizeData, "Generating memory usage report..."),
+        )
+    )
+
+    Default("sizedata")

platformio/builder/tools/pioide.py

@@ -25,11 +25,11 @@ from platformio.managers.core import get_core_package_dir
 from platformio.proc import exec_command, where_is_program
 
 
-def _dump_includes(env, projenv):
+def _dump_includes(env):
     includes = []
 
-    for item in projenv.get("CPPPATH", []):
-        includes.append(projenv.subst(item))
+    for item in env.get("CPPPATH", []):
+        includes.append(env.subst(item))
 
     # installed libs
     for lb in env.GetLibBuilders():
@@ -45,7 +45,7 @@ def _dump_includes(env, projenv):
             join(toolchain_dir, "*", "include*"),
             join(toolchain_dir, "*", "include", "c++", "*"),
             join(toolchain_dir, "*", "include", "c++", "*", "*-*-*"),
-            join(toolchain_dir, "lib", "gcc", "*", "*", "include*")
+            join(toolchain_dir, "lib", "gcc", "*", "*", "include*"),
         ]
         for g in toolchain_incglobs:
             includes.extend(glob(g))
@@ -54,9 +54,7 @@ def _dump_includes(env, projenv):
     if unity_dir:
         includes.append(unity_dir)
 
-    includes.extend(
-        [env.subst("$PROJECTINCLUDE_DIR"),
-         env.subst("$PROJECTSRC_DIR")])
+    includes.extend([env.subst("$PROJECT_INCLUDE_DIR"), env.subst("$PROJECT_SRC_DIR")])
 
     # remove duplicates
     result = []
@@ -71,15 +69,15 @@ def _get_gcc_defines(env):
     items = []
     try:
         sysenv = environ.copy()
-        sysenv['PATH'] = str(env['ENV']['PATH'])
-        result = exec_command("echo | %s -dM -E -" % env.subst("$CC"),
-                              env=sysenv,
-                              shell=True)
+        sysenv["PATH"] = str(env["ENV"]["PATH"])
+        result = exec_command(
+            "echo | %s -dM -E -" % env.subst("$CC"), env=sysenv, shell=True
+        )
     except OSError:
         return items
-    if result['returncode'] != 0:
+    if result["returncode"] != 0:
         return items
-    for line in result['out'].split("\n"):
+    for line in result["out"].split("\n"):
         tokens = line.strip().split(" ", 2)
         if not tokens or tokens[0] != "#define":
             continue
@@ -94,17 +92,22 @@ def _dump_defines(env):
     defines = []
     # global symbols
     for item in processDefines(env.get("CPPDEFINES", [])):
-        defines.append(env.subst(item).replace('\\', ''))
+        defines.append(env.subst(item).replace("\\", ""))
 
     # special symbol for Atmel AVR MCU
-    if env['PIOPLATFORM'] == "atmelavr":
+    if env["PIOPLATFORM"] == "atmelavr":
         board_mcu = env.get("BOARD_MCU")
         if not board_mcu and "BOARD" in env:
             board_mcu = env.BoardConfig().get("build.mcu")
         if board_mcu:
             defines.append(
-                str("__AVR_%s__" % board_mcu.upper().replace(
-                    "ATMEGA", "ATmega").replace("ATTINY", "ATtiny")))
+                str(
+                    "__AVR_%s__"
+                    % board_mcu.upper()
+                    .replace("ATMEGA", "ATmega")
+                    .replace("ATTINY", "ATtiny")
+                )
+            )
 
     # built-in GCC marcos
     # if env.GetCompilerType() == "gcc":
@@ -135,38 +138,27 @@ def _get_svd_path(env):
     return None
 
 
-def DumpIDEData(env, projenv):
+def DumpIDEData(env):
     LINTCCOM = "$CFLAGS $CCFLAGS $CPPFLAGS"
     LINTCXXCOM = "$CXXFLAGS $CCFLAGS $CPPFLAGS"
 
     data = {
-        "env_name":
-        env['PIOENV'],
+        "env_name": env["PIOENV"],
         "libsource_dirs": [env.subst(l) for l in env.GetLibSourceDirs()],
-        "defines":
-        _dump_defines(env),
-        "includes":
-        _dump_includes(env, projenv),
-        "cc_flags":
-        env.subst(LINTCCOM),
-        "cxx_flags":
-        env.subst(LINTCXXCOM),
-        "cc_path":
-        where_is_program(env.subst("$CC"), env.subst("${ENV['PATH']}")),
-        "cxx_path":
-        where_is_program(env.subst("$CXX"), env.subst("${ENV['PATH']}")),
-        "gdb_path":
-        where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")),
-        "prog_path":
-        env.subst("$PROG_PATH"),
-        "flash_extra_images": [{
-            "offset": item[0],
-            "path": env.subst(item[1])
-        } for item in env.get("FLASH_EXTRA_IMAGES", [])],
-        "svd_path":
-        _get_svd_path(env),
-        "compiler_type":
-        env.GetCompilerType()
+        "defines": _dump_defines(env),
+        "includes": _dump_includes(env),
+        "cc_flags": env.subst(LINTCCOM),
+        "cxx_flags": env.subst(LINTCXXCOM),
+        "cc_path": where_is_program(env.subst("$CC"), env.subst("${ENV['PATH']}")),
+        "cxx_path": where_is_program(env.subst("$CXX"), env.subst("${ENV['PATH']}")),
+        "gdb_path": where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")),
+        "prog_path": env.subst("$PROG_PATH"),
+        "flash_extra_images": [
+            {"offset": item[0], "path": env.subst(item[1])}
+            for item in env.get("FLASH_EXTRA_IMAGES", [])
+        ],
+        "svd_path": _get_svd_path(env),
+        "compiler_type": env.GetCompilerType(),
     }
 
     env_ = env.Clone()
@@ -180,10 +172,7 @@ def DumpIDEData(env, projenv):
             _new_defines.append(item)
     env_.Replace(CPPDEFINES=_new_defines)
 
-    data.update({
-        "cc_flags": env_.subst(LINTCCOM),
-        "cxx_flags": env_.subst(LINTCXXCOM)
-    })
+    data.update({"cc_flags": env_.subst(LINTCCOM), "cxx_flags": env_.subst(LINTCXXCOM)})
 
     return data
 

platformio/builder/tools/piolib.py

@@ -17,13 +17,11 @@
 
 from __future__ import absolute_import
 
-import codecs
 import hashlib
 import os
 import re
 import sys
-from os.path import (basename, commonprefix, expanduser, isdir, isfile, join,
-                     realpath, sep)
+from os.path import basename, commonprefix, isdir, isfile, join, realpath, sep
 
 import click
 import SCons.Scanner  # pylint: disable=import-error
@@ -33,13 +31,13 @@ from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 
 from platformio import exception, fs, util
 from platformio.builder.tools import platformio as piotool
-from platformio.compat import (WINDOWS, get_file_contents, hashlib_encode_data,
-                               string_types)
+from platformio.compat import WINDOWS, hashlib_encode_data, string_types
 from platformio.managers.lib import LibraryManager
+from platformio.package.manifest.parser import ManifestParserFactory
+from platformio.project.options import ProjectOptions
 
 
 class LibBuilderFactory(object):
-
     @staticmethod
     def new(env, path, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))):
         clsname = "UnknownLibBuilder"
@@ -47,31 +45,30 @@ class LibBuilderFactory(object):
             clsname = "PlatformIOLibBuilder"
         else:
             used_frameworks = LibBuilderFactory.get_used_frameworks(env, path)
-            common_frameworks = (set(env.get("PIOFRAMEWORK", []))
-                                 & set(used_frameworks))
+            common_frameworks = set(env.get("PIOFRAMEWORK", [])) & set(used_frameworks)
             if common_frameworks:
                 clsname = "%sLibBuilder" % list(common_frameworks)[0].title()
             elif used_frameworks:
                 clsname = "%sLibBuilder" % used_frameworks[0].title()
 
-        obj = getattr(sys.modules[__name__], clsname)(env,
-                                                      path,
-                                                      verbose=verbose)
+        obj = getattr(sys.modules[__name__], clsname)(env, path, verbose=verbose)
         assert isinstance(obj, LibBuilderBase)
         return obj
 
     @staticmethod
     def get_used_frameworks(env, path):
         if any(
-                isfile(join(path, fname))
-                for fname in ("library.properties", "keywords.txt")):
+            isfile(join(path, fname))
+            for fname in ("library.properties", "keywords.txt")
+        ):
             return ["arduino"]
 
         if isfile(join(path, "module.json")):
             return ["mbed"]
 
-        include_re = re.compile(r'^#include\s+(<|")(Arduino|mbed)\.h(<|")',
-                                flags=re.MULTILINE)
+        include_re = re.compile(
+            r'^#include\s+(<|")(Arduino|mbed)\.h(<|")', flags=re.MULTILINE
+        )
 
         # check source files
         for root, _, files in os.walk(path, followlinks=True):
@@ -79,9 +76,10 @@ class LibBuilderFactory(object):
                 return ["mbed"]
             for fname in files:
                 if not fs.path_endswith_ext(
-                        fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT):
+                    fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT
+                ):
                     continue
-                content = get_file_contents(join(root, fname))
+                content = fs.get_file_contents(join(root, fname))
                 if not content:
                     continue
                 if "Arduino.h" in content and include_re.search(content):
@@ -93,12 +91,6 @@ class LibBuilderFactory(object):
 
 class LibBuilderBase(object):
 
-    LDF_MODES = ["off", "chain", "deep", "chain+", "deep+"]
-    LDF_MODE_DEFAULT = "chain"
-
-    COMPAT_MODES = ["off", "soft", "strict"]
-    COMPAT_MODE_DEFAULT = "soft"
-
     CLASSIC_SCANNER = SCons.Scanner.C.CScanner()
     CCONDITIONAL_SCANNER = SCons.Scanner.C.CConditionalScanner()
     # Max depth of nested includes:
@@ -124,7 +116,7 @@ class LibBuilderBase(object):
         self._processed_files = list()
 
         # reset source filter, could be overridden with extra script
-        self.env['SRC_FILTER'] = ""
+        self.env["SRC_FILTER"] = ""
 
         # process extra options and append to build environment
         self.process_extra_options()
@@ -153,7 +145,8 @@ class LibBuilderBase(object):
     @property
     def dependencies(self):
         return LibraryManager.normalize_dependencies(
-            self._manifest.get("dependencies", []))
+            self._manifest.get("dependencies", [])
+        )
 
     @property
     def src_filter(self):
@@ -161,7 +154,7 @@ class LibBuilderBase(object):
             "-<example%s>" % os.sep,
             "-<examples%s>" % os.sep,
             "-<test%s>" % os.sep,
-            "-<tests%s>" % os.sep
+            "-<tests%s>" % os.sep,
         ]
 
     @property
@@ -172,8 +165,7 @@ class LibBuilderBase(object):
 
     @property
     def src_dir(self):
-        return (join(self.path, "src")
-                if isdir(join(self.path, "src")) else self.path)
+        return join(self.path, "src") if isdir(join(self.path, "src")) else self.path
 
     def get_include_dirs(self):
         items = []
@@ -214,40 +206,41 @@ class LibBuilderBase(object):
 
     @property
     def lib_archive(self):
-        return self.env.GetProjectOption("lib_archive", True)
+        return self.env.GetProjectOption("lib_archive")
 
     @property
     def lib_ldf_mode(self):
-        return self.env.GetProjectOption("lib_ldf_mode", self.LDF_MODE_DEFAULT)
+        return self.env.GetProjectOption("lib_ldf_mode")
 
     @staticmethod
     def validate_ldf_mode(mode):
+        ldf_modes = ProjectOptions["env.lib_ldf_mode"].type.choices
         if isinstance(mode, string_types):
             mode = mode.strip().lower()
-        if mode in LibBuilderBase.LDF_MODES:
+        if mode in ldf_modes:
             return mode
         try:
-            return LibBuilderBase.LDF_MODES[int(mode)]
+            return ldf_modes[int(mode)]
         except (IndexError, ValueError):
             pass
-        return LibBuilderBase.LDF_MODE_DEFAULT
+        return ProjectOptions["env.lib_ldf_mode"].default
 
     @property
     def lib_compat_mode(self):
-        return self.env.GetProjectOption("lib_compat_mode",
-                                         self.COMPAT_MODE_DEFAULT)
+        return self.env.GetProjectOption("lib_compat_mode")
 
     @staticmethod
     def validate_compat_mode(mode):
+        compat_modes = ProjectOptions["env.lib_compat_mode"].type.choices
         if isinstance(mode, string_types):
             mode = mode.strip().lower()
-        if mode in LibBuilderBase.COMPAT_MODES:
+        if mode in compat_modes:
             return mode
         try:
-            return LibBuilderBase.COMPAT_MODES[int(mode)]
+            return compat_modes[int(mode)]
         except (IndexError, ValueError):
             pass
-        return LibBuilderBase.COMPAT_MODE_DEFAULT
+        return ProjectOptions["env.lib_compat_mode"].default
 
     def is_platforms_compatible(self, platforms):
         return True
@@ -263,11 +256,10 @@ class LibBuilderBase(object):
             self.env.ProcessFlags(self.build_flags)
             if self.extra_script:
                 self.env.SConscriptChdir(1)
-                self.env.SConscript(realpath(self.extra_script),
-                                    exports={
-                                        "env": self.env,
-                                        "pio_lib_builder": self
-                                    })
+                self.env.SConscript(
+                    realpath(self.extra_script),
+                    exports={"env": self.env, "pio_lib_builder": self},
+                )
             self.env.ProcessUnFlags(self.build_unflags)
 
     def process_dependencies(self):
@@ -276,7 +268,7 @@ class LibBuilderBase(object):
         for item in self.dependencies:
             found = False
             for lb in self.env.GetLibBuilders():
-                if item['name'] != lb.name:
+                if item["name"] != lb.name:
                     continue
                 found = True
                 if lb not in self.depbuilders:
@@ -284,37 +276,48 @@ class LibBuilderBase(object):
                 break
 
             if not found and self.verbose:
-                sys.stderr.write("Warning: Ignored `%s` dependency for `%s` "
-                                 "library\n" % (item['name'], self.name))
+                sys.stderr.write(
+                    "Warning: Ignored `%s` dependency for `%s` "
+                    "library\n" % (item["name"], self.name)
+                )
 
     def get_search_files(self):
         items = [
-            join(self.src_dir, item) for item in self.env.MatchSourceFiles(
-                self.src_dir, self.src_filter)
+            join(self.src_dir, item)
+            for item in self.env.MatchSourceFiles(self.src_dir, self.src_filter)
         ]
         include_dir = self.include_dir
         if include_dir:
-            items.extend([
-                join(include_dir, item)
-                for item in self.env.MatchSourceFiles(include_dir)
-            ])
+            items.extend(
+                [
+                    join(include_dir, item)
+                    for item in self.env.MatchSourceFiles(include_dir)
+                ]
+            )
         return items
 
     def _get_found_includes(  # pylint: disable=too-many-branches
-            self, search_files=None):
+        self, search_files=None
+    ):
         # all include directories
         if not LibBuilderBase._INCLUDE_DIRS_CACHE:
-            LibBuilderBase._INCLUDE_DIRS_CACHE = []
+            LibBuilderBase._INCLUDE_DIRS_CACHE = [
+                self.env.Dir(d)
+                for d in ProjectAsLibBuilder(
+                    self.envorigin, "$PROJECT_DIR"
+                ).get_include_dirs()
+            ]
             for lb in self.env.GetLibBuilders():
                 LibBuilderBase._INCLUDE_DIRS_CACHE.extend(
-                    [self.env.Dir(d) for d in lb.get_include_dirs()])
+                    [self.env.Dir(d) for d in lb.get_include_dirs()]
+                )
 
         # append self include directories
         include_dirs = [self.env.Dir(d) for d in self.get_include_dirs()]
         include_dirs.extend(LibBuilderBase._INCLUDE_DIRS_CACHE)
 
         result = []
-        for path in (search_files or []):
+        for path in search_files or []:
             if path in self._processed_files:
                 continue
             self._processed_files.append(path)
@@ -325,21 +328,27 @@ class LibBuilderBase(object):
                     self.env.File(path),
                     self.env,
                     tuple(include_dirs),
-                    depth=self.CCONDITIONAL_SCANNER_DEPTH)
+                    depth=self.CCONDITIONAL_SCANNER_DEPTH,
+                )
                 # mark candidates already processed via Conditional Scanner
-                self._processed_files.extend([
-                    c.get_abspath() for c in candidates
-                    if c.get_abspath() not in self._processed_files
-                ])
+                self._processed_files.extend(
+                    [
+                        c.get_abspath()
+                        for c in candidates
+                        if c.get_abspath() not in self._processed_files
+                    ]
+                )
             except Exception as e:  # pylint: disable=broad-except
                 if self.verbose and "+" in self.lib_ldf_mode:
                     sys.stderr.write(
                         "Warning! Classic Pre Processor is used for `%s`, "
-                        "advanced has failed with `%s`\n" % (path, e))
+                        "advanced has failed with `%s`\n" % (path, e)
+                    )
                 candidates = LibBuilderBase.CLASSIC_SCANNER(
-                    self.env.File(path), self.env, tuple(include_dirs))
+                    self.env.File(path), self.env, tuple(include_dirs)
+                )
 
-            # print(path, map(lambda n: n.get_abspath(), candidates))
+            # print(path, [c.get_abspath() for c in candidates])
             for item in candidates:
                 if item not in result:
                     result.append(item)
@@ -348,7 +357,7 @@ class LibBuilderBase(object):
                 _h_path = item.get_abspath()
                 if not fs.path_endswith_ext(_h_path, piotool.SRC_HEADER_EXT):
                     continue
-                _f_part = _h_path[:_h_path.rindex(".")]
+                _f_part = _h_path[: _h_path.rindex(".")]
                 for ext in piotool.SRC_C_EXT:
                     if not isfile("%s.%s" % (_f_part, ext)):
                         continue
@@ -359,7 +368,6 @@ class LibBuilderBase(object):
         return result
 
     def depend_recursive(self, lb, search_files=None):
-
         def _already_depends(_lb):
             if self in _lb.depbuilders:
                 return True
@@ -372,9 +380,10 @@ class LibBuilderBase(object):
         if self != lb:
             if _already_depends(lb):
                 if self.verbose:
-                    sys.stderr.write("Warning! Circular dependencies detected "
-                                     "between `%s` and `%s`\n" %
-                                     (self.path, lb.path))
+                    sys.stderr.write(
+                        "Warning! Circular dependencies detected "
+                        "between `%s` and `%s`\n" % (self.path, lb.path)
+                    )
                 self._circular_deps.append(lb)
             elif lb not in self._depbuilders:
                 self._depbuilders.append(lb)
@@ -431,11 +440,10 @@ class LibBuilderBase(object):
 
         if self.lib_archive:
             libs.append(
-                self.env.BuildLibrary(self.build_dir, self.src_dir,
-                                      self.src_filter))
+                self.env.BuildLibrary(self.build_dir, self.src_dir, self.src_filter)
+            )
         else:
-            self.env.BuildSources(self.build_dir, self.src_dir,
-                                  self.src_filter)
+            self.env.BuildSources(self.build_dir, self.src_dir, self.src_filter)
         return libs
 
 
@@ -444,19 +452,11 @@ class UnknownLibBuilder(LibBuilderBase):
 
 
 class ArduinoLibBuilder(LibBuilderBase):
-
     def load_manifest(self):
-        manifest = {}
-        if not isfile(join(self.path, "library.properties")):
-            return manifest
         manifest_path = join(self.path, "library.properties")
-        with codecs.open(manifest_path, encoding="utf-8") as fp:
-            for line in fp.readlines():
-                if "=" not in line:
-                    continue
-                key, value = line.split("=", 1)
-                manifest[key.strip()] = value.strip()
-        return manifest
+        if not isfile(manifest_path):
+            return {}
+        return ManifestParserFactory.new_from_file(manifest_path).as_dict()
 
     def get_include_dirs(self):
         include_dirs = LibBuilderBase.get_include_dirs(self)
@@ -500,35 +500,18 @@ class ArduinoLibBuilder(LibBuilderBase):
         return util.items_in_list(frameworks, ["arduino", "energia"])
 
     def is_platforms_compatible(self, platforms):
-        platforms_map = {
-            "avr": ["atmelavr"],
-            "sam": ["atmelsam"],
-            "samd": ["atmelsam"],
-            "esp8266": ["espressif8266"],
-            "esp32": ["espressif32"],
-            "arc32": ["intel_arc32"],
-            "stm32": ["ststm32"],
-            "nrf5": ["nordicnrf51", "nordicnrf52"]
-        }
-        items = []
-        for arch in self._manifest.get("architectures", "").split(","):
-            arch = arch.strip().lower()
-            if arch == "*":
-                items = "*"
-                break
-            if arch in platforms_map:
-                items.extend(platforms_map[arch])
+        items = self._manifest.get("platforms", [])
         if not items:
             return LibBuilderBase.is_platforms_compatible(self, platforms)
         return util.items_in_list(platforms, items)
 
 
 class MbedLibBuilder(LibBuilderBase):
-
     def load_manifest(self):
-        if not isfile(join(self.path, "module.json")):
+        manifest_path = join(self.path, "module.json")
+        if not isfile(manifest_path):
             return {}
-        return fs.load_json(join(self.path, "module.json"))
+        return ManifestParserFactory.new_from_file(manifest_path).as_dict()
 
     @property
     def include_dir(self):
@@ -583,8 +566,7 @@ class MbedLibBuilder(LibBuilderBase):
             mbed_config_path = join(self.env.subst(p), "mbed_config.h")
             if isfile(mbed_config_path):
                 break
-            else:
-                mbed_config_path = None
+            mbed_config_path = None
         if not mbed_config_path:
             return None
 
@@ -611,14 +593,15 @@ class MbedLibBuilder(LibBuilderBase):
         # default macros
         for macro in manifest.get("macros", []):
             macro = self._mbed_normalize_macro(macro)
-            macros[macro['name']] = macro
+            macros[macro["name"]] = macro
 
         # configuration items
         for key, options in manifest.get("config", {}).items():
             if "value" not in options:
                 continue
-            macros[key] = dict(name=options.get("macro_name"),
-                               value=options.get("value"))
+            macros[key] = dict(
+                name=options.get("macro_name"), value=options.get("value")
+            )
 
         # overrode items per target
         for target, options in manifest.get("target_overrides", {}).items():
@@ -626,25 +609,23 @@ class MbedLibBuilder(LibBuilderBase):
                 continue
             for macro in options.get("target.macros_add", []):
                 macro = self._mbed_normalize_macro(macro)
-                macros[macro['name']] = macro
+                macros[macro["name"]] = macro
             for key, value in options.items():
                 if not key.startswith("target.") and key in macros:
-                    macros[key]['value'] = value
+                    macros[key]["value"] = value
 
         # normalize macro names
         for key, macro in macros.items():
-            if not macro['name']:
-                macro['name'] = key
-                if "." not in macro['name']:
-                    macro['name'] = "%s.%s" % (manifest.get("name"),
-                                               macro['name'])
-                macro['name'] = re.sub(r"[^a-z\d]+",
-                                       "_",
-                                       macro['name'],
-                                       flags=re.I).upper()
-                macro['name'] = "MBED_CONF_" + macro['name']
-            if isinstance(macro['value'], bool):
-                macro['value'] = 1 if macro['value'] else 0
+            if not macro["name"]:
+                macro["name"] = key
+                if "." not in macro["name"]:
+                    macro["name"] = "%s.%s" % (manifest.get("name"), macro["name"])
+                macro["name"] = re.sub(
+                    r"[^a-z\d]+", "_", macro["name"], flags=re.I
+                ).upper()
+                macro["name"] = "MBED_CONF_" + macro["name"]
+            if isinstance(macro["value"], bool):
+                macro["value"] = 1 if macro["value"] else 0
 
         return {macro["name"]: macro["value"] for macro in macros.values()}
 
@@ -654,13 +635,13 @@ class MbedLibBuilder(LibBuilderBase):
             for line in fp.readlines():
                 line = line.strip()
                 if line == "#endif":
-                    lines.append(
-                        "// PlatformIO Library Dependency Finder (LDF)")
-                    lines.extend([
-                        "#define %s %s" %
-                        (name, value if value is not None else "")
-                        for name, value in macros.items()
-                    ])
+                    lines.append("// PlatformIO Library Dependency Finder (LDF)")
+                    lines.extend(
+                        [
+                            "#define %s %s" % (name, value if value is not None else "")
+                            for name, value in macros.items()
+                        ]
+                    )
                     lines.append("")
                 if not line.startswith("#define"):
                     lines.append(line)
@@ -674,22 +655,13 @@ class MbedLibBuilder(LibBuilderBase):
 
 
 class PlatformIOLibBuilder(LibBuilderBase):
-
     def load_manifest(self):
-        assert isfile(join(self.path, "library.json"))
-        manifest = fs.load_json(join(self.path, "library.json"))
-        assert "name" in manifest
+        manifest_path = join(self.path, "library.json")
+        if not isfile(manifest_path):
+            return {}
+        return ManifestParserFactory.new_from_file(manifest_path).as_dict()
 
-        # replace "espressif" old name dev/platform with ESP8266
-        if "platforms" in manifest:
-            manifest['platforms'] = [
-                "espressif8266" if p == "espressif" else p
-                for p in util.items_to_list(manifest['platforms'])
-            ]
-
-        return manifest
-
-    def _is_arduino_manifest(self):
+    def _has_arduino_manifest(self):
         return isfile(join(self.path, "library.properties"))
 
     @property
@@ -710,9 +682,9 @@ class PlatformIOLibBuilder(LibBuilderBase):
     def src_filter(self):
         if "srcFilter" in self._manifest.get("build", {}):
             return self._manifest.get("build").get("srcFilter")
-        if self.env['SRC_FILTER']:
-            return self.env['SRC_FILTER']
-        if self._is_arduino_manifest():
+        if self.env["SRC_FILTER"]:
+            return self.env["SRC_FILTER"]
+        if self._has_arduino_manifest():
             return ArduinoLibBuilder.src_filter.fget(self)
         return LibBuilderBase.src_filter.fget(self)
 
@@ -736,11 +708,13 @@ class PlatformIOLibBuilder(LibBuilderBase):
 
     @property
     def lib_archive(self):
-        global_value = self.env.GetProjectOption("lib_archive")
-        if global_value is not None:
+        unique_value = "_not_declared_%s" % id(self)
+        global_value = self.env.GetProjectOption("lib_archive", unique_value)
+        if global_value != unique_value:
             return global_value
         return self._manifest.get("build", {}).get(
-            "libArchive", LibBuilderBase.lib_archive.fget(self))
+            "libArchive", LibBuilderBase.lib_archive.fget(self)
+        )
 
     @property
     def lib_ldf_mode(self):
@@ -748,7 +722,10 @@ class PlatformIOLibBuilder(LibBuilderBase):
             self.env.GetProjectOption(
                 "lib_ldf_mode",
                 self._manifest.get("build", {}).get(
-                    "libLDFMode", LibBuilderBase.lib_ldf_mode.fget(self))))
+                    "libLDFMode", LibBuilderBase.lib_ldf_mode.fget(self)
+                ),
+            )
+        )
 
     @property
     def lib_compat_mode(self):
@@ -756,8 +733,10 @@ class PlatformIOLibBuilder(LibBuilderBase):
             self.env.GetProjectOption(
                 "lib_compat_mode",
                 self._manifest.get("build", {}).get(
-                    "libCompatMode",
-                    LibBuilderBase.lib_compat_mode.fget(self))))
+                    "libCompatMode", LibBuilderBase.lib_compat_mode.fget(self)
+                ),
+            )
+        )
 
     def is_platforms_compatible(self, platforms):
         items = self._manifest.get("platforms")
@@ -775,9 +754,12 @@ class PlatformIOLibBuilder(LibBuilderBase):
         include_dirs = LibBuilderBase.get_include_dirs(self)
 
         # backwards compatibility with PlatformIO 2.0
-        if ("build" not in self._manifest and self._is_arduino_manifest()
-                and not isdir(join(self.path, "src"))
-                and isdir(join(self.path, "utility"))):
+        if (
+            "build" not in self._manifest
+            and self._has_arduino_manifest()
+            and not isdir(join(self.path, "src"))
+            and isdir(join(self.path, "utility"))
+        ):
             include_dirs.append(join(self.path, "utility"))
 
         for path in self.env.get("CPPPATH", []):
@@ -788,25 +770,24 @@ class PlatformIOLibBuilder(LibBuilderBase):
 
 
 class ProjectAsLibBuilder(LibBuilderBase):
-
     def __init__(self, env, *args, **kwargs):
         # backup original value, will be reset in base.__init__
         project_src_filter = env.get("SRC_FILTER")
         super(ProjectAsLibBuilder, self).__init__(env, *args, **kwargs)
-        self.env['SRC_FILTER'] = project_src_filter
+        self.env["SRC_FILTER"] = project_src_filter
 
     @property
     def include_dir(self):
-        include_dir = self.env.subst("$PROJECTINCLUDE_DIR")
+        include_dir = self.env.subst("$PROJECT_INCLUDE_DIR")
         return include_dir if isdir(include_dir) else None
 
     @property
     def src_dir(self):
-        return self.env.subst("$PROJECTSRC_DIR")
+        return self.env.subst("$PROJECT_SRC_DIR")
 
     def get_include_dirs(self):
         include_dirs = []
-        project_include_dir = self.env.subst("$PROJECTINCLUDE_DIR")
+        project_include_dir = self.env.subst("$PROJECT_INCLUDE_DIR")
         if isdir(project_include_dir):
             include_dirs.append(project_include_dir)
         for include_dir in LibBuilderBase.get_include_dirs(self):
@@ -819,11 +800,14 @@ class ProjectAsLibBuilder(LibBuilderBase):
         items = LibBuilderBase.get_search_files(self)
         # test files
         if "__test" in COMMAND_LINE_TARGETS:
-            items.extend([
-                join("$PROJECTTEST_DIR",
-                     item) for item in self.env.MatchSourceFiles(
-                         "$PROJECTTEST_DIR", "$PIOTEST_SRC_FILTER")
-            ])
+            items.extend(
+                [
+                    join("$PROJECT_TEST_DIR", item)
+                    for item in self.env.MatchSourceFiles(
+                        "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER"
+                    )
+                ]
+            )
         return items
 
     @property
@@ -836,8 +820,7 @@ class ProjectAsLibBuilder(LibBuilderBase):
 
     @property
     def src_filter(self):
-        return (self.env.get("SRC_FILTER")
-                or LibBuilderBase.src_filter.fget(self))
+        return self.env.get("SRC_FILTER") or LibBuilderBase.src_filter.fget(self)
 
     @property
     def dependencies(self):
@@ -848,7 +831,6 @@ class ProjectAsLibBuilder(LibBuilderBase):
         pass
 
     def install_dependencies(self):
-
         def _is_builtin(uri):
             for lb in self.env.GetLibBuilders():
                 if lb.name == uri:
@@ -871,8 +853,7 @@ class ProjectAsLibBuilder(LibBuilderBase):
                 not_found_uri.append(uri)
 
         did_install = False
-        lm = LibraryManager(
-            self.env.subst(join("$PROJECTLIBDEPS_DIR", "$PIOENV")))
+        lm = LibraryManager(self.env.subst(join("$PROJECT_LIBDEPS_DIR", "$PIOENV")))
         for uri in not_found_uri:
             try:
                 lm.install(uri)
@@ -923,28 +904,26 @@ class ProjectAsLibBuilder(LibBuilderBase):
 
 def GetLibSourceDirs(env):
     items = env.GetProjectOption("lib_extra_dirs", [])
-    items.extend(env['LIBSOURCE_DIRS'])
+    items.extend(env["LIBSOURCE_DIRS"])
     return [
-        env.subst(expanduser(item) if item.startswith("~") else item)
+        env.subst(fs.expanduser(item) if item.startswith("~") else item)
         for item in items
     ]
 
 
-def IsCompatibleLibBuilder(env,
-                           lb,
-                           verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))):
+def IsCompatibleLibBuilder(env, lb, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))):
     compat_mode = lb.lib_compat_mode
     if lb.name in env.GetProjectOption("lib_ignore", []):
         if verbose:
             sys.stderr.write("Ignored library %s\n" % lb.path)
         return None
-    if compat_mode == "strict" and not lb.is_platforms_compatible(
-            env['PIOPLATFORM']):
+    if compat_mode == "strict" and not lb.is_platforms_compatible(env["PIOPLATFORM"]):
         if verbose:
             sys.stderr.write("Platform incompatible library %s\n" % lb.path)
         return False
-    if (compat_mode in ("soft", "strict") and "PIOFRAMEWORK" in env
-            and not lb.is_frameworks_compatible(env.get("PIOFRAMEWORK", []))):
+    if compat_mode in ("soft", "strict") and not lb.is_frameworks_compatible(
+        env.get("PIOFRAMEWORK", [])
+    ):
         if verbose:
             sys.stderr.write("Framework incompatible library %s\n" % lb.path)
         return False
@@ -953,8 +932,10 @@ def IsCompatibleLibBuilder(env,
 
 def GetLibBuilders(env):  # pylint: disable=too-many-branches
     if DefaultEnvironment().get("__PIO_LIB_BUILDERS", None) is not None:
-        return sorted(DefaultEnvironment()['__PIO_LIB_BUILDERS'],
-                      key=lambda lb: 0 if lb.dependent else 1)
+        return sorted(
+            DefaultEnvironment()["__PIO_LIB_BUILDERS"],
+            key=lambda lb: 0 if lb.dependent else 1,
+        )
 
     DefaultEnvironment().Replace(__PIO_LIB_BUILDERS=[])
 
@@ -974,7 +955,8 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
             except exception.InvalidJSONFile:
                 if verbose:
                     sys.stderr.write(
-                        "Skip library with broken manifest: %s\n" % lib_dir)
+                        "Skip library with broken manifest: %s\n" % lib_dir
+                    )
                 continue
             if env.IsCompatibleLibBuilder(lb):
                 DefaultEnvironment().Append(__PIO_LIB_BUILDERS=[lb])
@@ -989,15 +971,15 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
 
     if verbose and found_incompat:
         sys.stderr.write(
-            "More details about \"Library Compatibility Mode\": "
+            'More details about "Library Compatibility Mode": '
             "https://docs.platformio.org/page/librarymanager/ldf.html#"
-            "ldf-compat-mode\n")
+            "ldf-compat-mode\n"
+        )
 
-    return DefaultEnvironment()['__PIO_LIB_BUILDERS']
+    return DefaultEnvironment()["__PIO_LIB_BUILDERS"]
 
 
 def ConfigureProjectLibBuilder(env):
-
     def _get_vcs_info(lb):
         path = LibraryManager.get_src_manifest_path(lb.path)
         return fs.load_json(path) if path else None
@@ -1022,40 +1004,42 @@ def ConfigureProjectLibBuilder(env):
                 title += " %s" % lb.version
             if vcs_info and vcs_info.get("version"):
                 title += " #%s" % vcs_info.get("version")
-            sys.stdout.write("%s|-- %s" % (margin, title))
+            click.echo("%s|-- %s" % (margin, title), nl=False)
             if int(ARGUMENTS.get("PIOVERBOSE", 0)):
                 if vcs_info:
-                    sys.stdout.write(" [%s]" % vcs_info.get("url"))
-                sys.stdout.write(" (")
-                sys.stdout.write(lb.path)
-                sys.stdout.write(")")
-            sys.stdout.write("\n")
+                    click.echo(" [%s]" % vcs_info.get("url"), nl=False)
+                click.echo(" (", nl=False)
+                click.echo(lb.path, nl=False)
+                click.echo(")", nl=False)
+            click.echo("")
             if lb.depbuilders:
                 _print_deps_tree(lb, level + 1)
 
     project = ProjectAsLibBuilder(env, "$PROJECT_DIR")
     ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project)
 
-    print("LDF: Library Dependency Finder -> http://bit.ly/configure-pio-ldf")
-    print("LDF Modes: Finder ~ %s, Compatibility ~ %s" %
-          (ldf_mode, project.lib_compat_mode))
+    click.echo("LDF: Library Dependency Finder -> http://bit.ly/configure-pio-ldf")
+    click.echo(
+        "LDF Modes: Finder ~ %s, Compatibility ~ %s"
+        % (ldf_mode, project.lib_compat_mode)
+    )
 
     project.install_dependencies()
 
     lib_builders = env.GetLibBuilders()
-    print("Found %d compatible libraries" % len(lib_builders))
+    click.echo("Found %d compatible libraries" % len(lib_builders))
 
-    print("Scanning dependencies...")
+    click.echo("Scanning dependencies...")
     project.search_deps_recursive()
 
     if ldf_mode.startswith("chain") and project.depbuilders:
         _correct_found_libs(lib_builders)
 
     if project.depbuilders:
-        print("Dependency Graph")
+        click.echo("Dependency Graph")
         _print_deps_tree(project)
     else:
-        print("No dependencies")
+        click.echo("No dependencies")
 
     return project
 

platformio/builder/tools/piomaxlen.py

@@ -18,16 +18,17 @@ from hashlib import md5
 from os import makedirs
 from os.path import isdir, isfile, join
 
+from platformio import fs
 from platformio.compat import WINDOWS, hashlib_encode_data
 
 # Windows CLI has limit with command length to 8192
 # Leave 2000 chars for flags and other options
-MAX_SOURCES_LENGTH = 6000
+MAX_LINE_LENGTH = 6000 if WINDOWS else 128072
 
 
 def long_sources_hook(env, sources):
     _sources = str(sources).replace("\\", "/")
-    if len(str(_sources)) < MAX_SOURCES_LENGTH:
+    if len(str(_sources)) < MAX_LINE_LENGTH:
         return sources
 
     # fix space in paths
@@ -43,7 +44,7 @@ def long_sources_hook(env, sources):
 
 def long_incflags_hook(env, incflags):
     _incflags = env.subst(incflags).replace("\\", "/")
-    if len(_incflags) < MAX_SOURCES_LENGTH:
+    if len(_incflags) < MAX_LINE_LENGTH:
         return incflags
 
     # fix space in paths
@@ -61,12 +62,12 @@ def _file_long_data(env, data):
     build_dir = env.subst("$BUILD_DIR")
     if not isdir(build_dir):
         makedirs(build_dir)
-    tmp_file = join(build_dir,
-                    "longcmd-%s" % md5(hashlib_encode_data(data)).hexdigest())
+    tmp_file = join(
+        build_dir, "longcmd-%s" % md5(hashlib_encode_data(data)).hexdigest()
+    )
     if isfile(tmp_file):
         return tmp_file
-    with open(tmp_file, "w") as fp:
-        fp.write(data)
+    fs.write_file_contents(tmp_file, data)
     return tmp_file
 
 
@@ -75,18 +76,17 @@ def exists(_):
 
 
 def generate(env):
-    if not WINDOWS:
-        return None
-
     env.Replace(_long_sources_hook=long_sources_hook)
     env.Replace(_long_incflags_hook=long_incflags_hook)
     coms = {}
     for key in ("ARCOM", "LINKCOM"):
         coms[key] = env.get(key, "").replace(
-            "$SOURCES", "${_long_sources_hook(__env__, SOURCES)}")
+            "$SOURCES", "${_long_sources_hook(__env__, SOURCES)}"
+        )
     for key in ("_CCCOMCOM", "ASPPCOM"):
         coms[key] = env.get(key, "").replace(
-            "$_CPPINCFLAGS", "${_long_incflags_hook(__env__, _CPPINCFLAGS)}")
+            "$_CPPINCFLAGS", "${_long_incflags_hook(__env__, _CPPINCFLAGS)}"
+        )
     env.Replace(**coms)
 
     return env

platformio/builder/tools/piomisc.py

@@ -25,7 +25,7 @@ from SCons.Action import Action  # pylint: disable=import-error
 from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 
 from platformio import fs, util
-from platformio.compat import get_file_contents, glob_escape
+from platformio.compat import glob_escape
 from platformio.managers.core import get_core_package_dir
 from platformio.proc import exec_command
 
@@ -39,7 +39,9 @@ class InoToCPPConverter(object):
         ([a-z_\d]+\s*)              # name of prototype
         \([a-z_,\.\*\&\[\]\s\d]*\)  # arguments
         )\s*(\{|;)                  # must end with `{` or `;`
-        """, re.X | re.M | re.I)
+        """,
+        re.X | re.M | re.I,
+    )
     DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I)
     PROTOPTRS_TPLRE = r"\([^&\(]*&(%s)[^\)]*\)"
 
@@ -60,10 +62,8 @@ class InoToCPPConverter(object):
         assert nodes
         lines = []
         for node in nodes:
-            contents = get_file_contents(node.get_path())
-            _lines = [
-                '# 1 "%s"' % node.get_path().replace("\\", "/"), contents
-            ]
+            contents = fs.get_file_contents(node.get_path())
+            _lines = ['# 1 "%s"' % node.get_path().replace("\\", "/"), contents]
             if self.is_main_node(contents):
                 lines = _lines + lines
                 self._main_ino = node.get_path()
@@ -78,21 +78,24 @@ class InoToCPPConverter(object):
     def process(self, contents):
         out_file = self._main_ino + ".cpp"
         assert self._gcc_preprocess(contents, out_file)
-        contents = get_file_contents(out_file)
+        contents = fs.get_file_contents(out_file)
         contents = self._join_multiline_strings(contents)
-        with open(out_file, "w") as fp:
-            fp.write(self.append_prototypes(contents))
+        fs.write_file_contents(
+            out_file, self.append_prototypes(contents), errors="backslashreplace"
+        )
         return out_file
 
     def _gcc_preprocess(self, contents, out_file):
         tmp_path = mkstemp()[1]
-        with open(tmp_path, "w") as fp:
-            fp.write(contents)
+        fs.write_file_contents(tmp_path, contents, errors="backslashreplace")
         self.env.Execute(
             self.env.VerboseAction(
                 '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format(
-                    out_file, tmp_path),
-                "Converting " + basename(out_file[:-4])))
+                    out_file, tmp_path
+                ),
+                "Converting " + basename(out_file[:-4]),
+            )
+        )
         atexit.register(_delete_file, tmp_path)
         return isfile(out_file)
 
@@ -114,14 +117,15 @@ class InoToCPPConverter(object):
                     stropen = True
                     newlines.append(line[:-1])
                     continue
-                elif stropen:
+                if stropen:
                     newlines[len(newlines) - 1] += line[:-1]
                     continue
             elif stropen and line.endswith(('",', '";')):
                 newlines[len(newlines) - 1] += line
                 stropen = False
-                newlines.append('#line %d "%s"' %
-                                (linenum, self._main_ino.replace("\\", "/")))
+                newlines.append(
+                    '#line %d "%s"' % (linenum, self._main_ino.replace("\\", "/"))
+                )
                 continue
 
             newlines.append(line)
@@ -141,8 +145,10 @@ class InoToCPPConverter(object):
         prototypes = []
         reserved_keywords = set(["if", "else", "while"])
         for match in self.PROTOTYPE_RE.finditer(contents):
-            if (set([match.group(2).strip(),
-                     match.group(3).strip()]) & reserved_keywords):
+            if (
+                set([match.group(2).strip(), match.group(3).strip()])
+                & reserved_keywords
+            ):
                 continue
             prototypes.append(match)
         return prototypes
@@ -162,11 +168,8 @@ class InoToCPPConverter(object):
         prototypes = self._parse_prototypes(contents) or []
 
         # skip already declared prototypes
-        declared = set(
-            m.group(1).strip() for m in prototypes if m.group(4) == ";")
-        prototypes = [
-            m for m in prototypes if m.group(1).strip() not in declared
-        ]
+        declared = set(m.group(1).strip() for m in prototypes if m.group(4) == ";")
+        prototypes = [m for m in prototypes if m.group(1).strip() not in declared]
 
         if not prototypes:
             return contents
@@ -175,23 +178,29 @@ class InoToCPPConverter(object):
         split_pos = prototypes[0].start()
         match_ptrs = re.search(
             self.PROTOPTRS_TPLRE % ("|".join(prototype_names)),
-            contents[:split_pos], re.M)
+            contents[:split_pos],
+            re.M,
+        )
         if match_ptrs:
             split_pos = contents.rfind("\n", 0, match_ptrs.start()) + 1
 
         result = []
         result.append(contents[:split_pos].strip())
         result.append("%s;" % ";\n".join([m.group(1) for m in prototypes]))
-        result.append('#line %d "%s"' % (self._get_total_lines(
-            contents[:split_pos]), self._main_ino.replace("\\", "/")))
+        result.append(
+            '#line %d "%s"'
+            % (
+                self._get_total_lines(contents[:split_pos]),
+                self._main_ino.replace("\\", "/"),
+            )
+        )
         result.append(contents[split_pos:].strip())
         return "\n".join(result)
 
 
 def ConvertInoToCpp(env):
-    src_dir = glob_escape(env.subst("$PROJECTSRC_DIR"))
-    ino_nodes = (env.Glob(join(src_dir, "*.ino")) +
-                 env.Glob(join(src_dir, "*.pde")))
+    src_dir = glob_escape(env.subst("$PROJECT_SRC_DIR"))
+    ino_nodes = env.Glob(join(src_dir, "*.ino")) + env.Glob(join(src_dir, "*.pde"))
     if not ino_nodes:
         return
     c = InoToCPPConverter(env)
@@ -214,13 +223,13 @@ def _get_compiler_type(env):
         return "gcc"
     try:
         sysenv = environ.copy()
-        sysenv['PATH'] = str(env['ENV']['PATH'])
+        sysenv["PATH"] = str(env["ENV"]["PATH"])
         result = exec_command([env.subst("$CC"), "-v"], env=sysenv)
     except OSError:
         return None
-    if result['returncode'] != 0:
+    if result["returncode"] != 0:
         return None
-    output = "".join([result['out'], result['err']]).lower()
+    output = "".join([result["out"], result["err"]]).lower()
     if "clang" in output and "LLVM" in output:
         return "clang"
     if "gcc" in output:
@@ -233,7 +242,6 @@ def GetCompilerType(env):
 
 
 def GetActualLDScript(env):
-
     def _lookup_in_ldpath(script):
         for d in env.get("LIBPATH", []):
             path = join(env.subst(d), script)
@@ -248,7 +256,7 @@ def GetActualLDScript(env):
         if f == "-T":
             script_in_next = True
             continue
-        elif script_in_next:
+        if script_in_next:
             script_in_next = False
             raw_script = f
         elif f.startswith("-Wl,-T"):
@@ -264,12 +272,13 @@ def GetActualLDScript(env):
 
     if script:
         sys.stderr.write(
-            "Error: Could not find '%s' LD script in LDPATH '%s'\n" %
-            (script, env.subst("$LIBPATH")))
+            "Error: Could not find '%s' LD script in LDPATH '%s'\n"
+            % (script, env.subst("$LIBPATH"))
+        )
         env.Exit(1)
 
     if not script and "LDSCRIPT_PATH" in env:
-        path = _lookup_in_ldpath(env['LDSCRIPT_PATH'])
+        path = _lookup_in_ldpath(env["LDSCRIPT_PATH"])
         if path:
             return path
 
@@ -292,35 +301,45 @@ def PioClean(env, clean_dir):
         for f in files:
             dst = join(root, f)
             remove(dst)
-            print("Removed %s" %
-                  (dst if clean_rel_path.startswith(".") else relpath(dst)))
+            print(
+                "Removed %s" % (dst if clean_rel_path.startswith(".") else relpath(dst))
+            )
     print("Done cleaning")
     fs.rmtree(clean_dir)
     env.Exit(0)
 
 
-def ProcessDebug(env):
-    if not env.subst("$PIODEBUGFLAGS"):
-        env.Replace(PIODEBUGFLAGS=["-Og", "-g3", "-ggdb3"])
-    env.Append(BUILD_FLAGS=list(env['PIODEBUGFLAGS']) +
-               ["-D__PLATFORMIO_BUILD_DEBUG__"])
-    unflags = ["-Os"]
-    for level in [0, 1, 2]:
-        for flag in ("O", "g", "ggdb"):
-            unflags.append("-%s%d" % (flag, level))
-    env.Append(BUILD_UNFLAGS=unflags)
+def ConfigureDebugFlags(env):
+    def _cleanup_debug_flags(scope):
+        if scope not in env:
+            return
+        unflags = ["-Os", "-g"]
+        for level in [0, 1, 2]:
+            for flag in ("O", "g", "ggdb"):
+                unflags.append("-%s%d" % (flag, level))
+        env[scope] = [f for f in env.get(scope, []) if f not in unflags]
+
+    env.Append(CPPDEFINES=["__PLATFORMIO_BUILD_DEBUG__"])
+
+    debug_flags = ["-Og", "-g2", "-ggdb2"]
+    for scope in ("ASFLAGS", "CCFLAGS", "LINKFLAGS"):
+        _cleanup_debug_flags(scope)
+        env.Append(**{scope: debug_flags})
 
 
-def ProcessTest(env):
-    env.Append(CPPDEFINES=["UNIT_TEST", "UNITY_INCLUDE_CONFIG_H"],
-               CPPPATH=[join("$BUILD_DIR", "UnityTestLib")])
-    unitylib = env.BuildLibrary(join("$BUILD_DIR", "UnityTestLib"),
-                                get_core_package_dir("tool-unity"))
+def ConfigureTestTarget(env):
+    env.Append(
+        CPPDEFINES=["UNIT_TEST", "UNITY_INCLUDE_CONFIG_H"],
+        CPPPATH=[join("$BUILD_DIR", "UnityTestLib")],
+    )
+    unitylib = env.BuildLibrary(
+        join("$BUILD_DIR", "UnityTestLib"), get_core_package_dir("tool-unity")
+    )
     env.Prepend(LIBS=[unitylib])
 
     src_filter = ["+<*.cpp>", "+<*.c>"]
     if "PIOTEST_RUNNING_NAME" in env:
-        src_filter.append("+<%s%s>" % (env['PIOTEST_RUNNING_NAME'], sep))
+        src_filter.append("+<%s%s>" % (env["PIOTEST_RUNNING_NAME"], sep))
     env.Replace(PIOTEST_SRC_FILTER=src_filter)
 
 
@@ -330,7 +349,7 @@ def GetExtraScripts(env, scope):
         if scope == "post" and ":" not in item:
             items.append(item)
         elif item.startswith("%s:" % scope):
-            items.append(item[len(scope) + 1:])
+            items.append(item[len(scope) + 1 :])
     if not items:
         return items
     with fs.cd(env.subst("$PROJECT_DIR")):
@@ -347,7 +366,7 @@ def generate(env):
     env.AddMethod(GetActualLDScript)
     env.AddMethod(VerboseAction)
     env.AddMethod(PioClean)
-    env.AddMethod(ProcessDebug)
-    env.AddMethod(ProcessTest)
+    env.AddMethod(ConfigureDebugFlags)
+    env.AddMethod(ConfigureTestTarget)
     env.AddMethod(GetExtraScripts)
     return env

platformio/builder/tools/pioplatform.py

@@ -33,8 +33,8 @@ def PioPlatform(env):
     variables = env.GetProjectOptions(as_dict=True)
     if "framework" in variables:
         # support PIO Core 3.0 dev/platforms
-        variables['pioframework'] = variables['framework']
-    p = PlatformFactory.newPlatform(env['PLATFORM_MANIFEST'])
+        variables["pioframework"] = variables["framework"]
+    p = PlatformFactory.newPlatform(env["PLATFORM_MANIFEST"])
     p.configure_default_packages(variables, COMMAND_LINE_TARGETS)
     return p
 
@@ -54,7 +54,7 @@ def BoardConfig(env, board=None):
 def GetFrameworkScript(env, framework):
     p = env.PioPlatform()
     assert p.frameworks and framework in p.frameworks
-    script_path = env.subst(p.frameworks[framework]['script'])
+    script_path = env.subst(p.frameworks[framework]["script"])
     if not isfile(script_path):
         script_path = join(p.get_dir(), script_path)
     return script_path
@@ -65,7 +65,7 @@ def LoadPioPlatform(env):
     installed_packages = p.get_installed_packages()
 
     # Ensure real platform name
-    env['PIOPLATFORM'] = p.name
+    env["PIOPLATFORM"] = p.name
 
     # Add toolchains and uploaders to $PATH and $*_LIBRARY_PATH
     systype = util.get_systype()
@@ -75,14 +75,13 @@ def LoadPioPlatform(env):
             continue
         pkg_dir = p.get_package_dir(name)
         env.PrependENVPath(
-            "PATH",
-            join(pkg_dir, "bin") if isdir(join(pkg_dir, "bin")) else pkg_dir)
-        if (not WINDOWS and isdir(join(pkg_dir, "lib"))
-                and type_ != "toolchain"):
+            "PATH", join(pkg_dir, "bin") if isdir(join(pkg_dir, "bin")) else pkg_dir
+        )
+        if not WINDOWS and isdir(join(pkg_dir, "lib")) and type_ != "toolchain":
             env.PrependENVPath(
-                "DYLD_LIBRARY_PATH"
-                if "darwin" in systype else "LD_LIBRARY_PATH",
-                join(pkg_dir, "lib"))
+                "DYLD_LIBRARY_PATH" if "darwin" in systype else "LD_LIBRARY_PATH",
+                join(pkg_dir, "lib"),
+            )
 
     # Platform specific LD Scripts
     if isdir(join(p.get_dir(), "ldscripts")):
@@ -94,16 +93,27 @@ def LoadPioPlatform(env):
     # update board manifest with overridden data from INI config
     board_config = env.BoardConfig()
     for option, value in env.GetProjectOptions():
-        if option.startswith("board_"):
-            board_config.update(option.lower()[6:], value)
+        if not option.startswith("board_"):
+            continue
+        option = option.lower()[6:]
+        try:
+            if isinstance(board_config.get(option), bool):
+                value = str(value).lower() in ("1", "yes", "true")
+            elif isinstance(board_config.get(option), int):
+                value = int(value)
+        except KeyError:
+            pass
+        board_config.update(option, value)
 
     # load default variables from board config
     for option_meta in ProjectOptions.values():
         if not option_meta.buildenvvar or option_meta.buildenvvar in env:
             continue
-        data_path = (option_meta.name[6:]
-                     if option_meta.name.startswith("board_") else
-                     option_meta.name.replace("_", "."))
+        data_path = (
+            option_meta.name[6:]
+            if option_meta.name.startswith("board_")
+            else option_meta.name.replace("_", ".")
+        )
         try:
             env[option_meta.buildenvvar] = board_config.get(data_path)
         except KeyError:
@@ -118,22 +128,25 @@ def PrintConfiguration(env):  # pylint: disable=too-many-statements
     board_config = env.BoardConfig() if "BOARD" in env else None
 
     def _get_configuration_data():
-        return None if not board_config else [
-            "CONFIGURATION:",
-            "https://docs.platformio.org/page/boards/%s/%s.html" %
-            (platform.name, board_config.id)
-        ]
+        return (
+            None
+            if not board_config
+            else [
+                "CONFIGURATION:",
+                "https://docs.platformio.org/page/boards/%s/%s.html"
+                % (platform.name, board_config.id),
+            ]
+        )
 
     def _get_plaform_data():
         data = ["PLATFORM: %s %s" % (platform.title, platform.version)]
-        src_manifest_path = platform.pm.get_src_manifest_path(
-            platform.get_dir())
+        src_manifest_path = platform.pm.get_src_manifest_path(platform.get_dir())
         if src_manifest_path:
             src_manifest = fs.load_json(src_manifest_path)
             if "version" in src_manifest:
-                data.append("#" + src_manifest['version'])
+                data.append("#" + src_manifest["version"])
             if int(ARGUMENTS.get("PIOVERBOSE", 0)):
-                data.append("(%s)" % src_manifest['url'])
+                data.append("(%s)" % src_manifest["url"])
         if board_config:
             data.extend([">", board_config.get("name")])
         return data
@@ -151,19 +164,22 @@ def PrintConfiguration(env):  # pylint: disable=too-many-statements
             return data
         ram = board_config.get("upload", {}).get("maximum_ram_size")
         flash = board_config.get("upload", {}).get("maximum_size")
-        data.append("%s RAM, %s Flash" %
-                    (fs.format_filesize(ram), fs.format_filesize(flash)))
+        data.append(
+            "%s RAM, %s Flash" % (fs.format_filesize(ram), fs.format_filesize(flash))
+        )
         return data
 
     def _get_debug_data():
-        debug_tools = board_config.get(
-            "debug", {}).get("tools") if board_config else None
+        debug_tools = (
+            board_config.get("debug", {}).get("tools") if board_config else None
+        )
         if not debug_tools:
             return None
         data = [
-            "DEBUG:", "Current",
-            "(%s)" % board_config.get_debug_tool_name(
-                env.GetProjectOption("debug_tool"))
+            "DEBUG:",
+            "Current",
+            "(%s)"
+            % board_config.get_debug_tool_name(env.GetProjectOption("debug_tool")),
         ]
         onboard = []
         external = []
@@ -187,21 +203,25 @@ def PrintConfiguration(env):  # pylint: disable=too-many-statements
             if not pkg_dir:
                 continue
             manifest = platform.pm.load_manifest(pkg_dir)
-            original_version = util.get_original_version(manifest['version'])
-            info = "%s %s" % (manifest['name'], manifest['version'])
+            original_version = util.get_original_version(manifest["version"])
+            info = "%s %s" % (manifest["name"], manifest["version"])
             extra = []
             if original_version:
                 extra.append(original_version)
             if "__src_url" in manifest and int(ARGUMENTS.get("PIOVERBOSE", 0)):
-                extra.append(manifest['__src_url'])
+                extra.append(manifest["__src_url"])
             if extra:
                 info += " (%s)" % ", ".join(extra)
             data.append(info)
         return ["PACKAGES:", ", ".join(data)]
 
-    for data in (_get_configuration_data(), _get_plaform_data(),
-                 _get_hardware_data(), _get_debug_data(),
-                 _get_packages_data()):
+    for data in (
+        _get_configuration_data(),
+        _get_plaform_data(),
+        _get_hardware_data(),
+        _get_debug_data(),
+        _get_packages_data(),
+    ):
         if data and len(data) > 1:
             print(" ".join(data))
 

platformio/builder/tools/pioproject.py

@@ -18,22 +18,25 @@ from platformio.project.config import ProjectConfig, ProjectOptions
 
 
 def GetProjectConfig(env):
-    return ProjectConfig.get_instance(env['PROJECT_CONFIG'])
+    return ProjectConfig.get_instance(env["PROJECT_CONFIG"])
 
 
 def GetProjectOptions(env, as_dict=False):
-    return env.GetProjectConfig().items(env=env['PIOENV'], as_dict=as_dict)
+    return env.GetProjectConfig().items(env=env["PIOENV"], as_dict=as_dict)
 
 
 def GetProjectOption(env, option, default=None):
-    return env.GetProjectConfig().get("env:" + env['PIOENV'], option, default)
+    return env.GetProjectConfig().get("env:" + env["PIOENV"], option, default)
 
 
 def LoadProjectOptions(env):
     for option, value in env.GetProjectOptions():
         option_meta = ProjectOptions.get("env." + option)
-        if (not option_meta or not option_meta.buildenvvar
-                or option_meta.buildenvvar in env):
+        if (
+            not option_meta
+            or not option_meta.buildenvvar
+            or option_meta.buildenvvar in env
+        ):
             continue
         env[option_meta.buildenvvar] = value
 

platformio/builder/tools/piosize.py

@@ -0,0 +1,254 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=too-many-locals
+
+from __future__ import absolute_import
+
+import sys
+from os import environ, makedirs, remove
+from os.path import isdir, join, splitdrive
+
+from elftools.elf.descriptions import describe_sh_flags
+from elftools.elf.elffile import ELFFile
+
+from platformio.compat import dump_json_to_unicode
+from platformio.proc import exec_command
+from platformio.util import get_systype
+
+
+def _run_tool(cmd, env, tool_args):
+    sysenv = environ.copy()
+    sysenv["PATH"] = str(env["ENV"]["PATH"])
+
+    build_dir = env.subst("$BUILD_DIR")
+    if not isdir(build_dir):
+        makedirs(build_dir)
+    tmp_file = join(build_dir, "size-data-longcmd.txt")
+
+    with open(tmp_file, "w") as fp:
+        fp.write("\n".join(tool_args))
+
+    cmd.append("@" + tmp_file)
+    result = exec_command(cmd, env=sysenv)
+    remove(tmp_file)
+
+    return result
+
+
+def _get_symbol_locations(env, elf_path, addrs):
+    if not addrs:
+        return {}
+    cmd = [env.subst("$CC").replace("-gcc", "-addr2line"), "-e", elf_path]
+    result = _run_tool(cmd, env, addrs)
+    locations = [line for line in result["out"].split("\n") if line]
+    assert len(addrs) == len(locations)
+
+    return dict(zip(addrs, [l.strip() for l in locations]))
+
+
+def _get_demangled_names(env, mangled_names):
+    if not mangled_names:
+        return {}
+    result = _run_tool(
+        [env.subst("$CC").replace("-gcc", "-c++filt")], env, mangled_names
+    )
+    demangled_names = [line for line in result["out"].split("\n") if line]
+    assert len(mangled_names) == len(demangled_names)
+
+    return dict(
+        zip(
+            mangled_names,
+            [dn.strip().replace("::__FUNCTION__", "") for dn in demangled_names],
+        )
+    )
+
+
+def _determine_section(sections, symbol_addr):
+    for section, info in sections.items():
+        if not _is_flash_section(info) and not _is_ram_section(info):
+            continue
+        if symbol_addr in range(info["start_addr"], info["start_addr"] + info["size"]):
+            return section
+    return "unknown"
+
+
+def _is_ram_section(section):
+    return (
+        section.get("type", "") in ("SHT_NOBITS", "SHT_PROGBITS")
+        and section.get("flags", "") == "WA"
+    )
+
+
+def _is_flash_section(section):
+    return section.get("type", "") == "SHT_PROGBITS" and "A" in section.get("flags", "")
+
+
+def _is_valid_symbol(symbol_name, symbol_type, symbol_address):
+    return symbol_name and symbol_address != 0 and symbol_type != "STT_NOTYPE"
+
+
+def _collect_sections_info(elffile):
+    sections = {}
+    for section in elffile.iter_sections():
+        if section.is_null() or section.name.startswith(".debug"):
+            continue
+
+        section_type = section["sh_type"]
+        section_flags = describe_sh_flags(section["sh_flags"])
+        section_size = section.data_size
+
+        sections[section.name] = {
+            "size": section_size,
+            "start_addr": section["sh_addr"],
+            "type": section_type,
+            "flags": section_flags,
+        }
+
+    return sections
+
+
+def _collect_symbols_info(env, elffile, elf_path, sections):
+    symbols = []
+
+    symbol_section = elffile.get_section_by_name(".symtab")
+    if symbol_section.is_null():
+        sys.stderr.write("Couldn't find symbol table. Is ELF file stripped?")
+        env.Exit(1)
+
+    sysenv = environ.copy()
+    sysenv["PATH"] = str(env["ENV"]["PATH"])
+
+    symbol_addrs = []
+    mangled_names = []
+    for s in symbol_section.iter_symbols():
+        symbol_info = s.entry["st_info"]
+        symbol_addr = s["st_value"]
+        symbol_size = s["st_size"]
+        symbol_type = symbol_info["type"]
+
+        if not _is_valid_symbol(s.name, symbol_type, symbol_addr):
+            continue
+
+        symbol = {
+            "addr": symbol_addr,
+            "bind": symbol_info["bind"],
+            "name": s.name,
+            "type": symbol_type,
+            "size": symbol_size,
+            "section": _determine_section(sections, symbol_addr),
+        }
+
+        if s.name.startswith("_Z"):
+            mangled_names.append(s.name)
+
+        symbol_addrs.append(hex(symbol_addr))
+        symbols.append(symbol)
+
+    symbol_locations = _get_symbol_locations(env, elf_path, symbol_addrs)
+    demangled_names = _get_demangled_names(env, mangled_names)
+    for symbol in symbols:
+        if symbol["name"].startswith("_Z"):
+            symbol["demangled_name"] = demangled_names.get(symbol["name"])
+        location = symbol_locations.get(hex(symbol["addr"]))
+        if not location or "?" in location:
+            continue
+        if "windows" in get_systype():
+            drive, tail = splitdrive(location)
+            location = join(drive.upper(), tail)
+        symbol["file"] = location
+        symbol["line"] = 0
+        if ":" in location:
+            file_, line = location.rsplit(":", 1)
+            if line.isdigit():
+                symbol["file"] = file_
+                symbol["line"] = int(line)
+    return symbols
+
+
+def _calculate_firmware_size(sections):
+    flash_size = ram_size = 0
+    for section_info in sections.values():
+        if _is_flash_section(section_info):
+            flash_size += section_info.get("size", 0)
+        if _is_ram_section(section_info):
+            ram_size += section_info.get("size", 0)
+
+    return ram_size, flash_size
+
+
+def DumpSizeData(_, target, source, env):  # pylint: disable=unused-argument
+    data = {"device": {}, "memory": {}, "version": 1}
+
+    board = env.BoardConfig()
+    if board:
+        data["device"] = {
+            "mcu": board.get("build.mcu", ""),
+            "cpu": board.get("build.cpu", ""),
+            "frequency": board.get("build.f_cpu"),
+            "flash": int(board.get("upload.maximum_size", 0)),
+            "ram": int(board.get("upload.maximum_ram_size", 0)),
+        }
+        if data["device"]["frequency"] and data["device"]["frequency"].endswith("L"):
+            data["device"]["frequency"] = int(data["device"]["frequency"][0:-1])
+
+    elf_path = env.subst("$PIOMAINPROG")
+
+    with open(elf_path, "rb") as fp:
+        elffile = ELFFile(fp)
+
+        if not elffile.has_dwarf_info():
+            sys.stderr.write("Elf file doesn't contain DWARF information")
+            env.Exit(1)
+
+        sections = _collect_sections_info(elffile)
+        firmware_ram, firmware_flash = _calculate_firmware_size(sections)
+        data["memory"]["total"] = {
+            "ram_size": firmware_ram,
+            "flash_size": firmware_flash,
+            "sections": sections,
+        }
+
+        files = dict()
+        for symbol in _collect_symbols_info(env, elffile, elf_path, sections):
+            file_path = symbol.get("file") or "unknown"
+            if not files.get(file_path, {}):
+                files[file_path] = {"symbols": [], "ram_size": 0, "flash_size": 0}
+
+            symbol_size = symbol.get("size", 0)
+            section = sections.get(symbol.get("section", ""), {})
+            if _is_ram_section(section):
+                files[file_path]["ram_size"] += symbol_size
+            if _is_flash_section(section):
+                files[file_path]["flash_size"] += symbol_size
+
+            files[file_path]["symbols"].append(symbol)
+
+        data["memory"]["files"] = list()
+        for k, v in files.items():
+            file_data = {"path": k}
+            file_data.update(v)
+            data["memory"]["files"].append(file_data)
+
+    with open(join(env.subst("$BUILD_DIR"), "sizedata.json"), "w") as fp:
+        fp.write(dump_json_to_unicode(data))
+
+
+def exists(_):
+    return True
+
+
+def generate(env):
+    env.AddMethod(DumpSizeData)
+    return env

platformio/builder/tools/pioupload.py

@@ -60,9 +60,9 @@ def WaitForNewSerialPort(env, before):
     prev_port = env.subst("$UPLOAD_PORT")
     new_port = None
     elapsed = 0
-    before = [p['port'] for p in before]
+    before = [p["port"] for p in before]
     while elapsed < 5 and new_port is None:
-        now = [p['port'] for p in util.get_serial_ports()]
+        now = [p["port"] for p in util.get_serial_ports()]
         for p in now:
             if p not in before:
                 new_port = p
@@ -84,10 +84,12 @@ def WaitForNewSerialPort(env, before):
         sleep(1)
 
     if not new_port:
-        sys.stderr.write("Error: Couldn't find a board on the selected port. "
-                         "Check that you have the correct port selected. "
-                         "If it is correct, try pressing the board's reset "
-                         "button after initiating the upload.\n")
+        sys.stderr.write(
+            "Error: Couldn't find a board on the selected port. "
+            "Check that you have the correct port selected. "
+            "If it is correct, try pressing the board's reset "
+            "button after initiating the upload.\n"
+        )
         env.Exit(1)
 
     return new_port
@@ -99,8 +101,8 @@ def AutodetectUploadPort(*args, **kwargs):
     def _get_pattern():
         if "UPLOAD_PORT" not in env:
             return None
-        if set(["*", "?", "[", "]"]) & set(env['UPLOAD_PORT']):
-            return env['UPLOAD_PORT']
+        if set(["*", "?", "[", "]"]) & set(env["UPLOAD_PORT"]):
+            return env["UPLOAD_PORT"]
         return None
 
     def _is_match_pattern(port):
@@ -112,17 +114,13 @@ def AutodetectUploadPort(*args, **kwargs):
     def _look_for_mbed_disk():
         msdlabels = ("mbed", "nucleo", "frdm", "microbit")
         for item in util.get_logical_devices():
-            if item['path'].startswith("/net") or not _is_match_pattern(
-                    item['path']):
+            if item["path"].startswith("/net") or not _is_match_pattern(item["path"]):
                 continue
-            mbed_pages = [
-                join(item['path'], n) for n in ("mbed.htm", "mbed.html")
-            ]
+            mbed_pages = [join(item["path"], n) for n in ("mbed.htm", "mbed.html")]
             if any(isfile(p) for p in mbed_pages):
-                return item['path']
-            if item['name'] \
-                    and any(l in item['name'].lower() for l in msdlabels):
-                return item['path']
+                return item["path"]
+            if item["name"] and any(l in item["name"].lower() for l in msdlabels):
+                return item["path"]
         return None
 
     def _look_for_serial_port():
@@ -132,17 +130,17 @@ def AutodetectUploadPort(*args, **kwargs):
         if "BOARD" in env and "build.hwids" in env.BoardConfig():
             board_hwids = env.BoardConfig().get("build.hwids")
         for item in util.get_serial_ports(filter_hwid=True):
-            if not _is_match_pattern(item['port']):
+            if not _is_match_pattern(item["port"]):
                 continue
-            port = item['port']
+            port = item["port"]
             if upload_protocol.startswith("blackmagic"):
                 if WINDOWS and port.startswith("COM") and len(port) > 4:
                     port = "\\\\.\\%s" % port
-                if "GDB" in item['description']:
+                if "GDB" in item["description"]:
                     return port
             for hwid in board_hwids:
                 hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
-                if hwid_str in item['hwid']:
+                if hwid_str in item["hwid"]:
                     return port
         return port
 
@@ -150,9 +148,9 @@ def AutodetectUploadPort(*args, **kwargs):
         print(env.subst("Use manually specified: $UPLOAD_PORT"))
         return
 
-    if (env.subst("$UPLOAD_PROTOCOL") == "mbed"
-            or ("mbed" in env.subst("$PIOFRAMEWORK")
-                and not env.subst("$UPLOAD_PROTOCOL"))):
+    if env.subst("$UPLOAD_PROTOCOL") == "mbed" or (
+        "mbed" in env.subst("$PIOFRAMEWORK") and not env.subst("$UPLOAD_PROTOCOL")
+    ):
         env.Replace(UPLOAD_PORT=_look_for_mbed_disk())
     else:
         try:
@@ -168,7 +166,8 @@ def AutodetectUploadPort(*args, **kwargs):
             "Error: Please specify `upload_port` for environment or use "
             "global `--upload-port` option.\n"
             "For some development platforms it can be a USB flash "
-            "drive (i.e. /media/<user>/<device name>)\n")
+            "drive (i.e. /media/<user>/<device name>)\n"
+        )
         env.Exit(1)
 
 
@@ -179,16 +178,17 @@ def UploadToDisk(_, target, source, env):
         fpath = join(env.subst("$BUILD_DIR"), "%s.%s" % (progname, ext))
         if not isfile(fpath):
             continue
-        copyfile(fpath,
-                 join(env.subst("$UPLOAD_PORT"), "%s.%s" % (progname, ext)))
-    print("Firmware has been successfully uploaded.\n"
-          "(Some boards may require manual hard reset)")
+        copyfile(fpath, join(env.subst("$UPLOAD_PORT"), "%s.%s" % (progname, ext)))
+    print(
+        "Firmware has been successfully uploaded.\n"
+        "(Some boards may require manual hard reset)"
+    )
 
 
 def CheckUploadSize(_, target, source, env):
     check_conditions = [
         env.get("BOARD"),
-        env.get("SIZETOOL") or env.get("SIZECHECKCMD")
+        env.get("SIZETOOL") or env.get("SIZECHECKCMD"),
     ]
     if not all(check_conditions):
         return
@@ -198,9 +198,11 @@ def CheckUploadSize(_, target, source, env):
         return
 
     def _configure_defaults():
-        env.Replace(SIZECHECKCMD="$SIZETOOL -B -d $SOURCES",
-                    SIZEPROGREGEXP=r"^(\d+)\s+(\d+)\s+\d+\s",
-                    SIZEDATAREGEXP=r"^\d+\s+(\d+)\s+(\d+)\s+\d+")
+        env.Replace(
+            SIZECHECKCMD="$SIZETOOL -B -d $SOURCES",
+            SIZEPROGREGEXP=r"^(\d+)\s+(\d+)\s+\d+\s",
+            SIZEDATAREGEXP=r"^\d+\s+(\d+)\s+(\d+)\s+\d+",
+        )
 
     def _get_size_output():
         cmd = env.get("SIZECHECKCMD")
@@ -210,11 +212,11 @@ def CheckUploadSize(_, target, source, env):
             cmd = cmd.split()
         cmd = [arg.replace("$SOURCES", str(source[0])) for arg in cmd if arg]
         sysenv = environ.copy()
-        sysenv['PATH'] = str(env['ENV']['PATH'])
+        sysenv["PATH"] = str(env["ENV"]["PATH"])
         result = exec_command(env.subst(cmd), env=sysenv)
-        if result['returncode'] != 0:
+        if result["returncode"] != 0:
             return None
-        return result['out'].strip()
+        return result["out"].strip()
 
     def _calculate_size(output, pattern):
         if not output or not pattern:
@@ -238,7 +240,8 @@ def CheckUploadSize(_, target, source, env):
         if used_blocks > blocks_per_progress:
             used_blocks = blocks_per_progress
         return "[{:{}}] {: 6.1%} (used {:d} bytes from {:d} bytes)".format(
-            "=" * used_blocks, blocks_per_progress, percent_raw, value, total)
+            "=" * used_blocks, blocks_per_progress, percent_raw, value, total
+        )
 
     if not env.get("SIZECHECKCMD") and not env.get("SIZEPROGREGEXP"):
         _configure_defaults()
@@ -246,12 +249,11 @@ def CheckUploadSize(_, target, source, env):
     program_size = _calculate_size(output, env.get("SIZEPROGREGEXP"))
     data_size = _calculate_size(output, env.get("SIZEDATAREGEXP"))
 
-    print("Memory Usage -> http://bit.ly/pio-memory-usage")
+    print('Advanced Memory Usage is available via "PlatformIO Home > Project Inspect"')
     if data_max_size and data_size > -1:
         print("DATA:    %s" % _format_availale_bytes(data_size, data_max_size))
     if program_size > -1:
-        print("PROGRAM: %s" %
-              _format_availale_bytes(program_size, program_max_size))
+        print("PROGRAM: %s" % _format_availale_bytes(program_size, program_max_size))
     if int(ARGUMENTS.get("PIOVERBOSE", 0)):
         print(output)
 
@@ -262,9 +264,10 @@ def CheckUploadSize(_, target, source, env):
     #         "than maximum allowed (%s bytes)\n" % (data_size, data_max_size))
     #     env.Exit(1)
     if program_size > program_max_size:
-        sys.stderr.write("Error: The program size (%d bytes) is greater "
-                         "than maximum allowed (%s bytes)\n" %
-                         (program_size, program_max_size))
+        sys.stderr.write(
+            "Error: The program size (%d bytes) is greater "
+            "than maximum allowed (%s bytes)\n" % (program_size, program_max_size)
+        )
         env.Exit(1)
 
 
@@ -272,8 +275,7 @@ def PrintUploadInfo(env):
     configured = env.subst("$UPLOAD_PROTOCOL")
     available = [configured] if configured else []
     if "BOARD" in env:
-        available.extend(env.BoardConfig().get("upload",
-                                               {}).get("protocols", []))
+        available.extend(env.BoardConfig().get("upload", {}).get("protocols", []))
     if available:
         print("AVAILABLE: %s" % ", ".join(sorted(set(available))))
     if configured:

platformio/builder/tools/platformio.py

@@ -14,11 +14,12 @@
 
 from __future__ import absolute_import
 
+import fnmatch
 import os
 import sys
-from os.path import basename, dirname, isdir, join, realpath
 
 from SCons import Builder, Util  # pylint: disable=import-error
+from SCons.Node import FS  # pylint: disable=import-error
 from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
 from SCons.Script import AlwaysBuild  # pylint: disable=import-error
 from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
@@ -54,7 +55,8 @@ def _build_project_deps(env):
             key: project_lib_builder.env.get(key)
             for key in ("LIBS", "LIBPATH", "LINKFLAGS")
             if project_lib_builder.env.get(key)
-        })
+        }
+    )
 
     projenv = env.Clone()
 
@@ -65,27 +67,34 @@ def _build_project_deps(env):
 
     is_test = "__test" in COMMAND_LINE_TARGETS
     if is_test:
-        projenv.BuildSources("$BUILDTEST_DIR", "$PROJECTTEST_DIR",
-                             "$PIOTEST_SRC_FILTER")
-    if not is_test or env.GetProjectOption("test_build_project_src", False):
-        projenv.BuildSources("$BUILDSRC_DIR", "$PROJECTSRC_DIR",
-                             env.get("SRC_FILTER"))
+        projenv.BuildSources(
+            "$BUILD_TEST_DIR", "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER"
+        )
+    if not is_test or env.GetProjectOption("test_build_project_src"):
+        projenv.BuildSources(
+            "$BUILD_SRC_DIR", "$PROJECT_SRC_DIR", env.get("SRC_FILTER")
+        )
 
     if not env.get("PIOBUILDFILES") and not COMMAND_LINE_TARGETS:
         sys.stderr.write(
             "Error: Nothing to build. Please put your source code files "
-            "to '%s' folder\n" % env.subst("$PROJECTSRC_DIR"))
+            "to '%s' folder\n" % env.subst("$PROJECT_SRC_DIR")
+        )
         env.Exit(1)
 
     Export("projenv")
 
 
 def BuildProgram(env):
-
     def _append_pio_macros():
-        env.AppendUnique(CPPDEFINES=[(
-            "PLATFORMIO",
-            int("{0:02d}{1:02d}{2:02d}".format(*pioversion_to_intstr())))])
+        env.AppendUnique(
+            CPPDEFINES=[
+                (
+                    "PLATFORMIO",
+                    int("{0:02d}{1:02d}{2:02d}".format(*pioversion_to_intstr())),
+                )
+            ]
+        )
 
     _append_pio_macros()
 
@@ -95,10 +104,6 @@ def BuildProgram(env):
     if not Util.case_sensitive_suffixes(".s", ".S"):
         env.Replace(AS="$CC", ASCOM="$ASPPCOM")
 
-    if ("debug" in COMMAND_LINE_TARGETS
-            or env.GetProjectOption("build_type") == "debug"):
-        env.ProcessDebug()
-
     # process extra flags from board
     if "BOARD" in env and "build.extra_flags" in env.BoardConfig():
         env.ProcessFlags(env.BoardConfig().get("build.extra_flags"))
@@ -109,37 +114,45 @@ def BuildProgram(env):
     # process framework scripts
     env.BuildFrameworks(env.get("PIOFRAMEWORK"))
 
-    # restore PIO macros if it was deleted by framework
-    _append_pio_macros()
+    is_build_type_debug = (
+        set(["debug", "sizedata"]) & set(COMMAND_LINE_TARGETS)
+        or env.GetProjectOption("build_type") == "debug"
+    )
+    if is_build_type_debug:
+        env.ConfigureDebugFlags()
 
     # remove specified flags
     env.ProcessUnFlags(env.get("BUILD_UNFLAGS"))
 
     if "__test" in COMMAND_LINE_TARGETS:
-        env.ProcessTest()
-
-    # build project with dependencies
-    _build_project_deps(env)
+        env.ConfigureTestTarget()
 
     # append into the beginning a main LD script
-    if (env.get("LDSCRIPT_PATH")
-            and not any("-Wl,-T" in f for f in env['LINKFLAGS'])):
-        env.Prepend(LINKFLAGS=["-T", "$LDSCRIPT_PATH"])
+    if env.get("LDSCRIPT_PATH") and not any("-Wl,-T" in f for f in env["LINKFLAGS"]):
+        env.Prepend(LINKFLAGS=["-T", env.subst("$LDSCRIPT_PATH")])
 
     # enable "cyclic reference" for linker
     if env.get("LIBS") and env.GetCompilerType() == "gcc":
         env.Prepend(_LIBFLAGS="-Wl,--start-group ")
         env.Append(_LIBFLAGS=" -Wl,--end-group")
 
-    program = env.Program(join("$BUILD_DIR", env.subst("$PROGNAME")),
-                          env['PIOBUILDFILES'])
+    # build project with dependencies
+    _build_project_deps(env)
+
+    program = env.Program(
+        os.path.join("$BUILD_DIR", env.subst("$PROGNAME")), env["PIOBUILDFILES"]
+    )
     env.Replace(PIOMAINPROG=program)
 
     AlwaysBuild(
         env.Alias(
-            "checkprogsize", program,
-            env.VerboseAction(env.CheckUploadSize,
-                              "Checking size $PIOMAINPROG")))
+            "checkprogsize",
+            program,
+            env.VerboseAction(env.CheckUploadSize, "Checking size $PIOMAINPROG"),
+        )
+    )
+
+    print("Building in %s mode" % ("debug" if is_build_type_debug else "release"))
 
     return program
 
@@ -155,30 +168,30 @@ def ParseFlagsExtended(env, flags):  # pylint: disable=too-many-branches
             result[key].extend(value)
 
     cppdefines = []
-    for item in result['CPPDEFINES']:
+    for item in result["CPPDEFINES"]:
         if not Util.is_Sequence(item):
             cppdefines.append(item)
             continue
         name, value = item[:2]
-        if '\"' in value:
-            value = value.replace('\"', '\\\"')
+        if '"' in value:
+            value = value.replace('"', '\\"')
         elif value.isdigit():
             value = int(value)
         elif value.replace(".", "", 1).isdigit():
             value = float(value)
         cppdefines.append((name, value))
-    result['CPPDEFINES'] = cppdefines
+    result["CPPDEFINES"] = cppdefines
 
     # fix relative CPPPATH & LIBPATH
     for k in ("CPPPATH", "LIBPATH"):
         for i, p in enumerate(result.get(k, [])):
-            if isdir(p):
-                result[k][i] = realpath(p)
+            if os.path.isdir(p):
+                result[k][i] = os.path.realpath(p)
 
     # fix relative path for "-include"
     for i, f in enumerate(result.get("CCFLAGS", [])):
         if isinstance(f, tuple) and f[0] == "-include":
-            result['CCFLAGS'][i] = (f[0], env.File(realpath(f[1].get_path())))
+            result["CCFLAGS"][i] = (f[0], env.File(os.path.realpath(f[1].get_path())))
 
     return result
 
@@ -191,14 +204,15 @@ def ProcessFlags(env, flags):  # pylint: disable=too-many-branches
     # Cancel any previous definition of name, either built in or
     # provided with a -U option // Issue #191
     undefines = [
-        u for u in env.get("CCFLAGS", [])
+        u
+        for u in env.get("CCFLAGS", [])
         if isinstance(u, string_types) and u.startswith("-U")
     ]
     if undefines:
         for undef in undefines:
-            env['CCFLAGS'].remove(undef)
-            if undef[2:] in env['CPPDEFINES']:
-                env['CPPDEFINES'].remove(undef[2:])
+            env["CCFLAGS"].remove(undef)
+            if undef[2:] in env["CPPDEFINES"]:
+                env["CPPDEFINES"].remove(undef[2:])
         env.Append(_CPPDEFFLAGS=" %s" % " ".join(undefines))
 
 
@@ -221,8 +235,7 @@ def ProcessUnFlags(env, flags):
             for current in env.get(key, []):
                 conditions = [
                     unflag == current,
-                    isinstance(current, (tuple, list))
-                    and unflag[0] == current[0]
+                    isinstance(current, (tuple, list)) and unflag[0] == current[0],
                 ]
                 if any(conditions):
                     env[key].remove(current)
@@ -231,15 +244,14 @@ def ProcessUnFlags(env, flags):
 def MatchSourceFiles(env, src_dir, src_filter=None):
     src_filter = env.subst(src_filter) if src_filter else None
     src_filter = src_filter or SRC_FILTER_DEFAULT
-    return fs.match_src_files(env.subst(src_dir), src_filter,
-                              SRC_BUILD_EXT + SRC_HEADER_EXT)
+    return fs.match_src_files(
+        env.subst(src_dir), src_filter, SRC_BUILD_EXT + SRC_HEADER_EXT
+    )
 
 
-def CollectBuildFiles(env,
-                      variant_dir,
-                      src_dir,
-                      src_filter=None,
-                      duplicate=False):
+def CollectBuildFiles(
+    env, variant_dir, src_dir, src_filter=None, duplicate=False
+):  # pylint: disable=too-many-locals
     sources = []
     variants = []
 
@@ -248,27 +260,44 @@ def CollectBuildFiles(env,
         src_dir = src_dir[:-1]
 
     for item in env.MatchSourceFiles(src_dir, src_filter):
-        _reldir = dirname(item)
-        _src_dir = join(src_dir, _reldir) if _reldir else src_dir
-        _var_dir = join(variant_dir, _reldir) if _reldir else variant_dir
+        _reldir = os.path.dirname(item)
+        _src_dir = os.path.join(src_dir, _reldir) if _reldir else src_dir
+        _var_dir = os.path.join(variant_dir, _reldir) if _reldir else variant_dir
 
         if _var_dir not in variants:
             variants.append(_var_dir)
             env.VariantDir(_var_dir, _src_dir, duplicate)
 
         if fs.path_endswith_ext(item, SRC_BUILD_EXT):
-            sources.append(env.File(join(_var_dir, basename(item))))
+            sources.append(env.File(os.path.join(_var_dir, os.path.basename(item))))
+
+    for callback, pattern in env.get("__PIO_BUILD_MIDDLEWARES", []):
+        tmp = []
+        for node in sources:
+            if pattern and not fnmatch.fnmatch(node.get_path(), pattern):
+                tmp.append(node)
+                continue
+            n = callback(node)
+            if n:
+                tmp.append(n)
+        sources = tmp
 
     return sources
 
 
+def AddBuildMiddleware(env, callback, pattern=None):
+    env.Append(__PIO_BUILD_MIDDLEWARES=[(callback, pattern)])
+
+
 def BuildFrameworks(env, frameworks):
     if not frameworks:
         return
 
     if "BOARD" not in env:
-        sys.stderr.write("Please specify `board` in `platformio.ini` to use "
-                         "with '%s' framework\n" % ", ".join(frameworks))
+        sys.stderr.write(
+            "Please specify `board` in `platformio.ini` to use "
+            "with '%s' framework\n" % ", ".join(frameworks)
+        )
         env.Exit(1)
 
     board_frameworks = env.BoardConfig().get("frameworks", [])
@@ -276,8 +305,7 @@ def BuildFrameworks(env, frameworks):
         if board_frameworks:
             frameworks.insert(0, board_frameworks[0])
         else:
-            sys.stderr.write(
-                "Error: Please specify `board` in `platformio.ini`\n")
+            sys.stderr.write("Error: Please specify `board` in `platformio.ini`\n")
             env.Exit(1)
 
     for f in frameworks:
@@ -290,22 +318,24 @@ def BuildFrameworks(env, frameworks):
         if f in board_frameworks:
             SConscript(env.GetFrameworkScript(f), exports="env")
         else:
-            sys.stderr.write(
-                "Error: This board doesn't support %s framework!\n" % f)
+            sys.stderr.write("Error: This board doesn't support %s framework!\n" % f)
             env.Exit(1)
 
 
 def BuildLibrary(env, variant_dir, src_dir, src_filter=None):
     env.ProcessUnFlags(env.get("BUILD_UNFLAGS"))
     return env.StaticLibrary(
-        env.subst(variant_dir),
-        env.CollectBuildFiles(variant_dir, src_dir, src_filter))
+        env.subst(variant_dir), env.CollectBuildFiles(variant_dir, src_dir, src_filter)
+    )
 
 
 def BuildSources(env, variant_dir, src_dir, src_filter=None):
     nodes = env.CollectBuildFiles(variant_dir, src_dir, src_filter)
     DefaultEnvironment().Append(
-        PIOBUILDFILES=[env.Object(node) for node in nodes])
+        PIOBUILDFILES=[
+            env.Object(node) if isinstance(node, FS.File) else node for node in nodes
+        ]
+    )
 
 
 def exists(_):
@@ -319,6 +349,7 @@ def generate(env):
     env.AddMethod(ProcessUnFlags)
     env.AddMethod(MatchSourceFiles)
     env.AddMethod(CollectBuildFiles)
+    env.AddMethod(AddBuildMiddleware)
     env.AddMethod(BuildFrameworks)
     env.AddMethod(BuildLibrary)
     env.AddMethod(BuildSources)

platformio/commands/__init__.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 import os
-from os.path import dirname, isfile, join
 
 import click
 
@@ -22,13 +21,21 @@ class PlatformioCLI(click.MultiCommand):
 
     leftover_args = []
 
+    def __init__(self, *args, **kwargs):
+        super(PlatformioCLI, self).__init__(*args, **kwargs)
+        self._pio_cmds_dir = os.path.dirname(__file__)
+
     @staticmethod
     def in_silence():
         args = PlatformioCLI.leftover_args
-        return args and any([
-            args[0] == "debug" and "--interpreter" in " ".join(args),
-            args[0] == "upgrade", "--json-output" in args, "--version" in args
-        ])
+        return args and any(
+            [
+                args[0] == "debug" and "--interpreter" in " ".join(args),
+                args[0] == "upgrade",
+                "--json-output" in args,
+                "--version" in args,
+            ]
+        )
 
     def invoke(self, ctx):
         PlatformioCLI.leftover_args = ctx.args
@@ -38,35 +45,23 @@ class PlatformioCLI(click.MultiCommand):
 
     def list_commands(self, ctx):
         cmds = []
-        cmds_dir = dirname(__file__)
-        for name in os.listdir(cmds_dir):
-            if name.startswith("__init__"):
+        for cmd_name in os.listdir(self._pio_cmds_dir):
+            if cmd_name.startswith("__init__"):
                 continue
-            if isfile(join(cmds_dir, name, "command.py")):
-                cmds.append(name)
-            elif name.endswith(".py"):
-                cmds.append(name[:-3])
+            if os.path.isfile(os.path.join(self._pio_cmds_dir, cmd_name, "command.py")):
+                cmds.append(cmd_name)
+            elif cmd_name.endswith(".py"):
+                cmds.append(cmd_name[:-3])
         cmds.sort()
         return cmds
 
     def get_command(self, ctx, cmd_name):
         mod = None
         try:
-            mod = __import__("platformio.commands." + cmd_name, None, None,
-                             ["cli"])
+            mod_path = "platformio.commands." + cmd_name
+            if os.path.isfile(os.path.join(self._pio_cmds_dir, cmd_name, "command.py")):
+                mod_path = "platformio.commands.%s.command" % cmd_name
+            mod = __import__(mod_path, None, None, ["cli"])
         except ImportError:
-            try:
-                return self._handle_obsolate_command(cmd_name)
-            except AttributeError:
-                raise click.UsageError('No such command "%s"' % cmd_name, ctx)
+            raise click.UsageError('No such command "%s"' % cmd_name, ctx)
         return mod.cli
-
-    @staticmethod
-    def _handle_obsolate_command(name):
-        if name == "platforms":
-            from platformio.commands import platform
-            return platform.cli
-        if name == "serialports":
-            from platformio.commands import device
-            return device.cli
-        raise AttributeError()

platformio/commands/boards.py

@@ -34,9 +34,9 @@ def cli(query, installed, json_output):  # pylint: disable=R0912
     for board in _get_boards(installed):
         if query and query.lower() not in json.dumps(board).lower():
             continue
-        if board['platform'] not in grpboards:
-            grpboards[board['platform']] = []
-        grpboards[board['platform']].append(board)
+        if board["platform"] not in grpboards:
+            grpboards[board["platform"]] = []
+        grpboards[board["platform"]].append(board)
 
     terminal_width, _ = click.get_terminal_size()
     for (platform, boards) in sorted(grpboards.items()):
@@ -50,11 +50,21 @@ def cli(query, installed, json_output):  # pylint: disable=R0912
 
 def print_boards(boards):
     click.echo(
-        tabulate([(click.style(b['id'], fg="cyan"), b['mcu'], "%dMHz" %
-                   (b['fcpu'] / 1000000), fs.format_filesize(
-                       b['rom']), fs.format_filesize(b['ram']), b['name'])
-                  for b in boards],
-                 headers=["ID", "MCU", "Frequency", "Flash", "RAM", "Name"]))
+        tabulate(
+            [
+                (
+                    click.style(b["id"], fg="cyan"),
+                    b["mcu"],
+                    "%dMHz" % (b["fcpu"] / 1000000),
+                    fs.format_filesize(b["rom"]),
+                    fs.format_filesize(b["ram"]),
+                    b["name"],
+                )
+                for b in boards
+            ],
+            headers=["ID", "MCU", "Frequency", "Flash", "RAM", "Name"],
+        )
+    )
 
 
 def _get_boards(installed=False):
@@ -66,7 +76,7 @@ def _print_boards_json(query, installed=False):
     result = []
     for board in _get_boards(installed):
         if query:
-            search_data = "%s %s" % (board['id'], json.dumps(board).lower())
+            search_data = "%s %s" % (board["id"], json.dumps(board).lower())
             if query.lower() not in search_data.lower():
                 continue
         result.append(board)

platformio/commands/check/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/commands/check/command.py

@@ -0,0 +1,316 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=too-many-arguments,too-many-locals,too-many-branches
+# pylint: disable=redefined-builtin,too-many-statements
+
+import os
+from collections import Counter
+from os.path import dirname, isfile
+from time import time
+
+import click
+from tabulate import tabulate
+
+from platformio import app, exception, fs, util
+from platformio.commands.check.defect import DefectItem
+from platformio.commands.check.tools import CheckToolFactory
+from platformio.compat import dump_json_to_unicode
+from platformio.project.config import ProjectConfig
+from platformio.project.helpers import find_project_dir_above, get_project_dir
+
+
+@click.command("check", short_help="Run a static analysis tool on code")
+@click.option("-e", "--environment", multiple=True)
+@click.option(
+    "-d",
+    "--project-dir",
+    default=os.getcwd,
+    type=click.Path(
+        exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True
+    ),
+)
+@click.option(
+    "-c",
+    "--project-conf",
+    type=click.Path(
+        exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True
+    ),
+)
+@click.option("--pattern", multiple=True)
+@click.option("--flags", multiple=True)
+@click.option(
+    "--severity", multiple=True, type=click.Choice(DefectItem.SEVERITY_LABELS.values())
+)
+@click.option("-s", "--silent", is_flag=True)
+@click.option("-v", "--verbose", is_flag=True)
+@click.option("--json-output", is_flag=True)
+@click.option(
+    "--fail-on-defect",
+    multiple=True,
+    type=click.Choice(DefectItem.SEVERITY_LABELS.values()),
+)
+def cli(
+    environment,
+    project_dir,
+    project_conf,
+    pattern,
+    flags,
+    severity,
+    silent,
+    verbose,
+    json_output,
+    fail_on_defect,
+):
+    app.set_session_var("custom_project_conf", project_conf)
+
+    # find project directory on upper level
+    if isfile(project_dir):
+        project_dir = find_project_dir_above(project_dir)
+
+    results = []
+    with fs.cd(project_dir):
+        config = ProjectConfig.get_instance(project_conf)
+        config.validate(environment)
+
+        default_envs = config.default_envs()
+        for envname in config.envs():
+            skipenv = any(
+                [
+                    environment and envname not in environment,
+                    not environment and default_envs and envname not in default_envs,
+                ]
+            )
+
+            env_options = config.items(env=envname, as_dict=True)
+            env_dump = []
+            for k, v in env_options.items():
+                if k not in ("platform", "framework", "board"):
+                    continue
+                env_dump.append(
+                    "%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v)
+                )
+
+            default_patterns = [
+                config.get_optional_dir("src"),
+                config.get_optional_dir("include"),
+            ]
+            tool_options = dict(
+                verbose=verbose,
+                silent=silent,
+                patterns=pattern or env_options.get("check_patterns", default_patterns),
+                flags=flags or env_options.get("check_flags"),
+                severity=[DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_HIGH]]
+                if silent
+                else severity or config.get("env:" + envname, "check_severity"),
+            )
+
+            for tool in config.get("env:" + envname, "check_tool"):
+                if skipenv:
+                    results.append({"env": envname, "tool": tool})
+                    continue
+                if not silent and not json_output:
+                    print_processing_header(tool, envname, env_dump)
+
+                ct = CheckToolFactory.new(
+                    tool, project_dir, config, envname, tool_options
+                )
+
+                result = {"env": envname, "tool": tool, "duration": time()}
+                rc = ct.check(
+                    on_defect_callback=None
+                    if (json_output or verbose)
+                    else lambda defect: click.echo(repr(defect))
+                )
+
+                result["defects"] = ct.get_defects()
+                result["duration"] = time() - result["duration"]
+
+                result["succeeded"] = rc == 0
+                if fail_on_defect:
+                    result["succeeded"] = rc == 0 and not any(
+                        DefectItem.SEVERITY_LABELS[d.severity] in fail_on_defect
+                        for d in result["defects"]
+                    )
+                result["stats"] = collect_component_stats(result)
+                results.append(result)
+
+                if verbose:
+                    click.echo("\n".join(repr(d) for d in result["defects"]))
+
+                if not json_output and not silent:
+                    if rc != 0:
+                        click.echo(
+                            "Error: %s failed to perform check! Please "
+                            "examine tool output in verbose mode." % tool
+                        )
+                    elif not result["defects"]:
+                        click.echo("No defects found")
+                    print_processing_footer(result)
+
+        if json_output:
+            click.echo(dump_json_to_unicode(results_to_json(results)))
+        elif not silent:
+            print_check_summary(results)
+
+    command_failed = any(r.get("succeeded") is False for r in results)
+    if command_failed:
+        raise exception.ReturnErrorCode(1)
+
+
+def results_to_json(raw):
+    results = []
+    for item in raw:
+        if item.get("succeeded") is None:
+            continue
+        item.update(
+            {
+                "succeeded": bool(item.get("succeeded")),
+                "defects": [d.as_dict() for d in item.get("defects", [])],
+            }
+        )
+        results.append(item)
+
+    return results
+
+
+def print_processing_header(tool, envname, envdump):
+    click.echo(
+        "Checking %s > %s (%s)"
+        % (click.style(envname, fg="cyan", bold=True), tool, "; ".join(envdump))
+    )
+    terminal_width, _ = click.get_terminal_size()
+    click.secho("-" * terminal_width, bold=True)
+
+
+def print_processing_footer(result):
+    is_failed = not result.get("succeeded")
+    util.print_labeled_bar(
+        "[%s] Took %.2f seconds"
+        % (
+            (
+                click.style("FAILED", fg="red", bold=True)
+                if is_failed
+                else click.style("PASSED", fg="green", bold=True)
+            ),
+            result["duration"],
+        ),
+        is_error=is_failed,
+    )
+
+
+def collect_component_stats(result):
+    components = dict()
+
+    def _append_defect(component, defect):
+        if not components.get(component):
+            components[component] = Counter()
+        components[component].update({DefectItem.SEVERITY_LABELS[defect.severity]: 1})
+
+    for defect in result.get("defects", []):
+        component = dirname(defect.file) or defect.file
+        _append_defect(component, defect)
+
+        if component.startswith(get_project_dir()):
+            while os.sep in component:
+                component = dirname(component)
+                _append_defect(component, defect)
+
+    return components
+
+
+def print_defects_stats(results):
+    if not results:
+        return
+
+    component_stats = {}
+    for r in results:
+        for k, v in r.get("stats", {}).items():
+            if not component_stats.get(k):
+                component_stats[k] = Counter()
+            component_stats[k].update(r["stats"][k])
+
+    if not component_stats:
+        return
+
+    severity_labels = list(DefectItem.SEVERITY_LABELS.values())
+    severity_labels.reverse()
+    tabular_data = list()
+    for k, v in component_stats.items():
+        tool_defect = [v.get(s, 0) for s in severity_labels]
+        tabular_data.append([k] + tool_defect)
+
+    total = ["Total"] + [sum(d) for d in list(zip(*tabular_data))[1:]]
+    tabular_data.sort()
+    tabular_data.append([])  # Empty line as delimiter
+    tabular_data.append(total)
+
+    headers = ["Component"]
+    headers.extend([l.upper() for l in severity_labels])
+    headers = [click.style(h, bold=True) for h in headers]
+    click.echo(tabulate(tabular_data, headers=headers, numalign="center"))
+    click.echo()
+
+
+def print_check_summary(results):
+    click.echo()
+
+    tabular_data = []
+    succeeded_nums = 0
+    failed_nums = 0
+    duration = 0
+
+    print_defects_stats(results)
+
+    for result in results:
+        duration += result.get("duration", 0)
+        if result.get("succeeded") is False:
+            failed_nums += 1
+            status_str = click.style("FAILED", fg="red")
+        elif result.get("succeeded") is None:
+            status_str = "IGNORED"
+        else:
+            succeeded_nums += 1
+            status_str = click.style("PASSED", fg="green")
+
+        tabular_data.append(
+            (
+                click.style(result["env"], fg="cyan"),
+                result["tool"],
+                status_str,
+                util.humanize_duration_time(result.get("duration")),
+            )
+        )
+
+    click.echo(
+        tabulate(
+            tabular_data,
+            headers=[
+                click.style(s, bold=True)
+                for s in ("Environment", "Tool", "Status", "Duration")
+            ],
+        ),
+        err=failed_nums,
+    )
+
+    util.print_labeled_bar(
+        "%s%d succeeded in %s"
+        % (
+            "%d failed, " % failed_nums if failed_nums else "",
+            succeeded_nums,
+            util.humanize_duration_time(duration),
+        ),
+        is_error=failed_nums,
+        fg="red" if failed_nums else "green",
+    )

platformio/commands/check/defect.py

@@ -0,0 +1,95 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from os.path import abspath, relpath
+
+import click
+
+from platformio.project.helpers import get_project_dir
+
+# pylint: disable=too-many-instance-attributes, redefined-builtin
+# pylint: disable=too-many-arguments
+
+
+class DefectItem(object):
+
+    SEVERITY_HIGH = 1
+    SEVERITY_MEDIUM = 2
+    SEVERITY_LOW = 4
+    SEVERITY_LABELS = {4: "low", 2: "medium", 1: "high"}
+
+    def __init__(
+        self,
+        severity,
+        category,
+        message,
+        file="unknown",
+        line=0,
+        column=0,
+        id=None,
+        callstack=None,
+        cwe=None,
+    ):
+        assert severity in (self.SEVERITY_HIGH, self.SEVERITY_MEDIUM, self.SEVERITY_LOW)
+        self.severity = severity
+        self.category = category
+        self.message = message
+        self.line = int(line)
+        self.column = int(column)
+        self.callstack = callstack
+        self.cwe = cwe
+        self.id = id
+        self.file = file
+        if file.startswith(get_project_dir()):
+            self.file = relpath(file, get_project_dir())
+
+    def __repr__(self):
+        defect_color = None
+        if self.severity == self.SEVERITY_HIGH:
+            defect_color = "red"
+        elif self.severity == self.SEVERITY_MEDIUM:
+            defect_color = "yellow"
+
+        format_str = "{file}:{line}: [{severity}:{category}] {message} {id}"
+        return format_str.format(
+            severity=click.style(self.SEVERITY_LABELS[self.severity], fg=defect_color),
+            category=click.style(self.category.lower(), fg=defect_color),
+            file=click.style(self.file, bold=True),
+            message=self.message,
+            line=self.line,
+            id="%s" % "[%s]" % self.id if self.id else "",
+        )
+
+    def __or__(self, defect):
+        return self.severity | defect.severity
+
+    @staticmethod
+    def severity_to_int(label):
+        for key, value in DefectItem.SEVERITY_LABELS.items():
+            if label == value:
+                return key
+        raise Exception("Unknown severity label -> %s" % label)
+
+    def as_dict(self):
+        return {
+            "severity": self.SEVERITY_LABELS[self.severity],
+            "category": self.category,
+            "message": self.message,
+            "file": abspath(self.file),
+            "line": self.line,
+            "column": self.column,
+            "callstack": self.callstack,
+            "id": self.id,
+            "cwe": self.cwe,
+        }

platformio/commands/check/tools/__init__.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from platformio import exception
+from platformio.commands.check.tools.clangtidy import ClangtidyCheckTool
+from platformio.commands.check.tools.cppcheck import CppcheckCheckTool
+
+
+class CheckToolFactory(object):
+    @staticmethod
+    def new(tool, project_dir, config, envname, options):
+        cls = None
+        if tool == "cppcheck":
+            cls = CppcheckCheckTool
+        elif tool == "clangtidy":
+            cls = ClangtidyCheckTool
+        else:
+            raise exception.PlatformioException("Unknown check tool `%s`" % tool)
+        return cls(project_dir, config, envname, options)

platformio/commands/check/tools/base.py

@@ -0,0 +1,171 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import glob
+import os
+
+import click
+
+from platformio import fs, proc
+from platformio.commands.check.defect import DefectItem
+from platformio.project.helpers import get_project_dir, load_project_ide_data
+
+
+class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
+    def __init__(self, project_dir, config, envname, options):
+        self.config = config
+        self.envname = envname
+        self.options = options
+        self.cpp_defines = []
+        self.cpp_flags = []
+        self.cpp_includes = []
+
+        self._defects = []
+        self._on_defect_callback = None
+        self._bad_input = False
+        self._load_cpp_data(project_dir, envname)
+
+        # detect all defects by default
+        if not self.options.get("severity"):
+            self.options["severity"] = [
+                DefectItem.SEVERITY_LOW,
+                DefectItem.SEVERITY_MEDIUM,
+                DefectItem.SEVERITY_HIGH,
+            ]
+        # cast to severity by ids
+        self.options["severity"] = [
+            s if isinstance(s, int) else DefectItem.severity_to_int(s)
+            for s in self.options["severity"]
+        ]
+
+    def _load_cpp_data(self, project_dir, envname):
+        data = load_project_ide_data(project_dir, envname)
+        if not data:
+            return
+        self.cpp_flags = data.get("cxx_flags", "").split(" ")
+        self.cpp_includes = data.get("includes", [])
+        self.cpp_defines = data.get("defines", [])
+        self.cpp_defines.extend(self._get_toolchain_defines(data.get("cc_path")))
+
+    def get_flags(self, tool):
+        result = []
+        flags = self.options.get("flags") or []
+        for flag in flags:
+            if ":" not in flag:
+                result.extend([f for f in flag.split(" ") if f])
+            elif flag.startswith("%s:" % tool):
+                result.extend([f for f in flag.split(":", 1)[1].split(" ") if f])
+
+        return result
+
+    @staticmethod
+    def _get_toolchain_defines(cc_path):
+        defines = []
+        result = proc.exec_command("echo | %s -dM -E -x c++ -" % cc_path, shell=True)
+
+        for line in result["out"].split("\n"):
+            tokens = line.strip().split(" ", 2)
+            if not tokens or tokens[0] != "#define":
+                continue
+            if len(tokens) > 2:
+                defines.append("%s=%s" % (tokens[1], tokens[2]))
+            else:
+                defines.append(tokens[1])
+
+        return defines
+
+    @staticmethod
+    def is_flag_set(flag, flags):
+        return any(flag in f for f in flags)
+
+    def get_defects(self):
+        return self._defects
+
+    def configure_command(self):
+        raise NotImplementedError
+
+    def on_tool_output(self, line):
+        line = self.tool_output_filter(line)
+        if not line:
+            return
+
+        defect = self.parse_defect(line)
+
+        if not isinstance(defect, DefectItem):
+            if self.options.get("verbose"):
+                click.echo(line)
+            return
+
+        if defect.severity not in self.options["severity"]:
+            return
+
+        self._defects.append(defect)
+        if self._on_defect_callback:
+            self._on_defect_callback(defect)
+
+    @staticmethod
+    def tool_output_filter(line):
+        return line
+
+    @staticmethod
+    def parse_defect(raw_line):
+        return raw_line
+
+    def clean_up(self):
+        pass
+
+    def get_project_target_files(self):
+        allowed_extensions = (".h", ".hpp", ".c", ".cc", ".cpp", ".ino")
+        result = []
+
+        def _add_file(path):
+            if not path.endswith(allowed_extensions):
+                return
+            result.append(os.path.abspath(path))
+
+        for pattern in self.options["patterns"]:
+            for item in glob.glob(pattern):
+                if not os.path.isdir(item):
+                    _add_file(item)
+                for root, _, files in os.walk(item, followlinks=True):
+                    for f in files:
+                        _add_file(os.path.join(root, f))
+
+        return result
+
+    def get_source_language(self):
+        with fs.cd(get_project_dir()):
+            for _, __, files in os.walk(self.config.get_optional_dir("src")):
+                for name in files:
+                    if "." not in name:
+                        continue
+                    if os.path.splitext(name)[1].lower() in (".cpp", ".cxx", ".ino"):
+                        return "c++"
+            return "c"
+
+    def check(self, on_defect_callback=None):
+        self._on_defect_callback = on_defect_callback
+        cmd = self.configure_command()
+        if self.options.get("verbose"):
+            click.echo(" ".join(cmd))
+
+        proc.exec_command(
+            cmd,
+            stdout=proc.LineBufferedAsyncPipe(self.on_tool_output),
+            stderr=proc.LineBufferedAsyncPipe(self.on_tool_output),
+        )
+
+        self.clean_up()
+
+        return self._bad_input

platformio/commands/check/tools/clangtidy.py

@@ -0,0 +1,67 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+from os.path import join
+
+from platformio.commands.check.defect import DefectItem
+from platformio.commands.check.tools.base import CheckToolBase
+from platformio.managers.core import get_core_package_dir
+
+
+class ClangtidyCheckTool(CheckToolBase):
+    def tool_output_filter(self, line):
+        if not self.options.get("verbose") and "[clang-diagnostic-error]" in line:
+            return ""
+
+        if "[CommonOptionsParser]" in line:
+            self._bad_input = True
+            return line
+
+        if any(d in line for d in ("note: ", "error: ", "warning: ")):
+            return line
+
+        return ""
+
+    def parse_defect(self, raw_line):
+        match = re.match(r"^(.*):(\d+):(\d+):\s+([^:]+):\s(.+)\[([^]]+)\]$", raw_line)
+        if not match:
+            return raw_line
+
+        file_, line, column, category, message, defect_id = match.groups()
+
+        severity = DefectItem.SEVERITY_LOW
+        if category == "error":
+            severity = DefectItem.SEVERITY_HIGH
+        elif category == "warning":
+            severity = DefectItem.SEVERITY_MEDIUM
+
+        return DefectItem(severity, category, message, file_, line, column, defect_id)
+
+    def configure_command(self):
+        tool_path = join(get_core_package_dir("tool-clangtidy"), "clang-tidy")
+
+        cmd = [tool_path, "--quiet"]
+        flags = self.get_flags("clangtidy")
+        if not self.is_flag_set("--checks", flags):
+            cmd.append("--checks=*")
+
+        cmd.extend(flags)
+        cmd.extend(self.get_project_target_files())
+        cmd.append("--")
+
+        cmd.extend(["-D%s" % d for d in self.cpp_defines])
+        cmd.extend(["-I%s" % inc for inc in self.cpp_includes])
+
+        return cmd

platformio/commands/check/tools/cppcheck.py

@@ -0,0 +1,158 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from os import remove
+from os.path import isfile, join
+from tempfile import NamedTemporaryFile
+
+from platformio.commands.check.defect import DefectItem
+from platformio.commands.check.tools.base import CheckToolBase
+from platformio.managers.core import get_core_package_dir
+
+
+class CppcheckCheckTool(CheckToolBase):
+    def __init__(self, *args, **kwargs):
+        self._tmp_files = []
+        self.defect_fields = [
+            "severity",
+            "message",
+            "file",
+            "line",
+            "column",
+            "callstack",
+            "cwe",
+            "id",
+        ]
+        super(CppcheckCheckTool, self).__init__(*args, **kwargs)
+
+    def tool_output_filter(self, line):
+        if (
+            not self.options.get("verbose")
+            and "--suppress=unmatchedSuppression:" in line
+        ):
+            return ""
+
+        if any(
+            msg in line
+            for msg in (
+                "No C or C++ source files found",
+                "unrecognized command line option",
+            )
+        ):
+            self._bad_input = True
+
+        return line
+
+    def parse_defect(self, raw_line):
+        if "<&PIO&>" not in raw_line or any(
+            f not in raw_line for f in self.defect_fields
+        ):
+            return None
+
+        args = dict()
+        for field in raw_line.split("<&PIO&>"):
+            field = field.strip().replace('"', "")
+            name, value = field.split("=", 1)
+            args[name] = value
+
+        args["category"] = args["severity"]
+        if args["severity"] == "error":
+            args["severity"] = DefectItem.SEVERITY_HIGH
+        elif args["severity"] == "warning":
+            args["severity"] = DefectItem.SEVERITY_MEDIUM
+        else:
+            args["severity"] = DefectItem.SEVERITY_LOW
+
+        return DefectItem(**args)
+
+    def configure_command(self):
+        tool_path = join(get_core_package_dir("tool-cppcheck"), "cppcheck")
+
+        cmd = [
+            tool_path,
+            "--error-exitcode=1",
+            "--verbose" if self.options.get("verbose") else "--quiet",
+        ]
+
+        cmd.append(
+            '--template="%s"'
+            % "<&PIO&>".join(["{0}={{{0}}}".format(f) for f in self.defect_fields])
+        )
+
+        flags = self.get_flags("cppcheck")
+        if not flags:
+            # by default user can suppress reporting individual defects
+            # directly in code // cppcheck-suppress warningID
+            cmd.append("--inline-suppr")
+        if not self.is_flag_set("--platform", flags):
+            cmd.append("--platform=unspecified")
+        if not self.is_flag_set("--enable", flags):
+            enabled_checks = [
+                "warning",
+                "style",
+                "performance",
+                "portability",
+                "unusedFunction",
+            ]
+            cmd.append("--enable=%s" % ",".join(enabled_checks))
+
+        if not self.is_flag_set("--language", flags):
+            if self.get_source_language() == "c++":
+                cmd.append("--language=c++")
+
+                if not self.is_flag_set("--std", flags):
+                    for f in self.cpp_flags:
+                        if "-std" in f:
+                            # Standards with GNU extensions are not allowed
+                            cmd.append("-" + f.replace("gnu", "c"))
+
+        cmd.extend(["-D%s" % d for d in self.cpp_defines])
+        cmd.extend(flags)
+
+        cmd.append("--file-list=%s" % self._generate_src_file())
+        cmd.append("--includes-file=%s" % self._generate_inc_file())
+
+        core_dir = self.config.get_optional_dir("core")
+        cmd.append("--suppress=*:%s*" % core_dir)
+        cmd.append("--suppress=unmatchedSuppression:%s*" % core_dir)
+
+        return cmd
+
+    def _create_tmp_file(self, data):
+        with NamedTemporaryFile("w", delete=False) as fp:
+            fp.write(data)
+            self._tmp_files.append(fp.name)
+            return fp.name
+
+    def _generate_src_file(self):
+        src_files = [
+            f for f in self.get_project_target_files() if not f.endswith((".h", ".hpp"))
+        ]
+        return self._create_tmp_file("\n".join(src_files))
+
+    def _generate_inc_file(self):
+        return self._create_tmp_file("\n".join(self.cpp_includes))
+
+    def clean_up(self):
+        for f in self._tmp_files:
+            if isfile(f):
+                remove(f)
+
+        # delete temporary dump files generated by addons
+        if not self.is_flag_set("--addon", self.get_flags("cppcheck")):
+            return
+        for f in self.get_project_target_files():
+            dump_file = f + ".dump"
+            if isfile(dump_file):
+                remove(dump_file)

platformio/commands/ci.py

@@ -14,7 +14,7 @@
 
 from glob import glob
 from os import getenv, makedirs, remove
-from os.path import abspath, basename, expanduser, isdir, isfile, join
+from os.path import abspath, basename, isdir, isfile, join
 from shutil import copyfile, copytree
 from tempfile import mkdtemp
 
@@ -23,7 +23,7 @@ import click
 from platformio import app, fs
 from platformio.commands.init import cli as cmd_init
 from platformio.commands.init import validate_boards
-from platformio.commands.run import cli as cmd_run
+from platformio.commands.run.command import cli as cmd_run
 from platformio.compat import glob_escape
 from platformio.exception import CIBuildEnvsEmpty
 from platformio.project.config import ProjectConfig
@@ -34,7 +34,7 @@ def validate_path(ctx, param, value):  # pylint: disable=unused-argument
     value = list(value)
     for i, p in enumerate(value):
         if p.startswith("~"):
-            value[i] = expanduser(p)
+            value[i] = fs.expanduser(p)
         value[i] = abspath(value[i])
         if not glob(value[i]):
             invalid_path = p
@@ -48,37 +48,37 @@ def validate_path(ctx, param, value):  # pylint: disable=unused-argument
 
 @click.command("ci", short_help="Continuous Integration")
 @click.argument("src", nargs=-1, callback=validate_path)
-@click.option("-l",
-              "--lib",
-              multiple=True,
-              callback=validate_path,
-              metavar="DIRECTORY")
+@click.option("-l", "--lib", multiple=True, callback=validate_path, metavar="DIRECTORY")
 @click.option("--exclude", multiple=True)
-@click.option("-b",
-              "--board",
-              multiple=True,
-              metavar="ID",
-              callback=validate_boards)
-@click.option("--build-dir",
-              default=mkdtemp,
-              type=click.Path(file_okay=False,
-                              dir_okay=True,
-                              writable=True,
-                              resolve_path=True))
+@click.option("-b", "--board", multiple=True, metavar="ID", callback=validate_boards)
+@click.option(
+    "--build-dir",
+    default=mkdtemp,
+    type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True),
+)
 @click.option("--keep-build-dir", is_flag=True)
-@click.option("-c",
-              "--project-conf",
-              type=click.Path(exists=True,
-                              file_okay=True,
-                              dir_okay=False,
-                              readable=True,
-                              resolve_path=True))
+@click.option(
+    "-c",
+    "--project-conf",
+    type=click.Path(
+        exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True
+    ),
+)
 @click.option("-O", "--project-option", multiple=True)
 @click.option("-v", "--verbose", is_flag=True)
 @click.pass_context
 def cli(  # pylint: disable=too-many-arguments, too-many-branches
-        ctx, src, lib, exclude, board, build_dir, keep_build_dir, project_conf,
-        project_option, verbose):
+    ctx,
+    src,
+    lib,
+    exclude,
+    board,
+    build_dir,
+    keep_build_dir,
+    project_conf,
+    project_option,
+    verbose,
+):
 
     if not src and getenv("PLATFORMIO_CI_SRC"):
         src = validate_path(ctx, None, getenv("PLATFORMIO_CI_SRC").split(":"))
@@ -110,10 +110,9 @@ def cli(  # pylint: disable=too-many-arguments, too-many-branches
             _exclude_contents(build_dir, exclude)
 
         # initialise project
-        ctx.invoke(cmd_init,
-                   project_dir=build_dir,
-                   board=board,
-                   project_option=project_option)
+        ctx.invoke(
+            cmd_init, project_dir=build_dir, board=board, project_option=project_option
+        )
 
         # process project
         ctx.invoke(cmd_run, project_dir=build_dir, verbose=verbose)
@@ -127,27 +126,27 @@ def _copy_contents(dst_dir, contents):
 
     for path in contents:
         if isdir(path):
-            items['dirs'].add(path)
+            items["dirs"].add(path)
         elif isfile(path):
-            items['files'].add(path)
+            items["files"].add(path)
 
     dst_dir_name = basename(dst_dir)
 
-    if dst_dir_name == "src" and len(items['dirs']) == 1:
-        copytree(list(items['dirs']).pop(), dst_dir, symlinks=True)
+    if dst_dir_name == "src" and len(items["dirs"]) == 1:
+        copytree(list(items["dirs"]).pop(), dst_dir, symlinks=True)
     else:
         if not isdir(dst_dir):
             makedirs(dst_dir)
-        for d in items['dirs']:
+        for d in items["dirs"]:
             copytree(d, join(dst_dir, basename(d)), symlinks=True)
 
-    if not items['files']:
+    if not items["files"]:
         return
 
     if dst_dir_name == "lib":
         dst_dir = join(dst_dir, mkdtemp(dir=dst_dir))
 
-    for f in items['files']:
+    for f in items["files"]:
         dst_file = join(dst_dir, basename(f))
         if f == dst_file:
             continue

platformio/commands/debug/__init__.py

@@ -11,5 +11,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from platformio.commands.debug.command import cli

platformio/commands/debug/client.py

@@ -30,7 +30,7 @@ from platformio import app, exception, fs, proc, util
 from platformio.commands.debug import helpers, initcfgs
 from platformio.commands.debug.process import BaseProcess
 from platformio.commands.debug.server import DebugServer
-from platformio.compat import hashlib_encode_data
+from platformio.compat import hashlib_encode_data, is_bytes
 from platformio.project.helpers import get_project_cache_dir
 from platformio.telemetry import MeasurementProtocol
 
@@ -53,8 +53,7 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
 
         if not isdir(get_project_cache_dir()):
             os.makedirs(get_project_cache_dir())
-        self._gdbsrc_dir = mkdtemp(dir=get_project_cache_dir(),
-                                   prefix=".piodebug-")
+        self._gdbsrc_dir = mkdtemp(dir=get_project_cache_dir(), prefix=".piodebug-")
 
         self._target_is_run = False
         self._last_server_activity = 0
@@ -70,39 +69,40 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
             "PROG_PATH": prog_path,
             "PROG_DIR": dirname(prog_path),
             "PROG_NAME": basename(splitext(prog_path)[0]),
-            "DEBUG_PORT": self.debug_options['port'],
-            "UPLOAD_PROTOCOL": self.debug_options['upload_protocol'],
-            "INIT_BREAK": self.debug_options['init_break'] or "",
-            "LOAD_CMDS": "\n".join(self.debug_options['load_cmds'] or []),
+            "DEBUG_PORT": self.debug_options["port"],
+            "UPLOAD_PROTOCOL": self.debug_options["upload_protocol"],
+            "INIT_BREAK": self.debug_options["init_break"] or "",
+            "LOAD_CMDS": "\n".join(self.debug_options["load_cmds"] or []),
         }
 
         self._debug_server.spawn(patterns)
 
-        if not patterns['DEBUG_PORT']:
-            patterns['DEBUG_PORT'] = self._debug_server.get_debug_port()
+        if not patterns["DEBUG_PORT"]:
+            patterns["DEBUG_PORT"] = self._debug_server.get_debug_port()
         self.generate_pioinit(self._gdbsrc_dir, patterns)
 
         # start GDB client
         args = [
             "piogdb",
             "-q",
-            "--directory", self._gdbsrc_dir,
-            "--directory", self.project_dir,
-            "-l", "10"
-        ]  # yapf: disable
+            "--directory",
+            self._gdbsrc_dir,
+            "--directory",
+            self.project_dir,
+            "-l",
+            "10",
+        ]
         args.extend(self.args)
         if not gdb_path:
             raise exception.DebugInvalidOptions("GDB client is not configured")
         gdb_data_dir = self._get_data_dir(gdb_path)
         if gdb_data_dir:
             args.extend(["--data-directory", gdb_data_dir])
-        args.append(patterns['PROG_PATH'])
+        args.append(patterns["PROG_PATH"])
 
-        return reactor.spawnProcess(self,
-                                    gdb_path,
-                                    args,
-                                    path=self.project_dir,
-                                    env=os.environ)
+        return reactor.spawnProcess(
+            self, gdb_path, args, path=self.project_dir, env=os.environ
+        )
 
     @staticmethod
     def _get_data_dir(gdb_path):
@@ -112,8 +112,9 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
         return gdb_data_dir if isdir(gdb_data_dir) else None
 
     def generate_pioinit(self, dst_dir, patterns):
-        server_exe = (self.debug_options.get("server")
-                      or {}).get("executable", "").lower()
+        server_exe = (
+            (self.debug_options.get("server") or {}).get("executable", "").lower()
+        )
         if "jlink" in server_exe:
             cfg = initcfgs.GDB_JLINK_INIT_CONFIG
         elif "st-util" in server_exe:
@@ -122,43 +123,43 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
             cfg = initcfgs.GDB_MSPDEBUG_INIT_CONFIG
         elif "qemu" in server_exe:
             cfg = initcfgs.GDB_QEMU_INIT_CONFIG
-        elif self.debug_options['require_debug_port']:
+        elif self.debug_options["require_debug_port"]:
             cfg = initcfgs.GDB_BLACKMAGIC_INIT_CONFIG
         else:
             cfg = initcfgs.GDB_DEFAULT_INIT_CONFIG
         commands = cfg.split("\n")
 
-        if self.debug_options['init_cmds']:
-            commands = self.debug_options['init_cmds']
-        commands.extend(self.debug_options['extra_cmds'])
+        if self.debug_options["init_cmds"]:
+            commands = self.debug_options["init_cmds"]
+        commands.extend(self.debug_options["extra_cmds"])
 
-        if not any("define pio_reset_target" in cmd for cmd in commands):
+        if not any("define pio_reset_run_target" in cmd for cmd in commands):
             commands = [
-                "define pio_reset_target",
-                "   echo Warning! Undefined pio_reset_target command\\n",
-                "   mon reset",
-                "end"
-            ] + commands  # yapf: disable
+                "define pio_reset_run_target",
+                "   echo Warning! Undefined pio_reset_run_target command\\n",
+                "   monitor reset",
+                "end",
+            ] + commands
         if not any("define pio_reset_halt_target" in cmd for cmd in commands):
             commands = [
                 "define pio_reset_halt_target",
                 "   echo Warning! Undefined pio_reset_halt_target command\\n",
-                "   mon reset halt",
-                "end"
-            ] + commands  # yapf: disable
+                "   monitor reset halt",
+                "end",
+            ] + commands
         if not any("define pio_restart_target" in cmd for cmd in commands):
             commands += [
                 "define pio_restart_target",
                 "   pio_reset_halt_target",
                 "   $INIT_BREAK",
-                "   %s" % ("continue" if patterns['INIT_BREAK'] else "next"),
-                "end"
-            ]  # yapf: disable
+                "   %s" % ("continue" if patterns["INIT_BREAK"] else "next"),
+                "end",
+            ]
 
         banner = [
             "echo PlatformIO Unified Debugger -> http://bit.ly/pio-debug\\n",
-            "echo PlatformIO: debug_tool = %s\\n" % self.debug_options['tool'],
-            "echo PlatformIO: Initializing remote target...\\n"
+            "echo PlatformIO: debug_tool = %s\\n" % self.debug_options["tool"],
+            "echo PlatformIO: Initializing remote target...\\n",
         ]
         footer = ["echo %s\\n" % self.INIT_COMPLETED_BANNER]
         commands = banner + commands + footer
@@ -192,7 +193,7 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
         if b"-gdb-exit" in data or data.strip() in (b"q", b"quit"):
             # Allow terminating via SIGINT/CTRL+C
             signal.signal(signal.SIGINT, signal.default_int_handler)
-            self.transport.write(b"pio_reset_target\n")
+            self.transport.write(b"pio_reset_run_target\n")
         self.transport.write(data)
 
     def processEnded(self, reason):  # pylint: disable=unused-argument
@@ -214,8 +215,7 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
         self._handle_error(data)
         # go to init break automatically
         if self.INIT_COMPLETED_BANNER.encode() in data:
-            self._auto_continue_timer = task.LoopingCall(
-                self._auto_exec_continue)
+            self._auto_continue_timer = task.LoopingCall(self._auto_exec_continue)
             self._auto_continue_timer.start(0.1)
 
     def errReceived(self, data):
@@ -223,10 +223,9 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
         self._handle_error(data)
 
     def console_log(self, msg):
-        if helpers.is_mi_mode(self.args):
-            self.outReceived(('~"%s\\n"\n' % msg).encode())
-        else:
-            self.outReceived(("%s\n" % msg).encode())
+        if helpers.is_gdbmi_mode():
+            msg = helpers.escape_gdbmi_stream("~", msg)
+        self.outReceived(msg if is_bytes(msg) else msg.encode())
 
     def _auto_exec_continue(self):
         auto_exec_delay = 0.5  # in seconds
@@ -236,29 +235,34 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
             self._auto_continue_timer.stop()
         self._auto_continue_timer = None
 
-        if not self.debug_options['init_break'] or self._target_is_run:
+        if not self.debug_options["init_break"] or self._target_is_run:
             return
         self.console_log(
-            "PlatformIO: Resume the execution to `debug_init_break = %s`" %
-            self.debug_options['init_break'])
-        self.console_log("PlatformIO: More configuration options -> "
-                         "http://bit.ly/pio-debug")
-        self.transport.write(b"0-exec-continue\n" if helpers.
-                             is_mi_mode(self.args) else b"continue\n")
+            "PlatformIO: Resume the execution to `debug_init_break = %s`\n"
+            % self.debug_options["init_break"]
+        )
+        self.console_log(
+            "PlatformIO: More configuration options -> http://bit.ly/pio-debug\n"
+        )
+        self.transport.write(
+            b"0-exec-continue\n" if helpers.is_gdbmi_mode() else b"continue\n"
+        )
         self._target_is_run = True
 
     def _handle_error(self, data):
-        if (self.PIO_SRC_NAME.encode() not in data
-                or b"Error in sourced" not in data):
+        if self.PIO_SRC_NAME.encode() not in data or b"Error in sourced" not in data:
             return
         configuration = {"debug": self.debug_options, "env": self.env_options}
         exd = re.sub(r'\\(?!")', "/", json.dumps(configuration))
-        exd = re.sub(r'"(?:[a-z]\:)?((/[^"/]+)+)"',
-                     lambda m: '"%s"' % join(*m.group(1).split("/")[-2:]), exd,
-                     re.I | re.M)
+        exd = re.sub(
+            r'"(?:[a-z]\:)?((/[^"/]+)+)"',
+            lambda m: '"%s"' % join(*m.group(1).split("/")[-2:]),
+            exd,
+            re.I | re.M,
+        )
         mp = MeasurementProtocol()
-        mp['exd'] = "DebugGDBPioInitError: %s" % exd
-        mp['exf'] = 1
+        mp["exd"] = "DebugGDBPioInitError: %s" % exd
+        mp["exf"] = 1
         mp.send("exception")
         self.transport.loseConnection()
 

platformio/commands/debug/command.py

@@ -17,43 +17,45 @@
 
 import os
 import signal
-from os.path import isfile, join
+from os.path import isfile
 
 import click
 
-from platformio import exception, fs, proc, util
+from platformio import app, exception, fs, proc, util
 from platformio.commands.debug import helpers
 from platformio.managers.core import inject_contrib_pysite
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import (is_platformio_project,
-                                        load_project_ide_data)
+from platformio.project.helpers import is_platformio_project, load_project_ide_data
 
 
-@click.command("debug",
-               context_settings=dict(ignore_unknown_options=True),
-               short_help="PIO Unified Debugger")
-@click.option("-d",
-              "--project-dir",
-              default=os.getcwd,
-              type=click.Path(exists=True,
-                              file_okay=False,
-                              dir_okay=True,
-                              writable=True,
-                              resolve_path=True))
-@click.option("-c",
-              "--project-conf",
-              type=click.Path(exists=True,
-                              file_okay=True,
-                              dir_okay=False,
-                              readable=True,
-                              resolve_path=True))
+@click.command(
+    "debug",
+    context_settings=dict(ignore_unknown_options=True),
+    short_help="PIO Unified Debugger",
+)
+@click.option(
+    "-d",
+    "--project-dir",
+    default=os.getcwd,
+    type=click.Path(
+        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
+    ),
+)
+@click.option(
+    "-c",
+    "--project-conf",
+    type=click.Path(
+        exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True
+    ),
+)
 @click.option("--environment", "-e", metavar="<environment>")
 @click.option("--verbose", "-v", is_flag=True)
 @click.option("--interface", type=click.Choice(["gdb"]))
 @click.argument("__unprocessed", nargs=-1, type=click.UNPROCESSED)
 @click.pass_context
-def cli(ctx, project_dir, project_conf, environment, verbose, interface,
-        __unprocessed):
+def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unprocessed):
+    app.set_session_var("custom_project_conf", project_conf)
+
     # use env variables from Eclipse or CLion
     for sysenv in ("CWD", "PWD", "PLATFORMIO_PROJECT_DIR"):
         if is_platformio_project(project_dir):
@@ -62,8 +64,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface,
             project_dir = os.getenv(sysenv)
 
     with fs.cd(project_dir):
-        config = ProjectConfig.get_instance(
-            project_conf or join(project_dir, "platformio.ini"))
+        config = ProjectConfig.get_instance(project_conf)
         config.validate(envs=[environment] if environment else None)
 
         env_name = environment or helpers.get_default_debug_env(config)
@@ -74,76 +75,81 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface,
         assert debug_options
 
     if not interface:
-        return helpers.predebug_project(ctx, project_dir, env_name, False,
-                                        verbose)
+        return helpers.predebug_project(ctx, project_dir, env_name, False, verbose)
 
     configuration = load_project_ide_data(project_dir, env_name)
     if not configuration:
-        raise exception.DebugInvalidOptions(
-            "Could not load debug configuration")
+        raise exception.DebugInvalidOptions("Could not load debug configuration")
 
     if "--version" in __unprocessed:
-        result = proc.exec_command([configuration['gdb_path'], "--version"])
-        if result['returncode'] == 0:
-            return click.echo(result['out'])
-        raise exception.PlatformioException("\n".join(
-            [result['out'], result['err']]))
+        result = proc.exec_command([configuration["gdb_path"], "--version"])
+        if result["returncode"] == 0:
+            return click.echo(result["out"])
+        raise exception.PlatformioException("\n".join([result["out"], result["err"]]))
 
     try:
         fs.ensure_udev_rules()
     except exception.InvalidUdevRules as e:
-        for line in str(e).split("\n") + [""]:
-            click.echo(
-                ('~"%s\\n"' if helpers.is_mi_mode(__unprocessed) else "%s") %
-                line)
+        click.echo(
+            helpers.escape_gdbmi_stream("~", str(e) + "\n")
+            if helpers.is_gdbmi_mode()
+            else str(e) + "\n",
+            nl=False,
+        )
 
-    debug_options['load_cmds'] = helpers.configure_esp32_load_cmds(
-        debug_options, configuration)
+    debug_options["load_cmds"] = helpers.configure_esp32_load_cmds(
+        debug_options, configuration
+    )
 
     rebuild_prog = False
-    preload = debug_options['load_cmds'] == ["preload"]
-    load_mode = debug_options['load_mode']
+    preload = debug_options["load_cmds"] == ["preload"]
+    load_mode = debug_options["load_mode"]
     if load_mode == "always":
-        rebuild_prog = (
-            preload
-            or not helpers.has_debug_symbols(configuration['prog_path']))
+        rebuild_prog = preload or not helpers.has_debug_symbols(
+            configuration["prog_path"]
+        )
     elif load_mode == "modified":
-        rebuild_prog = (
-            helpers.is_prog_obsolete(configuration['prog_path'])
-            or not helpers.has_debug_symbols(configuration['prog_path']))
+        rebuild_prog = helpers.is_prog_obsolete(
+            configuration["prog_path"]
+        ) or not helpers.has_debug_symbols(configuration["prog_path"])
     else:
-        rebuild_prog = not isfile(configuration['prog_path'])
+        rebuild_prog = not isfile(configuration["prog_path"])
 
     if preload or (not rebuild_prog and load_mode != "always"):
         # don't load firmware through debug server
-        debug_options['load_cmds'] = []
+        debug_options["load_cmds"] = []
 
     if rebuild_prog:
-        if helpers.is_mi_mode(__unprocessed):
-            click.echo('~"Preparing firmware for debugging...\\n"')
-            output = helpers.GDBBytesIO()
-            with util.capture_std_streams(output):
-                helpers.predebug_project(ctx, project_dir, env_name, preload,
-                                         verbose)
-            output.close()
+        if helpers.is_gdbmi_mode():
+            click.echo(
+                helpers.escape_gdbmi_stream(
+                    "~", "Preparing firmware for debugging...\n"
+                ),
+                nl=False,
+            )
+            stream = helpers.GDBMIConsoleStream()
+            with util.capture_std_streams(stream):
+                helpers.predebug_project(ctx, project_dir, env_name, preload, verbose)
+            stream.close()
         else:
             click.echo("Preparing firmware for debugging...")
-            helpers.predebug_project(ctx, project_dir, env_name, preload,
-                                     verbose)
+            helpers.predebug_project(ctx, project_dir, env_name, preload, verbose)
 
         # save SHA sum of newly created prog
         if load_mode == "modified":
-            helpers.is_prog_obsolete(configuration['prog_path'])
+            helpers.is_prog_obsolete(configuration["prog_path"])
 
-    if not isfile(configuration['prog_path']):
+    if not isfile(configuration["prog_path"]):
         raise exception.DebugInvalidOptions("Program/firmware is missed")
 
     # run debugging client
     inject_contrib_pysite()
+
+    # pylint: disable=import-outside-toplevel
     from platformio.commands.debug.client import GDBClient, reactor
 
     client = GDBClient(project_dir, __unprocessed, debug_options, env_options)
-    client.spawn(configuration['gdb_path'], configuration['prog_path'])
+    client.spawn(configuration["gdb_path"], configuration["prog_path"])
 
     signal.signal(signal.SIGINT, lambda *args, **kwargs: None)
     reactor.run()

platformio/commands/debug/helpers.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import re
 import sys
 import time
 from fnmatch import fnmatch
@@ -20,28 +21,46 @@ from io import BytesIO
 from os.path import isfile
 
 from platformio import exception, fs, util
-from platformio.commands.platform import \
-    platform_install as cmd_platform_install
-from platformio.commands.run import cli as cmd_run
+from platformio.commands import PlatformioCLI
+from platformio.commands.platform import platform_install as cmd_platform_install
+from platformio.commands.run.command import cli as cmd_run
+from platformio.compat import is_bytes
 from platformio.managers.platform import PlatformFactory
 from platformio.project.config import ProjectConfig
+from platformio.project.options import ProjectOptions
 
 
-class GDBBytesIO(BytesIO):  # pylint: disable=too-few-public-methods
+class GDBMIConsoleStream(BytesIO):  # pylint: disable=too-few-public-methods
 
     STDOUT = sys.stdout
 
     def write(self, text):
-        if "\n" in text:
-            for line in text.strip().split("\n"):
-                self.STDOUT.write('~"%s\\n"\n' % line)
-        else:
-            self.STDOUT.write('~"%s"' % text)
+        self.STDOUT.write(escape_gdbmi_stream("~", text))
         self.STDOUT.flush()
 
 
-def is_mi_mode(args):
-    return "--interpreter" in " ".join(args)
+def is_gdbmi_mode():
+    return "--interpreter" in " ".join(PlatformioCLI.leftover_args)
+
+
+def escape_gdbmi_stream(prefix, stream):
+    bytes_stream = False
+    if is_bytes(stream):
+        bytes_stream = True
+        stream = stream.decode()
+
+    if not stream:
+        return b"" if bytes_stream else ""
+
+    ends_nl = stream.endswith("\n")
+    stream = re.sub(r"\\+", "\\\\\\\\", stream)
+    stream = stream.replace('"', '\\"')
+    stream = stream.replace("\n", "\\n")
+    stream = '%s"%s"' % (prefix, stream)
+    if ends_nl:
+        stream += "\n"
+
+    return stream.encode() if bytes_stream else stream
 
 
 def get_default_debug_env(config):
@@ -57,41 +76,41 @@ def get_default_debug_env(config):
 
 
 def predebug_project(ctx, project_dir, env_name, preload, verbose):
-    ctx.invoke(cmd_run,
-               project_dir=project_dir,
-               environment=[env_name],
-               target=["debug"] + (["upload"] if preload else []),
-               verbose=verbose)
+    ctx.invoke(
+        cmd_run,
+        project_dir=project_dir,
+        environment=[env_name],
+        target=["debug"] + (["upload"] if preload else []),
+        verbose=verbose,
+    )
     if preload:
         time.sleep(5)
 
 
 def validate_debug_options(cmd_ctx, env_options):
-
     def _cleanup_cmds(items):
         items = ProjectConfig.parse_multi_values(items)
-        return [
-            "$LOAD_CMDS" if item == "$LOAD_CMD" else item for item in items
-        ]
+        return ["$LOAD_CMDS" if item == "$LOAD_CMD" else item for item in items]
 
     try:
-        platform = PlatformFactory.newPlatform(env_options['platform'])
+        platform = PlatformFactory.newPlatform(env_options["platform"])
     except exception.UnknownPlatform:
-        cmd_ctx.invoke(cmd_platform_install,
-                       platforms=[env_options['platform']],
-                       skip_default_package=True)
-        platform = PlatformFactory.newPlatform(env_options['platform'])
+        cmd_ctx.invoke(
+            cmd_platform_install,
+            platforms=[env_options["platform"]],
+            skip_default_package=True,
+        )
+        platform = PlatformFactory.newPlatform(env_options["platform"])
 
-    board_config = platform.board_config(env_options['board'])
+    board_config = platform.board_config(env_options["board"])
     tool_name = board_config.get_debug_tool_name(env_options.get("debug_tool"))
-    tool_settings = board_config.get("debug", {}).get("tools",
-                                                      {}).get(tool_name, {})
+    tool_settings = board_config.get("debug", {}).get("tools", {}).get(tool_name, {})
     server_options = None
 
     # specific server per a system
     if isinstance(tool_settings.get("server", {}), list):
-        for item in tool_settings['server'][:]:
-            tool_settings['server'] = item
+        for item in tool_settings["server"][:]:
+            tool_settings["server"] = item
             if util.get_systype() in item.get("system", []):
                 break
 
@@ -100,76 +119,98 @@ def validate_debug_options(cmd_ctx, env_options):
         server_options = {
             "cwd": None,
             "executable": None,
-            "arguments": env_options.get("debug_server")
+            "arguments": env_options.get("debug_server"),
         }
-        server_options['executable'] = server_options['arguments'][0]
-        server_options['arguments'] = server_options['arguments'][1:]
+        server_options["executable"] = server_options["arguments"][0]
+        server_options["arguments"] = server_options["arguments"][1:]
     elif "server" in tool_settings:
-        server_package = tool_settings['server'].get("package")
-        server_package_dir = platform.get_package_dir(
-            server_package) if server_package else None
+        server_package = tool_settings["server"].get("package")
+        server_package_dir = (
+            platform.get_package_dir(server_package) if server_package else None
+        )
         if server_package and not server_package_dir:
-            platform.install_packages(with_packages=[server_package],
-                                      skip_default_package=True,
-                                      silent=True)
+            platform.install_packages(
+                with_packages=[server_package], skip_default_package=True, silent=True
+            )
             server_package_dir = platform.get_package_dir(server_package)
         server_options = dict(
             cwd=server_package_dir if server_package else None,
-            executable=tool_settings['server'].get("executable"),
+            executable=tool_settings["server"].get("executable"),
             arguments=[
                 a.replace("$PACKAGE_DIR", server_package_dir)
-                if server_package_dir else a
-                for a in tool_settings['server'].get("arguments", [])
-            ])
+                if server_package_dir
+                else a
+                for a in tool_settings["server"].get("arguments", [])
+            ],
+        )
 
     extra_cmds = _cleanup_cmds(env_options.get("debug_extra_cmds"))
     extra_cmds.extend(_cleanup_cmds(tool_settings.get("extra_cmds")))
     result = dict(
         tool=tool_name,
         upload_protocol=env_options.get(
-            "upload_protocol",
-            board_config.get("upload", {}).get("protocol")),
+            "upload_protocol", board_config.get("upload", {}).get("protocol")
+        ),
         load_cmds=_cleanup_cmds(
             env_options.get(
                 "debug_load_cmds",
-                tool_settings.get("load_cmds",
-                                  tool_settings.get("load_cmd", "load")))),
-        load_mode=env_options.get("debug_load_mode",
-                                  tool_settings.get("load_mode", "always")),
+                tool_settings.get(
+                    "load_cmds",
+                    tool_settings.get(
+                        "load_cmd", ProjectOptions["env.debug_load_cmds"].default
+                    ),
+                ),
+            )
+        ),
+        load_mode=env_options.get(
+            "debug_load_mode",
+            tool_settings.get(
+                "load_mode", ProjectOptions["env.debug_load_mode"].default
+            ),
+        ),
         init_break=env_options.get(
-            "debug_init_break", tool_settings.get("init_break",
-                                                  "tbreak main")),
+            "debug_init_break",
+            tool_settings.get(
+                "init_break", ProjectOptions["env.debug_init_break"].default
+            ),
+        ),
         init_cmds=_cleanup_cmds(
-            env_options.get("debug_init_cmds",
-                            tool_settings.get("init_cmds"))),
+            env_options.get("debug_init_cmds", tool_settings.get("init_cmds"))
+        ),
         extra_cmds=extra_cmds,
         require_debug_port=tool_settings.get("require_debug_port", False),
         port=reveal_debug_port(
             env_options.get("debug_port", tool_settings.get("port")),
-            tool_name, tool_settings),
-        server=server_options)
+            tool_name,
+            tool_settings,
+        ),
+        server=server_options,
+    )
     return result
 
 
 def configure_esp32_load_cmds(debug_options, configuration):
     ignore_conds = [
-        debug_options['load_cmds'] != ["load"],
+        debug_options["load_cmds"] != ["load"],
         "xtensa-esp32" not in configuration.get("cc_path", ""),
-        not configuration.get("flash_extra_images"), not all([
-            isfile(item['path'])
-            for item in configuration.get("flash_extra_images")
-        ])
+        not configuration.get("flash_extra_images"),
+        not all(
+            [isfile(item["path"]) for item in configuration.get("flash_extra_images")]
+        ),
     ]
     if any(ignore_conds):
-        return debug_options['load_cmds']
+        return debug_options["load_cmds"]
 
     mon_cmds = [
         'monitor program_esp32 "{{{path}}}" {offset} verify'.format(
-            path=fs.to_unix_path(item['path']), offset=item['offset'])
+            path=fs.to_unix_path(item["path"]), offset=item["offset"]
+        )
         for item in configuration.get("flash_extra_images")
     ]
-    mon_cmds.append('monitor program_esp32 "{%s.bin}" 0x10000 verify' %
-                    fs.to_unix_path(configuration['prog_path'][:-4]))
+    mon_cmds.append(
+        'monitor program_esp32 "{%s.bin}" 0x10000 verify'
+        % fs.to_unix_path(configuration["prog_path"][:-4])
+    )
     return mon_cmds
 
 
@@ -181,7 +222,7 @@ def has_debug_symbols(prog_path):
         b".debug_abbrev": False,
         b" -Og": False,
         b" -g": False,
-        b"__PLATFORMIO_BUILD_DEBUG__": False
+        b"__PLATFORMIO_BUILD_DEBUG__": False,
     }
     with open(prog_path, "rb") as fp:
         last_data = b""
@@ -210,19 +251,16 @@ def is_prog_obsolete(prog_path):
                 break
             shasum.update(data)
     new_digest = shasum.hexdigest()
-    old_digest = None
-    if isfile(prog_hash_path):
-        with open(prog_hash_path, "r") as fp:
-            old_digest = fp.read()
+    old_digest = (
+        fs.get_file_contents(prog_hash_path) if isfile(prog_hash_path) else None
+    )
     if new_digest == old_digest:
         return False
-    with open(prog_hash_path, "w") as fp:
-        fp.write(new_digest)
+    fs.write_file_contents(prog_hash_path, new_digest)
     return True
 
 
 def reveal_debug_port(env_debug_port, tool_name, tool_settings):
-
     def _get_pattern():
         if not env_debug_port:
             return None
@@ -238,18 +276,21 @@ def reveal_debug_port(env_debug_port, tool_name, tool_settings):
 
     def _look_for_serial_port(hwids):
         for item in util.get_serialports(filter_hwid=True):
-            if not _is_match_pattern(item['port']):
+            if not _is_match_pattern(item["port"]):
                 continue
-            port = item['port']
+            port = item["port"]
             if tool_name.startswith("blackmagic"):
-                if "windows" in util.get_systype() and \
-                        port.startswith("COM") and len(port) > 4:
+                if (
+                    "windows" in util.get_systype()
+                    and port.startswith("COM")
+                    and len(port) > 4
+                ):
                     port = "\\\\.\\%s" % port
-                if "GDB" in item['description']:
+                if "GDB" in item["description"]:
                     return port
             for hwid in hwids:
                 hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
-                if hwid_str in item['hwid']:
+                if hwid_str in item["hwid"]:
                     return port
         return None
 
@@ -261,5 +302,6 @@ def reveal_debug_port(env_debug_port, tool_name, tool_settings):
     debug_port = _look_for_serial_port(tool_settings.get("hwids", []))
     if not debug_port:
         raise exception.DebugInvalidOptions(
-            "Please specify `debug_port` for environment")
+            "Please specify `debug_port` for environment"
+        )
     return debug_port

platformio/commands/debug/initcfgs.py

@@ -17,50 +17,51 @@ define pio_reset_halt_target
     monitor reset halt
 end
 
-define pio_reset_target
+define pio_reset_run_target
     monitor reset
 end
 
 target extended-remote $DEBUG_PORT
-$INIT_BREAK
-pio_reset_halt_target
-$LOAD_CMDS
 monitor init
+$LOAD_CMDS
 pio_reset_halt_target
+$INIT_BREAK
 """
 
 GDB_STUTIL_INIT_CONFIG = """
 define pio_reset_halt_target
-    monitor halt
     monitor reset
+    monitor halt
 end
 
-define pio_reset_target
+define pio_reset_run_target
     monitor reset
 end
 
 target extended-remote $DEBUG_PORT
-$INIT_BREAK
-pio_reset_halt_target
 $LOAD_CMDS
 pio_reset_halt_target
+$INIT_BREAK
 """
 
 GDB_JLINK_INIT_CONFIG = """
 define pio_reset_halt_target
-    monitor halt
     monitor reset
+    monitor halt
 end
 
-define pio_reset_target
+define pio_reset_run_target
+    monitor clrbp
     monitor reset
+    monitor go
 end
 
 target extended-remote $DEBUG_PORT
-$INIT_BREAK
-pio_reset_halt_target
+monitor clrbp
+monitor speed auto
 $LOAD_CMDS
 pio_reset_halt_target
+$INIT_BREAK
 """
 
 GDB_BLACKMAGIC_INIT_CONFIG = """
@@ -74,7 +75,7 @@ define pio_reset_halt_target
     set language auto
 end
 
-define pio_reset_target
+define pio_reset_run_target
     pio_reset_halt_target
 end
 
@@ -82,8 +83,8 @@ target extended-remote $DEBUG_PORT
 monitor swdp_scan
 attach 1
 set mem inaccessible-by-default off
-$INIT_BREAK
 $LOAD_CMDS
+$INIT_BREAK
 
 set language c
 set *0xE000ED0C = 0x05FA0004
@@ -98,14 +99,14 @@ GDB_MSPDEBUG_INIT_CONFIG = """
 define pio_reset_halt_target
 end
 
-define pio_reset_target
+define pio_reset_run_target
 end
 
 target extended-remote $DEBUG_PORT
-$INIT_BREAK
 monitor erase
 $LOAD_CMDS
 pio_reset_halt_target
+$INIT_BREAK
 """
 
 GDB_QEMU_INIT_CONFIG = """
@@ -113,12 +114,12 @@ define pio_reset_halt_target
     monitor system_reset
 end
 
-define pio_reset_target
-    pio_reset_halt_target
+define pio_reset_run_target
+    monitor system_reset
 end
 
 target extended-remote $DEBUG_PORT
-$INIT_BREAK
 $LOAD_CMDS
 pio_reset_halt_target
+$INIT_BREAK
 """

platformio/commands/debug/process.py

@@ -32,7 +32,7 @@ class BaseProcess(protocol.ProcessProtocol, object):
     COMMON_PATTERNS = {
         "PLATFORMIO_HOME_DIR": get_project_core_dir(),
         "PLATFORMIO_CORE_DIR": get_project_core_dir(),
-        "PYTHONEXE": get_pythonexe_path()
+        "PYTHONEXE": get_pythonexe_path(),
     }
 
     def apply_patterns(self, source, patterns=None):
@@ -52,8 +52,7 @@ class BaseProcess(protocol.ProcessProtocol, object):
         if isinstance(source, string_types):
             source = _replace(source)
         elif isinstance(source, (list, dict)):
-            items = enumerate(source) if isinstance(source,
-                                                    list) else source.items()
+            items = enumerate(source) if isinstance(source, list) else source.items()
             for key, value in items:
                 if isinstance(value, string_types):
                     source[key] = _replace(value)
@@ -67,9 +66,9 @@ class BaseProcess(protocol.ProcessProtocol, object):
             with open(LOG_FILE, "ab") as fp:
                 fp.write(data)
         while data:
-            chunk = data[:self.STDOUT_CHUNK_SIZE]
+            chunk = data[: self.STDOUT_CHUNK_SIZE]
             click.echo(chunk, nl=False)
-            data = data[self.STDOUT_CHUNK_SIZE:]
+            data = data[self.STDOUT_CHUNK_SIZE :]
 
     @staticmethod
     def errReceived(data):

platformio/commands/debug/server.py

@@ -19,12 +19,12 @@ from twisted.internet import error  # pylint: disable=import-error
 from twisted.internet import reactor  # pylint: disable=import-error
 
 from platformio import exception, fs, util
+from platformio.commands.debug.helpers import escape_gdbmi_stream, is_gdbmi_mode
 from platformio.commands.debug.process import BaseProcess
 from platformio.proc import where_is_program
 
 
 class DebugServer(BaseProcess):
-
     def __init__(self, debug_options, env_options):
         self.debug_options = debug_options
         self.env_options = env_options
@@ -39,13 +39,16 @@ class DebugServer(BaseProcess):
         if not server:
             return None
         server = self.apply_patterns(server, patterns)
-        server_executable = server['executable']
+        server_executable = server["executable"]
         if not server_executable:
             return None
-        if server['cwd']:
-            server_executable = join(server['cwd'], server_executable)
-        if ("windows" in systype and not server_executable.endswith(".exe")
-                and isfile(server_executable + ".exe")):
+        if server["cwd"]:
+            server_executable = join(server["cwd"], server_executable)
+        if (
+            "windows" in systype
+            and not server_executable.endswith(".exe")
+            and isfile(server_executable + ".exe")
+        ):
             server_executable = server_executable + ".exe"
 
         if not isfile(server_executable):
@@ -55,48 +58,56 @@ class DebugServer(BaseProcess):
                 "\nCould not launch Debug Server '%s'. Please check that it "
                 "is installed and is included in a system PATH\n\n"
                 "See documentation or contact contact@platformio.org:\n"
-                "http://docs.platformio.org/page/plus/debugging.html\n" %
-                server_executable)
+                "http://docs.platformio.org/page/plus/debugging.html\n"
+                % server_executable
+            )
 
         self._debug_port = ":3333"
-        openocd_pipe_allowed = all([
-            not self.debug_options['port'],
-            "openocd" in server_executable
-        ])  # yapf: disable
+        openocd_pipe_allowed = all(
+            [not self.debug_options["port"], "openocd" in server_executable]
+        )
         if openocd_pipe_allowed:
             args = []
-            if server['cwd']:
-                args.extend(["-s", server['cwd']])
-            args.extend([
-                "-c", "gdb_port pipe; tcl_port disabled; telnet_port disabled"
-            ])
-            args.extend(server['arguments'])
+            if server["cwd"]:
+                args.extend(["-s", server["cwd"]])
+            args.extend(
+                ["-c", "gdb_port pipe; tcl_port disabled; telnet_port disabled"]
+            )
+            args.extend(server["arguments"])
             str_args = " ".join(
-                [arg if arg.startswith("-") else '"%s"' % arg for arg in args])
+                [arg if arg.startswith("-") else '"%s"' % arg for arg in args]
+            )
             self._debug_port = '| "%s" %s' % (server_executable, str_args)
             self._debug_port = fs.to_unix_path(self._debug_port)
         else:
             env = os.environ.copy()
             # prepend server "lib" folder to LD path
-            if ("windows" not in systype and server['cwd']
-                    and isdir(join(server['cwd'], "lib"))):
-                ld_key = ("DYLD_LIBRARY_PATH"
-                          if "darwin" in systype else "LD_LIBRARY_PATH")
-                env[ld_key] = join(server['cwd'], "lib")
+            if (
+                "windows" not in systype
+                and server["cwd"]
+                and isdir(join(server["cwd"], "lib"))
+            ):
+                ld_key = (
+                    "DYLD_LIBRARY_PATH" if "darwin" in systype else "LD_LIBRARY_PATH"
+                )
+                env[ld_key] = join(server["cwd"], "lib")
                 if os.environ.get(ld_key):
-                    env[ld_key] = "%s:%s" % (env[ld_key],
-                                             os.environ.get(ld_key))
+                    env[ld_key] = "%s:%s" % (env[ld_key], os.environ.get(ld_key))
             # prepend BIN to PATH
-            if server['cwd'] and isdir(join(server['cwd'], "bin")):
-                env['PATH'] = "%s%s%s" % (
-                    join(server['cwd'], "bin"), os.pathsep,
-                    os.environ.get("PATH", os.environ.get("Path", "")))
+            if server["cwd"] and isdir(join(server["cwd"], "bin")):
+                env["PATH"] = "%s%s%s" % (
+                    join(server["cwd"], "bin"),
+                    os.pathsep,
+                    os.environ.get("PATH", os.environ.get("Path", "")),
+                )
 
             self._transport = reactor.spawnProcess(
                 self,
-                server_executable, [server_executable] + server['arguments'],
-                path=server['cwd'],
-                env=env)
+                server_executable,
+                [server_executable] + server["arguments"],
+                path=server["cwd"],
+                env=env,
+            )
             if "mspdebug" in server_executable.lower():
                 self._debug_port = ":2000"
             elif "jlink" in server_executable.lower():
@@ -109,6 +120,11 @@ class DebugServer(BaseProcess):
     def get_debug_port(self):
         return self._debug_port
 
+    def outReceived(self, data):
+        super(DebugServer, self).outReceived(
+            escape_gdbmi_stream("@", data) if is_gdbmi_mode() else data
+        )
+
     def processEnded(self, reason):
         self._process_ended = True
         super(DebugServer, self).processEnded(reason)

platformio/commands/device.py

@@ -15,12 +15,11 @@
 import sys
 from fnmatch import fnmatch
 from os import getcwd
-from os.path import join
 
 import click
 from serial.tools import miniterm
 
-from platformio import exception, util
+from platformio import exception, fs, util
 from platformio.compat import dump_json_to_unicode
 from platformio.project.config import ProjectConfig
 
@@ -36,27 +35,29 @@ def cli():
 @click.option("--mdns", is_flag=True, help="List multicast DNS services")
 @click.option("--json-output", is_flag=True)
 def device_list(  # pylint: disable=too-many-branches
-        serial, logical, mdns, json_output):
+    serial, logical, mdns, json_output
+):
     if not logical and not mdns:
         serial = True
     data = {}
     if serial:
-        data['serial'] = util.get_serial_ports()
+        data["serial"] = util.get_serial_ports()
     if logical:
-        data['logical'] = util.get_logical_devices()
+        data["logical"] = util.get_logical_devices()
     if mdns:
-        data['mdns'] = util.get_mdns_services()
+        data["mdns"] = util.get_mdns_services()
 
     single_key = list(data)[0] if len(list(data)) == 1 else None
 
     if json_output:
         return click.echo(
-            dump_json_to_unicode(data[single_key] if single_key else data))
+            dump_json_to_unicode(data[single_key] if single_key else data)
+        )
 
     titles = {
         "serial": "Serial Ports",
         "logical": "Logical Devices",
-        "mdns": "Multicast DNS Services"
+        "mdns": "Multicast DNS Services",
     }
 
     for key, value in data.items():
@@ -66,31 +67,38 @@ def device_list(  # pylint: disable=too-many-branches
 
         if key == "serial":
             for item in value:
-                click.secho(item['port'], fg="cyan")
-                click.echo("-" * len(item['port']))
-                click.echo("Hardware ID: %s" % item['hwid'])
-                click.echo("Description: %s" % item['description'])
+                click.secho(item["port"], fg="cyan")
+                click.echo("-" * len(item["port"]))
+                click.echo("Hardware ID: %s" % item["hwid"])
+                click.echo("Description: %s" % item["description"])
                 click.echo("")
 
         if key == "logical":
             for item in value:
-                click.secho(item['path'], fg="cyan")
-                click.echo("-" * len(item['path']))
-                click.echo("Name: %s" % item['name'])
+                click.secho(item["path"], fg="cyan")
+                click.echo("-" * len(item["path"]))
+                click.echo("Name: %s" % item["name"])
                 click.echo("")
 
         if key == "mdns":
             for item in value:
-                click.secho(item['name'], fg="cyan")
-                click.echo("-" * len(item['name']))
-                click.echo("Type: %s" % item['type'])
-                click.echo("IP: %s" % item['ip'])
-                click.echo("Port: %s" % item['port'])
-                if item['properties']:
-                    click.echo("Properties: %s" % ("; ".join([
-                        "%s=%s" % (k, v)
-                        for k, v in item['properties'].items()
-                    ])))
+                click.secho(item["name"], fg="cyan")
+                click.echo("-" * len(item["name"]))
+                click.echo("Type: %s" % item["type"])
+                click.echo("IP: %s" % item["ip"])
+                click.echo("Port: %s" % item["port"])
+                if item["properties"]:
+                    click.echo(
+                        "Properties: %s"
+                        % (
+                            "; ".join(
+                                [
+                                    "%s=%s" % (k, v)
+                                    for k, v in item["properties"].items()
+                                ]
+                            )
+                        )
+                    )
                 click.echo("")
 
         if single_key:
@@ -102,66 +110,72 @@ def device_list(  # pylint: disable=too-many-branches
 @cli.command("monitor", short_help="Monitor device (Serial)")
 @click.option("--port", "-p", help="Port, a number or a device name")
 @click.option("--baud", "-b", type=int, help="Set baud rate, default=9600")
-@click.option("--parity",
-              default="N",
-              type=click.Choice(["N", "E", "O", "S", "M"]),
-              help="Set parity, default=N")
-@click.option("--rtscts",
-              is_flag=True,
-              help="Enable RTS/CTS flow control, default=Off")
-@click.option("--xonxoff",
-              is_flag=True,
-              help="Enable software flow control, default=Off")
-@click.option("--rts",
-              default=None,
-              type=click.IntRange(0, 1),
-              help="Set initial RTS line state")
-@click.option("--dtr",
-              default=None,
-              type=click.IntRange(0, 1),
-              help="Set initial DTR line state")
+@click.option(
+    "--parity",
+    default="N",
+    type=click.Choice(["N", "E", "O", "S", "M"]),
+    help="Set parity, default=N",
+)
+@click.option("--rtscts", is_flag=True, help="Enable RTS/CTS flow control, default=Off")
+@click.option(
+    "--xonxoff", is_flag=True, help="Enable software flow control, default=Off"
+)
+@click.option(
+    "--rts", default=None, type=click.IntRange(0, 1), help="Set initial RTS line state"
+)
+@click.option(
+    "--dtr", default=None, type=click.IntRange(0, 1), help="Set initial DTR line state"
+)
 @click.option("--echo", is_flag=True, help="Enable local echo, default=Off")
-@click.option("--encoding",
-              default="UTF-8",
-              help="Set the encoding for the serial port (e.g. hexlify, "
-              "Latin1, UTF-8), default: UTF-8")
+@click.option(
+    "--encoding",
+    default="UTF-8",
+    help="Set the encoding for the serial port (e.g. hexlify, "
+    "Latin1, UTF-8), default: UTF-8",
+)
 @click.option("--filter", "-f", multiple=True, help="Add text transformation")
-@click.option("--eol",
-              default="CRLF",
-              type=click.Choice(["CR", "LF", "CRLF"]),
-              help="End of line mode, default=CRLF")
-@click.option("--raw",
-              is_flag=True,
-              help="Do not apply any encodings/transformations")
-@click.option("--exit-char",
-              type=int,
-              default=3,
-              help="ASCII code of special character that is used to exit "
-              "the application, default=3 (Ctrl+C)")
-@click.option("--menu-char",
-              type=int,
-              default=20,
-              help="ASCII code of special character that is used to "
-              "control miniterm (menu), default=20 (DEC)")
-@click.option("--quiet",
-              is_flag=True,
-              help="Diagnostics: suppress non-error messages, default=Off")
-@click.option("-d",
-              "--project-dir",
-              default=getcwd,
-              type=click.Path(exists=True,
-                              file_okay=False,
-                              dir_okay=True,
-                              resolve_path=True))
+@click.option(
+    "--eol",
+    default="CRLF",
+    type=click.Choice(["CR", "LF", "CRLF"]),
+    help="End of line mode, default=CRLF",
+)
+@click.option("--raw", is_flag=True, help="Do not apply any encodings/transformations")
+@click.option(
+    "--exit-char",
+    type=int,
+    default=3,
+    help="ASCII code of special character that is used to exit "
+    "the application, default=3 (Ctrl+C)",
+)
+@click.option(
+    "--menu-char",
+    type=int,
+    default=20,
+    help="ASCII code of special character that is used to "
+    "control miniterm (menu), default=20 (DEC)",
+)
+@click.option(
+    "--quiet",
+    is_flag=True,
+    help="Diagnostics: suppress non-error messages, default=Off",
+)
+@click.option(
+    "-d",
+    "--project-dir",
+    default=getcwd,
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+)
 @click.option(
     "-e",
     "--environment",
-    help="Load configuration from `platformio.ini` and specified environment")
+    help="Load configuration from `platformio.ini` and specified environment",
+)
 def device_monitor(**kwargs):  # pylint: disable=too-many-branches
     env_options = {}
     try:
-        env_options = get_project_options(kwargs['project_dir'],
-                                          kwargs['environment'])
+        with fs.cd(kwargs["project_dir"]):
+            env_options = get_project_options(kwargs["environment"])
         for k in ("port", "speed", "rts", "dtr"):
             k2 = "monitor_%s" % k
             if k == "speed":
@@ -173,10 +187,10 @@ def device_monitor(**kwargs):  # pylint: disable=too-many-branches
     except exception.NotPlatformIOProject:
         pass
 
-    if not kwargs['port']:
+    if not kwargs["port"]:
         ports = util.get_serial_ports(filter_hwid=True)
         if len(ports) == 1:
-            kwargs['port'] = ports[0]['port']
+            kwargs["port"] = ports[0]["port"]
 
     sys.argv = ["monitor"] + env_options.get("monitor_flags", [])
     for k, v in kwargs.items():
@@ -194,23 +208,25 @@ def device_monitor(**kwargs):  # pylint: disable=too-many-branches
         else:
             sys.argv.extend([k, str(v)])
 
-    if kwargs['port'] and (set(["*", "?", "[", "]"]) & set(kwargs['port'])):
+    if kwargs["port"] and (set(["*", "?", "[", "]"]) & set(kwargs["port"])):
         for item in util.get_serial_ports():
-            if fnmatch(item['port'], kwargs['port']):
-                kwargs['port'] = item['port']
+            if fnmatch(item["port"], kwargs["port"]):
+                kwargs["port"] = item["port"]
                 break
 
     try:
-        miniterm.main(default_port=kwargs['port'],
-                      default_baudrate=kwargs['baud'] or 9600,
-                      default_rts=kwargs['rts'],
-                      default_dtr=kwargs['dtr'])
+        miniterm.main(
+            default_port=kwargs["port"],
+            default_baudrate=kwargs["baud"] or 9600,
+            default_rts=kwargs["rts"],
+            default_dtr=kwargs["dtr"],
+        )
     except Exception as e:
         raise exception.MinitermException(e)
 
 
-def get_project_options(project_dir, environment=None):
-    config = ProjectConfig.get_instance(join(project_dir, "platformio.ini"))
+def get_project_options(environment=None):
+    config = ProjectConfig.get_instance()
     config.validate(envs=[environment] if environment else None)
     if not environment:
         default_envs = config.default_envs()

platformio/commands/home/__init__.py

@@ -11,5 +11,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from platformio.commands.home.command import cli

platformio/commands/home/command.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# pylint: disable=too-many-locals
+
 import mimetypes
 import socket
 from os.path import isdir
@@ -19,8 +21,7 @@ from os.path import isdir
 import click
 
 from platformio import exception
-from platformio.managers.core import (get_core_package_dir,
-                                      inject_contrib_pysite)
+from platformio.managers.core import get_core_package_dir, inject_contrib_pysite
 
 
 @click.command("home", short_help="PIO Home")
@@ -28,17 +29,30 @@ from platformio.managers.core import (get_core_package_dir,
 @click.option(
     "--host",
     default="127.0.0.1",
-    help="HTTP host, default=127.0.0.1. "
-    "You can open PIO Home for inbound connections with --host=0.0.0.0")
-@click.option("--no-open", is_flag=True)  # pylint: disable=too-many-locals
-def cli(port, host, no_open):
+    help=(
+        "HTTP host, default=127.0.0.1. You can open PIO Home for inbound "
+        "connections with --host=0.0.0.0"
+    ),
+)
+@click.option("--no-open", is_flag=True)
+@click.option(
+    "--shutdown-timeout",
+    default=0,
+    type=int,
+    help=(
+        "Automatically shutdown server on timeout (in seconds) when no clients "
+        "are connected. Default is 0 which means never auto shutdown"
+    ),
+)
+def cli(port, host, no_open, shutdown_timeout):
+    # pylint: disable=import-error, import-outside-toplevel
+
     # import contrib modules
     inject_contrib_pysite()
-    # pylint: disable=import-error
     from autobahn.twisted.resource import WebSocketResource
     from twisted.internet import reactor
     from twisted.web import server
-    # pylint: enable=import-error
+
     from platformio.commands.home.rpc.handlers.app import AppRPC
     from platformio.commands.home.rpc.handlers.ide import IDERPC
     from platformio.commands.home.rpc.handlers.misc import MiscRPC
@@ -48,7 +62,7 @@ def cli(port, host, no_open):
     from platformio.commands.home.rpc.server import JSONRPCServerFactory
     from platformio.commands.home.web import WebRoot
 
-    factory = JSONRPCServerFactory()
+    factory = JSONRPCServerFactory(shutdown_timeout)
     factory.addHandler(AppRPC(), namespace="app")
     factory.addHandler(IDERPC(), namespace="ide")
     factory.addHandler(MiscRPC(), namespace="misc")
@@ -89,14 +103,18 @@ def cli(port, host, no_open):
         else:
             reactor.callLater(1, lambda: click.launch(home_url))
 
-    click.echo("\n".join([
-        "",
-        "  ___I_",
-        " /\\-_--\\   PlatformIO Home",
-        "/  \\_-__\\",
-        "|[]| [] |  %s" % home_url,
-        "|__|____|______________%s" % ("_" * len(host)),
-    ]))
+    click.echo(
+        "\n".join(
+            [
+                "",
+                "  ___I_",
+                " /\\-_--\\   PlatformIO Home",
+                "/  \\_-__\\",
+                "|[]| [] |  %s" % home_url,
+                "|__|____|______________%s" % ("_" * len(host)),
+            ]
+        )
+    )
     click.echo("")
     click.echo("Open PIO Home in your browser by this URL => %s" % home_url)
 

platformio/commands/home/helpers.py

@@ -27,7 +27,6 @@ from platformio.proc import where_is_program
 
 
 class AsyncSession(requests.Session):
-
     def __init__(self, n=None, *args, **kwargs):
         if n:
             pool = reactor.getThreadPool()
@@ -51,7 +50,8 @@ def requests_session():
 @util.memoized(expire="60s")
 def get_core_fullpath():
     return where_is_program(
-        "platformio" + (".exe" if "windows" in util.get_systype() else ""))
+        "platformio" + (".exe" if "windows" in util.get_systype() else "")
+    )
 
 
 @util.memoized(expire="10s")
@@ -60,9 +60,7 @@ def is_twitter_blocked():
     timeout = 2
     try:
         if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")):
-            requests.get("http://%s" % ip,
-                         allow_redirects=False,
-                         timeout=timeout)
+            requests.get("http://%s" % ip, allow_redirects=False, timeout=timeout)
         else:
             socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((ip, 80))
         return False

platformio/commands/home/rpc/handlers/app.py

@@ -14,11 +14,10 @@
 
 from __future__ import absolute_import
 
-from os.path import expanduser, join
+from os.path import join
 
-from platformio import __version__, app, util
-from platformio.project.helpers import (get_project_core_dir,
-                                        is_platformio_project)
+from platformio import __version__, app, fs, util
+from platformio.project.helpers import get_project_core_dir, is_platformio_project
 
 
 class AppRPC(object):
@@ -26,8 +25,13 @@ class AppRPC(object):
     APPSTATE_PATH = join(get_project_core_dir(), "homestate.json")
 
     IGNORE_STORAGE_KEYS = [
-        "cid", "coreVersion", "coreSystype", "coreCaller", "coreSettings",
-        "homeDir", "projectsDir"
+        "cid",
+        "coreVersion",
+        "coreSystype",
+        "coreCaller",
+        "coreSettings",
+        "homeDir",
+        "projectsDir",
     ]
 
     @staticmethod
@@ -37,31 +41,28 @@ class AppRPC(object):
 
             # base data
             caller_id = app.get_session_var("caller_id")
-            storage['cid'] = app.get_cid()
-            storage['coreVersion'] = __version__
-            storage['coreSystype'] = util.get_systype()
-            storage['coreCaller'] = (str(caller_id).lower()
-                                     if caller_id else None)
-            storage['coreSettings'] = {
+            storage["cid"] = app.get_cid()
+            storage["coreVersion"] = __version__
+            storage["coreSystype"] = util.get_systype()
+            storage["coreCaller"] = str(caller_id).lower() if caller_id else None
+            storage["coreSettings"] = {
                 name: {
-                    "description": data['description'],
-                    "default_value": data['value'],
-                    "value": app.get_setting(name)
+                    "description": data["description"],
+                    "default_value": data["value"],
+                    "value": app.get_setting(name),
                 }
                 for name, data in app.DEFAULT_SETTINGS.items()
             }
 
-            storage['homeDir'] = expanduser("~")
-            storage['projectsDir'] = storage['coreSettings']['projects_dir'][
-                'value']
+            storage["homeDir"] = fs.expanduser("~")
+            storage["projectsDir"] = storage["coreSettings"]["projects_dir"]["value"]
 
             # skip non-existing recent projects
-            storage['recentProjects'] = [
-                p for p in storage.get("recentProjects", [])
-                if is_platformio_project(p)
+            storage["recentProjects"] = [
+                p for p in storage.get("recentProjects", []) if is_platformio_project(p)
             ]
 
-            state['storage'] = storage
+            state["storage"] = storage
             state.modified = False  # skip saving extra fields
             return state.as_dict()
 

platformio/commands/home/rpc/handlers/ide.py

@@ -19,20 +19,18 @@ from twisted.internet import defer  # pylint: disable=import-error
 
 
 class IDERPC(object):
-
     def __init__(self):
         self._queue = {}
 
-    def send_command(self, command, params, sid=0):
+    def send_command(self, sid, command, params):
         if not self._queue.get(sid):
             raise jsonrpc.exceptions.JSONRPCDispatchException(
-                code=4005, message="PIO Home IDE agent is not started")
+                code=4005, message="PIO Home IDE agent is not started"
+            )
         while self._queue[sid]:
-            self._queue[sid].pop().callback({
-                "id": time.time(),
-                "method": command,
-                "params": params
-            })
+            self._queue[sid].pop().callback(
+                {"id": time.time(), "method": command, "params": params}
+            )
 
     def listen_commands(self, sid=0):
         if sid not in self._queue:
@@ -40,5 +38,10 @@ class IDERPC(object):
         self._queue[sid].append(defer.Deferred())
         return self._queue[sid][-1]
 
-    def open_project(self, project_dir, sid=0):
-        return self.send_command("open_project", project_dir, sid)
+    def open_project(self, sid, project_dir):
+        return self.send_command(sid, "open_project", project_dir)
+
+    def open_text_document(self, sid, path, line=None, column=None):
+        return self.send_command(
+            sid, "open_text_document", dict(path=path, line=line, column=column)
+        )

platformio/commands/home/rpc/handlers/misc.py

@@ -22,33 +22,31 @@ from platformio.commands.home.rpc.handlers.os import OSRPC
 
 
 class MiscRPC(object):
-
-    def load_latest_tweets(self, username):
-        cache_key = "piohome_latest_tweets_" + str(username)
+    def load_latest_tweets(self, data_url):
+        cache_key = data_url
         cache_valid = "7d"
         with app.ContentCache() as cc:
             cache_data = cc.get(cache_key)
             if cache_data:
                 cache_data = json.loads(cache_data)
                 # automatically update cache in background every 12 hours
-                if cache_data['time'] < (time.time() - (3600 * 12)):
-                    reactor.callLater(5, self._preload_latest_tweets, username,
-                                      cache_key, cache_valid)
-                return cache_data['result']
+                if cache_data["time"] < (time.time() - (3600 * 12)):
+                    reactor.callLater(
+                        5, self._preload_latest_tweets, data_url, cache_key, cache_valid
+                    )
+                return cache_data["result"]
 
-        result = self._preload_latest_tweets(username, cache_key, cache_valid)
+        result = self._preload_latest_tweets(data_url, cache_key, cache_valid)
         return result
 
     @staticmethod
     @defer.inlineCallbacks
-    def _preload_latest_tweets(username, cache_key, cache_valid):
-        result = yield OSRPC.fetch_content(
-            "https://api.platformio.org/tweets/" + username)
-        result = json.loads(result)
+    def _preload_latest_tweets(data_url, cache_key, cache_valid):
+        result = json.loads((yield OSRPC.fetch_content(data_url)))
         with app.ContentCache() as cc:
-            cc.set(cache_key,
-                   json.dumps({
-                       "time": int(time.time()),
-                       "result": result
-                   }), cache_valid)
+            cc.set(
+                cache_key,
+                json.dumps({"time": int(time.time()), "result": result}),
+                cache_valid,
+            )
         defer.returnValue(result)

platformio/commands/home/rpc/handlers/os.py

@@ -19,30 +19,29 @@ import glob
 import os
 import shutil
 from functools import cmp_to_key
-from os.path import expanduser, isdir, isfile, join
+from os.path import isdir, isfile, join
 
 import click
 from twisted.internet import defer  # pylint: disable=import-error
 
-from platformio import app, util
+from platformio import app, fs, util
 from platformio.commands.home import helpers
 from platformio.compat import PY2, get_filesystem_encoding
 
 
 class OSRPC(object):
-
     @staticmethod
     @defer.inlineCallbacks
     def fetch_content(uri, data=None, headers=None, cache_valid=None):
         if not headers:
             headers = {
-                "User-Agent":
-                ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) "
-                 "AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 "
-                 "Safari/603.3.8")
+                "User-Agent": (
+                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) "
+                    "AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 "
+                    "Safari/603.3.8"
+                )
             }
-        cache_key = (app.ContentCache.key_from_args(uri, data)
-                     if cache_valid else None)
+        cache_key = app.ContentCache.key_from_args(uri, data) if cache_valid else None
         with app.ContentCache() as cc:
             if cache_key:
                 result = cc.get(cache_key)
@@ -66,7 +65,7 @@ class OSRPC(object):
         defer.returnValue(result)
 
     def request_content(self, uri, data=None, headers=None, cache_valid=None):
-        if uri.startswith('http'):
+        if uri.startswith("http"):
             return self.fetch_content(uri, data, headers, cache_valid)
         if not isfile(uri):
             return None
@@ -80,8 +79,12 @@ class OSRPC(object):
     @staticmethod
     def reveal_file(path):
         return click.launch(
-            path.encode(get_filesystem_encoding()) if PY2 else path,
-            locate=True)
+            path.encode(get_filesystem_encoding()) if PY2 else path, locate=True
+        )
+
+    @staticmethod
+    def open_file(path):
+        return click.launch(path.encode(get_filesystem_encoding()) if PY2 else path)
 
     @staticmethod
     def is_file(path):
@@ -109,13 +112,11 @@ class OSRPC(object):
             pathnames = [pathnames]
         result = set()
         for pathname in pathnames:
-            result |= set(
-                glob.glob(join(root, pathname) if root else pathname))
+            result |= set(glob.glob(join(root, pathname) if root else pathname))
         return list(result)
 
     @staticmethod
     def list_dir(path):
-
         def _cmp(x, y):
             if x[1] and not y[1]:
                 return -1
@@ -129,7 +130,7 @@ class OSRPC(object):
 
         items = []
         if path.startswith("~"):
-            path = expanduser(path)
+            path = fs.expanduser(path)
         if not isdir(path):
             return items
         for item in os.listdir(path):
@@ -146,7 +147,7 @@ class OSRPC(object):
     def get_logical_devices():
         items = []
         for item in util.get_logical_devices():
-            if item['name']:
-                item['name'] = item['name']
+            if item["name"]:
+                item["name"] = item["name"]
             items.append(item)
         return items

platformio/commands/home/rpc/handlers/piocore.py

@@ -27,8 +27,7 @@ from twisted.internet import utils  # pylint: disable=import-error
 
 from platformio import __main__, __version__, fs
 from platformio.commands.home import helpers
-from platformio.compat import (PY2, get_filesystem_encoding, is_bytes,
-                               string_types)
+from platformio.compat import PY2, get_filesystem_encoding, is_bytes, string_types
 
 try:
     from thread import get_ident as thread_get_ident
@@ -37,7 +36,6 @@ except ImportError:
 
 
 class MultiThreadingStdStream(object):
-
     def __init__(self, parent_stream):
         self._buffers = {thread_get_ident(): parent_stream}
 
@@ -54,7 +52,8 @@ class MultiThreadingStdStream(object):
         thread_id = thread_get_ident()
         self._ensure_thread_buffer(thread_id)
         return self._buffers[thread_id].write(
-            value.decode() if is_bytes(value) else value)
+            value.decode() if is_bytes(value) else value
+        )
 
     def get_value_and_reset(self):
         result = ""
@@ -68,7 +67,6 @@ class MultiThreadingStdStream(object):
 
 
 class PIOCoreRPC(object):
-
     @staticmethod
     def version():
         return __version__
@@ -104,16 +102,15 @@ class PIOCoreRPC(object):
             else:
                 result = yield PIOCoreRPC._call_inline(args, options)
                 try:
-                    defer.returnValue(
-                        PIOCoreRPC._process_result(result, to_json))
+                    defer.returnValue(PIOCoreRPC._process_result(result, to_json))
                 except ValueError:
                     # fall-back to subprocess method
                     result = yield PIOCoreRPC._call_subprocess(args, options)
-                    defer.returnValue(
-                        PIOCoreRPC._process_result(result, to_json))
+                    defer.returnValue(PIOCoreRPC._process_result(result, to_json))
         except Exception as e:  # pylint: disable=bare-except
             raise jsonrpc.exceptions.JSONRPCDispatchException(
-                code=4003, message="PIO Core Call Error", data=str(e))
+                code=4003, message="PIO Core Call Error", data=str(e)
+            )
 
     @staticmethod
     def _call_inline(args, options):
@@ -123,8 +120,11 @@ class PIOCoreRPC(object):
         def _thread_task():
             with fs.cd(cwd):
                 exit_code = __main__.main(["-c"] + args)
-            return (PIOCoreRPC.thread_stdout.get_value_and_reset(),
-                    PIOCoreRPC.thread_stderr.get_value_and_reset(), exit_code)
+            return (
+                PIOCoreRPC.thread_stdout.get_value_and_reset(),
+                PIOCoreRPC.thread_stderr.get_value_and_reset(),
+                exit_code,
+            )
 
         return threads.deferToThread(_thread_task)
 
@@ -135,8 +135,8 @@ class PIOCoreRPC(object):
             helpers.get_core_fullpath(),
             args,
             path=cwd,
-            env={k: v
-                 for k, v in os.environ.items() if "%" not in k})
+            env={k: v for k, v in os.environ.items() if "%" not in k},
+        )
 
     @staticmethod
     def _process_result(result, to_json=False):

platformio/commands/home/rpc/handlers/project.py

@@ -17,8 +17,7 @@ from __future__ import absolute_import
 import os
 import shutil
 import time
-from os.path import (basename, expanduser, getmtime, isdir, isfile, join,
-                     realpath, sep)
+from os.path import basename, getmtime, isdir, isfile, join, realpath, sep
 
 import jsonrpc  # pylint: disable=import-error
 
@@ -29,38 +28,75 @@ from platformio.compat import PY2, get_filesystem_encoding
 from platformio.ide.projectgenerator import ProjectGenerator
 from platformio.managers.platform import PlatformManager
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import (get_project_libdeps_dir,
-                                        get_project_src_dir,
-                                        is_platformio_project)
+from platformio.project.helpers import get_project_dir, is_platformio_project
+from platformio.project.options import get_config_options_schema
 
 
 class ProjectRPC(object):
+    @staticmethod
+    def config_call(init_kwargs, method, *args):
+        assert isinstance(init_kwargs, dict)
+        assert "path" in init_kwargs
+        project_dir = get_project_dir()
+        if isfile(init_kwargs["path"]):
+            project_dir = os.path.dirname(init_kwargs["path"])
+        with fs.cd(project_dir):
+            return getattr(ProjectConfig(**init_kwargs), method)(*args)
+
+    @staticmethod
+    def config_load(path):
+        return ProjectConfig(
+            path, parse_extra=False, expand_interpolations=False
+        ).as_tuple()
+
+    @staticmethod
+    def config_dump(path, data):
+        config = ProjectConfig(path, parse_extra=False, expand_interpolations=False)
+        config.update(data, clear=True)
+        return config.save()
+
+    @staticmethod
+    def config_update_description(path, text):
+        config = ProjectConfig(path, parse_extra=False, expand_interpolations=False)
+        if not config.has_section("platformio"):
+            config.add_section("platformio")
+        if text:
+            config.set("platformio", "description", text)
+        else:
+            if config.has_option("platformio", "description"):
+                config.remove_option("platformio", "description")
+            if not config.options("platformio"):
+                config.remove_section("platformio")
+        return config.save()
+
+    @staticmethod
+    def get_config_schema():
+        return get_config_options_schema()
 
     @staticmethod
     def _get_projects(project_dirs=None):
-
-        def _get_project_data(project_dir):
+        def _get_project_data():
             data = {"boards": [], "envLibdepsDirs": [], "libExtraDirs": []}
-            config = ProjectConfig(join(project_dir, "platformio.ini"))
-            libdeps_dir = get_project_libdeps_dir()
-
-            data['libExtraDirs'].extend(
-                config.get("platformio", "lib_extra_dirs", []))
+            config = ProjectConfig()
+            data["envs"] = config.envs()
+            data["description"] = config.get("platformio", "description")
+            data["libExtraDirs"].extend(config.get("platformio", "lib_extra_dirs", []))
 
+            libdeps_dir = config.get_optional_dir("libdeps")
             for section in config.sections():
                 if not section.startswith("env:"):
                     continue
-                data['envLibdepsDirs'].append(join(libdeps_dir, section[4:]))
+                data["envLibdepsDirs"].append(join(libdeps_dir, section[4:]))
                 if config.has_option(section, "board"):
-                    data['boards'].append(config.get(section, "board"))
-                data['libExtraDirs'].extend(
-                    config.get(section, "lib_extra_dirs", []))
+                    data["boards"].append(config.get(section, "board"))
+                data["libExtraDirs"].extend(config.get(section, "lib_extra_dirs", []))
 
             # skip non existing folders and resolve full path
             for key in ("envLibdepsDirs", "libExtraDirs"):
                 data[key] = [
-                    expanduser(d) if d.startswith("~") else realpath(d)
-                    for d in data[key] if isdir(d)
+                    fs.expanduser(d) if d.startswith("~") else realpath(d)
+                    for d in data[key]
+                    if isdir(d)
                 ]
 
             return data
@@ -69,7 +105,7 @@ class ProjectRPC(object):
             return (sep).join(path.split(sep)[-2:])
 
         if not project_dirs:
-            project_dirs = AppRPC.load_state()['storage']['recentProjects']
+            project_dirs = AppRPC.load_state()["storage"]["recentProjects"]
 
         result = []
         pm = PlatformManager()
@@ -78,36 +114,36 @@ class ProjectRPC(object):
             boards = []
             try:
                 with fs.cd(project_dir):
-                    data = _get_project_data(project_dir)
+                    data = _get_project_data()
             except exception.PlatformIOProjectException:
                 continue
 
             for board_id in data.get("boards", []):
                 name = board_id
                 try:
-                    name = pm.board_config(board_id)['name']
+                    name = pm.board_config(board_id)["name"]
                 except exception.PlatformioException:
                     pass
                 boards.append({"id": board_id, "name": name})
 
-            result.append({
-                "path":
-                project_dir,
-                "name":
-                _path_to_name(project_dir),
-                "modified":
-                int(getmtime(project_dir)),
-                "boards":
-                boards,
-                "envLibStorages": [{
-                    "name": basename(d),
-                    "path": d
-                } for d in data.get("envLibdepsDirs", [])],
-                "extraLibStorages": [{
-                    "name": _path_to_name(d),
-                    "path": d
-                } for d in data.get("libExtraDirs", [])]
-            })
+            result.append(
+                {
+                    "path": project_dir,
+                    "name": _path_to_name(project_dir),
+                    "modified": int(getmtime(project_dir)),
+                    "boards": boards,
+                    "description": data.get("description"),
+                    "envs": data.get("envs", []),
+                    "envLibStorages": [
+                        {"name": basename(d), "path": d}
+                        for d in data.get("envLibdepsDirs", [])
+                    ],
+                    "extraLibStorages": [
+                        {"name": _path_to_name(d), "path": d}
+                        for d in data.get("libExtraDirs", [])
+                    ],
+                }
+            )
         return result
 
     def get_projects(self, project_dirs=None):
@@ -117,7 +153,7 @@ class ProjectRPC(object):
     def get_project_examples():
         result = []
         for manifest in PlatformManager().get_installed():
-            examples_dir = join(manifest['__pkg_dir'], "examples")
+            examples_dir = join(manifest["__pkg_dir"], "examples")
             if not isdir(examples_dir):
                 continue
             items = []
@@ -126,28 +162,30 @@ class ProjectRPC(object):
                 try:
                     config = ProjectConfig(join(project_dir, "platformio.ini"))
                     config.validate(silent=True)
-                    project_description = config.get("platformio",
-                                                     "description")
+                    project_description = config.get("platformio", "description")
                 except exception.PlatformIOProjectException:
                     continue
 
                 path_tokens = project_dir.split(sep)
-                items.append({
-                    "name":
-                    "/".join(path_tokens[path_tokens.index("examples") + 1:]),
-                    "path":
-                    project_dir,
-                    "description":
-                    project_description
-                })
-            result.append({
-                "platform": {
-                    "title": manifest['title'],
-                    "version": manifest['version']
-                },
-                "items": sorted(items, key=lambda item: item['name'])
-            })
-        return sorted(result, key=lambda data: data['platform']['title'])
+                items.append(
+                    {
+                        "name": "/".join(
+                            path_tokens[path_tokens.index("examples") + 1 :]
+                        ),
+                        "path": project_dir,
+                        "description": project_description,
+                    }
+                )
+            result.append(
+                {
+                    "platform": {
+                        "title": manifest["title"],
+                        "version": manifest["version"],
+                    },
+                    "items": sorted(items, key=lambda item: item["name"]),
+                }
+            )
+        return sorted(result, key=lambda data: data["platform"]["title"])
 
     def init(self, board, framework, project_dir):
         assert project_dir
@@ -157,9 +195,11 @@ class ProjectRPC(object):
         args = ["init", "--board", board]
         if framework:
             args.extend(["--project-option", "framework = %s" % framework])
-        if (state['storage']['coreCaller'] and state['storage']['coreCaller']
-                in ProjectGenerator.get_supported_ides()):
-            args.extend(["--ide", state['storage']['coreCaller']])
+        if (
+            state["storage"]["coreCaller"]
+            and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides()
+        ):
+            args.extend(["--ide", state["storage"]["coreCaller"]])
         d = PIOCoreRPC.call(args, options={"cwd": project_dir})
         d.addCallback(self._generate_project_main, project_dir, framework)
         return d
@@ -168,89 +208,99 @@ class ProjectRPC(object):
     def _generate_project_main(_, project_dir, framework):
         main_content = None
         if framework == "arduino":
-            main_content = "\n".join([
-                "#include <Arduino.h>",
-                "",
-                "void setup() {",
-                "  // put your setup code here, to run once:",
-                "}",
-                "",
-                "void loop() {",
-                "  // put your main code here, to run repeatedly:",
-                "}"
-                ""
-            ])   # yapf: disable
+            main_content = "\n".join(
+                [
+                    "#include <Arduino.h>",
+                    "",
+                    "void setup() {",
+                    "  // put your setup code here, to run once:",
+                    "}",
+                    "",
+                    "void loop() {",
+                    "  // put your main code here, to run repeatedly:",
+                    "}",
+                    "",
+                ]
+            )
         elif framework == "mbed":
-            main_content = "\n".join([
-                "#include <mbed.h>",
-                "",
-                "int main() {",
-                "",
-                "  // put your setup code here, to run once:",
-                "",
-                "  while(1) {",
-                "    // put your main code here, to run repeatedly:",
-                "  }",
-                "}",
-                ""
-            ])   # yapf: disable
+            main_content = "\n".join(
+                [
+                    "#include <mbed.h>",
+                    "",
+                    "int main() {",
+                    "",
+                    "  // put your setup code here, to run once:",
+                    "",
+                    "  while(1) {",
+                    "    // put your main code here, to run repeatedly:",
+                    "  }",
+                    "}",
+                    "",
+                ]
+            )
         if not main_content:
             return project_dir
         with fs.cd(project_dir):
-            src_dir = get_project_src_dir()
+            config = ProjectConfig()
+            src_dir = config.get_optional_dir("src")
             main_path = join(src_dir, "main.cpp")
             if isfile(main_path):
                 return project_dir
             if not isdir(src_dir):
                 os.makedirs(src_dir)
-            with open(main_path, "w") as f:
-                f.write(main_content.strip())
+            fs.write_file_contents(main_path, main_content.strip())
         return project_dir
 
     def import_arduino(self, board, use_arduino_libs, arduino_project_dir):
         board = str(board)
         if arduino_project_dir and PY2:
-            arduino_project_dir = arduino_project_dir.encode(
-                get_filesystem_encoding())
+            arduino_project_dir = arduino_project_dir.encode(get_filesystem_encoding())
         # don't import PIO Project
         if is_platformio_project(arduino_project_dir):
             return arduino_project_dir
 
-        is_arduino_project = any([
-            isfile(
-                join(arduino_project_dir,
-                     "%s.%s" % (basename(arduino_project_dir), ext)))
-            for ext in ("ino", "pde")
-        ])
+        is_arduino_project = any(
+            [
+                isfile(
+                    join(
+                        arduino_project_dir,
+                        "%s.%s" % (basename(arduino_project_dir), ext),
+                    )
+                )
+                for ext in ("ino", "pde")
+            ]
+        )
         if not is_arduino_project:
             raise jsonrpc.exceptions.JSONRPCDispatchException(
-                code=4000,
-                message="Not an Arduino project: %s" % arduino_project_dir)
+                code=4000, message="Not an Arduino project: %s" % arduino_project_dir
+            )
 
         state = AppRPC.load_state()
-        project_dir = join(state['storage']['projectsDir'],
-                           time.strftime("%y%m%d-%H%M%S-") + board)
+        project_dir = join(
+            state["storage"]["projectsDir"], time.strftime("%y%m%d-%H%M%S-") + board
+        )
         if not isdir(project_dir):
             os.makedirs(project_dir)
         args = ["init", "--board", board]
         args.extend(["--project-option", "framework = arduino"])
         if use_arduino_libs:
-            args.extend([
-                "--project-option",
-                "lib_extra_dirs = ~/Documents/Arduino/libraries"
-            ])
-        if (state['storage']['coreCaller'] and state['storage']['coreCaller']
-                in ProjectGenerator.get_supported_ides()):
-            args.extend(["--ide", state['storage']['coreCaller']])
+            args.extend(
+                ["--project-option", "lib_extra_dirs = ~/Documents/Arduino/libraries"]
+            )
+        if (
+            state["storage"]["coreCaller"]
+            and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides()
+        ):
+            args.extend(["--ide", state["storage"]["coreCaller"]])
         d = PIOCoreRPC.call(args, options={"cwd": project_dir})
-        d.addCallback(self._finalize_arduino_import, project_dir,
-                      arduino_project_dir)
+        d.addCallback(self._finalize_arduino_import, project_dir, arduino_project_dir)
         return d
 
     @staticmethod
     def _finalize_arduino_import(_, project_dir, arduino_project_dir):
         with fs.cd(project_dir):
-            src_dir = get_project_src_dir()
+            config = ProjectConfig()
+            src_dir = config.get_optional_dir("src")
             if isdir(src_dir):
                 fs.rmtree(src_dir)
             shutil.copytree(arduino_project_dir, src_dir)
@@ -260,18 +310,21 @@ class ProjectRPC(object):
     def import_pio(project_dir):
         if not project_dir or not is_platformio_project(project_dir):
             raise jsonrpc.exceptions.JSONRPCDispatchException(
-                code=4001,
-                message="Not an PlatformIO project: %s" % project_dir)
+                code=4001, message="Not an PlatformIO project: %s" % project_dir
+            )
         new_project_dir = join(
-            AppRPC.load_state()['storage']['projectsDir'],
-            time.strftime("%y%m%d-%H%M%S-") + basename(project_dir))
+            AppRPC.load_state()["storage"]["projectsDir"],
+            time.strftime("%y%m%d-%H%M%S-") + basename(project_dir),
+        )
         shutil.copytree(project_dir, new_project_dir)
 
         state = AppRPC.load_state()
         args = ["init"]
-        if (state['storage']['coreCaller'] and state['storage']['coreCaller']
-                in ProjectGenerator.get_supported_ides()):
-            args.extend(["--ide", state['storage']['coreCaller']])
+        if (
+            state["storage"]["coreCaller"]
+            and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides()
+        ):
+            args.extend(["--ide", state["storage"]["coreCaller"]])
         d = PIOCoreRPC.call(args, options={"cwd": new_project_dir})
         d.addCallback(lambda _: new_project_dir)
         return d

platformio/commands/home/rpc/server.py

@@ -16,49 +16,58 @@
 
 import click
 import jsonrpc
-from autobahn.twisted.websocket import (WebSocketServerFactory,
-                                        WebSocketServerProtocol)
+from autobahn.twisted.websocket import WebSocketServerFactory, WebSocketServerProtocol
 from jsonrpc.exceptions import JSONRPCDispatchException
-from twisted.internet import defer
+from twisted.internet import defer, reactor
 
 from platformio.compat import PY2, dump_json_to_unicode, is_bytes
 
 
 class JSONRPCServerProtocol(WebSocketServerProtocol):
+    def onOpen(self):
+        self.factory.connection_nums += 1
+        if self.factory.shutdown_timer:
+            self.factory.shutdown_timer.cancel()
+            self.factory.shutdown_timer = None
+
+    def onClose(self, wasClean, code, reason):  # pylint: disable=unused-argument
+        self.factory.connection_nums -= 1
+        if self.factory.connection_nums == 0:
+            self.factory.shutdownByTimeout()
 
     def onMessage(self, payload, isBinary):  # pylint: disable=unused-argument
         # click.echo("> %s" % payload)
         response = jsonrpc.JSONRPCResponseManager.handle(
-            payload, self.factory.dispatcher).data
+            payload, self.factory.dispatcher
+        ).data
         # if error
         if "result" not in response:
             self.sendJSONResponse(response)
             return None
 
-        d = defer.maybeDeferred(lambda: response['result'])
+        d = defer.maybeDeferred(lambda: response["result"])
         d.addCallback(self._callback, response)
         d.addErrback(self._errback, response)
 
         return None
 
     def _callback(self, result, response):
-        response['result'] = result
+        response["result"] = result
         self.sendJSONResponse(response)
 
     def _errback(self, failure, response):
         if isinstance(failure.value, JSONRPCDispatchException):
             e = failure.value
         else:
-            e = JSONRPCDispatchException(code=4999,
-                                         message=failure.getErrorMessage())
+            e = JSONRPCDispatchException(code=4999, message=failure.getErrorMessage())
         del response["result"]
-        response['error'] = e.error._data  # pylint: disable=protected-access
+        response["error"] = e.error._data  # pylint: disable=protected-access
         self.sendJSONResponse(response)
 
     def sendJSONResponse(self, response):
         # click.echo("< %s" % response)
         if "error" in response:
-            click.secho("Error: %s" % response['error'], fg="red", err=True)
+            click.secho("Error: %s" % response["error"], fg="red", err=True)
         response = dump_json_to_unicode(response)
         if not PY2 and not is_bytes(response):
             response = response.encode("utf-8")
@@ -68,10 +77,25 @@ class JSONRPCServerProtocol(WebSocketServerProtocol):
 class JSONRPCServerFactory(WebSocketServerFactory):
 
     protocol = JSONRPCServerProtocol
+    connection_nums = 0
+    shutdown_timer = 0
 
-    def __init__(self):
+    def __init__(self, shutdown_timeout=0):
         super(JSONRPCServerFactory, self).__init__()
+        self.shutdown_timeout = shutdown_timeout
         self.dispatcher = jsonrpc.Dispatcher()
 
+    def shutdownByTimeout(self):
+        if self.shutdown_timeout < 1:
+            return
+
+        def _auto_shutdown_server():
+            click.echo("Automatically shutdown server on timeout")
+            reactor.stop()
+
+        self.shutdown_timer = reactor.callLater(
+            self.shutdown_timeout, _auto_shutdown_server
+        )
+
     def addHandler(self, handler, namespace):
         self.dispatcher.build_method_map(handler, prefix="%s." % namespace)

platformio/commands/home/web.py

@@ -17,14 +17,12 @@ from twisted.web import static  # pylint: disable=import-error
 
 
 class WebRoot(static.File):
-
     def render_GET(self, request):
         if request.args.get("__shutdown__", False):
             reactor.stop()
             return "Server has been stopped"
 
-        request.setHeader("cache-control",
-                          "no-cache, no-store, must-revalidate")
+        request.setHeader("cache-control", "no-cache, no-store, must-revalidate")
         request.setHeader("pragma", "no-cache")
         request.setHeader("expires", "0")
         return static.File.render_GET(self, request)

platformio/commands/init.py

@@ -20,16 +20,11 @@ from os.path import isdir, isfile, join
 import click
 
 from platformio import exception, fs
-from platformio.commands.platform import \
-    platform_install as cli_platform_install
+from platformio.commands.platform import platform_install as cli_platform_install
 from platformio.ide.projectgenerator import ProjectGenerator
 from platformio.managers.platform import PlatformManager
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import (get_project_include_dir,
-                                        get_project_lib_dir,
-                                        get_project_src_dir,
-                                        get_project_test_dir,
-                                        is_platformio_project)
+from platformio.project.helpers import is_platformio_project
 
 
 def validate_boards(ctx, param, value):  # pylint: disable=W0613
@@ -40,66 +35,66 @@ def validate_boards(ctx, param, value):  # pylint: disable=W0613
         except exception.UnknownBoard:
             raise click.BadParameter(
                 "`%s`. Please search for board ID using `platformio boards` "
-                "command" % id_)
+                "command" % id_
+            )
     return value
 
 
-@click.command("init",
-               short_help="Initialize PlatformIO project or update existing")
-@click.option("--project-dir",
-              "-d",
-              default=getcwd,
-              type=click.Path(exists=True,
-                              file_okay=False,
-                              dir_okay=True,
-                              writable=True,
-                              resolve_path=True))
-@click.option("-b",
-              "--board",
-              multiple=True,
-              metavar="ID",
-              callback=validate_boards)
-@click.option("--ide",
-              type=click.Choice(ProjectGenerator.get_supported_ides()))
+@click.command("init", short_help="Initialize PlatformIO project or update existing")
+@click.option(
+    "--project-dir",
+    "-d",
+    default=getcwd,
+    type=click.Path(
+        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
+    ),
+)
+@click.option("-b", "--board", multiple=True, metavar="ID", callback=validate_boards)
+@click.option("--ide", type=click.Choice(ProjectGenerator.get_supported_ides()))
 @click.option("-O", "--project-option", multiple=True)
 @click.option("--env-prefix", default="")
 @click.option("-s", "--silent", is_flag=True)
 @click.pass_context
 def cli(
-        ctx,  # pylint: disable=R0913
-        project_dir,
-        board,
-        ide,
-        project_option,
-        env_prefix,
-        silent):
+    ctx,  # pylint: disable=R0913
+    project_dir,
+    board,
+    ide,
+    project_option,
+    env_prefix,
+    silent,
+):
     if not silent:
         if project_dir == getcwd():
-            click.secho("\nThe current working directory",
-                        fg="yellow",
-                        nl=False)
+            click.secho("\nThe current working directory", fg="yellow", nl=False)
             click.secho(" %s " % project_dir, fg="cyan", nl=False)
             click.secho("will be used for the project.", fg="yellow")
             click.echo("")
 
-        click.echo("The next files/directories have been created in %s" %
-                   click.style(project_dir, fg="cyan"))
-        click.echo("%s - Put project header files here" %
-                   click.style("include", fg="cyan"))
-        click.echo("%s - Put here project specific (private) libraries" %
-                   click.style("lib", fg="cyan"))
-        click.echo("%s - Put project source files here" %
-                   click.style("src", fg="cyan"))
-        click.echo("%s - Project Configuration File" %
-                   click.style("platformio.ini", fg="cyan"))
+        click.echo(
+            "The next files/directories have been created in %s"
+            % click.style(project_dir, fg="cyan")
+        )
+        click.echo(
+            "%s - Put project header files here" % click.style("include", fg="cyan")
+        )
+        click.echo(
+            "%s - Put here project specific (private) libraries"
+            % click.style("lib", fg="cyan")
+        )
+        click.echo("%s - Put project source files here" % click.style("src", fg="cyan"))
+        click.echo(
+            "%s - Project Configuration File" % click.style("platformio.ini", fg="cyan")
+        )
 
     is_new_project = not is_platformio_project(project_dir)
     if is_new_project:
         init_base_project(project_dir)
 
     if board:
-        fill_project_envs(ctx, project_dir, board, project_option, env_prefix,
-                          ide is not None)
+        fill_project_envs(
+            ctx, project_dir, board, project_option, env_prefix, ide is not None
+        )
 
     if ide:
         pg = ProjectGenerator(project_dir, ide, board)
@@ -115,9 +110,9 @@ def cli(
     if ide:
         click.secho(
             "\nProject has been successfully %s including configuration files "
-            "for `%s` IDE." %
-            ("initialized" if is_new_project else "updated", ide),
-            fg="green")
+            "for `%s` IDE." % ("initialized" if is_new_project else "updated", ide),
+            fg="green",
+        )
     else:
         click.secho(
             "\nProject has been successfully %s! Useful commands:\n"
@@ -125,19 +120,21 @@ def cli(
             "`pio run --target upload` or `pio run -t upload` "
             "- upload firmware to a target\n"
             "`pio run --target clean` - clean project (remove compiled files)"
-            "\n`pio run --help` - additional information" %
-            ("initialized" if is_new_project else "updated"),
-            fg="green")
+            "\n`pio run --help` - additional information"
+            % ("initialized" if is_new_project else "updated"),
+            fg="green",
+        )
 
 
 def init_base_project(project_dir):
-    ProjectConfig(join(project_dir, "platformio.ini")).save()
     with fs.cd(project_dir):
+        config = ProjectConfig()
+        config.save()
         dir_to_readme = [
-            (get_project_src_dir(), None),
-            (get_project_include_dir(), init_include_readme),
-            (get_project_lib_dir(), init_lib_readme),
-            (get_project_test_dir(), init_test_readme),
+            (config.get_optional_dir("src"), None),
+            (config.get_optional_dir("include"), init_include_readme),
+            (config.get_optional_dir("lib"), init_lib_readme),
+            (config.get_optional_dir("test"), init_test_readme),
         ]
         for (path, cb) in dir_to_readme:
             if isdir(path):
@@ -148,8 +145,9 @@ def init_base_project(project_dir):
 
 
 def init_include_readme(include_dir):
-    with open(join(include_dir, "README"), "w") as f:
-        f.write("""
+    fs.write_file_contents(
+        join(include_dir, "README"),
+        """
 This directory is intended for project header files.
 
 A header file is a file containing C declarations and macro definitions
@@ -188,12 +186,15 @@ Read more about using header files in official GCC documentation:
 * Computed Includes
 
 https://gcc.gnu.org/onlinedocs/cpp/Header-Files.html
-""")
+""",
+    )
 
 
 def init_lib_readme(lib_dir):
-    with open(join(lib_dir, "README"), "w") as f:
-        f.write("""
+    # pylint: disable=line-too-long
+    fs.write_file_contents(
+        join(lib_dir, "README"),
+        """
 This directory is intended for project specific (private) libraries.
 PlatformIO will compile them to static libraries and link into executable file.
 
@@ -239,12 +240,14 @@ libraries scanning project source files.
 
 More information about PlatformIO Library Dependency Finder
 - https://docs.platformio.org/page/librarymanager/ldf.html
-""")
+""",
+    )
 
 
 def init_test_readme(test_dir):
-    with open(join(test_dir, "README"), "w") as f:
-        f.write("""
+    fs.write_file_contents(
+        join(test_dir, "README"),
+        """
 This directory is intended for PIO Unit Testing and project tests.
 
 Unit Testing is a software testing method by which individual units of
@@ -255,15 +258,17 @@ in the development cycle.
 
 More information about PIO Unit Testing:
 - https://docs.platformio.org/page/plus/unit-testing.html
-""")
+""",
+    )
 
 
 def init_ci_conf(project_dir):
     conf_path = join(project_dir, ".travis.yml")
     if isfile(conf_path):
         return
-    with open(conf_path, "w") as f:
-        f.write("""# Continuous Integration (CI) is the practice, in software
+    fs.write_file_contents(
+        conf_path,
+        """# Continuous Integration (CI) is the practice, in software
 # engineering, of merging all developer working copies with a shared mainline
 # several times a day < https://docs.platformio.org/page/ci/index.html >
 #
@@ -330,27 +335,24 @@ def init_ci_conf(project_dir):
 #
 # script:
 #     - platformio ci --lib="." --board=ID_1 --board=ID_2 --board=ID_N
-""")
+""",
+    )
 
 
 def init_cvs_ignore(project_dir):
     conf_path = join(project_dir, ".gitignore")
     if isfile(conf_path):
         return
-    with open(conf_path, "w") as fp:
-        fp.write(".pio\n")
+    fs.write_file_contents(conf_path, ".pio\n")
 
 
-def fill_project_envs(ctx, project_dir, board_ids, project_option, env_prefix,
-                      force_download):
-    config = ProjectConfig(join(project_dir, "platformio.ini"),
-                           parse_extra=False)
+def fill_project_envs(
+    ctx, project_dir, board_ids, project_option, env_prefix, force_download
+):
+    config = ProjectConfig(join(project_dir, "platformio.ini"), parse_extra=False)
     used_boards = []
     for section in config.sections():
-        cond = [
-            section.startswith("env:"),
-            config.has_option(section, "board")
-        ]
+        cond = [section.startswith("env:"), config.has_option(section, "board")]
         if all(cond):
             used_boards.append(config.get(section, "board"))
 
@@ -359,17 +361,17 @@ def fill_project_envs(ctx, project_dir, board_ids, project_option, env_prefix,
     modified = False
     for id_ in board_ids:
         board_config = pm.board_config(id_)
-        used_platforms.append(board_config['platform'])
+        used_platforms.append(board_config["platform"])
         if id_ in used_boards:
             continue
         used_boards.append(id_)
         modified = True
 
-        envopts = {"platform": board_config['platform'], "board": id_}
+        envopts = {"platform": board_config["platform"], "board": id_}
         # find default framework for board
         frameworks = board_config.get("frameworks")
         if frameworks:
-            envopts['framework'] = frameworks[0]
+            envopts["framework"] = frameworks[0]
 
         for item in project_option:
             if "=" not in item:
@@ -391,10 +393,9 @@ def fill_project_envs(ctx, project_dir, board_ids, project_option, env_prefix,
 
 
 def _install_dependent_platforms(ctx, platforms):
-    installed_platforms = [
-        p['name'] for p in PlatformManager().get_installed()
-    ]
+    installed_platforms = [p["name"] for p in PlatformManager().get_installed()]
     if set(platforms) <= set(installed_platforms):
         return
-    ctx.invoke(cli_platform_install,
-               platforms=list(set(platforms) - set(installed_platforms)))
+    ctx.invoke(
+        cli_platform_install, platforms=list(set(platforms) - set(installed_platforms))
+    )

platformio/commands/lib.py

@@ -14,24 +14,22 @@
 
 # pylint: disable=too-many-branches, too-many-locals
 
+import os
 import time
-from os.path import isdir, join
 
 import click
 import semantic_version
 from tabulate import tabulate
 
-from platformio import exception, fs, util
+from platformio import exception, util
 from platformio.commands import PlatformioCLI
 from platformio.compat import dump_json_to_unicode
-from platformio.managers.lib import (LibraryManager, get_builtin_libs,
-                                     is_builtin_lib)
+from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib
+from platformio.package.manifest.parser import ManifestParserFactory
+from platformio.package.manifest.schema import ManifestSchema, ManifestValidationError
 from platformio.proc import is_ci
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import (get_project_dir,
-                                        get_project_global_lib_dir,
-                                        get_project_libdeps_dir,
-                                        is_platformio_project)
+from platformio.project.helpers import get_project_dir, is_platformio_project
 
 try:
     from urllib.parse import quote
@@ -44,36 +42,43 @@ CTX_META_STORAGE_DIRS_KEY = __name__ + ".storage_dirs"
 CTX_META_STORAGE_LIBDEPS_KEY = __name__ + ".storage_lib_deps"
 
 
+def get_project_global_lib_dir():
+    return ProjectConfig.get_instance().get_optional_dir("globallib")
+
+
 @click.group(short_help="Library Manager")
-@click.option("-d",
-              "--storage-dir",
-              multiple=True,
-              default=None,
-              type=click.Path(exists=True,
-                              file_okay=False,
-                              dir_okay=True,
-                              writable=True,
-                              resolve_path=True),
-              help="Manage custom library storage")
-@click.option("-g",
-              "--global",
-              is_flag=True,
-              help="Manage global PlatformIO library storage")
+@click.option(
+    "-d",
+    "--storage-dir",
+    multiple=True,
+    default=None,
+    type=click.Path(
+        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
+    ),
+    help="Manage custom library storage",
+)
+@click.option(
+    "-g", "--global", is_flag=True, help="Manage global PlatformIO library storage"
+)
 @click.option(
     "-e",
     "--environment",
     multiple=True,
-    help=("Manage libraries for the specific project build environments "
-          "declared in `platformio.ini`"))
+    help=(
+        "Manage libraries for the specific project build environments "
+        "declared in `platformio.ini`"
+    ),
+)
 @click.pass_context
 def cli(ctx, **options):
     storage_cmds = ("install", "uninstall", "update", "list")
     # skip commands that don't need storage folder
-    if ctx.invoked_subcommand not in storage_cmds or \
-            (len(ctx.args) == 2 and ctx.args[1] in ("-h", "--help")):
+    if ctx.invoked_subcommand not in storage_cmds or (
+        len(ctx.args) == 2 and ctx.args[1] in ("-h", "--help")
+    ):
         return
-    storage_dirs = list(options['storage_dir'])
-    if options['global']:
+    storage_dirs = list(options["storage_dir"])
+    if options["global"]:
         storage_dirs.append(get_project_global_lib_dir())
     if not storage_dirs:
         if is_platformio_project():
@@ -84,15 +89,16 @@ def cli(ctx, **options):
                 "Warning! Global library storage is used automatically. "
                 "Please use `platformio lib --global %s` command to remove "
                 "this warning." % ctx.invoked_subcommand,
-                fg="yellow")
+                fg="yellow",
+            )
 
     if not storage_dirs:
-        raise exception.NotGlobalLibDir(get_project_dir(),
-                                        get_project_global_lib_dir(),
-                                        ctx.invoked_subcommand)
+        raise exception.NotGlobalLibDir(
+            get_project_dir(), get_project_global_lib_dir(), ctx.invoked_subcommand
+        )
 
     in_silence = PlatformioCLI.in_silence()
-    ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] = options['environment']
+    ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] = options["environment"]
     ctx.meta[CTX_META_INPUT_DIRS_KEY] = storage_dirs
     ctx.meta[CTX_META_STORAGE_DIRS_KEY] = []
     ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY] = {}
@@ -100,18 +106,17 @@ def cli(ctx, **options):
         if not is_platformio_project(storage_dir):
             ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir)
             continue
-        with fs.cd(storage_dir):
-            libdeps_dir = get_project_libdeps_dir()
-        config = ProjectConfig.get_instance(join(storage_dir,
-                                                 "platformio.ini"))
-        config.validate(options['environment'], silent=in_silence)
+        config = ProjectConfig.get_instance(os.path.join(storage_dir, "platformio.ini"))
+        config.validate(options["environment"], silent=in_silence)
+        libdeps_dir = config.get_optional_dir("libdeps")
         for env in config.envs():
-            if options['environment'] and env not in options['environment']:
+            if options["environment"] and env not in options["environment"]:
                 continue
-            storage_dir = join(libdeps_dir, env)
+            storage_dir = os.path.join(libdeps_dir, env)
             ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir)
             ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY][storage_dir] = config.get(
-                "env:" + env, "lib_deps", [])
+                "env:" + env, "lib_deps", []
+            )
 
 
 @cli.command("install", short_help="Install library")
@@ -119,21 +124,19 @@ def cli(ctx, **options):
 @click.option(
     "--save",
     is_flag=True,
-    help="Save installed libraries into the `platformio.ini` dependency list")
-@click.option("-s",
-              "--silent",
-              is_flag=True,
-              help="Suppress progress reporting")
-@click.option("--interactive",
-              is_flag=True,
-              help="Allow to make a choice for all prompts")
-@click.option("-f",
-              "--force",
-              is_flag=True,
-              help="Reinstall/redownload library if exists")
+    help="Save installed libraries into the `platformio.ini` dependency list",
+)
+@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting")
+@click.option(
+    "--interactive", is_flag=True, help="Allow to make a choice for all prompts"
+)
+@click.option(
+    "-f", "--force", is_flag=True, help="Reinstall/redownload library if exists"
+)
 @click.pass_context
 def lib_install(  # pylint: disable=too-many-arguments
-        ctx, libraries, save, silent, interactive, force):
+    ctx, libraries, save, silent, interactive, force
+):
     storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
     storage_libdeps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, [])
 
@@ -144,25 +147,22 @@ def lib_install(  # pylint: disable=too-many-arguments
         lm = LibraryManager(storage_dir)
         if libraries:
             for library in libraries:
-                pkg_dir = lm.install(library,
-                                     silent=silent,
-                                     interactive=interactive,
-                                     force=force)
+                pkg_dir = lm.install(
+                    library, silent=silent, interactive=interactive, force=force
+                )
                 installed_manifests[library] = lm.load_manifest(pkg_dir)
         elif storage_dir in storage_libdeps:
             builtin_lib_storages = None
             for library in storage_libdeps[storage_dir]:
                 try:
-                    pkg_dir = lm.install(library,
-                                         silent=silent,
-                                         interactive=interactive,
-                                         force=force)
+                    pkg_dir = lm.install(
+                        library, silent=silent, interactive=interactive, force=force
+                    )
                     installed_manifests[library] = lm.load_manifest(pkg_dir)
                 except exception.LibNotFound as e:
                     if builtin_lib_storages is None:
                         builtin_lib_storages = get_builtin_libs()
-                    if not silent or not is_builtin_lib(
-                            builtin_lib_storages, library):
+                    if not silent or not is_builtin_lib(builtin_lib_storages, library):
                         click.secho("Warning! %s" % e, fg="yellow")
 
     if not save or not libraries:
@@ -171,7 +171,7 @@ def lib_install(  # pylint: disable=too-many-arguments
     input_dirs = ctx.meta.get(CTX_META_INPUT_DIRS_KEY, [])
     project_environments = ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY]
     for input_dir in input_dirs:
-        config = ProjectConfig.get_instance(join(input_dir, "platformio.ini"))
+        config = ProjectConfig.get_instance(os.path.join(input_dir, "platformio.ini"))
         config.validate(project_environments)
         for env in config.envs():
             if project_environments and env not in project_environments:
@@ -183,8 +183,8 @@ def lib_install(  # pylint: disable=too-many-arguments
                     continue
                 manifest = installed_manifests[library]
                 try:
-                    assert library.lower() == manifest['name'].lower()
-                    assert semantic_version.Version(manifest['version'])
+                    assert library.lower() == manifest["name"].lower()
+                    assert semantic_version.Version(manifest["version"])
                     lib_deps.append("{name}@^{version}".format(**manifest))
                 except (AssertionError, ValueError):
                     lib_deps.append(library)
@@ -206,13 +206,15 @@ def lib_uninstall(ctx, libraries):
 
 @cli.command("update", short_help="Update installed libraries")
 @click.argument("libraries", required=False, nargs=-1, metavar="[LIBRARY...]")
-@click.option("-c",
-              "--only-check",
-              is_flag=True,
-              help="DEPRECATED. Please use `--dry-run` instead")
-@click.option("--dry-run",
-              is_flag=True,
-              help="Do not update, only check for the new versions")
+@click.option(
+    "-c",
+    "--only-check",
+    is_flag=True,
+    help="DEPRECATED. Please use `--dry-run` instead",
+)
+@click.option(
+    "--dry-run", is_flag=True, help="Do not update, only check for the new versions"
+)
 @click.option("--json-output", is_flag=True)
 @click.pass_context
 def lib_update(ctx, libraries, only_check, dry_run, json_output):
@@ -226,14 +228,12 @@ def lib_update(ctx, libraries, only_check, dry_run, json_output):
 
         _libraries = libraries
         if not _libraries:
-            _libraries = [
-                manifest['__pkg_dir'] for manifest in lm.get_installed()
-            ]
+            _libraries = [manifest["__pkg_dir"] for manifest in lm.get_installed()]
 
         if only_check and json_output:
             result = []
             for library in _libraries:
-                pkg_dir = library if isdir(library) else None
+                pkg_dir = library if os.path.isdir(library) else None
                 requirements = None
                 url = None
                 if not pkg_dir:
@@ -245,7 +245,7 @@ def lib_update(ctx, libraries, only_check, dry_run, json_output):
                 if not latest:
                     continue
                 manifest = lm.load_manifest(pkg_dir)
-                manifest['versionLatest'] = latest
+                manifest["versionLatest"] = latest
                 result.append(manifest)
             json_result[storage_dir] = result
         else:
@@ -254,8 +254,10 @@ def lib_update(ctx, libraries, only_check, dry_run, json_output):
 
     if json_output:
         return click.echo(
-            dump_json_to_unicode(json_result[storage_dirs[0]]
-                                 if len(storage_dirs) == 1 else json_result))
+            dump_json_to_unicode(
+                json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result
+            )
+        )
 
     return True
 
@@ -274,15 +276,17 @@ def lib_list(ctx, json_output):
         if json_output:
             json_result[storage_dir] = items
         elif items:
-            for item in sorted(items, key=lambda i: i['name']):
+            for item in sorted(items, key=lambda i: i["name"]):
                 print_lib_item(item)
         else:
             click.echo("No items found")
 
     if json_output:
         return click.echo(
-            dump_json_to_unicode(json_result[storage_dirs[0]]
-                                 if len(storage_dirs) == 1 else json_result))
+            dump_json_to_unicode(
+                json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result
+            )
+        )
 
     return True
 
@@ -298,9 +302,11 @@ def lib_list(ctx, json_output):
 @click.option("-f", "--framework", multiple=True)
 @click.option("-p", "--platform", multiple=True)
 @click.option("-i", "--header", multiple=True)
-@click.option("--noninteractive",
-              is_flag=True,
-              help="Do not prompt, automatically paginate with delay")
+@click.option(
+    "--noninteractive",
+    is_flag=True,
+    help="Do not prompt, automatically paginate with delay",
+)
 def lib_search(query, json_output, page, noninteractive, **filters):
     if not query:
         query = []
@@ -311,55 +317,61 @@ def lib_search(query, json_output, page, noninteractive, **filters):
         for value in values:
             query.append('%s:"%s"' % (key, value))
 
-    result = util.get_api_result("/v2/lib/search",
-                                 dict(query=" ".join(query), page=page),
-                                 cache_valid="1d")
+    result = util.get_api_result(
+        "/v2/lib/search", dict(query=" ".join(query), page=page), cache_valid="1d"
+    )
 
     if json_output:
         click.echo(dump_json_to_unicode(result))
         return
 
-    if result['total'] == 0:
+    if result["total"] == 0:
         click.secho(
             "Nothing has been found by your request\n"
             "Try a less-specific search or use truncation (or wildcard) "
             "operator",
             fg="yellow",
-            nl=False)
+            nl=False,
+        )
         click.secho(" *", fg="green")
         click.secho("For example: DS*, PCA*, DHT* and etc.\n", fg="yellow")
-        click.echo("For more examples and advanced search syntax, "
-                   "please use documentation:")
+        click.echo(
+            "For more examples and advanced search syntax, please use documentation:"
+        )
         click.secho(
             "https://docs.platformio.org/page/userguide/lib/cmd_search.html\n",
-            fg="cyan")
+            fg="cyan",
+        )
         return
 
-    click.secho("Found %d libraries:\n" % result['total'],
-                fg="green" if result['total'] else "yellow")
+    click.secho(
+        "Found %d libraries:\n" % result["total"],
+        fg="green" if result["total"] else "yellow",
+    )
 
     while True:
-        for item in result['items']:
+        for item in result["items"]:
             print_lib_item(item)
 
-        if (int(result['page']) * int(result['perpage']) >= int(
-                result['total'])):
+        if int(result["page"]) * int(result["perpage"]) >= int(result["total"]):
             break
 
         if noninteractive:
             click.echo()
-            click.secho("Loading next %d libraries... Press Ctrl+C to stop!" %
-                        result['perpage'],
-                        fg="yellow")
+            click.secho(
+                "Loading next %d libraries... Press Ctrl+C to stop!"
+                % result["perpage"],
+                fg="yellow",
+            )
             click.echo()
             time.sleep(5)
         elif not click.confirm("Show next libraries?"):
             break
-        result = util.get_api_result("/v2/lib/search", {
-            "query": " ".join(query),
-            "page": int(result['page']) + 1
-        },
-                                     cache_valid="1d")
+        result = util.get_api_result(
+            "/v2/lib/search",
+            {"query": " ".join(query), "page": int(result["page"]) + 1},
+            cache_valid="1d",
+        )
 
 
 @cli.command("builtin", short_help="List built-in libraries")
@@ -371,13 +383,13 @@ def lib_builtin(storage, json_output):
         return click.echo(dump_json_to_unicode(items))
 
     for storage_ in items:
-        if not storage_['items']:
+        if not storage_["items"]:
             continue
-        click.secho(storage_['name'], fg="green")
-        click.echo("*" * len(storage_['name']))
+        click.secho(storage_["name"], fg="green")
+        click.echo("*" * len(storage_["name"]))
         click.echo()
 
-        for item in sorted(storage_['items'], key=lambda i: i['name']):
+        for item in sorted(storage_["items"], key=lambda i: i["name"]):
             print_lib_item(item)
 
     return True
@@ -389,27 +401,29 @@ def lib_builtin(storage, json_output):
 def lib_show(library, json_output):
     lm = LibraryManager()
     name, requirements, _ = lm.parse_pkg_uri(library)
-    lib_id = lm.search_lib_id({
-        "name": name,
-        "requirements": requirements
-    },
-                              silent=json_output,
-                              interactive=not json_output)
+    lib_id = lm.search_lib_id(
+        {"name": name, "requirements": requirements},
+        silent=json_output,
+        interactive=not json_output,
+    )
     lib = util.get_api_result("/lib/info/%d" % lib_id, cache_valid="1d")
     if json_output:
         return click.echo(dump_json_to_unicode(lib))
 
-    click.secho(lib['name'], fg="cyan")
-    click.echo("=" * len(lib['name']))
-    click.secho("#ID: %d" % lib['id'], bold=True)
-    click.echo(lib['description'])
+    click.secho(lib["name"], fg="cyan")
+    click.echo("=" * len(lib["name"]))
+    click.secho("#ID: %d" % lib["id"], bold=True)
+    click.echo(lib["description"])
     click.echo()
 
     click.echo(
-        "Version: %s, released %s" %
-        (lib['version']['name'],
-         time.strftime("%c", util.parse_date(lib['version']['released']))))
-    click.echo("Manifest: %s" % lib['confurl'])
+        "Version: %s, released %s"
+        % (
+            lib["version"]["name"],
+            time.strftime("%c", util.parse_date(lib["version"]["released"])),
+        )
+    )
+    click.echo("Manifest: %s" % lib["confurl"])
     for key in ("homepage", "repository", "license"):
         if key not in lib or not lib[key]:
             continue
@@ -436,23 +450,33 @@ def lib_show(library, json_output):
     if _authors:
         blocks.append(("Authors", _authors))
 
-    blocks.append(("Keywords", lib['keywords']))
+    blocks.append(("Keywords", lib["keywords"]))
     for key in ("frameworks", "platforms"):
         if key not in lib or not lib[key]:
             continue
-        blocks.append(("Compatible %s" % key, [i['title'] for i in lib[key]]))
-    blocks.append(("Headers", lib['headers']))
-    blocks.append(("Examples", lib['examples']))
-    blocks.append(("Versions", [
-        "%s, released %s" %
-        (v['name'], time.strftime("%c", util.parse_date(v['released'])))
-        for v in lib['versions']
-    ]))
-    blocks.append(("Unique Downloads", [
-        "Today: %s" % lib['dlstats']['day'],
-        "Week: %s" % lib['dlstats']['week'],
-        "Month: %s" % lib['dlstats']['month']
-    ]))
+        blocks.append(("Compatible %s" % key, [i["title"] for i in lib[key]]))
+    blocks.append(("Headers", lib["headers"]))
+    blocks.append(("Examples", lib["examples"]))
+    blocks.append(
+        (
+            "Versions",
+            [
+                "%s, released %s"
+                % (v["name"], time.strftime("%c", util.parse_date(v["released"])))
+                for v in lib["versions"]
+            ],
+        )
+    )
+    blocks.append(
+        (
+            "Unique Downloads",
+            [
+                "Today: %s" % lib["dlstats"]["day"],
+                "Week: %s" % lib["dlstats"]["week"],
+                "Month: %s" % lib["dlstats"]["month"],
+            ],
+        )
+    )
 
     for (title, rows) in blocks:
         click.echo()
@@ -467,16 +491,22 @@ def lib_show(library, json_output):
 @cli.command("register", short_help="Register a new library")
 @click.argument("config_url")
 def lib_register(config_url):
-    if (not config_url.startswith("http://")
-            and not config_url.startswith("https://")):
+    if not config_url.startswith("http://") and not config_url.startswith("https://"):
         raise exception.InvalidLibConfURL(config_url)
 
-    result = util.get_api_result("/lib/register",
-                                 data=dict(config_url=config_url))
-    if "message" in result and result['message']:
-        click.secho(result['message'],
-                    fg="green" if "successed" in result and result['successed']
-                    else "red")
+    # Validate manifest
+    data, error = ManifestSchema(strict=False).load(
+        ManifestParserFactory.new_from_url(config_url).as_dict()
+    )
+    if error:
+        raise ManifestValidationError(error, data)
+
+    result = util.get_api_result("/lib/register", data=dict(config_url=config_url))
+    if "message" in result and result["message"]:
+        click.secho(
+            result["message"],
+            fg="green" if "successed" in result and result["successed"] else "red",
+        )
 
 
 @cli.command("stats", short_help="Library Registry Statistics")
@@ -488,46 +518,56 @@ def lib_stats(json_output):
         return click.echo(dump_json_to_unicode(result))
 
     for key in ("updated", "added"):
-        tabular_data = [(click.style(item['name'], fg="cyan"),
-                         time.strftime("%c", util.parse_date(item['date'])),
-                         "https://platformio.org/lib/show/%s/%s" %
-                         (item['id'], quote(item['name'])))
-                        for item in result.get(key, [])]
-        table = tabulate(tabular_data,
-                         headers=[
-                             click.style("RECENTLY " + key.upper(), bold=True),
-                             "Date", "URL"
-                         ])
+        tabular_data = [
+            (
+                click.style(item["name"], fg="cyan"),
+                time.strftime("%c", util.parse_date(item["date"])),
+                "https://platformio.org/lib/show/%s/%s"
+                % (item["id"], quote(item["name"])),
+            )
+            for item in result.get(key, [])
+        ]
+        table = tabulate(
+            tabular_data,
+            headers=[click.style("RECENTLY " + key.upper(), bold=True), "Date", "URL"],
+        )
         click.echo(table)
         click.echo()
 
     for key in ("lastkeywords", "topkeywords"):
-        tabular_data = [(click.style(name, fg="cyan"),
-                         "https://platformio.org/lib/search?query=" +
-                         quote("keyword:%s" % name))
-                        for name in result.get(key, [])]
+        tabular_data = [
+            (
+                click.style(name, fg="cyan"),
+                "https://platformio.org/lib/search?query=" + quote("keyword:%s" % name),
+            )
+            for name in result.get(key, [])
+        ]
         table = tabulate(
             tabular_data,
             headers=[
                 click.style(
-                    ("RECENT" if key == "lastkeywords" else "POPULAR") +
-                    " KEYWORDS",
-                    bold=True), "URL"
-            ])
+                    ("RECENT" if key == "lastkeywords" else "POPULAR") + " KEYWORDS",
+                    bold=True,
+                ),
+                "URL",
+            ],
+        )
         click.echo(table)
         click.echo()
 
-    for key, title in (("dlday", "Today"), ("dlweek", "Week"), ("dlmonth",
-                                                                "Month")):
-        tabular_data = [(click.style(item['name'], fg="cyan"),
-                         "https://platformio.org/lib/show/%s/%s" %
-                         (item['id'], quote(item['name'])))
-                        for item in result.get(key, [])]
-        table = tabulate(tabular_data,
-                         headers=[
-                             click.style("FEATURED: " + title.upper(),
-                                         bold=True), "URL"
-                         ])
+    for key, title in (("dlday", "Today"), ("dlweek", "Week"), ("dlmonth", "Month")):
+        tabular_data = [
+            (
+                click.style(item["name"], fg="cyan"),
+                "https://platformio.org/lib/show/%s/%s"
+                % (item["id"], quote(item["name"])),
+            )
+            for item in result.get(key, [])
+        ]
+        table = tabulate(
+            tabular_data,
+            headers=[click.style("FEATURED: " + title.upper(), bold=True), "URL"],
+        )
         click.echo(table)
         click.echo()
 
@@ -538,15 +578,16 @@ def print_storage_header(storage_dirs, storage_dir):
     if storage_dirs and storage_dirs[0] != storage_dir:
         click.echo("")
     click.echo(
-        click.style("Library Storage: ", bold=True) +
-        click.style(storage_dir, fg="blue"))
+        click.style("Library Storage: ", bold=True)
+        + click.style(storage_dir, fg="blue")
+    )
 
 
 def print_lib_item(item):
-    click.secho(item['name'], fg="cyan")
-    click.echo("=" * len(item['name']))
+    click.secho(item["name"], fg="cyan")
+    click.echo("=" * len(item["name"]))
     if "id" in item:
-        click.secho("#ID: %d" % item['id'], bold=True)
+        click.secho("#ID: %d" % item["id"], bold=True)
     if "description" in item or "url" in item:
         click.echo(item.get("description", item.get("url", "")))
     click.echo()
@@ -562,14 +603,26 @@ def print_lib_item(item):
     for key in ("frameworks", "platforms"):
         if key not in item:
             continue
-        click.echo("Compatible %s: %s" % (key, ", ".join(
-            [i['title'] if isinstance(i, dict) else i for i in item[key]])))
+        click.echo(
+            "Compatible %s: %s"
+            % (
+                key,
+                ", ".join(
+                    [i["title"] if isinstance(i, dict) else i for i in item[key]]
+                ),
+            )
+        )
 
     if "authors" in item or "authornames" in item:
-        click.echo("Authors: %s" % ", ".join(
-            item.get("authornames",
-                     [a.get("name", "") for a in item.get("authors", [])])))
+        click.echo(
+            "Authors: %s"
+            % ", ".join(
+                item.get(
+                    "authornames", [a.get("name", "") for a in item.get("authors", [])]
+                )
+            )
+        )
 
     if "__src_url" in item:
-        click.secho("Source: %s" % item['__src_url'])
+        click.secho("Source: %s" % item["__src_url"])
     click.echo()

platformio/commands/platform.py

@@ -29,24 +29,27 @@ def cli():
 
 def _print_platforms(platforms):
     for platform in platforms:
-        click.echo("{name} ~ {title}".format(name=click.style(platform['name'],
-                                                              fg="cyan"),
-                                             title=platform['title']))
-        click.echo("=" * (3 + len(platform['name'] + platform['title'])))
-        click.echo(platform['description'])
+        click.echo(
+            "{name} ~ {title}".format(
+                name=click.style(platform["name"], fg="cyan"), title=platform["title"]
+            )
+        )
+        click.echo("=" * (3 + len(platform["name"] + platform["title"])))
+        click.echo(platform["description"])
         click.echo()
         if "homepage" in platform:
-            click.echo("Home: %s" % platform['homepage'])
-        if "frameworks" in platform and platform['frameworks']:
-            click.echo("Frameworks: %s" % ", ".join(platform['frameworks']))
+            click.echo("Home: %s" % platform["homepage"])
+        if "frameworks" in platform and platform["frameworks"]:
+            click.echo("Frameworks: %s" % ", ".join(platform["frameworks"]))
         if "packages" in platform:
-            click.echo("Packages: %s" % ", ".join(platform['packages']))
+            click.echo("Packages: %s" % ", ".join(platform["packages"]))
         if "version" in platform:
             if "__src_url" in platform:
-                click.echo("Version: #%s (%s)" %
-                           (platform['version'], platform['__src_url']))
+                click.echo(
+                    "Version: #%s (%s)" % (platform["version"], platform["__src_url"])
+                )
             else:
-                click.echo("Version: " + platform['version'])
+                click.echo("Version: " + platform["version"])
         click.echo()
 
 
@@ -54,7 +57,7 @@ def _get_registry_platforms():
     platforms = util.get_api_result("/platforms", cache_valid="7d")
     pm = PlatformManager()
     for platform in platforms or []:
-        platform['versions'] = pm.get_all_repo_versions(platform['name'])
+        platform["versions"] = pm.get_all_repo_versions(platform["name"])
     return platforms
 
 
@@ -65,22 +68,22 @@ def _get_platform_data(*args, **kwargs):
         return _get_registry_platform_data(*args, **kwargs)
 
 
-def _get_installed_platform_data(platform,
-                                 with_boards=True,
-                                 expose_packages=True):
+def _get_installed_platform_data(platform, with_boards=True, expose_packages=True):
     p = PlatformFactory.newPlatform(platform)
-    data = dict(name=p.name,
-                title=p.title,
-                description=p.description,
-                version=p.version,
-                homepage=p.homepage,
-                repository=p.repository_url,
-                url=p.vendor_url,
-                docs=p.docs_url,
-                license=p.license,
-                forDesktop=not p.is_embedded(),
-                frameworks=sorted(list(p.frameworks) if p.frameworks else []),
-                packages=list(p.packages) if p.packages else [])
+    data = dict(
+        name=p.name,
+        title=p.title,
+        description=p.description,
+        version=p.version,
+        homepage=p.homepage,
+        repository=p.repository_url,
+        url=p.vendor_url,
+        docs=p.docs_url,
+        license=p.license,
+        forDesktop=not p.is_embedded(),
+        frameworks=sorted(list(p.frameworks) if p.frameworks else []),
+        packages=list(p.packages) if p.packages else [],
+    )
 
     # if dump to API
     # del data['version']
@@ -94,18 +97,20 @@ def _get_installed_platform_data(platform,
             data[key] = manifest[key]
 
     if with_boards:
-        data['boards'] = [c.get_brief_data() for c in p.get_boards().values()]
+        data["boards"] = [c.get_brief_data() for c in p.get_boards().values()]
 
-    if not data['packages'] or not expose_packages:
+    if not data["packages"] or not expose_packages:
         return data
 
-    data['packages'] = []
+    data["packages"] = []
     installed_pkgs = p.get_installed_packages()
     for name, opts in p.packages.items():
-        item = dict(name=name,
-                    type=p.get_package_type(name),
-                    requirements=opts.get("version"),
-                    optional=opts.get("optional") is True)
+        item = dict(
+            name=name,
+            type=p.get_package_type(name),
+            requirements=opts.get("version"),
+            optional=opts.get("optional") is True,
+        )
         if name in installed_pkgs:
             for key, value in installed_pkgs[name].items():
                 if key not in ("url", "version", "description"):
@@ -113,40 +118,42 @@ def _get_installed_platform_data(platform,
                 item[key] = value
                 if key == "version":
                     item["originalVersion"] = util.get_original_version(value)
-        data['packages'].append(item)
+        data["packages"].append(item)
 
     return data
 
 
 def _get_registry_platform_data(  # pylint: disable=unused-argument
-        platform,
-        with_boards=True,
-        expose_packages=True):
+    platform, with_boards=True, expose_packages=True
+):
     _data = None
     for p in _get_registry_platforms():
-        if p['name'] == platform:
+        if p["name"] == platform:
             _data = p
             break
 
     if not _data:
         return None
 
-    data = dict(name=_data['name'],
-                title=_data['title'],
-                description=_data['description'],
-                homepage=_data['homepage'],
-                repository=_data['repository'],
-                url=_data['url'],
-                license=_data['license'],
-                forDesktop=_data['forDesktop'],
-                frameworks=_data['frameworks'],
-                packages=_data['packages'],
-                versions=_data['versions'])
+    data = dict(
+        name=_data["name"],
+        title=_data["title"],
+        description=_data["description"],
+        homepage=_data["homepage"],
+        repository=_data["repository"],
+        url=_data["url"],
+        license=_data["license"],
+        forDesktop=_data["forDesktop"],
+        frameworks=_data["frameworks"],
+        packages=_data["packages"],
+        versions=_data["versions"],
+    )
 
     if with_boards:
-        data['boards'] = [
-            board for board in PlatformManager().get_registered_boards()
-            if board['platform'] == _data['name']
+        data["boards"] = [
+            board
+            for board in PlatformManager().get_registered_boards()
+            if board["platform"] == _data["name"]
         ]
 
     return data
@@ -164,9 +171,10 @@ def platform_search(query, json_output):
         if query and query.lower() not in search_data.lower():
             continue
         platforms.append(
-            _get_registry_platform_data(platform['name'],
-                                        with_boards=False,
-                                        expose_packages=False))
+            _get_registry_platform_data(
+                platform["name"], with_boards=False, expose_packages=False
+            )
+        )
 
     if json_output:
         click.echo(dump_json_to_unicode(platforms))
@@ -185,15 +193,15 @@ def platform_frameworks(query, json_output):
         search_data = dump_json_to_unicode(framework)
         if query and query.lower() not in search_data.lower():
             continue
-        framework['homepage'] = ("https://platformio.org/frameworks/" +
-                                 framework['name'])
-        framework['platforms'] = [
-            platform['name'] for platform in _get_registry_platforms()
-            if framework['name'] in platform['frameworks']
+        framework["homepage"] = "https://platformio.org/frameworks/" + framework["name"]
+        framework["platforms"] = [
+            platform["name"]
+            for platform in _get_registry_platforms()
+            if framework["name"] in platform["frameworks"]
         ]
         frameworks.append(framework)
 
-    frameworks = sorted(frameworks, key=lambda manifest: manifest['name'])
+    frameworks = sorted(frameworks, key=lambda manifest: manifest["name"])
     if json_output:
         click.echo(dump_json_to_unicode(frameworks))
     else:
@@ -207,11 +215,12 @@ def platform_list(json_output):
     pm = PlatformManager()
     for manifest in pm.get_installed():
         platforms.append(
-            _get_installed_platform_data(manifest['__pkg_dir'],
-                                         with_boards=False,
-                                         expose_packages=False))
+            _get_installed_platform_data(
+                manifest["__pkg_dir"], with_boards=False, expose_packages=False
+            )
+        )
 
-    platforms = sorted(platforms, key=lambda manifest: manifest['name'])
+    platforms = sorted(platforms, key=lambda manifest: manifest["name"])
     if json_output:
         click.echo(dump_json_to_unicode(platforms))
     else:
@@ -228,55 +237,58 @@ def platform_show(platform, json_output):  # pylint: disable=too-many-branches
     if json_output:
         return click.echo(dump_json_to_unicode(data))
 
-    click.echo("{name} ~ {title}".format(name=click.style(data['name'],
-                                                          fg="cyan"),
-                                         title=data['title']))
-    click.echo("=" * (3 + len(data['name'] + data['title'])))
-    click.echo(data['description'])
+    click.echo(
+        "{name} ~ {title}".format(
+            name=click.style(data["name"], fg="cyan"), title=data["title"]
+        )
+    )
+    click.echo("=" * (3 + len(data["name"] + data["title"])))
+    click.echo(data["description"])
     click.echo()
     if "version" in data:
-        click.echo("Version: %s" % data['version'])
-    if data['homepage']:
-        click.echo("Home: %s" % data['homepage'])
-    if data['repository']:
-        click.echo("Repository: %s" % data['repository'])
-    if data['url']:
-        click.echo("Vendor: %s" % data['url'])
-    if data['license']:
-        click.echo("License: %s" % data['license'])
-    if data['frameworks']:
-        click.echo("Frameworks: %s" % ", ".join(data['frameworks']))
+        click.echo("Version: %s" % data["version"])
+    if data["homepage"]:
+        click.echo("Home: %s" % data["homepage"])
+    if data["repository"]:
+        click.echo("Repository: %s" % data["repository"])
+    if data["url"]:
+        click.echo("Vendor: %s" % data["url"])
+    if data["license"]:
+        click.echo("License: %s" % data["license"])
+    if data["frameworks"]:
+        click.echo("Frameworks: %s" % ", ".join(data["frameworks"]))
 
-    if not data['packages']:
+    if not data["packages"]:
         return None
 
-    if not isinstance(data['packages'][0], dict):
-        click.echo("Packages: %s" % ", ".join(data['packages']))
+    if not isinstance(data["packages"][0], dict):
+        click.echo("Packages: %s" % ", ".join(data["packages"]))
     else:
         click.echo()
         click.secho("Packages", bold=True)
         click.echo("--------")
-        for item in data['packages']:
+        for item in data["packages"]:
             click.echo()
-            click.echo("Package %s" % click.style(item['name'], fg="yellow"))
-            click.echo("-" * (8 + len(item['name'])))
-            if item['type']:
-                click.echo("Type: %s" % item['type'])
-            click.echo("Requirements: %s" % item['requirements'])
-            click.echo("Installed: %s" %
-                       ("Yes" if item.get("version") else "No (optional)"))
+            click.echo("Package %s" % click.style(item["name"], fg="yellow"))
+            click.echo("-" * (8 + len(item["name"])))
+            if item["type"]:
+                click.echo("Type: %s" % item["type"])
+            click.echo("Requirements: %s" % item["requirements"])
+            click.echo(
+                "Installed: %s" % ("Yes" if item.get("version") else "No (optional)")
+            )
             if "version" in item:
-                click.echo("Version: %s" % item['version'])
+                click.echo("Version: %s" % item["version"])
             if "originalVersion" in item:
-                click.echo("Original version: %s" % item['originalVersion'])
+                click.echo("Original version: %s" % item["originalVersion"])
             if "description" in item:
-                click.echo("Description: %s" % item['description'])
+                click.echo("Description: %s" % item["description"])
 
-    if data['boards']:
+    if data["boards"]:
         click.echo()
         click.secho("Boards", bold=True)
         click.echo("------")
-        print_boards(data['boards'])
+        print_boards(data["boards"])
 
     return True
 
@@ -290,20 +302,26 @@ def platform_show(platform, json_output):  # pylint: disable=too-many-branches
     "-f",
     "--force",
     is_flag=True,
-    help="Reinstall/redownload dev/platform and its packages if exist")
-def platform_install(platforms, with_package, without_package,
-                     skip_default_package, force):
+    help="Reinstall/redownload dev/platform and its packages if exist",
+)
+def platform_install(
+    platforms, with_package, without_package, skip_default_package, force
+):
     pm = PlatformManager()
     for platform in platforms:
-        if pm.install(name=platform,
-                      with_packages=with_package,
-                      without_packages=without_package,
-                      skip_default_package=skip_default_package,
-                      force=force):
-            click.secho("The platform '%s' has been successfully installed!\n"
-                        "The rest of packages will be installed automatically "
-                        "depending on your build environment." % platform,
-                        fg="green")
+        if pm.install(
+            name=platform,
+            with_packages=with_package,
+            without_packages=without_package,
+            skip_default_package=skip_default_package,
+            force=force,
+        ):
+            click.secho(
+                "The platform '%s' has been successfully installed!\n"
+                "The rest of packages will be installed automatically "
+                "depending on your build environment." % platform,
+                fg="green",
+            )
 
 
 @cli.command("uninstall", short_help="Uninstall development platform")
@@ -312,35 +330,39 @@ def platform_uninstall(platforms):
     pm = PlatformManager()
     for platform in platforms:
         if pm.uninstall(platform):
-            click.secho("The platform '%s' has been successfully "
-                        "uninstalled!" % platform,
-                        fg="green")
+            click.secho(
+                "The platform '%s' has been successfully uninstalled!" % platform,
+                fg="green",
+            )
 
 
 @cli.command("update", short_help="Update installed development platforms")
 @click.argument("platforms", nargs=-1, required=False, metavar="[PLATFORM...]")
-@click.option("-p",
-              "--only-packages",
-              is_flag=True,
-              help="Update only the platform packages")
-@click.option("-c",
-              "--only-check",
-              is_flag=True,
-              help="DEPRECATED. Please use `--dry-run` instead")
-@click.option("--dry-run",
-              is_flag=True,
-              help="Do not update, only check for the new versions")
+@click.option(
+    "-p", "--only-packages", is_flag=True, help="Update only the platform packages"
+)
+@click.option(
+    "-c",
+    "--only-check",
+    is_flag=True,
+    help="DEPRECATED. Please use `--dry-run` instead",
+)
+@click.option(
+    "--dry-run", is_flag=True, help="Do not update, only check for the new versions"
+)
 @click.option("--json-output", is_flag=True)
 def platform_update(  # pylint: disable=too-many-locals
-        platforms, only_packages, only_check, dry_run, json_output):
+    platforms, only_packages, only_check, dry_run, json_output
+):
     pm = PlatformManager()
     pkg_dir_to_name = {}
     if not platforms:
         platforms = []
         for manifest in pm.get_installed():
-            platforms.append(manifest['__pkg_dir'])
-            pkg_dir_to_name[manifest['__pkg_dir']] = manifest.get(
-                "title", manifest['name'])
+            platforms.append(manifest["__pkg_dir"])
+            pkg_dir_to_name[manifest["__pkg_dir"]] = manifest.get(
+                "title", manifest["name"]
+            )
 
     only_check = dry_run or only_check
 
@@ -356,14 +378,16 @@ def platform_update(  # pylint: disable=too-many-locals
             if not pkg_dir:
                 continue
             latest = pm.outdated(pkg_dir, requirements)
-            if (not latest and not PlatformFactory.newPlatform(
-                    pkg_dir).are_outdated_packages()):
+            if (
+                not latest
+                and not PlatformFactory.newPlatform(pkg_dir).are_outdated_packages()
+            ):
                 continue
-            data = _get_installed_platform_data(pkg_dir,
-                                                with_boards=False,
-                                                expose_packages=False)
+            data = _get_installed_platform_data(
+                pkg_dir, with_boards=False, expose_packages=False
+            )
             if latest:
-                data['versionLatest'] = latest
+                data["versionLatest"] = latest
             result.append(data)
         return click.echo(dump_json_to_unicode(result))
 
@@ -371,8 +395,9 @@ def platform_update(  # pylint: disable=too-many-locals
     app.clean_cache()
     for platform in platforms:
         click.echo(
-            "Platform %s" %
-            click.style(pkg_dir_to_name.get(platform, platform), fg="cyan"))
+            "Platform %s"
+            % click.style(pkg_dir_to_name.get(platform, platform), fg="cyan")
+        )
         click.echo("--------")
         pm.update(platform, only_packages=only_packages, only_check=only_check)
         click.echo()

platformio/commands/remote.py

@@ -23,7 +23,6 @@ import click
 
 from platformio import exception, fs
 from platformio.commands.device import device_monitor as cmd_device_monitor
-from platformio.compat import get_file_contents
 from platformio.managers.core import pioplus_call
 
 # pylint: disable=unused-argument
@@ -43,13 +42,12 @@ def remote_agent():
 @remote_agent.command("start", short_help="Start agent")
 @click.option("-n", "--name")
 @click.option("-s", "--share", multiple=True, metavar="E-MAIL")
-@click.option("-d",
-              "--working-dir",
-              envvar="PLATFORMIO_REMOTE_AGENT_DIR",
-              type=click.Path(file_okay=False,
-                              dir_okay=True,
-                              writable=True,
-                              resolve_path=True))
+@click.option(
+    "-d",
+    "--working-dir",
+    envvar="PLATFORMIO_REMOTE_AGENT_DIR",
+    type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True),
+)
 def remote_agent_start(**kwargs):
     pioplus_call(sys.argv[1:])
 
@@ -64,15 +62,16 @@ def remote_agent_list():
     pioplus_call(sys.argv[1:])
 
 
-@cli.command("update",
-             short_help="Update installed Platforms, Packages and Libraries")
-@click.option("-c",
-              "--only-check",
-              is_flag=True,
-              help="DEPRECATED. Please use `--dry-run` instead")
-@click.option("--dry-run",
-              is_flag=True,
-              help="Do not update, only check for the new versions")
+@cli.command("update", short_help="Update installed Platforms, Packages and Libraries")
+@click.option(
+    "-c",
+    "--only-check",
+    is_flag=True,
+    help="DEPRECATED. Please use `--dry-run` instead",
+)
+@click.option(
+    "--dry-run", is_flag=True, help="Do not update, only check for the new versions"
+)
 def remote_update(only_check, dry_run):
     pioplus_call(sys.argv[1:])
 
@@ -81,14 +80,14 @@ def remote_update(only_check, dry_run):
 @click.option("-e", "--environment", multiple=True)
 @click.option("-t", "--target", multiple=True)
 @click.option("--upload-port")
-@click.option("-d",
-              "--project-dir",
-              default=getcwd,
-              type=click.Path(exists=True,
-                              file_okay=True,
-                              dir_okay=True,
-                              writable=True,
-                              resolve_path=True))
+@click.option(
+    "-d",
+    "--project-dir",
+    default=getcwd,
+    type=click.Path(
+        exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True
+    ),
+)
 @click.option("--disable-auto-clean", is_flag=True)
 @click.option("-r", "--force-remote", is_flag=True)
 @click.option("-s", "--silent", is_flag=True)
@@ -102,14 +101,14 @@ def remote_run(**kwargs):
 @click.option("--ignore", "-i", multiple=True, metavar="<pattern>")
 @click.option("--upload-port")
 @click.option("--test-port")
-@click.option("-d",
-              "--project-dir",
-              default=getcwd,
-              type=click.Path(exists=True,
-                              file_okay=False,
-                              dir_okay=True,
-                              writable=True,
-                              resolve_path=True))
+@click.option(
+    "-d",
+    "--project-dir",
+    default=getcwd,
+    type=click.Path(
+        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
+    ),
+)
 @click.option("-r", "--force-remote", is_flag=True)
 @click.option("--without-building", is_flag=True)
 @click.option("--without-uploading", is_flag=True)
@@ -131,58 +130,61 @@ def device_list(json_output):
 
 @remote_device.command("monitor", short_help="Monitor remote device")
 @click.option("--port", "-p", help="Port, a number or a device name")
-@click.option("--baud",
-              "-b",
-              type=int,
-              default=9600,
-              help="Set baud rate, default=9600")
-@click.option("--parity",
-              default="N",
-              type=click.Choice(["N", "E", "O", "S", "M"]),
-              help="Set parity, default=N")
-@click.option("--rtscts",
-              is_flag=True,
-              help="Enable RTS/CTS flow control, default=Off")
-@click.option("--xonxoff",
-              is_flag=True,
-              help="Enable software flow control, default=Off")
-@click.option("--rts",
-              default=None,
-              type=click.IntRange(0, 1),
-              help="Set initial RTS line state")
-@click.option("--dtr",
-              default=None,
-              type=click.IntRange(0, 1),
-              help="Set initial DTR line state")
+@click.option(
+    "--baud", "-b", type=int, default=9600, help="Set baud rate, default=9600"
+)
+@click.option(
+    "--parity",
+    default="N",
+    type=click.Choice(["N", "E", "O", "S", "M"]),
+    help="Set parity, default=N",
+)
+@click.option("--rtscts", is_flag=True, help="Enable RTS/CTS flow control, default=Off")
+@click.option(
+    "--xonxoff", is_flag=True, help="Enable software flow control, default=Off"
+)
+@click.option(
+    "--rts", default=None, type=click.IntRange(0, 1), help="Set initial RTS line state"
+)
+@click.option(
+    "--dtr", default=None, type=click.IntRange(0, 1), help="Set initial DTR line state"
+)
 @click.option("--echo", is_flag=True, help="Enable local echo, default=Off")
-@click.option("--encoding",
-              default="UTF-8",
-              help="Set the encoding for the serial port (e.g. hexlify, "
-              "Latin1, UTF-8), default: UTF-8")
+@click.option(
+    "--encoding",
+    default="UTF-8",
+    help="Set the encoding for the serial port (e.g. hexlify, "
+    "Latin1, UTF-8), default: UTF-8",
+)
 @click.option("--filter", "-f", multiple=True, help="Add text transformation")
-@click.option("--eol",
-              default="CRLF",
-              type=click.Choice(["CR", "LF", "CRLF"]),
-              help="End of line mode, default=CRLF")
-@click.option("--raw",
-              is_flag=True,
-              help="Do not apply any encodings/transformations")
-@click.option("--exit-char",
-              type=int,
-              default=3,
-              help="ASCII code of special character that is used to exit "
-              "the application, default=3 (Ctrl+C)")
-@click.option("--menu-char",
-              type=int,
-              default=20,
-              help="ASCII code of special character that is used to "
-              "control miniterm (menu), default=20 (DEC)")
-@click.option("--quiet",
-              is_flag=True,
-              help="Diagnostics: suppress non-error messages, default=Off")
+@click.option(
+    "--eol",
+    default="CRLF",
+    type=click.Choice(["CR", "LF", "CRLF"]),
+    help="End of line mode, default=CRLF",
+)
+@click.option("--raw", is_flag=True, help="Do not apply any encodings/transformations")
+@click.option(
+    "--exit-char",
+    type=int,
+    default=3,
+    help="ASCII code of special character that is used to exit "
+    "the application, default=3 (Ctrl+C)",
+)
+@click.option(
+    "--menu-char",
+    type=int,
+    default=20,
+    help="ASCII code of special character that is used to "
+    "control miniterm (menu), default=20 (DEC)",
+)
+@click.option(
+    "--quiet",
+    is_flag=True,
+    help="Diagnostics: suppress non-error messages, default=Off",
+)
 @click.pass_context
 def device_monitor(ctx, **kwargs):
-
     def _tx_target(sock_dir):
         try:
             pioplus_call(sys.argv[1:] + ["--sock", sock_dir])
@@ -192,13 +194,13 @@ def device_monitor(ctx, **kwargs):
     sock_dir = mkdtemp(suffix="pioplus")
     sock_file = join(sock_dir, "sock")
     try:
-        t = threading.Thread(target=_tx_target, args=(sock_dir, ))
+        t = threading.Thread(target=_tx_target, args=(sock_dir,))
         t.start()
         while t.is_alive() and not isfile(sock_file):
             sleep(0.1)
         if not t.is_alive():
             return
-        kwargs['port'] = get_file_contents(sock_file)
+        kwargs["port"] = fs.get_file_contents(sock_file)
         ctx.invoke(cmd_device_monitor, **kwargs)
         t.join(2)
     finally:

platformio/commands/run/__init__.py

@@ -11,5 +11,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from platformio.commands.run.command import cli

platformio/commands/run/command.py

@@ -14,21 +14,19 @@
 
 from multiprocessing import cpu_count
 from os import getcwd
-from os.path import isfile, join
+from os.path import isfile
 from time import time
 
 import click
 from tabulate import tabulate
 
-from platformio import exception, fs, util
+from platformio import app, exception, fs, util
 from platformio.commands.device import device_monitor as cmd_device_monitor
-from platformio.commands.run.helpers import (clean_build_dir,
-                                             handle_legacy_libdeps)
+from platformio.commands.run.helpers import clean_build_dir, handle_legacy_libdeps
 from platformio.commands.run.processor import EnvironmentProcessor
 from platformio.commands.test.processor import CTX_META_TEST_IS_RUNNING
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import (find_project_dir_above,
-                                        get_project_build_dir)
+from platformio.project.helpers import find_project_dir_above
 
 # pylint: disable=too-many-arguments,too-many-locals,too-many-branches
 
@@ -42,34 +40,49 @@ except NotImplementedError:
 @click.option("-e", "--environment", multiple=True)
 @click.option("-t", "--target", multiple=True)
 @click.option("--upload-port")
-@click.option("-d",
-              "--project-dir",
-              default=getcwd,
-              type=click.Path(exists=True,
-                              file_okay=True,
-                              dir_okay=True,
-                              writable=True,
-                              resolve_path=True))
-@click.option("-c",
-              "--project-conf",
-              type=click.Path(exists=True,
-                              file_okay=True,
-                              dir_okay=False,
-                              readable=True,
-                              resolve_path=True))
-@click.option("-j",
-              "--jobs",
-              type=int,
-              default=DEFAULT_JOB_NUMS,
-              help=("Allow N jobs at once. "
-                    "Default is a number of CPUs in a system (N=%d)" %
-                    DEFAULT_JOB_NUMS))
+@click.option(
+    "-d",
+    "--project-dir",
+    default=getcwd,
+    type=click.Path(
+        exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True
+    ),
+)
+@click.option(
+    "-c",
+    "--project-conf",
+    type=click.Path(
+        exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True
+    ),
+)
+@click.option(
+    "-j",
+    "--jobs",
+    type=int,
+    default=DEFAULT_JOB_NUMS,
+    help=(
+        "Allow N jobs at once. "
+        "Default is a number of CPUs in a system (N=%d)" % DEFAULT_JOB_NUMS
+    ),
+)
 @click.option("-s", "--silent", is_flag=True)
 @click.option("-v", "--verbose", is_flag=True)
 @click.option("--disable-auto-clean", is_flag=True)
 @click.pass_context
-def cli(ctx, environment, target, upload_port, project_dir, project_conf, jobs,
-        silent, verbose, disable_auto_clean):
+def cli(
+    ctx,
+    environment,
+    target,
+    upload_port,
+    project_dir,
+    project_conf,
+    jobs,
+    silent,
+    verbose,
+    disable_auto_clean,
+):
+    app.set_session_var("custom_project_conf", project_conf)
+
     # find project directory on upper level
     if isfile(project_dir):
         project_dir = find_project_dir_above(project_dir)
@@ -77,47 +90,58 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, jobs,
     is_test_running = CTX_META_TEST_IS_RUNNING in ctx.meta
 
     with fs.cd(project_dir):
-        config = ProjectConfig.get_instance(
-            project_conf or join(project_dir, "platformio.ini"))
+        config = ProjectConfig.get_instance(project_conf)
         config.validate(environment)
 
         # clean obsolete build dir
         if not disable_auto_clean:
+            build_dir = config.get_optional_dir("build")
             try:
-                clean_build_dir(get_project_build_dir(), config)
+                clean_build_dir(build_dir, config)
             except:  # pylint: disable=bare-except
                 click.secho(
                     "Can not remove temporary directory `%s`. Please remove "
-                    "it manually to avoid build issues" %
-                    get_project_build_dir(force=True),
-                    fg="yellow")
+                    "it manually to avoid build issues" % build_dir,
+                    fg="yellow",
+                )
 
         handle_legacy_libdeps(project_dir, config)
 
         default_envs = config.default_envs()
         results = []
         for env in config.envs():
-            skipenv = any([
-                environment and env not in environment, not environment
-                and default_envs and env not in default_envs
-            ])
+            skipenv = any(
+                [
+                    environment and env not in environment,
+                    not environment and default_envs and env not in default_envs,
+                ]
+            )
             if skipenv:
                 results.append({"env": env})
                 continue
 
             # print empty line between multi environment project
-            if not silent and any(
-                    r.get("succeeded") is not None for r in results):
+            if not silent and any(r.get("succeeded") is not None for r in results):
                 click.echo()
 
             results.append(
-                process_env(ctx, env, config, environment, target, upload_port,
-                            silent, verbose, jobs, is_test_running))
+                process_env(
+                    ctx,
+                    env,
+                    config,
+                    environment,
+                    target,
+                    upload_port,
+                    silent,
+                    verbose,
+                    jobs,
+                    is_test_running,
+                )
+            )
 
         command_failed = any(r.get("succeeded") is False for r in results)
 
-        if (not is_test_running and (command_failed or not silent)
-                and len(results) > 1):
+        if not is_test_running and (command_failed or not silent) and len(results) > 1:
             print_processing_summary(results)
 
         if command_failed:
@@ -125,24 +149,39 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, jobs,
         return True
 
 
-def process_env(ctx, name, config, environments, targets, upload_port, silent,
-                verbose, jobs, is_test_running):
+def process_env(
+    ctx,
+    name,
+    config,
+    environments,
+    targets,
+    upload_port,
+    silent,
+    verbose,
+    jobs,
+    is_test_running,
+):
     if not is_test_running and not silent:
         print_processing_header(name, config, verbose)
 
-    ep = EnvironmentProcessor(ctx, name, config, targets, upload_port, silent,
-                              verbose, jobs)
+    ep = EnvironmentProcessor(
+        ctx, name, config, targets, upload_port, silent, verbose, jobs
+    )
     result = {"env": name, "duration": time(), "succeeded": ep.process()}
-    result['duration'] = time() - result['duration']
+    result["duration"] = time() - result["duration"]
 
     # print footer on error or when is not unit testing
-    if not is_test_running and (not silent or not result['succeeded']):
+    if not is_test_running and (not silent or not result["succeeded"]):
         print_processing_footer(result)
 
-    if (result['succeeded'] and "monitor" in ep.get_build_targets()
-            and "nobuild" not in ep.get_build_targets()):
-        ctx.invoke(cmd_device_monitor,
-                   environment=environments[0] if environments else None)
+    if (
+        result["succeeded"]
+        and "monitor" in ep.get_build_targets()
+        and "nobuild" not in ep.get_build_targets()
+    ):
+        ctx.invoke(
+            cmd_device_monitor, environment=environments[0] if environments else None
+        )
 
     return result
 
@@ -151,10 +190,11 @@ def print_processing_header(env, config, verbose=False):
     env_dump = []
     for k, v in config.items(env=env):
         if verbose or k in ("platform", "framework", "board"):
-            env_dump.append("%s: %s" %
-                            (k, ", ".join(v) if isinstance(v, list) else v))
-    click.echo("Processing %s (%s)" %
-               (click.style(env, fg="cyan", bold=True), "; ".join(env_dump)))
+            env_dump.append("%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v))
+    click.echo(
+        "Processing %s (%s)"
+        % (click.style(env, fg="cyan", bold=True), "; ".join(env_dump))
+    )
     terminal_width, _ = click.get_terminal_size()
     click.secho("-" * terminal_width, bold=True)
 
@@ -162,10 +202,17 @@ def print_processing_header(env, config, verbose=False):
 def print_processing_footer(result):
     is_failed = not result.get("succeeded")
     util.print_labeled_bar(
-        "[%s] Took %.2f seconds" %
-        ((click.style("FAILED", fg="red", bold=True) if is_failed else
-          click.style("SUCCESS", fg="green", bold=True)), result['duration']),
-        is_error=is_failed)
+        "[%s] Took %.2f seconds"
+        % (
+            (
+                click.style("FAILED", fg="red", bold=True)
+                if is_failed
+                else click.style("SUCCESS", fg="green", bold=True)
+            ),
+            result["duration"],
+        ),
+        is_error=is_failed,
+    )
 
 
 def print_processing_summary(results):
@@ -186,20 +233,31 @@ def print_processing_summary(results):
             status_str = click.style("SUCCESS", fg="green")
 
         tabular_data.append(
-            (click.style(result['env'], fg="cyan"), status_str,
-             util.humanize_duration_time(result.get("duration"))))
+            (
+                click.style(result["env"], fg="cyan"),
+                status_str,
+                util.humanize_duration_time(result.get("duration")),
+            )
+        )
 
     click.echo()
-    click.echo(tabulate(tabular_data,
-                        headers=[
-                            click.style(s, bold=True)
-                            for s in ("Environment", "Status", "Duration")
-                        ]),
-               err=failed_nums)
+    click.echo(
+        tabulate(
+            tabular_data,
+            headers=[
+                click.style(s, bold=True) for s in ("Environment", "Status", "Duration")
+            ],
+        ),
+        err=failed_nums,
+    )
 
     util.print_labeled_bar(
-        "%s%d succeeded in %s" %
-        ("%d failed, " % failed_nums if failed_nums else "", succeeded_nums,
-         util.humanize_duration_time(duration)),
+        "%s%d succeeded in %s"
+        % (
+            "%d failed, " % failed_nums if failed_nums else "",
+            succeeded_nums,
+            util.humanize_duration_time(duration),
+        ),
         is_error=failed_nums,
-        fg="red" if failed_nums else "green")
+        fg="red" if failed_nums else "green",
+    )

platformio/commands/run/helpers.py

@@ -18,15 +18,14 @@ from os.path import isdir, isfile, join
 import click
 
 from platformio import fs
-from platformio.project.helpers import (compute_project_checksum,
-                                        get_project_dir,
-                                        get_project_libdeps_dir)
+from platformio.project.helpers import compute_project_checksum, get_project_dir
 
 
 def handle_legacy_libdeps(project_dir, config):
     legacy_libdeps_dir = join(project_dir, ".piolibdeps")
-    if (not isdir(legacy_libdeps_dir)
-            or legacy_libdeps_dir == get_project_libdeps_dir()):
+    if not isdir(legacy_libdeps_dir) or legacy_libdeps_dir == config.get_optional_dir(
+        "libdeps"
+    ):
         return
     if not config.has_section("env"):
         config.add_section("env")
@@ -39,7 +38,8 @@ def handle_legacy_libdeps(project_dir, config):
         " file using `lib_deps` option and remove `{0}` folder."
         "\nMore details -> http://docs.platformio.org/page/projectconf/"
         "section_env_library.html#lib-deps".format(legacy_libdeps_dir),
-        fg="yellow")
+        fg="yellow",
+    )
 
 
 def clean_build_dir(build_dir, config):
@@ -53,12 +53,9 @@ def clean_build_dir(build_dir, config):
 
     if isdir(build_dir):
         # check project structure
-        if isfile(checksum_file):
-            with open(checksum_file) as f:
-                if f.read() == checksum:
-                    return
+        if isfile(checksum_file) and fs.get_file_contents(checksum_file) == checksum:
+            return
         fs.rmtree(build_dir)
 
     makedirs(build_dir)
-    with open(checksum_file, "w") as f:
-        f.write(checksum)
+    fs.write_file_contents(checksum_file, checksum)

platformio/commands/run/processor.py

@@ -13,8 +13,7 @@
 # limitations under the License.
 
 from platformio import exception, telemetry
-from platformio.commands.platform import \
-    platform_install as cmd_platform_install
+from platformio.commands.platform import platform_install as cmd_platform_install
 from platformio.commands.test.processor import CTX_META_TEST_RUNNING_NAME
 from platformio.managers.platform import PlatformFactory
 
@@ -22,10 +21,9 @@ from platformio.managers.platform import PlatformFactory
 
 
 class EnvironmentProcessor(object):
-
     def __init__(  # pylint: disable=too-many-arguments
-            self, cmd_ctx, name, config, targets, upload_port, silent, verbose,
-            jobs):
+        self, cmd_ctx, name, config, targets, upload_port, silent, verbose, jobs
+    ):
         self.cmd_ctx = cmd_ctx
         self.name = name
         self.config = config
@@ -40,25 +38,28 @@ class EnvironmentProcessor(object):
         variables = {"pioenv": self.name, "project_config": self.config.path}
 
         if CTX_META_TEST_RUNNING_NAME in self.cmd_ctx.meta:
-            variables['piotest_running_name'] = self.cmd_ctx.meta[
-                CTX_META_TEST_RUNNING_NAME]
+            variables["piotest_running_name"] = self.cmd_ctx.meta[
+                CTX_META_TEST_RUNNING_NAME
+            ]
 
         if self.upload_port:
             # override upload port with a custom from CLI
-            variables['upload_port'] = self.upload_port
+            variables["upload_port"] = self.upload_port
         return variables
 
     def get_build_targets(self):
-        if self.targets:
-            return [t for t in self.targets]
-        return self.config.get("env:" + self.name, "targets", [])
+        return (
+            self.targets
+            if self.targets
+            else self.config.get("env:" + self.name, "targets", [])
+        )
 
     def process(self):
         if "platform" not in self.options:
             raise exception.UndefinedEnvPlatform(self.name)
 
         build_vars = self.get_build_variables()
-        build_targets = self.get_build_targets()
+        build_targets = list(self.get_build_targets())
 
         telemetry.on_run_environment(self.options, build_targets)
 
@@ -67,13 +68,14 @@ class EnvironmentProcessor(object):
             build_targets.remove("monitor")
 
         try:
-            p = PlatformFactory.newPlatform(self.options['platform'])
+            p = PlatformFactory.newPlatform(self.options["platform"])
         except exception.UnknownPlatform:
-            self.cmd_ctx.invoke(cmd_platform_install,
-                                platforms=[self.options['platform']],
-                                skip_default_package=True)
-            p = PlatformFactory.newPlatform(self.options['platform'])
+            self.cmd_ctx.invoke(
+                cmd_platform_install,
+                platforms=[self.options["platform"]],
+                skip_default_package=True,
+            )
+            p = PlatformFactory.newPlatform(self.options["platform"])
 
-        result = p.run(build_vars, build_targets, self.silent, self.verbose,
-                       self.jobs)
-        return result['returncode'] == 0
+        result = p.run(build_vars, build_targets, self.silent, self.verbose, self.jobs)
+        return result["returncode"] == 0

platformio/commands/settings.py

@@ -42,20 +42,24 @@ def settings_get(name):
         raw_value = app.get_setting(key)
         formatted_value = format_value(raw_value)
 
-        if raw_value != options['value']:
-            default_formatted_value = format_value(options['value'])
+        if raw_value != options["value"]:
+            default_formatted_value = format_value(options["value"])
             formatted_value += "%s" % (
-                "\n" if len(default_formatted_value) > 10 else " ")
-            formatted_value += "[%s]" % click.style(default_formatted_value,
-                                                    fg="yellow")
+                "\n" if len(default_formatted_value) > 10 else " "
+            )
+            formatted_value += "[%s]" % click.style(
+                default_formatted_value, fg="yellow"
+            )
 
         tabular_data.append(
-            (click.style(key,
-                         fg="cyan"), formatted_value, options['description']))
+            (click.style(key, fg="cyan"), formatted_value, options["description"])
+        )
 
     click.echo(
-        tabulate(tabular_data,
-                 headers=["Name", "Current value [Default]", "Description"]))
+        tabulate(
+            tabular_data, headers=["Name", "Current value [Default]", "Description"]
+        )
+    )
 
 
 @cli.command("set", short_help="Set new value for the setting")

platformio/commands/test/__init__.py

@@ -11,5 +11,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from platformio.commands.test.command import cli

platformio/commands/test/command.py

@@ -22,70 +22,91 @@ from time import time
 import click
 from tabulate import tabulate
 
-from platformio import exception, fs, util
+from platformio import app, exception, fs, util
 from platformio.commands.test.embedded import EmbeddedTestProcessor
 from platformio.commands.test.native import NativeTestProcessor
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import get_project_test_dir
 
 
 @click.command("test", short_help="Unit Testing")
 @click.option("--environment", "-e", multiple=True, metavar="<environment>")
-@click.option("--filter",
-              "-f",
-              multiple=True,
-              metavar="<pattern>",
-              help="Filter tests by a pattern")
-@click.option("--ignore",
-              "-i",
-              multiple=True,
-              metavar="<pattern>",
-              help="Ignore tests by a pattern")
+@click.option(
+    "--filter",
+    "-f",
+    multiple=True,
+    metavar="<pattern>",
+    help="Filter tests by a pattern",
+)
+@click.option(
+    "--ignore",
+    "-i",
+    multiple=True,
+    metavar="<pattern>",
+    help="Ignore tests by a pattern",
+)
 @click.option("--upload-port")
 @click.option("--test-port")
-@click.option("-d",
-              "--project-dir",
-              default=getcwd,
-              type=click.Path(exists=True,
-                              file_okay=False,
-                              dir_okay=True,
-                              writable=True,
-                              resolve_path=True))
-@click.option("-c",
-              "--project-conf",
-              type=click.Path(exists=True,
-                              file_okay=True,
-                              dir_okay=False,
-                              readable=True,
-                              resolve_path=True))
+@click.option(
+    "-d",
+    "--project-dir",
+    default=getcwd,
+    type=click.Path(
+        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
+    ),
+)
+@click.option(
+    "-c",
+    "--project-conf",
+    type=click.Path(
+        exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True
+    ),
+)
 @click.option("--without-building", is_flag=True)
 @click.option("--without-uploading", is_flag=True)
 @click.option("--without-testing", is_flag=True)
 @click.option("--no-reset", is_flag=True)
-@click.option("--monitor-rts",
-              default=None,
-              type=click.IntRange(0, 1),
-              help="Set initial RTS line state for Serial Monitor")
-@click.option("--monitor-dtr",
-              default=None,
-              type=click.IntRange(0, 1),
-              help="Set initial DTR line state for Serial Monitor")
+@click.option(
+    "--monitor-rts",
+    default=None,
+    type=click.IntRange(0, 1),
+    help="Set initial RTS line state for Serial Monitor",
+)
+@click.option(
+    "--monitor-dtr",
+    default=None,
+    type=click.IntRange(0, 1),
+    help="Set initial DTR line state for Serial Monitor",
+)
 @click.option("--verbose", "-v", is_flag=True)
 @click.pass_context
 def cli(  # pylint: disable=redefined-builtin
-        ctx, environment, ignore, filter, upload_port, test_port, project_dir,
-        project_conf, without_building, without_uploading, without_testing,
-        no_reset, monitor_rts, monitor_dtr, verbose):
+    ctx,
+    environment,
+    ignore,
+    filter,
+    upload_port,
+    test_port,
+    project_dir,
+    project_conf,
+    without_building,
+    without_uploading,
+    without_testing,
+    no_reset,
+    monitor_rts,
+    monitor_dtr,
+    verbose,
+):
+    app.set_session_var("custom_project_conf", project_conf)
+
     with fs.cd(project_dir):
-        test_dir = get_project_test_dir()
+        config = ProjectConfig.get_instance(project_conf)
+        config.validate(envs=environment)
+
+        test_dir = config.get_optional_dir("test")
         if not isdir(test_dir):
             raise exception.TestDirNotExists(test_dir)
         test_names = get_test_names(test_dir)
 
-        config = ProjectConfig.get_instance(
-            project_conf or join(project_dir, "platformio.ini"))
-        config.validate(envs=environment)
-
         click.echo("Verbose mode can be enabled via `-v, --verbose` option")
         click.secho("Collected %d items" % len(test_names), bold=True)
 
@@ -99,19 +120,16 @@ def cli(  # pylint: disable=redefined-builtin
                 # filter and ignore patterns
                 patterns = dict(filter=list(filter), ignore=list(ignore))
                 for key in patterns:
-                    patterns[key].extend(
-                        config.get(section, "test_%s" % key, []))
+                    patterns[key].extend(config.get(section, "test_%s" % key, []))
 
                 skip_conditions = [
                     environment and envname not in environment,
-                    not environment and default_envs
-                    and envname not in default_envs,
-                    testname != "*" and patterns['filter'] and
-                    not any([fnmatch(testname, p)
-                             for p in patterns['filter']]),
+                    not environment and default_envs and envname not in default_envs,
                     testname != "*"
-                    and any([fnmatch(testname, p)
-                             for p in patterns['ignore']]),
+                    and patterns["filter"]
+                    and not any([fnmatch(testname, p) for p in patterns["filter"]]),
+                    testname != "*"
+                    and any([fnmatch(testname, p) for p in patterns["ignore"]]),
                 ]
                 if any(skip_conditions):
                     results.append({"env": envname, "test": testname})
@@ -120,29 +138,36 @@ def cli(  # pylint: disable=redefined-builtin
                 click.echo()
                 print_processing_header(testname, envname)
 
-                cls = (NativeTestProcessor
-                       if config.get(section, "platform") == "native" else
-                       EmbeddedTestProcessor)
+                cls = (
+                    NativeTestProcessor
+                    if config.get(section, "platform") == "native"
+                    else EmbeddedTestProcessor
+                )
                 tp = cls(
-                    ctx, testname, envname,
-                    dict(project_config=config,
-                         project_dir=project_dir,
-                         upload_port=upload_port,
-                         test_port=test_port,
-                         without_building=without_building,
-                         without_uploading=without_uploading,
-                         without_testing=without_testing,
-                         no_reset=no_reset,
-                         monitor_rts=monitor_rts,
-                         monitor_dtr=monitor_dtr,
-                         verbose=verbose))
+                    ctx,
+                    testname,
+                    envname,
+                    dict(
+                        project_config=config,
+                        project_dir=project_dir,
+                        upload_port=upload_port,
+                        test_port=test_port,
+                        without_building=without_building,
+                        without_uploading=without_uploading,
+                        without_testing=without_testing,
+                        no_reset=no_reset,
+                        monitor_rts=monitor_rts,
+                        monitor_dtr=monitor_dtr,
+                        verbose=verbose,
+                    ),
+                )
                 result = {
                     "env": envname,
                     "test": testname,
                     "duration": time(),
-                    "succeeded": tp.process()
+                    "succeeded": tp.process(),
                 }
-                result['duration'] = time() - result['duration']
+                result["duration"] = time() - result["duration"]
                 results.append(result)
 
                 print_processing_footer(result)
@@ -168,8 +193,13 @@ def get_test_names(test_dir):
 
 
 def print_processing_header(test, env):
-    click.echo("Processing %s in %s environment" % (click.style(
-        test, fg="yellow", bold=True), click.style(env, fg="cyan", bold=True)))
+    click.echo(
+        "Processing %s in %s environment"
+        % (
+            click.style(test, fg="yellow", bold=True),
+            click.style(env, fg="cyan", bold=True),
+        )
+    )
     terminal_width, _ = click.get_terminal_size()
     click.secho("-" * terminal_width, bold=True)
 
@@ -177,10 +207,17 @@ def print_processing_header(test, env):
 def print_processing_footer(result):
     is_failed = not result.get("succeeded")
     util.print_labeled_bar(
-        "[%s] Took %.2f seconds" %
-        ((click.style("FAILED", fg="red", bold=True) if is_failed else
-          click.style("PASSED", fg="green", bold=True)), result['duration']),
-        is_error=is_failed)
+        "[%s] Took %.2f seconds"
+        % (
+            (
+                click.style("FAILED", fg="red", bold=True)
+                if is_failed
+                else click.style("PASSED", fg="green", bold=True)
+            ),
+            result["duration"],
+        ),
+        is_error=is_failed,
+    )
 
 
 def print_testing_summary(results):
@@ -203,20 +240,32 @@ def print_testing_summary(results):
             status_str = click.style("PASSED", fg="green")
 
         tabular_data.append(
-            (result['test'], click.style(result['env'], fg="cyan"), status_str,
-             util.humanize_duration_time(result.get("duration"))))
+            (
+                result["test"],
+                click.style(result["env"], fg="cyan"),
+                status_str,
+                util.humanize_duration_time(result.get("duration")),
+            )
+        )
 
-    click.echo(tabulate(tabular_data,
-                        headers=[
-                            click.style(s, bold=True)
-                            for s in ("Test", "Environment", "Status",
-                                      "Duration")
-                        ]),
-               err=failed_nums)
+    click.echo(
+        tabulate(
+            tabular_data,
+            headers=[
+                click.style(s, bold=True)
+                for s in ("Test", "Environment", "Status", "Duration")
+            ],
+        ),
+        err=failed_nums,
+    )
 
     util.print_labeled_bar(
-        "%s%d succeeded in %s" %
-        ("%d failed, " % failed_nums if failed_nums else "", succeeded_nums,
-         util.humanize_duration_time(duration)),
+        "%s%d succeeded in %s"
+        % (
+            "%d failed, " % failed_nums if failed_nums else "",
+            succeeded_nums,
+            util.humanize_duration_time(duration),
+        ),
         is_error=failed_nums,
-        fg="red" if failed_nums else "green")
+        fg="red" if failed_nums else "green",
+    )

platformio/commands/test/embedded.py

@@ -27,47 +27,50 @@ class EmbeddedTestProcessor(TestProcessorBase):
     SERIAL_TIMEOUT = 600
 
     def process(self):
-        if not self.options['without_building']:
+        if not self.options["without_building"]:
             self.print_progress("Building...")
             target = ["__test"]
-            if self.options['without_uploading']:
+            if self.options["without_uploading"]:
                 target.append("checkprogsize")
             if not self.build_or_upload(target):
                 return False
 
-        if not self.options['without_uploading']:
+        if not self.options["without_uploading"]:
             self.print_progress("Uploading...")
             target = ["upload"]
-            if self.options['without_building']:
+            if self.options["without_building"]:
                 target.append("nobuild")
             else:
                 target.append("__test")
             if not self.build_or_upload(target):
                 return False
 
-        if self.options['without_testing']:
+        if self.options["without_testing"]:
             return None
 
         self.print_progress("Testing...")
         return self.run()
 
     def run(self):
-        click.echo("If you don't see any output for the first 10 secs, "
-                   "please reset board (press reset button)")
+        click.echo(
+            "If you don't see any output for the first 10 secs, "
+            "please reset board (press reset button)"
+        )
         click.echo()
 
         try:
-            ser = serial.Serial(baudrate=self.get_baudrate(),
-                                timeout=self.SERIAL_TIMEOUT)
+            ser = serial.Serial(
+                baudrate=self.get_baudrate(), timeout=self.SERIAL_TIMEOUT
+            )
             ser.port = self.get_test_port()
-            ser.rts = self.options['monitor_rts']
-            ser.dtr = self.options['monitor_dtr']
+            ser.rts = self.options["monitor_rts"]
+            ser.dtr = self.options["monitor_dtr"]
             ser.open()
         except serial.SerialException as e:
             click.secho(str(e), fg="red", err=True)
             return False
 
-        if not self.options['no_reset']:
+        if not self.options["no_reset"]:
             ser.flushInput()
             ser.setDTR(False)
             ser.setRTS(False)
@@ -90,7 +93,7 @@ class EmbeddedTestProcessor(TestProcessorBase):
             if not line:
                 continue
             if isinstance(line, bytes):
-                line = line.decode("utf8")
+                line = line.decode("utf8", "ignore")
             self.on_run_out(line)
             if all([l in line for l in ("Tests", "Failures", "Ignored")]):
                 break
@@ -105,17 +108,16 @@ class EmbeddedTestProcessor(TestProcessorBase):
             return self.env_options.get("test_port")
 
         assert set(["platform", "board"]) & set(self.env_options.keys())
-        p = PlatformFactory.newPlatform(self.env_options['platform'])
-        board_hwids = p.board_config(self.env_options['board']).get(
-            "build.hwids", [])
+        p = PlatformFactory.newPlatform(self.env_options["platform"])
+        board_hwids = p.board_config(self.env_options["board"]).get("build.hwids", [])
         port = None
         elapsed = 0
         while elapsed < 5 and not port:
             for item in util.get_serialports():
-                port = item['port']
+                port = item["port"]
                 for hwid in board_hwids:
                     hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
-                    if hwid_str in item['hwid']:
+                    if hwid_str in item["hwid"]:
                         return port
 
             # check if port is already configured
@@ -131,5 +133,6 @@ class EmbeddedTestProcessor(TestProcessorBase):
         if not port:
             raise exception.PlatformioException(
                 "Please specify `test_port` for environment or use "
-                "global `--test-port` option.")
+                "global `--test-port` option."
+            )
         return port

platformio/commands/test/native.py

@@ -14,30 +14,28 @@
 
 from os.path import join
 
-from platformio import fs, proc
+from platformio import proc
 from platformio.commands.test.processor import TestProcessorBase
 from platformio.proc import LineBufferedAsyncPipe
-from platformio.project.helpers import get_project_build_dir
 
 
 class NativeTestProcessor(TestProcessorBase):
-
     def process(self):
-        if not self.options['without_building']:
+        if not self.options["without_building"]:
             self.print_progress("Building...")
             if not self.build_or_upload(["__test"]):
                 return False
-        if self.options['without_testing']:
+        if self.options["without_testing"]:
             return None
         self.print_progress("Testing...")
         return self.run()
 
     def run(self):
-        with fs.cd(self.options['project_dir']):
-            build_dir = get_project_build_dir()
+        build_dir = self.options["project_config"].get_optional_dir("build")
         result = proc.exec_command(
             [join(build_dir, self.env_name, "program")],
             stdout=LineBufferedAsyncPipe(self.on_run_out),
-            stderr=LineBufferedAsyncPipe(self.on_run_out))
+            stderr=LineBufferedAsyncPipe(self.on_run_out),
+        )
         assert "returncode" in result
-        return result['returncode'] == 0 and not self._run_failed
+        return result["returncode"] == 0 and not self._run_failed

platformio/commands/test/processor.py

@@ -19,8 +19,7 @@ from string import Template
 
 import click
 
-from platformio import exception
-from platformio.project.helpers import get_project_test_dir
+from platformio import exception, fs
 
 TRANSPORT_OPTIONS = {
     "arduino": {
@@ -29,7 +28,7 @@ TRANSPORT_OPTIONS = {
         "putchar": "Serial.write(c)",
         "flush": "Serial.flush()",
         "begin": "Serial.begin($baudrate)",
-        "end": "Serial.end()"
+        "end": "Serial.end()",
     },
     "mbed": {
         "include": "#include <mbed.h>",
@@ -37,7 +36,7 @@ TRANSPORT_OPTIONS = {
         "putchar": "pc.putc(c)",
         "flush": "",
         "begin": "pc.baud($baudrate)",
-        "end": ""
+        "end": "",
     },
     "espidf": {
         "include": "#include <stdio.h>",
@@ -45,7 +44,7 @@ TRANSPORT_OPTIONS = {
         "putchar": "putchar(c)",
         "flush": "fflush(stdout)",
         "begin": "",
-        "end": ""
+        "end": "",
     },
     "native": {
         "include": "#include <stdio.h>",
@@ -53,7 +52,7 @@ TRANSPORT_OPTIONS = {
         "putchar": "putchar(c)",
         "flush": "fflush(stdout)",
         "begin": "",
-        "end": ""
+        "end": "",
     },
     "custom": {
         "include": '#include "unittest_transport.h"',
@@ -61,8 +60,8 @@ TRANSPORT_OPTIONS = {
         "putchar": "unittest_uart_putchar(c)",
         "flush": "unittest_uart_flush()",
         "begin": "unittest_uart_begin()",
-        "end": "unittest_uart_end()"
-    }
+        "end": "unittest_uart_end()",
+    },
 }
 
 CTX_META_TEST_IS_RUNNING = __name__ + ".test_running"
@@ -79,8 +78,7 @@ class TestProcessorBase(object):
         self.test_name = testname
         self.options = options
         self.env_name = envname
-        self.env_options = options['project_config'].items(env=envname,
-                                                           as_dict=True)
+        self.env_options = options["project_config"].items(env=envname, as_dict=True)
         self._run_failed = False
         self._outputcpp_generated = False
 
@@ -90,10 +88,11 @@ class TestProcessorBase(object):
         elif "framework" in self.env_options:
             transport = self.env_options.get("framework")[0]
         if "test_transport" in self.env_options:
-            transport = self.env_options['test_transport']
+            transport = self.env_options["test_transport"]
         if transport not in TRANSPORT_OPTIONS:
             raise exception.PlatformioException(
-                "Unknown Unit Test transport `%s`" % transport)
+                "Unknown Unit Test transport `%s`" % transport
+            )
         return transport.lower()
 
     def get_baudrate(self):
@@ -104,21 +103,27 @@ class TestProcessorBase(object):
 
     def build_or_upload(self, target):
         if not self._outputcpp_generated:
-            self.generate_outputcpp(get_project_test_dir())
+            self.generate_outputcpp(
+                self.options["project_config"].get_optional_dir("test")
+            )
             self._outputcpp_generated = True
 
         if self.test_name != "*":
             self.cmd_ctx.meta[CTX_META_TEST_RUNNING_NAME] = self.test_name
 
         try:
-            from platformio.commands.run import cli as cmd_run
-            return self.cmd_ctx.invoke(cmd_run,
-                                       project_dir=self.options['project_dir'],
-                                       upload_port=self.options['upload_port'],
-                                       silent=not self.options['verbose'],
-                                       environment=[self.env_name],
-                                       disable_auto_clean="nobuild" in target,
-                                       target=target)
+            # pylint: disable=import-outside-toplevel
+            from platformio.commands.run.command import cli as cmd_run
+
+            return self.cmd_ctx.invoke(
+                cmd_run,
+                project_dir=self.options["project_dir"],
+                upload_port=self.options["upload_port"],
+                silent=not self.options["verbose"],
+                environment=[self.env_name],
+                disable_auto_clean="nobuild" in target,
+                target=target,
+            )
         except exception.ReturnErrorCode:
             return False
 
@@ -131,8 +136,7 @@ class TestProcessorBase(object):
     def on_run_out(self, line):
         line = line.strip()
         if line.endswith(":PASS"):
-            click.echo("%s\t[%s]" %
-                       (line[:-5], click.style("PASSED", fg="green")))
+            click.echo("%s\t[%s]" % (line[:-5], click.style("PASSED", fg="green")))
         elif ":FAIL" in line:
             self._run_failed = True
             click.echo("%s\t[%s]" % (line, click.style("FAILED", fg="red")))
@@ -142,36 +146,38 @@ class TestProcessorBase(object):
     def generate_outputcpp(self, test_dir):
         assert isdir(test_dir)
 
-        cpp_tpl = "\n".join([
-            "$include",
-            "#include <output_export.h>",
-            "",
-            "$object",
-            "",
-            "#ifdef __GNUC__",
-            "void output_start(unsigned int baudrate __attribute__((unused)))",
-            "#else",
-            "void output_start(unsigned int baudrate)",
-            "#endif",
-            "{",
-            "    $begin;",
-            "}",
-            "",
-            "void output_char(int c)",
-            "{",
-            "    $putchar;",
-            "}",
-            "",
-            "void output_flush(void)",
-            "{",
-            "    $flush;",
-            "}",
-            "",
-            "void output_complete(void)",
-            "{",
-            "   $end;",
-            "}"
-        ])  # yapf: disable
+        cpp_tpl = "\n".join(
+            [
+                "$include",
+                "#include <output_export.h>",
+                "",
+                "$object",
+                "",
+                "#ifdef __GNUC__",
+                "void output_start(unsigned int baudrate __attribute__((unused)))",
+                "#else",
+                "void output_start(unsigned int baudrate)",
+                "#endif",
+                "{",
+                "    $begin;",
+                "}",
+                "",
+                "void output_char(int c)",
+                "{",
+                "    $putchar;",
+                "}",
+                "",
+                "void output_flush(void)",
+                "{",
+                "    $flush;",
+                "}",
+                "",
+                "void output_complete(void)",
+                "{",
+                "   $end;",
+                "}",
+            ]
+        )
 
         def delete_tmptest_file(file_):
             try:
@@ -181,14 +187,13 @@ class TestProcessorBase(object):
                     click.secho(
                         "Warning: Could not remove temporary file '%s'. "
                         "Please remove it manually." % file_,
-                        fg="yellow")
+                        fg="yellow",
+                    )
 
-        tpl = Template(cpp_tpl).substitute(
-            TRANSPORT_OPTIONS[self.get_transport()])
+        tpl = Template(cpp_tpl).substitute(TRANSPORT_OPTIONS[self.get_transport()])
         data = Template(tpl).substitute(baudrate=self.get_baudrate())
 
         tmp_file = join(test_dir, "output_export.cpp")
-        with open(tmp_file, "w") as f:
-            f.write(data)
+        fs.write_file_contents(tmp_file, data)
 
         atexit.register(delete_tmptest_file, tmp_file)

platformio/commands/update.py

@@ -22,18 +22,19 @@ from platformio.managers.core import update_core_packages
 from platformio.managers.lib import LibraryManager
 
 
-@click.command("update",
-               short_help="Update installed platforms, packages and libraries")
-@click.option("--core-packages",
-              is_flag=True,
-              help="Update only the core packages")
-@click.option("-c",
-              "--only-check",
-              is_flag=True,
-              help="DEPRECATED. Please use `--dry-run` instead")
-@click.option("--dry-run",
-              is_flag=True,
-              help="Do not update, only check for the new versions")
+@click.command(
+    "update", short_help="Update installed platforms, packages and libraries"
+)
+@click.option("--core-packages", is_flag=True, help="Update only the core packages")
+@click.option(
+    "-c",
+    "--only-check",
+    is_flag=True,
+    help="DEPRECATED. Please use `--dry-run` instead",
+)
+@click.option(
+    "--dry-run", is_flag=True, help="Do not update, only check for the new versions"
+)
 @click.pass_context
 def cli(ctx, core_packages, only_check, dry_run):
     # cleanup lib search results, cached board and platform lists

platformio/commands/upgrade.py

@@ -19,27 +19,29 @@ from zipfile import ZipFile
 import click
 import requests
 
-from platformio import VERSION, __version__, exception, util
+from platformio import VERSION, __version__, app, exception, util
 from platformio.compat import WINDOWS
 from platformio.proc import exec_command, get_pythonexe_path
 from platformio.project.helpers import get_project_cache_dir
 
 
-@click.command("upgrade",
-               short_help="Upgrade PlatformIO to the latest version")
+@click.command("upgrade", short_help="Upgrade PlatformIO to the latest version")
 @click.option("--dev", is_flag=True, help="Use development branch")
 def cli(dev):
     if not dev and __version__ == get_latest_version():
         return click.secho(
             "You're up-to-date!\nPlatformIO %s is currently the "
             "newest version available." % __version__,
-            fg="green")
+            fg="green",
+        )
 
     click.secho("Please wait while upgrading PlatformIO ...", fg="yellow")
 
     to_develop = dev or not all(c.isdigit() for c in __version__ if c != ".")
-    cmds = (["pip", "install", "--upgrade",
-             get_pip_package(to_develop)], ["platformio", "--version"])
+    cmds = (
+        ["pip", "install", "--upgrade", get_pip_package(to_develop)],
+        ["platformio", "--version"],
+    )
 
     cmd = None
     r = {}
@@ -49,26 +51,30 @@ def cli(dev):
             r = exec_command(cmd)
 
             # try pip with disabled cache
-            if r['returncode'] != 0 and cmd[2] == "pip":
+            if r["returncode"] != 0 and cmd[2] == "pip":
                 cmd.insert(3, "--no-cache-dir")
                 r = exec_command(cmd)
 
-            assert r['returncode'] == 0
-        assert "version" in r['out']
-        actual_version = r['out'].strip().split("version", 1)[1].strip()
-        click.secho("PlatformIO has been successfully upgraded to %s" %
-                    actual_version,
-                    fg="green")
+            assert r["returncode"] == 0
+        assert "version" in r["out"]
+        actual_version = r["out"].strip().split("version", 1)[1].strip()
+        click.secho(
+            "PlatformIO has been successfully upgraded to %s" % actual_version,
+            fg="green",
+        )
         click.echo("Release notes: ", nl=False)
-        click.secho("https://docs.platformio.org/en/latest/history.html",
-                    fg="cyan")
+        click.secho("https://docs.platformio.org/en/latest/history.html", fg="cyan")
+        if app.get_session_var("caller_id"):
+            click.secho(
+                "Warning! Please restart IDE to affect PIO Home changes", fg="yellow"
+            )
     except Exception as e:  # pylint: disable=broad-except
         if not r:
             raise exception.UpgradeError("\n".join([str(cmd), str(e)]))
         permission_errors = ("permission denied", "not permitted")
-        if (any(m in r['err'].lower() for m in permission_errors)
-                and not WINDOWS):
-            click.secho("""
+        if any(m in r["err"].lower() for m in permission_errors) and not WINDOWS:
+            click.secho(
+                """
 -----------------
 Permission denied
 -----------------
@@ -78,10 +84,11 @@ You need the `sudo` permission to install Python packages. Try
 
 WARNING! Don't use `sudo` for the rest PlatformIO commands.
 """,
-                        fg="yellow",
-                        err=True)
+                fg="yellow",
+                err=True,
+            )
             raise exception.ReturnErrorCode(1)
-        raise exception.UpgradeError("\n".join([str(cmd), r['out'], r['err']]))
+        raise exception.UpgradeError("\n".join([str(cmd), r["out"], r["err"]]))
 
     return True
 
@@ -89,18 +96,17 @@ WARNING! Don't use `sudo` for the rest PlatformIO commands.
 def get_pip_package(to_develop):
     if not to_develop:
         return "platformio"
-    dl_url = ("https://github.com/platformio/"
-              "platformio-core/archive/develop.zip")
+    dl_url = "https://github.com/platformio/platformio-core/archive/develop.zip"
     cache_dir = get_project_cache_dir()
     if not os.path.isdir(cache_dir):
         os.makedirs(cache_dir)
     pkg_name = os.path.join(cache_dir, "piocoredevelop.zip")
     try:
         with open(pkg_name, "w") as fp:
-            r = exec_command(["curl", "-fsSL", dl_url],
-                             stdout=fp,
-                             universal_newlines=True)
-            assert r['returncode'] == 0
+            r = exec_command(
+                ["curl", "-fsSL", dl_url], stdout=fp, universal_newlines=True
+            )
+            assert r["returncode"] == 0
         # check ZIP structure
         with ZipFile(pkg_name) as zp:
             assert zp.testzip() is None
@@ -127,7 +133,8 @@ def get_develop_latest_version():
     r = requests.get(
         "https://raw.githubusercontent.com/platformio/platformio"
         "/develop/platformio/__init__.py",
-        headers=util.get_request_defheaders())
+        headers=util.get_request_defheaders(),
+    )
     r.raise_for_status()
     for line in r.text.split("\n"):
         line = line.strip()
@@ -145,7 +152,8 @@ def get_develop_latest_version():
 
 
 def get_pypi_latest_version():
-    r = requests.get("https://pypi.org/pypi/platformio/json",
-                     headers=util.get_request_defheaders())
+    r = requests.get(
+        "https://pypi.org/pypi/platformio/json", headers=util.get_request_defheaders()
+    )
     r.raise_for_status()
-    return r.json()['info']['version']
+    return r.json()["info"]["version"]

platformio/compat.py

@@ -12,24 +12,41 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# pylint: disable=unused-import
+# pylint: disable=unused-import, no-name-in-module, import-error,
+# pylint: disable=no-member, undefined-variable
 
+import inspect
 import json
+import locale
 import os
 import re
 import sys
 
 PY2 = sys.version_info[0] == 2
-CYGWIN = sys.platform.startswith('cygwin')
-WINDOWS = sys.platform.startswith('win')
+CYGWIN = sys.platform.startswith("cygwin")
+WINDOWS = sys.platform.startswith("win")
 
 
 def get_filesystem_encoding():
     return sys.getfilesystemencoding() or sys.getdefaultencoding()
 
 
+def get_locale_encoding():
+    return locale.getdefaultlocale()[1]
+
+
+def get_class_attributes(cls):
+    attributes = inspect.getmembers(cls, lambda a: not (inspect.isroutine(a)))
+    return {
+        a[0]: a[1]
+        for a in attributes
+        if not (a[0].startswith("__") and a[0].endswith("__"))
+    }
+
+
 if PY2:
-    # pylint: disable=undefined-variable
+    import imp
+
     string_types = (str, unicode)
 
     def is_bytes(x):
@@ -40,10 +57,6 @@ if PY2:
             return path
         return path.decode(get_filesystem_encoding()).encode("utf-8")
 
-    def get_file_contents(path):
-        with open(path) as f:
-            return f.read()
-
     def hashlib_encode_data(data):
         if is_bytes(data):
             return data
@@ -56,13 +69,12 @@ if PY2:
     def dump_json_to_unicode(obj):
         if isinstance(obj, unicode):
             return obj
-        return json.dumps(obj,
-                          encoding=get_filesystem_encoding(),
-                          ensure_ascii=False,
-                          sort_keys=True).encode("utf8")
+        return json.dumps(
+            obj, encoding=get_filesystem_encoding(), ensure_ascii=False, sort_keys=True
+        ).encode("utf8")
 
-    _magic_check = re.compile('([*?[])')
-    _magic_check_bytes = re.compile(b'([*?[])')
+    _magic_check = re.compile("([*?[])")
+    _magic_check_bytes = re.compile(b"([*?[])")
 
     def glob_escape(pathname):
         """Escape all special characters."""
@@ -72,14 +84,20 @@ if PY2:
         # escaped.
         drive, pathname = os.path.splitdrive(pathname)
         if isinstance(pathname, bytes):
-            pathname = _magic_check_bytes.sub(br'[\1]', pathname)
+            pathname = _magic_check_bytes.sub(br"[\1]", pathname)
         else:
-            pathname = _magic_check.sub(r'[\1]', pathname)
+            pathname = _magic_check.sub(r"[\1]", pathname)
         return drive + pathname
-else:
-    from glob import escape as glob_escape  # pylint: disable=no-name-in-module
 
-    string_types = (str, )
+    def load_python_module(name, pathname):
+        return imp.load_source(name, pathname)
+
+
+else:
+    import importlib.util
+    from glob import escape as glob_escape
+
+    string_types = (str,)
 
     def is_bytes(x):
         return isinstance(x, (bytes, memoryview, bytearray))
@@ -87,14 +105,6 @@ else:
     def path_to_unicode(path):
         return path
 
-    def get_file_contents(path):
-        try:
-            with open(path) as f:
-                return f.read()
-        except UnicodeDecodeError:
-            with open(path, encoding="latin-1") as f:
-                return f.read()
-
     def hashlib_encode_data(data):
         if is_bytes(data):
             return data
@@ -106,3 +116,9 @@ else:
         if isinstance(obj, string_types):
             return obj
         return json.dumps(obj, ensure_ascii=False, sort_keys=True)
+
+    def load_python_module(name, pathname):
+        spec = importlib.util.spec_from_file_location(name, pathname)
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+        return module

platformio/downloader.py

@@ -22,8 +22,11 @@ import click
 import requests
 
 from platformio import util
-from platformio.exception import (FDSHASumMismatch, FDSizeMismatch,
-                                  FDUnrecognizedStatusCode)
+from platformio.exception import (
+    FDSHASumMismatch,
+    FDSizeMismatch,
+    FDUnrecognizedStatusCode,
+)
 from platformio.proc import exec_command
 
 
@@ -34,17 +37,22 @@ class FileDownloader(object):
     def __init__(self, url, dest_dir=None):
         self._request = None
         # make connection
-        self._request = requests.get(url,
-                                     stream=True,
-                                     headers=util.get_request_defheaders(),
-                                     verify=version_info >= (2, 7, 9))
+        self._request = requests.get(
+            url,
+            stream=True,
+            headers=util.get_request_defheaders(),
+            verify=version_info >= (2, 7, 9),
+        )
         if self._request.status_code != 200:
             raise FDUnrecognizedStatusCode(self._request.status_code, url)
 
         disposition = self._request.headers.get("content-disposition")
         if disposition and "filename=" in disposition:
-            self._fname = disposition[disposition.index("filename=") +
-                                      9:].replace('"', "").replace("'", "")
+            self._fname = (
+                disposition[disposition.index("filename=") + 9 :]
+                .replace('"', "")
+                .replace("'", "")
+            )
         else:
             self._fname = [p for p in url.split("/") if p][-1]
         self._fname = str(self._fname)
@@ -64,7 +72,7 @@ class FileDownloader(object):
     def get_size(self):
         if "content-length" not in self._request.headers:
             return -1
-        return int(self._request.headers['content-length'])
+        return int(self._request.headers["content-length"])
 
     def start(self, with_progress=True):
         label = "Downloading"
@@ -101,11 +109,11 @@ class FileDownloader(object):
         dlsha1 = None
         try:
             result = exec_command(["sha1sum", self._destination])
-            dlsha1 = result['out']
+            dlsha1 = result["out"]
         except (OSError, ValueError):
             try:
                 result = exec_command(["shasum", "-a", "1", self._destination])
-                dlsha1 = result['out']
+                dlsha1 = result["out"]
             except (OSError, ValueError):
                 pass
         if not dlsha1:

platformio/exception.py

@@ -64,8 +64,10 @@ class IncompatiblePlatform(PlatformioException):
 
 class PlatformNotInstalledYet(PlatformioException):
 
-    MESSAGE = ("The platform '{0}' has not been installed yet. "
-               "Use `platformio platform install {0}` command")
+    MESSAGE = (
+        "The platform '{0}' has not been installed yet. "
+        "Use `platformio platform install {0}` command"
+    )
 
 
 class UnknownBoard(PlatformioException):
@@ -102,22 +104,27 @@ class MissingPackageManifest(PlatformIOPackageException):
 
 class UndefinedPackageVersion(PlatformIOPackageException):
 
-    MESSAGE = ("Could not find a version that satisfies the requirement '{0}'"
-               " for your system '{1}'")
+    MESSAGE = (
+        "Could not find a version that satisfies the requirement '{0}'"
+        " for your system '{1}'"
+    )
 
 
 class PackageInstallError(PlatformIOPackageException):
 
-    MESSAGE = ("Could not install '{0}' with version requirements '{1}' "
-               "for your system '{2}'.\n\n"
-               "Please try this solution -> http://bit.ly/faq-package-manager")
+    MESSAGE = (
+        "Could not install '{0}' with version requirements '{1}' "
+        "for your system '{2}'.\n\n"
+        "Please try this solution -> http://bit.ly/faq-package-manager"
+    )
 
 
 class ExtractArchiveItemError(PlatformIOPackageException):
 
     MESSAGE = (
         "Could not extract `{0}` to `{1}`. Try to disable antivirus "
-        "tool or check this solution -> http://bit.ly/faq-package-manager")
+        "tool or check this solution -> http://bit.ly/faq-package-manager"
+    )
 
 
 class UnsupportedArchiveType(PlatformIOPackageException):
@@ -132,14 +139,17 @@ class FDUnrecognizedStatusCode(PlatformIOPackageException):
 
 class FDSizeMismatch(PlatformIOPackageException):
 
-    MESSAGE = ("The size ({0:d} bytes) of downloaded file '{1}' "
-               "is not equal to remote size ({2:d} bytes)")
+    MESSAGE = (
+        "The size ({0:d} bytes) of downloaded file '{1}' "
+        "is not equal to remote size ({2:d} bytes)"
+    )
 
 
 class FDSHASumMismatch(PlatformIOPackageException):
 
-    MESSAGE = ("The 'sha1' sum '{0}' of downloaded file '{1}' "
-               "is not equal to remote '{2}'")
+    MESSAGE = (
+        "The 'sha1' sum '{0}' of downloaded file '{1}' is not equal to remote '{2}'"
+    )
 
 
 #
@@ -156,12 +166,13 @@ class NotPlatformIOProject(PlatformIOProjectException):
     MESSAGE = (
         "Not a PlatformIO project. `platformio.ini` file has not been "
         "found in current working directory ({0}). To initialize new project "
-        "please use `platformio init` command")
+        "please use `platformio init` command"
+    )
 
 
 class InvalidProjectConf(PlatformIOProjectException):
 
-    MESSAGE = ("Invalid '{0}' (project configuration file): '{1}'")
+    MESSAGE = "Invalid '{0}' (project configuration file): '{1}'"
 
 
 class UndefinedEnvPlatform(PlatformIOProjectException):
@@ -191,9 +202,11 @@ class ProjectOptionValueError(PlatformIOProjectException):
 
 class LibNotFound(PlatformioException):
 
-    MESSAGE = ("Library `{0}` has not been found in PlatformIO Registry.\n"
-               "You can ignore this message, if `{0}` is a built-in library "
-               "(included in framework, SDK). E.g., SPI, Wire, etc.")
+    MESSAGE = (
+        "Library `{0}` has not been found in PlatformIO Registry.\n"
+        "You can ignore this message, if `{0}` is a built-in library "
+        "(included in framework, SDK). E.g., SPI, Wire, etc."
+    )
 
 
 class NotGlobalLibDir(UserSideException):
@@ -203,7 +216,8 @@ class NotGlobalLibDir(UserSideException):
         "To manage libraries in global storage `{1}`,\n"
         "please use `platformio lib --global {2}` or specify custom storage "
         "`platformio lib --storage-dir /path/to/storage/ {2}`.\n"
-        "Check `platformio lib --help` for details.")
+        "Check `platformio lib --help` for details."
+    )
 
 
 class InvalidLibConfURL(PlatformioException):
@@ -224,7 +238,8 @@ class MissedUdevRules(InvalidUdevRules):
 
     MESSAGE = (
         "Warning! Please install `99-platformio-udev.rules`. \nMode details: "
-        "https://docs.platformio.org/en/latest/faq.html#platformio-udev-rules")
+        "https://docs.platformio.org/en/latest/faq.html#platformio-udev-rules"
+    )
 
 
 class OutdatedUdevRules(InvalidUdevRules):
@@ -232,7 +247,8 @@ class OutdatedUdevRules(InvalidUdevRules):
     MESSAGE = (
         "Warning! Your `{0}` are outdated. Please update or reinstall them."
         "\n Mode details: https://docs.platformio.org"
-        "/en/latest/faq.html#platformio-udev-rules")
+        "/en/latest/faq.html#platformio-udev-rules"
+    )
 
 
 #
@@ -260,7 +276,8 @@ class InternetIsOffline(UserSideException):
     MESSAGE = (
         "You are not connected to the Internet.\n"
         "If you build a project first time, we need Internet connection "
-        "to install all dependencies and toolchains.")
+        "to install all dependencies and toolchains."
+    )
 
 
 class BuildScriptNotFound(PlatformioException):
@@ -285,9 +302,11 @@ class InvalidJSONFile(PlatformioException):
 
 class CIBuildEnvsEmpty(PlatformioException):
 
-    MESSAGE = ("Can't find PlatformIO build environments.\n"
-               "Please specify `--board` or path to `platformio.ini` with "
-               "predefined environments using `--project-conf` option")
+    MESSAGE = (
+        "Can't find PlatformIO build environments.\n"
+        "Please specify `--board` or path to `platformio.ini` with "
+        "predefined environments using `--project-conf` option"
+    )
 
 
 class UpgradeError(PlatformioException):
@@ -307,13 +326,16 @@ class HomeDirPermissionsError(PlatformioException):
         "current user and PlatformIO can not store configuration data.\n"
         "Please check the permissions and owner of that directory.\n"
         "Otherwise, please remove manually `{0}` directory and PlatformIO "
-        "will create new from the current user.")
+        "will create new from the current user."
+    )
 
 
 class CygwinEnvDetected(PlatformioException):
 
-    MESSAGE = ("PlatformIO does not work within Cygwin environment. "
-               "Use native Terminal instead.")
+    MESSAGE = (
+        "PlatformIO does not work within Cygwin environment. "
+        "Use native Terminal instead."
+    )
 
 
 class DebugSupportError(PlatformioException):
@@ -322,7 +344,8 @@ class DebugSupportError(PlatformioException):
         "Currently, PlatformIO does not support debugging for `{0}`.\n"
         "Please request support at https://github.com/platformio/"
         "platformio-core/issues \nor visit -> https://docs.platformio.org"
-        "/page/plus/debugging.html")
+        "/page/plus/debugging.html"
+    )
 
 
 class DebugInvalidOptions(PlatformioException):
@@ -331,8 +354,10 @@ class DebugInvalidOptions(PlatformioException):
 
 class TestDirNotExists(PlatformioException):
 
-    MESSAGE = "A test folder '{0}' does not exist.\nPlease create 'test' "\
-              "directory in project's root and put a test set.\n"\
-              "More details about Unit "\
-              "Testing: http://docs.platformio.org/page/plus/"\
-              "unit-testing.html"
+    MESSAGE = (
+        "A test folder '{0}' does not exist.\nPlease create 'test' "
+        "directory in project's root and put a test set.\n"
+        "More details about Unit "
+        "Testing: http://docs.platformio.org/page/plus/"
+        "unit-testing.html"
+    )

platformio/fs.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import io
 import json
 import os
 import re
@@ -23,11 +24,10 @@ from glob import glob
 import click
 
 from platformio import exception
-from platformio.compat import WINDOWS, get_file_contents, glob_escape
+from platformio.compat import WINDOWS, glob_escape
 
 
 class cd(object):
-
     def __init__(self, new_path):
         self.new_path = new_path
         self.prev_path = os.getcwd()
@@ -49,6 +49,30 @@ def get_source_dir():
     return os.path.dirname(curpath)
 
 
+def get_file_contents(path):
+    try:
+        with open(path) as fp:
+            return fp.read()
+    except UnicodeDecodeError:
+        click.secho(
+            "Unicode decode error has occurred, please remove invalid "
+            "(non-ASCII or non-UTF8) characters from %s file" % path,
+            fg="yellow",
+            err=True,
+        )
+        with io.open(path, encoding="latin-1") as fp:
+            return fp.read()
+
+
+def write_file_contents(path, contents, errors=None):
+    try:
+        with open(path, "w") as fp:
+            return fp.write(contents)
+    except UnicodeEncodeError:
+        with io.open(path, "w", encoding="latin-1", errors=errors) as fp:
+            return fp.write(contents)
+
+
 def load_json(file_path):
     try:
         with open(file_path, "r") as f:
@@ -65,34 +89,37 @@ def format_filesize(filesize):
     if filesize < base:
         return "%d%s" % (filesize, suffix)
     for i, suffix in enumerate("KMGTPEZY"):
-        unit = base**(i + 2)
+        unit = base ** (i + 2)
         if filesize >= unit:
             continue
-        if filesize % (base**(i + 1)):
+        if filesize % (base ** (i + 1)):
             return "%.2f%sB" % ((base * filesize / unit), suffix)
         break
     return "%d%sB" % ((base * filesize / unit), suffix)
 
 
 def ensure_udev_rules():
-    from platformio.util import get_systype
+    from platformio.util import get_systype  # pylint: disable=import-outside-toplevel
 
     def _rules_to_set(rules_path):
-        return set(l.strip() for l in get_file_contents(rules_path).split("\n")
-                   if l.strip() and not l.startswith("#"))
+        return set(
+            l.strip()
+            for l in get_file_contents(rules_path).split("\n")
+            if l.strip() and not l.startswith("#")
+        )
 
     if "linux" not in get_systype():
         return None
     installed_rules = [
         "/etc/udev/rules.d/99-platformio-udev.rules",
-        "/lib/udev/rules.d/99-platformio-udev.rules"
+        "/lib/udev/rules.d/99-platformio-udev.rules",
     ]
     if not any(os.path.isfile(p) for p in installed_rules):
         raise exception.MissedUdevRules
 
     origin_path = os.path.abspath(
-        os.path.join(get_source_dir(), "..", "scripts",
-                     "99-platformio-udev.rules"))
+        os.path.join(get_source_dir(), "..", "scripts", "99-platformio-udev.rules")
+    )
     if not os.path.isfile(origin_path):
         return None
 
@@ -117,7 +144,6 @@ def path_endswith_ext(path, extensions):
 
 
 def match_src_files(src_dir, src_filter=None, src_exts=None):
-
     def _append_build_item(items, item, src_dir):
         if not src_exts or path_endswith_ext(item, src_exts):
             items.add(item.replace(src_dir + os.sep, ""))
@@ -135,8 +161,7 @@ def match_src_files(src_dir, src_filter=None, src_exts=None):
             if os.path.isdir(item):
                 for root, _, files in os.walk(item, followlinks=True):
                     for f in files:
-                        _append_build_item(items, os.path.join(root, f),
-                                           src_dir)
+                        _append_build_item(items, os.path.join(root, f), src_dir)
             else:
                 _append_build_item(items, item, src_dir)
         if action == "+":
@@ -152,8 +177,16 @@ def to_unix_path(path):
     return re.sub(r"[\\]+", "/", path)
 
 
-def rmtree(path):
+def expanduser(path):
+    """
+    Be compatible with Python 3.8, on Windows skip HOME and check for USERPROFILE
+    """
+    if not WINDOWS or not path.startswith("~") or "USERPROFILE" not in os.environ:
+        return os.path.expanduser(path)
+    return os.environ["USERPROFILE"] + path[1:]
 
+
+def rmtree(path):
     def _onerror(func, path, __):
         try:
             st_mode = os.stat(path).st_mode
@@ -161,9 +194,10 @@ def rmtree(path):
                 os.chmod(path, st_mode | stat.S_IWRITE)
             func(path)
         except Exception as e:  # pylint: disable=broad-except
-            click.secho("%s \nPlease manually remove the file `%s`" %
-                        (str(e), path),
-                        fg="red",
-                        err=True)
+            click.secho(
+                "%s \nPlease manually remove the file `%s`" % (str(e), path),
+                fg="red",
+                err=True,
+            )
 
     return shutil.rmtree(path, onerror=_onerror)

platformio/ide/projectgenerator.py

@@ -12,28 +12,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import codecs
+import io
 import os
 import sys
-from os.path import abspath, basename, expanduser, isdir, isfile, join, relpath
+from os.path import abspath, basename, isdir, isfile, join, relpath
 
 import bottle
 
 from platformio import fs, util
-from platformio.compat import get_file_contents
 from platformio.proc import where_is_program
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import (get_project_lib_dir,
-                                        get_project_libdeps_dir,
-                                        get_project_src_dir,
-                                        load_project_ide_data)
+from platformio.project.helpers import load_project_ide_data
 
 
 class ProjectGenerator(object):
-
     def __init__(self, project_dir, ide, boards):
-        self.config = ProjectConfig.get_instance(
-            join(project_dir, "platformio.ini"))
+        self.config = ProjectConfig.get_instance(join(project_dir, "platformio.ini"))
         self.config.validate()
         self.project_dir = project_dir
         self.ide = str(ide)
@@ -42,8 +36,7 @@ class ProjectGenerator(object):
     @staticmethod
     def get_supported_ides():
         tpls_dir = join(fs.get_source_dir(), "ide", "tpls")
-        return sorted(
-            [d for d in os.listdir(tpls_dir) if isdir(join(tpls_dir, d))])
+        return sorted([d for d in os.listdir(tpls_dir) if isdir(join(tpls_dir, d))])
 
     def get_best_envname(self, boards=None):
         envname = None
@@ -71,29 +64,30 @@ class ProjectGenerator(object):
             "project_name": basename(self.project_dir),
             "project_dir": self.project_dir,
             "env_name": self.env_name,
-            "user_home_dir": abspath(expanduser("~")),
-            "platformio_path":
-                sys.argv[0] if isfile(sys.argv[0])
-                else where_is_program("platformio"),
+            "user_home_dir": abspath(fs.expanduser("~")),
+            "platformio_path": sys.argv[0]
+            if isfile(sys.argv[0])
+            else where_is_program("platformio"),
             "env_path": os.getenv("PATH"),
-            "env_pathsep": os.pathsep
-        }   # yapf: disable
+            "env_pathsep": os.pathsep,
+        }
 
         # default env configuration
         tpl_vars.update(self.config.items(env=self.env_name, as_dict=True))
         # build data
-        tpl_vars.update(
-            load_project_ide_data(self.project_dir, self.env_name) or {})
+        tpl_vars.update(load_project_ide_data(self.project_dir, self.env_name) or {})
 
         with fs.cd(self.project_dir):
-            tpl_vars.update({
-                "src_files": self.get_src_files(),
-                "project_src_dir": get_project_src_dir(),
-                "project_lib_dir": get_project_lib_dir(),
-                "project_libdeps_dir": join(
-                    get_project_libdeps_dir(), self.env_name)
-
-            })  # yapf: disable
+            tpl_vars.update(
+                {
+                    "src_files": self.get_src_files(),
+                    "project_src_dir": self.config.get_optional_dir("src"),
+                    "project_lib_dir": self.config.get_optional_dir("lib"),
+                    "project_libdeps_dir": join(
+                        self.config.get_optional_dir("libdeps"), self.env_name
+                    ),
+                }
+            )
 
         for key, value in tpl_vars.items():
             if key.endswith(("_path", "_dir")):
@@ -103,13 +97,13 @@ class ProjectGenerator(object):
                 continue
             tpl_vars[key] = [fs.to_unix_path(inc) for inc in tpl_vars[key]]
 
-        tpl_vars['to_unix_path'] = fs.to_unix_path
+        tpl_vars["to_unix_path"] = fs.to_unix_path
         return tpl_vars
 
     def get_src_files(self):
         result = []
         with fs.cd(self.project_dir):
-            for root, _, files in os.walk(get_project_src_dir()):
+            for root, _, files in os.walk(self.config.get_optional_dir("src")):
                 for f in files:
                     result.append(relpath(join(root, f)))
         return result
@@ -142,11 +136,11 @@ class ProjectGenerator(object):
 
     @staticmethod
     def _render_tpl(tpl_path, tpl_vars):
-        return bottle.template(get_file_contents(tpl_path), **tpl_vars)
+        return bottle.template(fs.get_file_contents(tpl_path), **tpl_vars)
 
     @staticmethod
     def _merge_contents(dst_path, contents):
         if basename(dst_path) == ".gitignore" and isfile(dst_path):
             return
-        with codecs.open(dst_path, "w", encoding="utf8") as fp:
+        with io.open(dst_path, "w", encoding="utf8") as fp:
             fp.write(contents)

platformio/ide/tpls/atom/.gcc-flags.json.tpl

@@ -1,4 +1,4 @@
-% _defines = " ".join(["-D%s" % d for d in defines])
+% _defines = " ".join(["-D%s" % d.replace(" ", "\\\\ ") for d in defines])
 {
   "execPath": "{{ cxx_path }}",
   "gccDefaultCFlags": "-fsyntax-only {{! cc_flags.replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}",

platformio/ide/tpls/clion/CMakeLists.txt.tpl

@@ -6,7 +6,7 @@
 # The `CMakeListsUser.txt` will not be overwritten by PlatformIO.
 
 cmake_minimum_required(VERSION 3.2)
-project({{project_name}})
+project("{{project_name}}")
 
 include(CMakeListsPrivate.txt)
 
@@ -62,6 +62,12 @@ add_custom_target(
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 )
 
+add_custom_target(
+    PLATFORMIO_BUILD_DEBUG ALL
+    COMMAND ${PLATFORMIO_CMD} -f -c clion run --target debug "$<$<NOT:$<CONFIG:All>>:-e${CMAKE_BUILD_TYPE}>"
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+)
+
 add_custom_target(
     PLATFORMIO_UPDATE_ALL ALL
     COMMAND ${PLATFORMIO_CMD} -f -c clion update
@@ -80,4 +86,4 @@ add_custom_target(
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 )
 
-add_executable(${PROJECT_NAME} ${SRC_LIST})
+add_executable(Z_DUMMY_TARGET ${SRC_LIST})

platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl

@@ -31,6 +31,9 @@ set(CMAKE_CONFIGURATION_TYPES "{{ env_name }}" CACHE STRING "" FORCE)
 % end
 
 set(PLATFORMIO_CMD "{{ _normalize_path(platformio_path) }}")
+% if svd_path:
+set(SVD_PATH "{{ _normalize_path(svd_path) }}")
+% end
 
 SET(CMAKE_C_COMPILER "{{ _normalize_path(cc_path) }}")
 SET(CMAKE_CXX_COMPILER "{{ _normalize_path(cxx_path) }}")
@@ -57,8 +60,9 @@ if (CMAKE_BUILD_TYPE MATCHES "{{ env_name }}")
 %end
 endif()
 
-% leftover_envs = set(envs) ^ set([env_name])
+% leftover_envs = list(set(envs) ^ set([env_name]))
 %
+% ide_data = {}
 % if leftover_envs:
 %   ide_data = load_project_ide_data(project_dir, leftover_envs)
 % end

platformio/ide/tpls/emacs/.ccls.tpl

@@ -0,0 +1,22 @@
+% import re
+% STD_RE = re.compile(r"\-std=[a-z\+]+(\d+)")
+% cc_stds = STD_RE.findall(cc_flags)
+% cxx_stds = STD_RE.findall(cxx_flags)
+%
+%
+clang
+
+% if cc_stds:
+{{"%c"}} -std=c{{ cc_stds[-1] }}
+% end
+% if cxx_stds:
+{{"%cpp"}} -std=c++{{ cxx_stds[-1] }}
+% end
+
+% for include in includes:
+-I{{ include }}
+% end
+
+% for define in defines:
+-D{{ define }}
+% end

platformio/ide/tpls/vim/.gcc-flags.json.tpl

@@ -1,9 +1,9 @@
-% _defines = " ".join(["-D%s" % d for d in defines])
+% _defines = " ".join(["-D%s" % d.replace(" ", "\\\\ ") for d in defines])
 {
   "execPath": "{{ cxx_path }}",
   "gccDefaultCFlags": "-fsyntax-only {{! cc_flags.replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}",
   "gccDefaultCppFlags": "-fsyntax-only {{! cxx_flags.replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}",
   "gccErrorLimit": 15,
-  "gccIncludePaths": "{{! ','.join("'{}'".format(inc) for inc in includes)}}",
+  "gccIncludePaths": "{{ ','.join(includes) }}",
   "gccSuppressWarnings": false
 }

platformio/ide/tpls/visualstudio/platformio.vcxproj.filters.tpl

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <ItemGroup>
     <Filter Include="Source Files">

platformio/ide/tpls/visualstudio/platformio.vcxproj.tpl

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup>
     <Path>{{env_path}}</Path>

platformio/lockfile.py

@@ -26,10 +26,12 @@ LOCKFILE_INTERFACE_MSVCRT = 2
 
 try:
     import fcntl
+
     LOCKFILE_CURRENT_INTERFACE = LOCKFILE_INTERFACE_FCNTL
 except ImportError:
     try:
         import msvcrt
+
         LOCKFILE_CURRENT_INTERFACE = LOCKFILE_INTERFACE_MSVCRT
     except ImportError:
         LOCKFILE_CURRENT_INTERFACE = None
@@ -40,7 +42,6 @@ class LockFileExists(Exception):
 
 
 class LockFile(object):
-
     def __init__(self, path, timeout=LOCKFILE_TIMEOUT, delay=LOCKFILE_DELAY):
         self.timeout = timeout
         self.delay = delay

platformio/maintenance.py

@@ -49,12 +49,16 @@ def on_platformio_end(ctx, result):  # pylint: disable=unused-argument
         check_platformio_upgrade()
         check_internal_updates(ctx, "platforms")
         check_internal_updates(ctx, "libraries")
-    except (exception.InternetIsOffline, exception.GetLatestVersionError,
-            exception.APIRequestError):
+    except (
+        exception.InternetIsOffline,
+        exception.GetLatestVersionError,
+        exception.APIRequestError,
+    ):
         click.secho(
             "Failed to check for PlatformIO upgrades. "
             "Please check your Internet connection.",
-            fg="red")
+            fg="red",
+        )
 
 
 def on_platformio_exception(e):
@@ -78,15 +82,17 @@ def set_caller(caller=None):
 
 
 class Upgrader(object):
-
     def __init__(self, from_version, to_version):
         self.from_version = semantic_version.Version.coerce(
-            util.pepver_to_semver(from_version))
+            util.pepver_to_semver(from_version)
+        )
         self.to_version = semantic_version.Version.coerce(
-            util.pepver_to_semver(to_version))
+            util.pepver_to_semver(to_version)
+        )
 
-        self._upgraders = [(semantic_version.Version("3.5.0-a.2"),
-                            self._update_dev_platforms)]
+        self._upgraders = [
+            (semantic_version.Version("3.5.0-a.2"), self._update_dev_platforms)
+        ]
 
     def run(self, ctx):
         if self.from_version > self.to_version:
@@ -114,19 +120,21 @@ def after_upgrade(ctx):
 
     if last_version == "0.0.0":
         app.set_state_item("last_version", __version__)
-    elif semantic_version.Version.coerce(util.pepver_to_semver(
-            last_version)) > semantic_version.Version.coerce(
-                util.pepver_to_semver(__version__)):
+    elif semantic_version.Version.coerce(
+        util.pepver_to_semver(last_version)
+    ) > semantic_version.Version.coerce(util.pepver_to_semver(__version__)):
         click.secho("*" * terminal_width, fg="yellow")
-        click.secho("Obsolete PIO Core v%s is used (previous was %s)" %
-                    (__version__, last_version),
-                    fg="yellow")
-        click.secho("Please remove multiple PIO Cores from a system:",
-                    fg="yellow")
+        click.secho(
+            "Obsolete PIO Core v%s is used (previous was %s)"
+            % (__version__, last_version),
+            fg="yellow",
+        )
+        click.secho("Please remove multiple PIO Cores from a system:", fg="yellow")
         click.secho(
             "https://docs.platformio.org/page/faq.html"
             "#multiple-pio-cores-in-a-system",
-            fg="cyan")
+            fg="cyan",
+        )
         click.secho("*" * terminal_width, fg="yellow")
         return
     else:
@@ -139,37 +147,53 @@ def after_upgrade(ctx):
         u = Upgrader(last_version, __version__)
         if u.run(ctx):
             app.set_state_item("last_version", __version__)
-            click.secho("PlatformIO has been successfully upgraded to %s!\n" %
-                        __version__,
-                        fg="green")
-            telemetry.on_event(category="Auto",
-                               action="Upgrade",
-                               label="%s > %s" % (last_version, __version__))
+            click.secho(
+                "PlatformIO has been successfully upgraded to %s!\n" % __version__,
+                fg="green",
+            )
+            telemetry.on_event(
+                category="Auto",
+                action="Upgrade",
+                label="%s > %s" % (last_version, __version__),
+            )
         else:
             raise exception.UpgradeError("Auto upgrading...")
         click.echo("")
 
     # PlatformIO banner
     click.echo("*" * terminal_width)
-    click.echo("If you like %s, please:" %
-               (click.style("PlatformIO", fg="cyan")))
-    click.echo("- %s us on Twitter to stay up-to-date "
-               "on the latest project news > %s" %
-               (click.style("follow", fg="cyan"),
-                click.style("https://twitter.com/PlatformIO_Org", fg="cyan")))
+    click.echo("If you like %s, please:" % (click.style("PlatformIO", fg="cyan")))
     click.echo(
-        "- %s it on GitHub > %s" %
-        (click.style("star", fg="cyan"),
-         click.style("https://github.com/platformio/platformio", fg="cyan")))
+        "- %s us on Twitter to stay up-to-date "
+        "on the latest project news > %s"
+        % (
+            click.style("follow", fg="cyan"),
+            click.style("https://twitter.com/PlatformIO_Org", fg="cyan"),
+        )
+    )
+    click.echo(
+        "- %s it on GitHub > %s"
+        % (
+            click.style("star", fg="cyan"),
+            click.style("https://github.com/platformio/platformio", fg="cyan"),
+        )
+    )
     if not getenv("PLATFORMIO_IDE"):
         click.echo(
-            "- %s PlatformIO IDE for IoT development > %s" %
-            (click.style("try", fg="cyan"),
-             click.style("https://platformio.org/platformio-ide", fg="cyan")))
+            "- %s PlatformIO IDE for embedded development > %s"
+            % (
+                click.style("try", fg="cyan"),
+                click.style("https://platformio.org/platformio-ide", fg="cyan"),
+            )
+        )
     if not is_ci():
-        click.echo("- %s us with PlatformIO Plus > %s" %
-                   (click.style("support", fg="cyan"),
-                    click.style("https://pioplus.com", fg="cyan")))
+        click.echo(
+            "- %s us with PlatformIO Plus > %s"
+            % (
+                click.style("support", fg="cyan"),
+                click.style("https://pioplus.com", fg="cyan"),
+            )
+        )
 
     click.echo("*" * terminal_width)
     click.echo("")
@@ -181,7 +205,7 @@ def check_platformio_upgrade():
     if (time() - interval) < last_check.get("platformio_upgrade", 0):
         return
 
-    last_check['platformio_upgrade'] = int(time())
+    last_check["platformio_upgrade"] = int(time())
     app.set_state_item("last_check", last_check)
 
     util.internet_on(raise_exception=True)
@@ -190,23 +214,23 @@ def check_platformio_upgrade():
     update_core_packages(silent=True)
 
     latest_version = get_latest_version()
-    if semantic_version.Version.coerce(util.pepver_to_semver(
-            latest_version)) <= semantic_version.Version.coerce(
-                util.pepver_to_semver(__version__)):
+    if semantic_version.Version.coerce(
+        util.pepver_to_semver(latest_version)
+    ) <= semantic_version.Version.coerce(util.pepver_to_semver(__version__)):
         return
 
     terminal_width, _ = click.get_terminal_size()
 
     click.echo("")
     click.echo("*" * terminal_width)
-    click.secho("There is a new version %s of PlatformIO available.\n"
-                "Please upgrade it via `" % latest_version,
-                fg="yellow",
-                nl=False)
+    click.secho(
+        "There is a new version %s of PlatformIO available.\n"
+        "Please upgrade it via `" % latest_version,
+        fg="yellow",
+        nl=False,
+    )
     if getenv("PLATFORMIO_IDE"):
-        click.secho("PlatformIO IDE Menu: Upgrade PlatformIO",
-                    fg="cyan",
-                    nl=False)
+        click.secho("PlatformIO IDE Menu: Upgrade PlatformIO", fg="cyan", nl=False)
         click.secho("`.", fg="yellow")
     elif join("Cellar", "platformio") in fs.get_source_dir():
         click.secho("brew update && brew upgrade", fg="cyan", nl=False)
@@ -217,8 +241,7 @@ def check_platformio_upgrade():
         click.secho("pip install -U platformio", fg="cyan", nl=False)
         click.secho("` command.", fg="yellow")
     click.secho("Changes: ", fg="yellow", nl=False)
-    click.secho("https://docs.platformio.org/en/latest/history.html",
-                fg="cyan")
+    click.secho("https://docs.platformio.org/en/latest/history.html", fg="cyan")
     click.echo("*" * terminal_width)
     click.echo("")
 
@@ -229,7 +252,7 @@ def check_internal_updates(ctx, what):
     if (time() - interval) < last_check.get(what + "_update", 0):
         return
 
-    last_check[what + '_update'] = int(time())
+    last_check[what + "_update"] = int(time())
     app.set_state_item("last_check", last_check)
 
     util.internet_on(raise_exception=True)
@@ -237,15 +260,17 @@ def check_internal_updates(ctx, what):
     pm = PlatformManager() if what == "platforms" else LibraryManager()
     outdated_items = []
     for manifest in pm.get_installed():
-        if manifest['name'] in outdated_items:
+        if manifest["name"] in outdated_items:
             continue
         conds = [
-            pm.outdated(manifest['__pkg_dir']), what == "platforms"
+            pm.outdated(manifest["__pkg_dir"]),
+            what == "platforms"
             and PlatformFactory.newPlatform(
-                manifest['__pkg_dir']).are_outdated_packages()
+                manifest["__pkg_dir"]
+            ).are_outdated_packages(),
         ]
         if any(conds):
-            outdated_items.append(manifest['name'])
+            outdated_items.append(manifest["name"])
 
     if not outdated_items:
         return
@@ -254,26 +279,32 @@ def check_internal_updates(ctx, what):
 
     click.echo("")
     click.echo("*" * terminal_width)
-    click.secho("There are the new updates for %s (%s)" %
-                (what, ", ".join(outdated_items)),
-                fg="yellow")
+    click.secho(
+        "There are the new updates for %s (%s)" % (what, ", ".join(outdated_items)),
+        fg="yellow",
+    )
 
     if not app.get_setting("auto_update_" + what):
         click.secho("Please update them via ", fg="yellow", nl=False)
-        click.secho("`platformio %s update`" %
-                    ("lib --global" if what == "libraries" else "platform"),
-                    fg="cyan",
-                    nl=False)
+        click.secho(
+            "`platformio %s update`"
+            % ("lib --global" if what == "libraries" else "platform"),
+            fg="cyan",
+            nl=False,
+        )
         click.secho(" command.\n", fg="yellow")
         click.secho(
             "If you want to manually check for the new versions "
             "without updating, please use ",
             fg="yellow",
-            nl=False)
-        click.secho("`platformio %s update --dry-run`" %
-                    ("lib --global" if what == "libraries" else "platform"),
-                    fg="cyan",
-                    nl=False)
+            nl=False,
+        )
+        click.secho(
+            "`platformio %s update --dry-run`"
+            % ("lib --global" if what == "libraries" else "platform"),
+            fg="cyan",
+            nl=False,
+        )
         click.secho(" command.", fg="yellow")
     else:
         click.secho("Please wait while updating %s ..." % what, fg="yellow")
@@ -284,9 +315,7 @@ def check_internal_updates(ctx, what):
             ctx.invoke(cmd_lib_update, libraries=outdated_items)
         click.echo()
 
-        telemetry.on_event(category="Auto",
-                           action="Update",
-                           label=what.title())
+        telemetry.on_event(category="Auto", action="Update", label=what.title())
 
     click.echo("*" * terminal_width)
     click.echo("")

platformio/managers/core.py

@@ -21,15 +21,16 @@ from platformio import __version__, exception, fs
 from platformio.compat import PY2, WINDOWS
 from platformio.managers.package import PackageManager
 from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path
-from platformio.project.helpers import get_project_packages_dir
+from platformio.project.config import ProjectConfig
 
 CORE_PACKAGES = {
-    "contrib-piohome": "^2.3.2",
-    "contrib-pysite":
-    "~2.%d%d.190418" % (sys.version_info[0], sys.version_info[1]),
-    "tool-pioplus": "^2.5.2",
+    "contrib-piohome": "~3.0.0",
+    "contrib-pysite": "~2.%d%d.0" % (sys.version_info[0], sys.version_info[1]),
+    "tool-pioplus": "^2.5.8",
     "tool-unity": "~1.20403.0",
-    "tool-scons": "~2.20501.7" if PY2 else "~3.30101.0"
+    "tool-scons": "~2.20501.7" if PY2 else "~3.30101.0",
+    "tool-cppcheck": "~1.189.0",
+    "tool-clangtidy": "^1.80000.0",
 }
 
 PIOPLUS_AUTO_UPDATES_MAX = 100
@@ -38,20 +39,21 @@ PIOPLUS_AUTO_UPDATES_MAX = 100
 
 
 class CorePackageManager(PackageManager):
-
     def __init__(self):
-        super(CorePackageManager, self).__init__(get_project_packages_dir(), [
-            "https://dl.bintray.com/platformio/dl-packages/manifest.json",
-            "http%s://dl.platformio.org/packages/manifest.json" %
-            ("" if sys.version_info < (2, 7, 9) else "s")
-        ])
+        config = ProjectConfig.get_instance()
+        packages_dir = config.get_optional_dir("packages")
+        super(CorePackageManager, self).__init__(
+            packages_dir,
+            [
+                "https://dl.bintray.com/platformio/dl-packages/manifest.json",
+                "http%s://dl.platformio.org/packages/manifest.json"
+                % ("" if sys.version_info < (2, 7, 9) else "s"),
+            ],
+        )
 
     def install(  # pylint: disable=keyword-arg-before-vararg
-            self,
-            name,
-            requirements=None,
-            *args,
-            **kwargs):
+        self, name, requirements=None, *args, **kwargs
+    ):
         PackageManager.install(self, name, requirements, *args, **kwargs)
         self.cleanup_packages()
         return self.get_package_dir(name, requirements)
@@ -68,12 +70,12 @@ class CorePackageManager(PackageManager):
             pkg_dir = self.get_package_dir(name, requirements)
             if not pkg_dir:
                 continue
-            best_pkg_versions[name] = self.load_manifest(pkg_dir)['version']
+            best_pkg_versions[name] = self.load_manifest(pkg_dir)["version"]
         for manifest in self.get_installed():
-            if manifest['name'] not in best_pkg_versions:
+            if manifest["name"] not in best_pkg_versions:
                 continue
-            if manifest['version'] != best_pkg_versions[manifest['name']]:
-                self.uninstall(manifest['__pkg_dir'], after_update=True)
+            if manifest["version"] != best_pkg_versions[manifest["name"]]:
+                self.uninstall(manifest["__pkg_dir"], after_update=True)
         self.cache_reset()
         return True
 
@@ -101,7 +103,8 @@ def update_core_packages(only_check=False, silent=False):
 
 
 def inject_contrib_pysite():
-    from site import addsitedir
+    from site import addsitedir  # pylint: disable=import-outside-toplevel
+
     contrib_pysite_dir = get_core_package_dir("contrib-pysite")
     if contrib_pysite_dir in sys.path:
         return
@@ -114,16 +117,18 @@ def pioplus_call(args, **kwargs):
         raise exception.PlatformioException(
             "PlatformIO Core Plus v%s does not run under Python version %s.\n"
             "Minimum supported version is 2.7.6, please upgrade Python.\n"
-            "Python 3 is not yet supported.\n" % (__version__, sys.version))
+            "Python 3 is not yet supported.\n" % (__version__, sys.version)
+        )
 
     pioplus_path = join(get_core_package_dir("tool-pioplus"), "pioplus")
     pythonexe_path = get_pythonexe_path()
-    os.environ['PYTHONEXEPATH'] = pythonexe_path
-    os.environ['PYTHONPYSITEDIR'] = get_core_package_dir("contrib-pysite")
-    os.environ['PIOCOREPYSITEDIR'] = dirname(fs.get_source_dir() or "")
-    if dirname(pythonexe_path) not in os.environ['PATH'].split(os.pathsep):
-        os.environ['PATH'] = (os.pathsep).join(
-            [dirname(pythonexe_path), os.environ['PATH']])
+    os.environ["PYTHONEXEPATH"] = pythonexe_path
+    os.environ["PYTHONPYSITEDIR"] = get_core_package_dir("contrib-pysite")
+    os.environ["PIOCOREPYSITEDIR"] = dirname(fs.get_source_dir() or "")
+    if dirname(pythonexe_path) not in os.environ["PATH"].split(os.pathsep):
+        os.environ["PATH"] = (os.pathsep).join(
+            [dirname(pythonexe_path), os.environ["PATH"]]
+        )
     copy_pythonpath_to_osenv()
     code = subprocess.call([pioplus_path] + args, **kwargs)
 

platformio/managers/lib.py

@@ -16,7 +16,6 @@
 # pylint: disable=too-many-return-statements
 
 import json
-import re
 from glob import glob
 from os.path import isdir, join
 
@@ -27,7 +26,7 @@ from platformio import app, exception, util
 from platformio.compat import glob_escape, string_types
 from platformio.managers.package import BasePkgManager
 from platformio.managers.platform import PlatformFactory, PlatformManager
-from platformio.project.helpers import get_project_global_lib_dir
+from platformio.project.config import ProjectConfig
 
 
 class LibraryManager(BasePkgManager):
@@ -35,16 +34,14 @@ class LibraryManager(BasePkgManager):
     FILE_CACHE_VALID = "30d"  # 1 month
 
     def __init__(self, package_dir=None):
-        if not package_dir:
-            package_dir = get_project_global_lib_dir()
-        super(LibraryManager, self).__init__(package_dir)
+        self.config = ProjectConfig.get_instance()
+        super(LibraryManager, self).__init__(
+            package_dir or self.config.get_optional_dir("globallib")
+        )
 
     @property
     def manifest_names(self):
-        return [
-            ".library.json", "library.json", "library.properties",
-            "module.json"
-        ]
+        return [".library.json", "library.json", "library.properties", "module.json"]
 
     def get_manifest_path(self, pkg_dir):
         path = BasePkgManager.get_manifest_path(self, pkg_dir)
@@ -64,75 +61,6 @@ class LibraryManager(BasePkgManager):
 
         return None
 
-    def load_manifest(self, pkg_dir):
-        manifest = BasePkgManager.load_manifest(self, pkg_dir)
-        if not manifest:
-            return manifest
-
-        # if Arduino library.properties
-        if "sentence" in manifest:
-            manifest['frameworks'] = ["arduino"]
-            manifest['description'] = manifest['sentence']
-            del manifest['sentence']
-
-        if "author" in manifest:
-            if isinstance(manifest['author'], dict):
-                manifest['authors'] = [manifest['author']]
-            else:
-                manifest['authors'] = [{"name": manifest['author']}]
-            del manifest['author']
-
-        if "authors" in manifest and not isinstance(manifest['authors'], list):
-            manifest['authors'] = [manifest['authors']]
-
-        if "keywords" not in manifest:
-            keywords = []
-            for keyword in re.split(r"[\s/]+",
-                                    manifest.get("category", "Uncategorized")):
-                keyword = keyword.strip()
-                if not keyword:
-                    continue
-                keywords.append(keyword.lower())
-            manifest['keywords'] = keywords
-            if "category" in manifest:
-                del manifest['category']
-
-        # don't replace VCS URL
-        if "url" in manifest and "description" in manifest:
-            manifest['homepage'] = manifest['url']
-            del manifest['url']
-
-        if "architectures" in manifest:
-            platforms = []
-            platforms_map = {
-                "avr": "atmelavr",
-                "sam": "atmelsam",
-                "samd": "atmelsam",
-                "esp8266": "espressif8266",
-                "esp32": "espressif32",
-                "arc32": "intel_arc32"
-            }
-            for arch in manifest['architectures'].split(","):
-                arch = arch.strip()
-                if arch == "*":
-                    platforms = "*"
-                    break
-                if arch in platforms_map:
-                    platforms.append(platforms_map[arch])
-            manifest['platforms'] = platforms
-            del manifest['architectures']
-
-        # convert listed items via comma to array
-        for key in ("keywords", "frameworks", "platforms"):
-            if key not in manifest or \
-                    not isinstance(manifest[key], string_types):
-                continue
-            manifest[key] = [
-                i.strip() for i in manifest[key].split(",") if i.strip()
-            ]
-
-        return manifest
-
     @staticmethod
     def normalize_dependencies(dependencies):
         if not dependencies:
@@ -153,13 +81,10 @@ class LibraryManager(BasePkgManager):
                 if item[k] == "*":
                     del item[k]
                 elif isinstance(item[k], string_types):
-                    item[k] = [
-                        i.strip() for i in item[k].split(",") if i.strip()
-                    ]
+                    item[k] = [i.strip() for i in item[k].split(",") if i.strip()]
         return items
 
     def max_satisfying_repo_version(self, versions, requirements=None):
-
         def _cmp_dates(datestr1, datestr2):
             date1 = util.parse_date(datestr1)
             date2 = util.parse_date(datestr2)
@@ -169,61 +94,66 @@ class LibraryManager(BasePkgManager):
 
         semver_spec = None
         try:
-            semver_spec = semantic_version.SimpleSpec(
-                requirements) if requirements else None
+            semver_spec = (
+                semantic_version.SimpleSpec(requirements) if requirements else None
+            )
         except ValueError:
             pass
 
         item = {}
 
         for v in versions:
-            semver_new = self.parse_semver_version(v['name'])
+            semver_new = self.parse_semver_version(v["name"])
             if semver_spec:
                 if not semver_new or semver_new not in semver_spec:
                     continue
-                if not item or self.parse_semver_version(
-                        item['name']) < semver_new:
+                if not item or self.parse_semver_version(item["name"]) < semver_new:
                     item = v
             elif requirements:
-                if requirements == v['name']:
+                if requirements == v["name"]:
                     return v
 
             else:
-                if not item or _cmp_dates(item['released'],
-                                          v['released']) == -1:
+                if not item or _cmp_dates(item["released"], v["released"]) == -1:
                     item = v
         return item
 
     def get_latest_repo_version(self, name, requirements, silent=False):
         item = self.max_satisfying_repo_version(
-            util.get_api_result("/lib/info/%d" % self.search_lib_id(
-                {
-                    "name": name,
-                    "requirements": requirements
-                }, silent=silent),
-                                cache_valid="1h")['versions'], requirements)
-        return item['name'] if item else None
+            util.get_api_result(
+                "/lib/info/%d"
+                % self.search_lib_id(
+                    {"name": name, "requirements": requirements}, silent=silent
+                ),
+                cache_valid="1h",
+            )["versions"],
+            requirements,
+        )
+        return item["name"] if item else None
 
     def _install_from_piorepo(self, name, requirements):
         assert name.startswith("id="), name
         version = self.get_latest_repo_version(name, requirements)
         if not version:
-            raise exception.UndefinedPackageVersion(requirements or "latest",
-                                                    util.get_systype())
-        dl_data = util.get_api_result("/lib/download/" + str(name[3:]),
-                                      dict(version=version),
-                                      cache_valid="30d")
+            raise exception.UndefinedPackageVersion(
+                requirements or "latest", util.get_systype()
+            )
+        dl_data = util.get_api_result(
+            "/lib/download/" + str(name[3:]), dict(version=version), cache_valid="30d"
+        )
         assert dl_data
 
         return self._install_from_url(
-            name, dl_data['url'].replace("http://", "https://")
-            if app.get_setting("strict_ssl") else dl_data['url'], requirements)
+            name,
+            dl_data["url"].replace("http://", "https://")
+            if app.get_setting("strict_ssl")
+            else dl_data["url"],
+            requirements,
+        )
 
     def search_lib_id(  # pylint: disable=too-many-branches
-            self,
-            filters,
-            silent=False,
-            interactive=False):
+        self, filters, silent=False, interactive=False
+    ):
         assert isinstance(filters, dict)
         assert "name" in filters
 
@@ -234,8 +164,10 @@ class LibraryManager(BasePkgManager):
 
         # looking in PIO Library Registry
         if not silent:
-            click.echo("Looking for %s library in registry" %
-                       click.style(filters['name'], fg="cyan"))
+            click.echo(
+                "Looking for %s library in registry"
+                % click.style(filters["name"], fg="cyan")
+            )
         query = []
         for key in filters:
             if key not in ("name", "authors", "frameworks", "platforms"):
@@ -244,25 +176,30 @@ class LibraryManager(BasePkgManager):
             if not isinstance(values, list):
                 values = [v.strip() for v in values.split(",") if v]
             for value in values:
-                query.append('%s:"%s"' %
-                             (key[:-1] if key.endswith("s") else key, value))
+                query.append(
+                    '%s:"%s"' % (key[:-1] if key.endswith("s") else key, value)
+                )
 
         lib_info = None
-        result = util.get_api_result("/v2/lib/search",
-                                     dict(query=" ".join(query)),
-                                     cache_valid="1h")
-        if result['total'] == 1:
-            lib_info = result['items'][0]
-        elif result['total'] > 1:
+        result = util.get_api_result(
+            "/v2/lib/search", dict(query=" ".join(query)), cache_valid="1h"
+        )
+        if result["total"] == 1:
+            lib_info = result["items"][0]
+        elif result["total"] > 1:
             if silent and not interactive:
-                lib_info = result['items'][0]
+                lib_info = result["items"][0]
             else:
-                click.secho("Conflict: More than one library has been found "
-                            "by request %s:" % json.dumps(filters),
-                            fg="yellow",
-                            err=True)
+                click.secho(
+                    "Conflict: More than one library has been found "
+                    "by request %s:" % json.dumps(filters),
+                    fg="yellow",
+                    err=True,
+                )
+                # pylint: disable=import-outside-toplevel
                 from platformio.commands.lib import print_lib_item
-                for item in result['items']:
+
+                for item in result["items"]:
                     print_lib_item(item)
 
                 if not interactive:
@@ -270,36 +207,39 @@ class LibraryManager(BasePkgManager):
                         "Automatically chose the first available library "
                         "(use `--interactive` option to make a choice)",
                         fg="yellow",
-                        err=True)
-                    lib_info = result['items'][0]
+                        err=True,
+                    )
+                    lib_info = result["items"][0]
                 else:
-                    deplib_id = click.prompt("Please choose library ID",
-                                             type=click.Choice([
-                                                 str(i['id'])
-                                                 for i in result['items']
-                                             ]))
-                    for item in result['items']:
-                        if item['id'] == int(deplib_id):
+                    deplib_id = click.prompt(
+                        "Please choose library ID",
+                        type=click.Choice([str(i["id"]) for i in result["items"]]),
+                    )
+                    for item in result["items"]:
+                        if item["id"] == int(deplib_id):
                             lib_info = item
                             break
 
         if not lib_info:
             if list(filters) == ["name"]:
-                raise exception.LibNotFound(filters['name'])
+                raise exception.LibNotFound(filters["name"])
             raise exception.LibNotFound(str(filters))
         if not silent:
-            click.echo("Found: %s" % click.style(
-                "https://platformio.org/lib/show/{id}/{name}".format(
-                    **lib_info),
-                fg="blue"))
-        return int(lib_info['id'])
+            click.echo(
+                "Found: %s"
+                % click.style(
+                    "https://platformio.org/lib/show/{id}/{name}".format(**lib_info),
+                    fg="blue",
+                )
+            )
+        return int(lib_info["id"])
 
     def _get_lib_id_from_installed(self, filters):
-        if filters['name'].startswith("id="):
-            return int(filters['name'][3:])
+        if filters["name"].startswith("id="):
+            return int(filters["name"][3:])
         package_dir = self.get_package_dir(
-            filters['name'], filters.get("requirements",
-                                         filters.get("version")))
+            filters["name"], filters.get("requirements", filters.get("version"))
+        )
         if not package_dir:
             return None
         manifest = self.load_manifest(package_dir)
@@ -311,52 +251,55 @@ class LibraryManager(BasePkgManager):
                 continue
             if key not in manifest:
                 return None
-            if not util.items_in_list(util.items_to_list(filters[key]),
-                                      util.items_to_list(manifest[key])):
+            if not util.items_in_list(
+                util.items_to_list(filters[key]), util.items_to_list(manifest[key])
+            ):
                 return None
 
         if "authors" in filters:
             if "authors" not in manifest:
                 return None
-            manifest_authors = manifest['authors']
+            manifest_authors = manifest["authors"]
             if not isinstance(manifest_authors, list):
                 manifest_authors = [manifest_authors]
             manifest_authors = [
-                a['name'] for a in manifest_authors
+                a["name"]
+                for a in manifest_authors
                 if isinstance(a, dict) and "name" in a
             ]
-            filter_authors = filters['authors']
+            filter_authors = filters["authors"]
             if not isinstance(filter_authors, list):
                 filter_authors = [filter_authors]
             if not set(filter_authors) <= set(manifest_authors):
                 return None
 
-        return int(manifest['id'])
+        return int(manifest["id"])
 
     def install(  # pylint: disable=arguments-differ
-            self,
-            name,
-            requirements=None,
-            silent=False,
-            after_update=False,
-            interactive=False,
-            force=False):
+        self,
+        name,
+        requirements=None,
+        silent=False,
+        after_update=False,
+        interactive=False,
+        force=False,
+    ):
         _name, _requirements, _url = self.parse_pkg_uri(name, requirements)
         if not _url:
             name = "id=%d" % self.search_lib_id(
-                {
-                    "name": _name,
-                    "requirements": _requirements
-                },
+                {"name": _name, "requirements": _requirements},
                 silent=silent,
-                interactive=interactive)
+                interactive=interactive,
+            )
             requirements = _requirements
-        pkg_dir = BasePkgManager.install(self,
-                                         name,
-                                         requirements,
-                                         silent=silent,
-                                         after_update=after_update,
-                                         force=force)
+        pkg_dir = BasePkgManager.install(
+            self,
+            name,
+            requirements,
+            silent=silent,
+            after_update=after_update,
+            force=force,
+        )
 
         if not pkg_dir:
             return None
@@ -369,7 +312,7 @@ class LibraryManager(BasePkgManager):
             click.secho("Installing dependencies", fg="yellow")
 
         builtin_lib_storages = None
-        for filters in self.normalize_dependencies(manifest['dependencies']):
+        for filters in self.normalize_dependencies(manifest["dependencies"]):
             assert "name" in filters
 
             # avoid circle dependencies
@@ -381,35 +324,42 @@ class LibraryManager(BasePkgManager):
             self.INSTALL_HISTORY.append(history_key)
 
             if any(s in filters.get("version", "") for s in ("\\", "/")):
-                self.install("{name}={version}".format(**filters),
-                             silent=silent,
-                             after_update=after_update,
-                             interactive=interactive,
-                             force=force)
+                self.install(
+                    "{name}={version}".format(**filters),
+                    silent=silent,
+                    after_update=after_update,
+                    interactive=interactive,
+                    force=force,
+                )
             else:
                 try:
                     lib_id = self.search_lib_id(filters, silent, interactive)
                 except exception.LibNotFound as e:
                     if builtin_lib_storages is None:
                         builtin_lib_storages = get_builtin_libs()
-                    if not silent or is_builtin_lib(builtin_lib_storages,
-                                                    filters['name']):
+                    if not silent or is_builtin_lib(
+                        builtin_lib_storages, filters["name"]
+                    ):
                         click.secho("Warning! %s" % e, fg="yellow")
                     continue
 
                 if filters.get("version"):
-                    self.install(lib_id,
-                                 filters.get("version"),
-                                 silent=silent,
-                                 after_update=after_update,
-                                 interactive=interactive,
-                                 force=force)
+                    self.install(
+                        lib_id,
+                        filters.get("version"),
+                        silent=silent,
+                        after_update=after_update,
+                        interactive=interactive,
+                        force=force,
+                    )
                 else:
-                    self.install(lib_id,
-                                 silent=silent,
-                                 after_update=after_update,
-                                 interactive=interactive,
-                                 force=force)
+                    self.install(
+                        lib_id,
+                        silent=silent,
+                        after_update=after_update,
+                        interactive=interactive,
+                        force=force,
+                    )
         return pkg_dir
 
 
@@ -418,21 +368,23 @@ def get_builtin_libs(storage_names=None):
     storage_names = storage_names or []
     pm = PlatformManager()
     for manifest in pm.get_installed():
-        p = PlatformFactory.newPlatform(manifest['__pkg_dir'])
+        p = PlatformFactory.newPlatform(manifest["__pkg_dir"])
         for storage in p.get_lib_storages():
-            if storage_names and storage['name'] not in storage_names:
+            if storage_names and storage["name"] not in storage_names:
                 continue
-            lm = LibraryManager(storage['path'])
-            items.append({
-                "name": storage['name'],
-                "path": storage['path'],
-                "items": lm.get_installed()
-            })
+            lm = LibraryManager(storage["path"])
+            items.append(
+                {
+                    "name": storage["name"],
+                    "path": storage["path"],
+                    "items": lm.get_installed(),
+                }
+            )
     return items
 
 
 def is_builtin_lib(storages, name):
     for storage in storages or []:
-        if any(l.get("name") == name for l in storage['items']):
+        if any(l.get("name") == name for l in storage["items"]):
             return True
     return False

platformio/managers/package.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import codecs
 import hashlib
 import json
 import os
@@ -29,6 +28,10 @@ from platformio import __version__, app, exception, fs, telemetry, util
 from platformio.compat import hashlib_encode_data
 from platformio.downloader import FileDownloader
 from platformio.lockfile import LockFile
+from platformio.package.manifest.parser import (
+    ManifestParserError,
+    ManifestParserFactory,
+)
 from platformio.unpacker import FileUnpacker
 from platformio.vcsclient import VCSClientFactory
 
@@ -36,7 +39,6 @@ from platformio.vcsclient import VCSClientFactory
 
 
 class PackageRepoIterator(object):
-
     def __init__(self, package, repositories):
         assert isinstance(repositories, list)
         self.package = package
@@ -77,7 +79,7 @@ class PkgRepoMixin(object):
 
     @staticmethod
     def is_system_compatible(valid_systems):
-        if valid_systems in (None, "all", "*"):
+        if not valid_systems or "*" in valid_systems:
             return True
         if not isinstance(valid_systems, list):
             valid_systems = list([valid_systems])
@@ -87,8 +89,9 @@ class PkgRepoMixin(object):
         item = None
         reqspec = None
         try:
-            reqspec = semantic_version.SimpleSpec(
-                requirements) if requirements else None
+            reqspec = (
+                semantic_version.SimpleSpec(requirements) if requirements else None
+            )
         except ValueError:
             pass
 
@@ -99,33 +102,32 @@ class PkgRepoMixin(object):
             #     if PkgRepoMixin.PIO_VERSION not in requirements.SimpleSpec(
             #             v['engines']['platformio']):
             #         continue
-            specver = semantic_version.Version(v['version'])
+            specver = semantic_version.Version(v["version"])
             if reqspec and specver not in reqspec:
                 continue
-            if not item or semantic_version.Version(item['version']) < specver:
+            if not item or semantic_version.Version(item["version"]) < specver:
                 item = v
         return item
 
     def get_latest_repo_version(  # pylint: disable=unused-argument
-            self,
-            name,
-            requirements,
-            silent=False):
+        self, name, requirements, silent=False
+    ):
         version = None
         for versions in PackageRepoIterator(name, self.repositories):
             pkgdata = self.max_satisfying_repo_version(versions, requirements)
             if not pkgdata:
                 continue
-            if not version or semantic_version.compare(pkgdata['version'],
-                                                       version) == 1:
-                version = pkgdata['version']
+            if (
+                not version
+                or semantic_version.compare(pkgdata["version"], version) == 1
+            ):
+                version = pkgdata["version"]
         return version
 
     def get_all_repo_versions(self, name):
         result = []
         for versions in PackageRepoIterator(name, self.repositories):
-            result.extend(
-                [semantic_version.Version(v['version']) for v in versions])
+            result.extend([semantic_version.Version(v["version"]) for v in versions])
         return [str(v) for v in sorted(set(result))]
 
 
@@ -154,7 +156,8 @@ class PkgInstallerMixin(object):
         if result:
             return result
         result = [
-            join(src_dir, name) for name in sorted(os.listdir(src_dir))
+            join(src_dir, name)
+            for name in sorted(os.listdir(src_dir))
             if isdir(join(src_dir, name))
         ]
         self.cache_set(cache_key, result)
@@ -189,14 +192,17 @@ class PkgInstallerMixin(object):
                 click.secho(
                     "Error: Please read http://bit.ly/package-manager-ioerror",
                     fg="red",
-                    err=True)
+                    err=True,
+                )
                 raise e
 
         if sha1:
             fd.verify(sha1)
         dst_path = fd.get_filepath()
-        if not self.FILE_CACHE_VALID or getsize(
-                dst_path) > PkgInstallerMixin.FILE_CACHE_MAX_SIZE:
+        if (
+            not self.FILE_CACHE_VALID
+            or getsize(dst_path) > PkgInstallerMixin.FILE_CACHE_MAX_SIZE
+        ):
             return dst_path
 
         with app.ContentCache() as cc:
@@ -232,15 +238,15 @@ class PkgInstallerMixin(object):
         return None
 
     @staticmethod
-    def parse_pkg_uri(  # pylint: disable=too-many-branches
-            text, requirements=None):
+    def parse_pkg_uri(text, requirements=None):  # pylint: disable=too-many-branches
         text = str(text)
         name, url = None, None
 
         # Parse requirements
         req_conditions = [
-            "@" in text, not requirements, ":" not in text
-            or text.rfind("/") < text.rfind("@")
+            "@" in text,
+            not requirements,
+            ":" not in text or text.rfind("/") < text.rfind("@"),
         ]
         if all(req_conditions):
             text, requirements = text.rsplit("@", 1)
@@ -259,17 +265,16 @@ class PkgInstallerMixin(object):
         elif "/" in text or "\\" in text:
             git_conditions = [
                 # Handle GitHub URL (https://github.com/user/package)
-                text.startswith("https://github.com/") and not text.endswith(
-                    (".zip", ".tar.gz")),
-                (text.split("#", 1)[0]
-                 if "#" in text else text).endswith(".git")
+                text.startswith("https://github.com/")
+                and not text.endswith((".zip", ".tar.gz")),
+                (text.split("#", 1)[0] if "#" in text else text).endswith(".git"),
             ]
             hg_conditions = [
                 # Handle Developer Mbed URL
                 # (https://developer.mbed.org/users/user/code/package/)
                 # (https://os.mbed.com/users/user/code/package/)
                 text.startswith("https://developer.mbed.org"),
-                text.startswith("https://os.mbed.com")
+                text.startswith("https://os.mbed.com"),
             ]
             if any(git_conditions):
                 url = "git+" + text
@@ -296,9 +301,9 @@ class PkgInstallerMixin(object):
 
     @staticmethod
     def get_install_dirname(manifest):
-        name = re.sub(r"[^\da-z\_\-\. ]", "_", manifest['name'], flags=re.I)
+        name = re.sub(r"[^\da-z\_\-\. ]", "_", manifest["name"], flags=re.I)
         if "id" in manifest:
-            name += "_ID%d" % manifest['id']
+            name += "_ID%d" % manifest["id"]
         return str(name)
 
     @classmethod
@@ -322,10 +327,9 @@ class PkgInstallerMixin(object):
         return None
 
     def manifest_exists(self, pkg_dir):
-        return self.get_manifest_path(pkg_dir) or \
-            self.get_src_manifest_path(pkg_dir)
+        return self.get_manifest_path(pkg_dir) or self.get_src_manifest_path(pkg_dir)
 
-    def load_manifest(self, pkg_dir):
+    def load_manifest(self, pkg_dir):  # pylint: disable=too-many-branches
         cache_key = "load_manifest-%s" % pkg_dir
         result = self.cache_get(cache_key)
         if result:
@@ -341,31 +345,26 @@ class PkgInstallerMixin(object):
         if not manifest_path and not src_manifest_path:
             return None
 
-        if manifest_path and manifest_path.endswith(".json"):
-            manifest = fs.load_json(manifest_path)
-        elif manifest_path and manifest_path.endswith(".properties"):
-            with codecs.open(manifest_path, encoding="utf-8") as fp:
-                for line in fp.readlines():
-                    if "=" not in line:
-                        continue
-                    key, value = line.split("=", 1)
-                    manifest[key.strip()] = value.strip()
+        try:
+            manifest = ManifestParserFactory.new_from_file(manifest_path).as_dict()
+        except ManifestParserError:
+            pass
 
         if src_manifest:
             if "version" in src_manifest:
-                manifest['version'] = src_manifest['version']
-            manifest['__src_url'] = src_manifest['url']
+                manifest["version"] = src_manifest["version"]
+            manifest["__src_url"] = src_manifest["url"]
             # handle a custom package name
-            autogen_name = self.parse_pkg_uri(manifest['__src_url'])[0]
-            if "name" not in manifest or autogen_name != src_manifest['name']:
-                manifest['name'] = src_manifest['name']
+            autogen_name = self.parse_pkg_uri(manifest["__src_url"])[0]
+            if "name" not in manifest or autogen_name != src_manifest["name"]:
+                manifest["name"] = src_manifest["name"]
 
         if "name" not in manifest:
-            manifest['name'] = basename(pkg_dir)
+            manifest["name"] = basename(pkg_dir)
         if "version" not in manifest:
-            manifest['version'] = "0.0.0"
+            manifest["version"] = "0.0.0"
 
-        manifest['__pkg_dir'] = pkg_dir
+        manifest["__pkg_dir"] = pkg_dir
         self.cache_set(cache_key, manifest)
         return manifest
 
@@ -390,25 +389,24 @@ class PkgInstallerMixin(object):
                     continue
             elif pkg_id and manifest.get("id") != pkg_id:
                 continue
-            elif not pkg_id and manifest['name'] != name:
+            elif not pkg_id and manifest["name"] != name:
                 continue
             elif not PkgRepoMixin.is_system_compatible(manifest.get("system")):
                 continue
 
             # strict version or VCS HASH
-            if requirements and requirements == manifest['version']:
+            if requirements and requirements == manifest["version"]:
                 return manifest
 
             try:
-                if requirements and not semantic_version.SimpleSpec(
-                        requirements).match(
-                            self.parse_semver_version(manifest['version'],
-                                                      raise_exception=True)):
+                if requirements and not semantic_version.SimpleSpec(requirements).match(
+                    self.parse_semver_version(manifest["version"], raise_exception=True)
+                ):
                     continue
-                elif not best or (self.parse_semver_version(
-                        manifest['version'], raise_exception=True) >
-                                  self.parse_semver_version(
-                                      best['version'], raise_exception=True)):
+                if not best or (
+                    self.parse_semver_version(manifest["version"], raise_exception=True)
+                    > self.parse_semver_version(best["version"], raise_exception=True)
+                ):
                     best = manifest
             except ValueError:
                 pass
@@ -417,12 +415,15 @@ class PkgInstallerMixin(object):
 
     def get_package_dir(self, name, requirements=None, url=None):
         manifest = self.get_package(name, requirements, url)
-        return manifest.get("__pkg_dir") if manifest and isdir(
-            manifest.get("__pkg_dir")) else None
+        return (
+            manifest.get("__pkg_dir")
+            if manifest and isdir(manifest.get("__pkg_dir"))
+            else None
+        )
 
     def get_package_by_dir(self, pkg_dir):
         for manifest in self.get_installed():
-            if manifest['__pkg_dir'] == abspath(pkg_dir):
+            if manifest["__pkg_dir"] == abspath(pkg_dir):
                 return manifest
         return None
 
@@ -443,9 +444,9 @@ class PkgInstallerMixin(object):
             if not pkgdata:
                 continue
             try:
-                pkg_dir = self._install_from_url(name, pkgdata['url'],
-                                                 requirements,
-                                                 pkgdata.get("sha1"))
+                pkg_dir = self._install_from_url(
+                    name, pkgdata["url"], requirements, pkgdata.get("sha1")
+                )
                 break
             except Exception as e:  # pylint: disable=broad-except
                 click.secho("Warning! Package Mirror: %s" % e, fg="yellow")
@@ -455,16 +456,12 @@ class PkgInstallerMixin(object):
             util.internet_on(raise_exception=True)
             raise exception.UnknownPackage(name)
         if not pkgdata:
-            raise exception.UndefinedPackageVersion(requirements or "latest",
-                                                    util.get_systype())
+            raise exception.UndefinedPackageVersion(
+                requirements or "latest", util.get_systype()
+            )
         return pkg_dir
 
-    def _install_from_url(self,
-                          name,
-                          url,
-                          requirements=None,
-                          sha1=None,
-                          track=False):
+    def _install_from_url(self, name, url, requirements=None, sha1=None, track=False):
         tmp_dir = mkdtemp("-package", self.TMP_FOLDER_PREFIX, self.package_dir)
         src_manifest_dir = None
         src_manifest = {"name": name, "url": url, "requirements": requirements}
@@ -486,7 +483,7 @@ class PkgInstallerMixin(object):
                 vcs = VCSClientFactory.newClient(tmp_dir, url)
                 assert vcs.export()
                 src_manifest_dir = vcs.storage_dir
-                src_manifest['version'] = vcs.get_current_revision()
+                src_manifest["version"] = vcs.get_current_revision()
 
             _tmp_dir = tmp_dir
             if not src_manifest_dir:
@@ -515,7 +512,8 @@ class PkgInstallerMixin(object):
             json.dump(_data, fp)
 
     def _install_from_tmp_dir(  # pylint: disable=too-many-branches
-            self, tmp_dir, requirements=None):
+        self, tmp_dir, requirements=None
+    ):
         tmp_manifest = self.load_manifest(tmp_dir)
         assert set(["name", "version"]) <= set(tmp_manifest)
 
@@ -523,28 +521,30 @@ class PkgInstallerMixin(object):
         pkg_dir = join(self.package_dir, pkg_dirname)
         cur_manifest = self.load_manifest(pkg_dir)
 
-        tmp_semver = self.parse_semver_version(tmp_manifest['version'])
+        tmp_semver = self.parse_semver_version(tmp_manifest["version"])
         cur_semver = None
         if cur_manifest:
-            cur_semver = self.parse_semver_version(cur_manifest['version'])
+            cur_semver = self.parse_semver_version(cur_manifest["version"])
 
         # package should satisfy requirements
         if requirements:
-            mismatch_error = (
-                "Package version %s doesn't satisfy requirements %s" %
-                (tmp_manifest['version'], requirements))
+            mismatch_error = "Package version %s doesn't satisfy requirements %s" % (
+                tmp_manifest["version"],
+                requirements,
+            )
             try:
                 assert tmp_semver and tmp_semver in semantic_version.SimpleSpec(
-                    requirements), mismatch_error
+                    requirements
+                ), mismatch_error
             except (AssertionError, ValueError):
-                assert tmp_manifest['version'] == requirements, mismatch_error
+                assert tmp_manifest["version"] == requirements, mismatch_error
 
         # check if package already exists
         if cur_manifest:
             # 0-overwrite, 1-rename, 2-fix to a version
             action = 0
             if "__src_url" in cur_manifest:
-                if cur_manifest['__src_url'] != tmp_manifest.get("__src_url"):
+                if cur_manifest["__src_url"] != tmp_manifest.get("__src_url"):
                     action = 1
             elif "__src_url" in tmp_manifest:
                 action = 2
@@ -556,25 +556,25 @@ class PkgInstallerMixin(object):
 
             # rename
             if action == 1:
-                target_dirname = "%s@%s" % (pkg_dirname,
-                                            cur_manifest['version'])
+                target_dirname = "%s@%s" % (pkg_dirname, cur_manifest["version"])
                 if "__src_url" in cur_manifest:
                     target_dirname = "%s@src-%s" % (
                         pkg_dirname,
                         hashlib.md5(
-                            hashlib_encode_data(
-                                cur_manifest['__src_url'])).hexdigest())
+                            hashlib_encode_data(cur_manifest["__src_url"])
+                        ).hexdigest(),
+                    )
                 shutil.move(pkg_dir, join(self.package_dir, target_dirname))
             # fix to a version
             elif action == 2:
-                target_dirname = "%s@%s" % (pkg_dirname,
-                                            tmp_manifest['version'])
+                target_dirname = "%s@%s" % (pkg_dirname, tmp_manifest["version"])
                 if "__src_url" in tmp_manifest:
                     target_dirname = "%s@src-%s" % (
                         pkg_dirname,
                         hashlib.md5(
-                            hashlib_encode_data(
-                                tmp_manifest['__src_url'])).hexdigest())
+                            hashlib_encode_data(tmp_manifest["__src_url"])
+                        ).hexdigest(),
+                    )
                 pkg_dir = join(self.package_dir, target_dirname)
 
         # remove previous/not-satisfied package
@@ -622,9 +622,9 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
 
         if "__src_url" in manifest:
             try:
-                vcs = VCSClientFactory.newClient(pkg_dir,
-                                                 manifest['__src_url'],
-                                                 silent=True)
+                vcs = VCSClientFactory.newClient(
+                    pkg_dir, manifest["__src_url"], silent=True
+                )
             except (AttributeError, exception.PlatformioException):
                 return None
             if not vcs.can_be_updated:
@@ -633,10 +633,10 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
         else:
             try:
                 latest = self.get_latest_repo_version(
-                    "id=%d" %
-                    manifest['id'] if "id" in manifest else manifest['name'],
+                    "id=%d" % manifest["id"] if "id" in manifest else manifest["name"],
                     requirements,
-                    silent=True)
+                    silent=True,
+                )
             except (exception.PlatformioException, ValueError):
                 return None
 
@@ -646,21 +646,17 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
         up_to_date = False
         try:
             assert "__src_url" not in manifest
-            up_to_date = (self.parse_semver_version(manifest['version'],
-                                                    raise_exception=True) >=
-                          self.parse_semver_version(latest,
-                                                    raise_exception=True))
+            up_to_date = self.parse_semver_version(
+                manifest["version"], raise_exception=True
+            ) >= self.parse_semver_version(latest, raise_exception=True)
         except (AssertionError, ValueError):
-            up_to_date = latest == manifest['version']
+            up_to_date = latest == manifest["version"]
 
         return False if up_to_date else latest
 
-    def install(self,
-                name,
-                requirements=None,
-                silent=False,
-                after_update=False,
-                force=False):
+    def install(
+        self, name, requirements=None, silent=False, after_update=False, force=False
+    ):
         pkg_dir = None
         # interprocess lock
         with LockFile(self.package_dir):
@@ -690,34 +686,38 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
                 if not silent:
                     click.secho(
                         "{name} @ {version} is already installed".format(
-                            **self.load_manifest(package_dir)),
-                        fg="yellow")
+                            **self.load_manifest(package_dir)
+                        ),
+                        fg="yellow",
+                    )
                 return package_dir
 
             if url:
-                pkg_dir = self._install_from_url(name,
-                                                 url,
-                                                 requirements,
-                                                 track=True)
+                pkg_dir = self._install_from_url(name, url, requirements, track=True)
             else:
                 pkg_dir = self._install_from_piorepo(name, requirements)
 
             if not pkg_dir or not self.manifest_exists(pkg_dir):
-                raise exception.PackageInstallError(name, requirements or "*",
-                                                    util.get_systype())
+                raise exception.PackageInstallError(
+                    name, requirements or "*", util.get_systype()
+                )
 
             manifest = self.load_manifest(pkg_dir)
             assert manifest
 
             if not after_update:
-                telemetry.on_event(category=self.__class__.__name__,
-                                   action="Install",
-                                   label=manifest['name'])
+                telemetry.on_event(
+                    category=self.__class__.__name__,
+                    action="Install",
+                    label=manifest["name"],
+                )
 
             click.secho(
                 "{name} @ {version} has been successfully installed!".format(
-                    **manifest),
-                fg="green")
+                    **manifest
+                ),
+                fg="green",
+            )
 
         return pkg_dir
 
@@ -729,18 +729,20 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
             if isdir(package) and self.get_package_by_dir(package):
                 pkg_dir = package
             else:
-                name, requirements, url = self.parse_pkg_uri(
-                    package, requirements)
+                name, requirements, url = self.parse_pkg_uri(package, requirements)
                 pkg_dir = self.get_package_dir(name, requirements, url)
 
             if not pkg_dir:
-                raise exception.UnknownPackage("%s @ %s" %
-                                               (package, requirements or "*"))
+                raise exception.UnknownPackage(
+                    "%s @ %s" % (package, requirements or "*")
+                )
 
             manifest = self.load_manifest(pkg_dir)
-            click.echo("Uninstalling %s @ %s: \t" % (click.style(
-                manifest['name'], fg="cyan"), manifest['version']),
-                       nl=False)
+            click.echo(
+                "Uninstalling %s @ %s: \t"
+                % (click.style(manifest["name"], fg="cyan"), manifest["version"]),
+                nl=False,
+            )
 
             if islink(pkg_dir):
                 os.unlink(pkg_dir)
@@ -749,19 +751,21 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
             self.cache_reset()
 
             # unfix package with the same name
-            pkg_dir = self.get_package_dir(manifest['name'])
+            pkg_dir = self.get_package_dir(manifest["name"])
             if pkg_dir and "@" in pkg_dir:
                 shutil.move(
-                    pkg_dir,
-                    join(self.package_dir, self.get_install_dirname(manifest)))
+                    pkg_dir, join(self.package_dir, self.get_install_dirname(manifest))
+                )
                 self.cache_reset()
 
             click.echo("[%s]" % click.style("OK", fg="green"))
 
             if not after_update:
-                telemetry.on_event(category=self.__class__.__name__,
-                                   action="Uninstall",
-                                   label=manifest['name'])
+                telemetry.on_event(
+                    category=self.__class__.__name__,
+                    action="Uninstall",
+                    label=manifest["name"],
+                )
 
         return True
 
@@ -773,16 +777,19 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
             pkg_dir = self.get_package_dir(*self.parse_pkg_uri(package))
 
         if not pkg_dir:
-            raise exception.UnknownPackage("%s @ %s" %
-                                           (package, requirements or "*"))
+            raise exception.UnknownPackage("%s @ %s" % (package, requirements or "*"))
 
         manifest = self.load_manifest(pkg_dir)
-        name = manifest['name']
+        name = manifest["name"]
 
-        click.echo("{} {:<40} @ {:<15}".format(
-            "Checking" if only_check else "Updating",
-            click.style(manifest['name'], fg="cyan"), manifest['version']),
-                   nl=False)
+        click.echo(
+            "{} {:<40} @ {:<15}".format(
+                "Checking" if only_check else "Updating",
+                click.style(manifest["name"], fg="cyan"),
+                manifest["version"],
+            ),
+            nl=False,
+        )
         if not util.internet_on():
             click.echo("[%s]" % (click.style("Off-line", fg="yellow")))
             return None
@@ -799,22 +806,22 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
             return True
 
         if "__src_url" in manifest:
-            vcs = VCSClientFactory.newClient(pkg_dir, manifest['__src_url'])
+            vcs = VCSClientFactory.newClient(pkg_dir, manifest["__src_url"])
             assert vcs.update()
-            self._update_src_manifest(dict(version=vcs.get_current_revision()),
-                                      vcs.storage_dir)
+            self._update_src_manifest(
+                dict(version=vcs.get_current_revision()), vcs.storage_dir
+            )
         else:
             self.uninstall(pkg_dir, after_update=True)
             self.install(name, latest, after_update=True)
 
-        telemetry.on_event(category=self.__class__.__name__,
-                           action="Update",
-                           label=manifest['name'])
+        telemetry.on_event(
+            category=self.__class__.__name__, action="Update", label=manifest["name"]
+        )
         return True
 
 
 class PackageManager(BasePkgManager):
-
     @property
     def manifest_names(self):
         return ["package.json"]

platformio/managers/platform.py

@@ -12,27 +12,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# pylint: disable=too-many-public-methods, too-many-instance-attributes
+
 import base64
 import os
 import re
 import sys
-from imp import load_source
 from os.path import basename, dirname, isdir, isfile, join
 
 import click
 import semantic_version
 
-from platformio import __version__, app, exception, fs, util
-from platformio.compat import PY2, hashlib_encode_data, is_bytes
+from platformio import __version__, app, exception, fs, proc, util
+from platformio.compat import PY2, hashlib_encode_data, is_bytes, load_python_module
 from platformio.managers.core import get_core_package_dir
 from platformio.managers.package import BasePkgManager, PackageManager
-from platformio.proc import (BuildAsyncPipe, copy_pythonpath_to_osenv,
-                             exec_command, get_pythonexe_path)
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import (get_project_boards_dir,
-                                        get_project_core_dir,
-                                        get_project_packages_dir,
-                                        get_project_platforms_dir)
 
 try:
     from urllib.parse import quote
@@ -41,16 +36,18 @@ except ImportError:
 
 
 class PlatformManager(BasePkgManager):
-
     def __init__(self, package_dir=None, repositories=None):
         if not repositories:
             repositories = [
                 "https://dl.bintray.com/platformio/dl-platforms/manifest.json",
                 "{0}://dl.platformio.org/platforms/manifest.json".format(
-                    "https" if app.get_setting("strict_ssl") else "http")
+                    "https" if app.get_setting("strict_ssl") else "http"
+                ),
             ]
-        BasePkgManager.__init__(self, package_dir
-                                or get_project_platforms_dir(), repositories)
+        self.config = ProjectConfig.get_instance()
+        BasePkgManager.__init__(
+            self, package_dir or self.config.get_optional_dir("platforms"), repositories
+        )
 
     @property
     def manifest_names(self):
@@ -65,21 +62,21 @@ class PlatformManager(BasePkgManager):
                 return manifest_path
         return None
 
-    def install(self,
-                name,
-                requirements=None,
-                with_packages=None,
-                without_packages=None,
-                skip_default_package=False,
-                after_update=False,
-                silent=False,
-                force=False,
-                **_):  # pylint: disable=too-many-arguments, arguments-differ
-        platform_dir = BasePkgManager.install(self,
-                                              name,
-                                              requirements,
-                                              silent=silent,
-                                              force=force)
+    def install(
+        self,
+        name,
+        requirements=None,
+        with_packages=None,
+        without_packages=None,
+        skip_default_package=False,
+        after_update=False,
+        silent=False,
+        force=False,
+        **_
+    ):  # pylint: disable=too-many-arguments, arguments-differ
+        platform_dir = BasePkgManager.install(
+            self, name, requirements, silent=silent, force=force
+        )
         p = PlatformFactory.newPlatform(platform_dir)
 
         # don't cleanup packages or install them after update
@@ -87,11 +84,13 @@ class PlatformManager(BasePkgManager):
         if after_update:
             return True
 
-        p.install_packages(with_packages,
-                           without_packages,
-                           skip_default_package,
-                           silent=silent,
-                           force=force)
+        p.install_packages(
+            with_packages,
+            without_packages,
+            skip_default_package,
+            silent=silent,
+            force=force,
+        )
         return self.cleanup_packages(list(p.packages))
 
     def uninstall(self, package, requirements=None, after_update=False):
@@ -115,11 +114,8 @@ class PlatformManager(BasePkgManager):
         return self.cleanup_packages(list(p.packages))
 
     def update(  # pylint: disable=arguments-differ
-            self,
-            package,
-            requirements=None,
-            only_check=False,
-            only_packages=False):
+        self, package, requirements=None, only_check=False, only_packages=False
+    ):
         if isdir(package):
             pkg_dir = package
         else:
@@ -143,8 +139,9 @@ class PlatformManager(BasePkgManager):
         self.cleanup_packages(list(p.packages))
 
         if missed_pkgs:
-            p.install_packages(with_packages=list(missed_pkgs),
-                               skip_default_package=True)
+            p.install_packages(
+                with_packages=list(missed_pkgs), skip_default_package=True
+            )
 
         return True
 
@@ -152,20 +149,22 @@ class PlatformManager(BasePkgManager):
         self.cache_reset()
         deppkgs = {}
         for manifest in PlatformManager().get_installed():
-            p = PlatformFactory.newPlatform(manifest['__pkg_dir'])
+            p = PlatformFactory.newPlatform(manifest["__pkg_dir"])
             for pkgname, pkgmanifest in p.get_installed_packages().items():
                 if pkgname not in deppkgs:
                     deppkgs[pkgname] = set()
-                deppkgs[pkgname].add(pkgmanifest['version'])
+                deppkgs[pkgname].add(pkgmanifest["version"])
 
-        pm = PackageManager(get_project_packages_dir())
+        pm = PackageManager(self.config.get_optional_dir("packages"))
         for manifest in pm.get_installed():
-            if manifest['name'] not in names:
+            if manifest["name"] not in names:
                 continue
-            if (manifest['name'] not in deppkgs
-                    or manifest['version'] not in deppkgs[manifest['name']]):
+            if (
+                manifest["name"] not in deppkgs
+                or manifest["version"] not in deppkgs[manifest["name"]]
+            ):
                 try:
-                    pm.uninstall(manifest['__pkg_dir'], after_update=True)
+                    pm.uninstall(manifest["__pkg_dir"], after_update=True)
                 except exception.UnknownPackage:
                     pass
 
@@ -176,7 +175,7 @@ class PlatformManager(BasePkgManager):
     def get_installed_boards(self):
         boards = []
         for manifest in self.get_installed():
-            p = PlatformFactory.newPlatform(manifest['__pkg_dir'])
+            p = PlatformFactory.newPlatform(manifest["__pkg_dir"])
             for config in p.get_boards().values():
                 board = config.get_brief_data()
                 if board not in boards:
@@ -189,30 +188,31 @@ class PlatformManager(BasePkgManager):
 
     def get_all_boards(self):
         boards = self.get_installed_boards()
-        know_boards = ["%s:%s" % (b['platform'], b['id']) for b in boards]
+        know_boards = ["%s:%s" % (b["platform"], b["id"]) for b in boards]
         try:
             for board in self.get_registered_boards():
-                key = "%s:%s" % (board['platform'], board['id'])
+                key = "%s:%s" % (board["platform"], board["id"])
                 if key not in know_boards:
                     boards.append(board)
         except (exception.APIRequestError, exception.InternetIsOffline):
             pass
-        return sorted(boards, key=lambda b: b['name'])
+        return sorted(boards, key=lambda b: b["name"])
 
     def board_config(self, id_, platform=None):
         for manifest in self.get_installed_boards():
-            if manifest['id'] == id_ and (not platform
-                                          or manifest['platform'] == platform):
+            if manifest["id"] == id_ and (
+                not platform or manifest["platform"] == platform
+            ):
                 return manifest
         for manifest in self.get_registered_boards():
-            if manifest['id'] == id_ and (not platform
-                                          or manifest['platform'] == platform):
+            if manifest["id"] == id_ and (
+                not platform or manifest["platform"] == platform
+            ):
                 return manifest
         raise exception.UnknownBoard(id_)
 
 
 class PlatformFactory(object):
-
     @staticmethod
     def get_clsname(name):
         name = re.sub(r"[^\da-z\_]+", "", name, flags=re.I)
@@ -220,13 +220,10 @@ class PlatformFactory(object):
 
     @staticmethod
     def load_module(name, path):
-        module = None
         try:
-            module = load_source("platformio.managers.platform.%s" % name,
-                                 path)
+            return load_python_module("platformio.managers.platform.%s" % name, path)
         except ImportError:
             raise exception.UnknownPlatform(name)
-        return module
 
     @classmethod
     def newPlatform(cls, name, requirements=None):
@@ -234,28 +231,29 @@ class PlatformFactory(object):
         platform_dir = None
         if isdir(name):
             platform_dir = name
-            name = pm.load_manifest(platform_dir)['name']
+            name = pm.load_manifest(platform_dir)["name"]
         elif name.endswith("platform.json") and isfile(name):
             platform_dir = dirname(name)
-            name = fs.load_json(name)['name']
+            name = fs.load_json(name)["name"]
         else:
             name, requirements, url = pm.parse_pkg_uri(name, requirements)
             platform_dir = pm.get_package_dir(name, requirements, url)
             if platform_dir:
-                name = pm.load_manifest(platform_dir)['name']
+                name = pm.load_manifest(platform_dir)["name"]
 
         if not platform_dir:
             raise exception.UnknownPlatform(
-                name if not requirements else "%s@%s" % (name, requirements))
+                name if not requirements else "%s@%s" % (name, requirements)
+            )
 
         platform_cls = None
         if isfile(join(platform_dir, "platform.py")):
             platform_cls = getattr(
                 cls.load_module(name, join(platform_dir, "platform.py")),
-                cls.get_clsname(name))
+                cls.get_clsname(name),
+            )
         else:
-            platform_cls = type(str(cls.get_clsname(name)), (PlatformBase, ),
-                                {})
+            platform_cls = type(str(cls.get_clsname(name)), (PlatformBase,), {})
 
         _instance = platform_cls(join(platform_dir, "platform.json"))
         assert isinstance(_instance, PlatformBase)
@@ -263,14 +261,14 @@ class PlatformFactory(object):
 
 
 class PlatformPackagesMixin(object):
-
     def install_packages(  # pylint: disable=too-many-arguments
-            self,
-            with_packages=None,
-            without_packages=None,
-            skip_default_package=False,
-            silent=False,
-            force=False):
+        self,
+        with_packages=None,
+        without_packages=None,
+        skip_default_package=False,
+        silent=False,
+        force=False,
+    ):
         with_packages = set(self.find_pkg_names(with_packages or []))
         without_packages = set(self.find_pkg_names(without_packages or []))
 
@@ -283,12 +281,13 @@ class PlatformPackagesMixin(object):
             version = opts.get("version", "")
             if name in without_packages:
                 continue
-            elif (name in with_packages or
-                  not (skip_default_package or opts.get("optional", False))):
+            if name in with_packages or not (
+                skip_default_package or opts.get("optional", False)
+            ):
                 if ":" in version:
-                    self.pm.install("%s=%s" % (name, version),
-                                    silent=silent,
-                                    force=force)
+                    self.pm.install(
+                        "%s=%s" % (name, version), silent=silent, force=force
+                    )
                 else:
                     self.pm.install(name, version, silent=silent, force=force)
 
@@ -305,9 +304,12 @@ class PlatformPackagesMixin(object):
                     result.append(_name)
                     found = True
 
-            if (self.frameworks and candidate.startswith("framework-")
-                    and candidate[10:] in self.frameworks):
-                result.append(self.frameworks[candidate[10:]]['package'])
+            if (
+                self.frameworks
+                and candidate.startswith("framework-")
+                and candidate[10:] in self.frameworks
+            ):
+                result.append(self.frameworks[candidate[10:]]["package"])
                 found = True
 
             if not found:
@@ -320,7 +322,7 @@ class PlatformPackagesMixin(object):
             requirements = self.packages[name].get("version", "")
             if ":" in requirements:
                 _, requirements, __ = self.pm.parse_pkg_uri(requirements)
-            self.pm.update(manifest['__pkg_dir'], requirements, only_check)
+            self.pm.update(manifest["__pkg_dir"], requirements, only_check)
 
     def get_installed_packages(self):
         items = {}
@@ -335,7 +337,7 @@ class PlatformPackagesMixin(object):
             requirements = self.packages[name].get("version", "")
             if ":" in requirements:
                 _, requirements, __ = self.pm.parse_pkg_uri(requirements)
-            if self.pm.outdated(manifest['__pkg_dir'], requirements):
+            if self.pm.outdated(manifest["__pkg_dir"], requirements):
                 return True
         return False
 
@@ -343,7 +345,8 @@ class PlatformPackagesMixin(object):
         version = self.packages[name].get("version", "")
         if ":" in version:
             return self.pm.get_package_dir(
-                *self.pm.parse_pkg_uri("%s=%s" % (name, version)))
+                *self.pm.parse_pkg_uri("%s=%s" % (name, version))
+            )
         return self.pm.get_package_dir(name, version)
 
     def get_package_version(self, name):
@@ -368,15 +371,15 @@ class PlatformRunMixin(object):
         return value.decode() if is_bytes(value) else value
 
     def run(  # pylint: disable=too-many-arguments
-            self, variables, targets, silent, verbose, jobs):
+        self, variables, targets, silent, verbose, jobs
+    ):
         assert isinstance(variables, dict)
         assert isinstance(targets, list)
 
-        config = ProjectConfig.get_instance(variables['project_config'])
-        options = config.items(env=variables['pioenv'], as_dict=True)
+        options = self.config.items(env=variables["pioenv"], as_dict=True)
         if "framework" in options:
             # support PIO Core 3.0 dev/platforms
-            options['pioframework'] = options['framework']
+            options["pioframework"] = options["framework"]
         self.configure_default_packages(options, targets)
         self.install_packages(silent=True)
 
@@ -386,12 +389,12 @@ class PlatformRunMixin(object):
         if "clean" in targets:
             targets = ["-c", "."]
 
-        variables['platform_manifest'] = self.manifest_path
+        variables["platform_manifest"] = self.manifest_path
 
         if "build_script" not in variables:
-            variables['build_script'] = self.get_build_script()
-        if not isfile(variables['build_script']):
-            raise exception.BuildScriptNotFound(variables['build_script'])
+            variables["build_script"] = self.get_build_script()
+        if not isfile(variables["build_script"]):
+            raise exception.BuildScriptNotFound(variables["build_script"])
 
         result = self._run_scons(variables, targets, jobs)
         assert "returncode" in result
@@ -400,16 +403,18 @@ class PlatformRunMixin(object):
 
     def _run_scons(self, variables, targets, jobs):
         args = [
-            get_pythonexe_path(),
+            proc.get_pythonexe_path(),
             join(get_core_package_dir("tool-scons"), "script", "scons"),
-            "-Q", "--warn=no-no-parallel-support",
-            "--jobs", str(jobs),
-            "--sconstruct", join(fs.get_source_dir(), "builder", "main.py")
-        ]  # yapf: disable
+            "-Q",
+            "--warn=no-no-parallel-support",
+            "--jobs",
+            str(jobs),
+            "--sconstruct",
+            join(fs.get_source_dir(), "builder", "main.py"),
+        ]
         args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0))
         # pylint: disable=protected-access
-        args.append("ISATTY=%d" %
-                    (1 if click._compat.isatty(sys.stdout) else 0))
+        args.append("ISATTY=%d" % (1 if click._compat.isatty(sys.stdout) else 0))
         args += targets
 
         # encode and append variables
@@ -423,15 +428,25 @@ class PlatformRunMixin(object):
             except IOError:
                 pass
 
-        copy_pythonpath_to_osenv()
-        result = exec_command(
-            args,
-            stdout=BuildAsyncPipe(
-                line_callback=self._on_stdout_line,
-                data_callback=lambda data: _write_and_flush(sys.stdout, data)),
-            stderr=BuildAsyncPipe(
-                line_callback=self._on_stderr_line,
-                data_callback=lambda data: _write_and_flush(sys.stderr, data)))
+        proc.copy_pythonpath_to_osenv()
+        if click._compat.isatty(sys.stdout):
+            result = proc.exec_command(
+                args,
+                stdout=proc.BuildAsyncPipe(
+                    line_callback=self._on_stdout_line,
+                    data_callback=lambda data: _write_and_flush(sys.stdout, data),
+                ),
+                stderr=proc.BuildAsyncPipe(
+                    line_callback=self._on_stderr_line,
+                    data_callback=lambda data: _write_and_flush(sys.stderr, data),
+                ),
+            )
+        else:
+            result = proc.exec_command(
+                args,
+                stdout=proc.LineBufferedAsyncPipe(line_callback=self._on_stdout_line),
+                stderr=proc.LineBufferedAsyncPipe(line_callback=self._on_stderr_line),
+            )
         return result
 
     def _on_stdout_line(self, line):
@@ -447,7 +462,7 @@ class PlatformRunMixin(object):
         b_pos = line.rfind(": No such file or directory")
         if a_pos == -1 or b_pos == -1:
             return
-        self._echo_missed_dependency(line[a_pos + 12:b_pos].strip())
+        self._echo_missed_dependency(line[a_pos + 12 : b_pos].strip())
 
     def _echo_line(self, line, level):
         if line.startswith("scons: "):
@@ -472,18 +487,20 @@ class PlatformRunMixin(object):
 * Web  > {link}
 *
 {dots}
-""".format(filename=filename,
-           filename_styled=click.style(filename, fg="cyan"),
-           link=click.style(
-               "https://platformio.org/lib/search?query=header:%s" %
-               quote(filename, safe=""),
-               fg="blue"),
-           dots="*" * (56 + len(filename)))
+""".format(
+            filename=filename,
+            filename_styled=click.style(filename, fg="cyan"),
+            link=click.style(
+                "https://platformio.org/lib/search?query=header:%s"
+                % quote(filename, safe=""),
+                fg="blue",
+            ),
+            dots="*" * (56 + len(filename)),
+        )
         click.echo(banner, err=True)
 
 
-class PlatformBase(  # pylint: disable=too-many-public-methods
-        PlatformPackagesMixin, PlatformRunMixin):
+class PlatformBase(PlatformPackagesMixin, PlatformRunMixin):
 
     PIO_VERSION = semantic_version.Version(util.pepver_to_semver(__version__))
     _BOARDS_CACHE = {}
@@ -497,9 +514,10 @@ class PlatformBase(  # pylint: disable=too-many-public-methods
         self._manifest = fs.load_json(manifest_path)
         self._custom_packages = None
 
-        self.pm = PackageManager(get_project_packages_dir(),
-                                 self.package_repositories)
-
+        self.config = ProjectConfig.get_instance()
+        self.pm = PackageManager(
+            self.config.get_optional_dir("packages"), self.package_repositories
+        )
         # if self.engines and "platformio" in self.engines:
         #     if self.PIO_VERSION not in semantic_version.SimpleSpec(
         #             self.engines['platformio']):
@@ -508,19 +526,19 @@ class PlatformBase(  # pylint: disable=too-many-public-methods
 
     @property
     def name(self):
-        return self._manifest['name']
+        return self._manifest["name"]
 
     @property
     def title(self):
-        return self._manifest['title']
+        return self._manifest["title"]
 
     @property
     def description(self):
-        return self._manifest['description']
+        return self._manifest["description"]
 
     @property
     def version(self):
-        return self._manifest['version']
+        return self._manifest["version"]
 
     @property
     def homepage(self):
@@ -561,7 +579,7 @@ class PlatformBase(  # pylint: disable=too-many-public-methods
     @property
     def packages(self):
         packages = self._manifest.get("packages", {})
-        for item in (self._custom_packages or []):
+        for item in self._custom_packages or []:
             name = item
             version = "*"
             if "@" in item:
@@ -569,10 +587,7 @@ class PlatformBase(  # pylint: disable=too-many-public-methods
             name = name.strip()
             if name not in packages:
                 packages[name] = {}
-            packages[name].update({
-                "version": version.strip(),
-                "optional": False
-            })
+            packages[name].update({"version": version.strip(), "optional": False})
         return packages
 
     def get_dir(self):
@@ -591,20 +606,18 @@ class PlatformBase(  # pylint: disable=too-many-public-methods
         return False
 
     def get_boards(self, id_=None):
-
         def _append_board(board_id, manifest_path):
             config = PlatformBoardConfig(manifest_path)
             if "platform" in config and config.get("platform") != self.name:
                 return
-            if "platforms" in config \
-                    and self.name not in config.get("platforms"):
+            if "platforms" in config and self.name not in config.get("platforms"):
                 return
-            config.manifest['platform'] = self.name
+            config.manifest["platform"] = self.name
             self._BOARDS_CACHE[board_id] = config
 
         bdirs = [
-            get_project_boards_dir(),
-            join(get_project_core_dir(), "boards"),
+            self.config.get_optional_dir("boards"),
+            join(self.config.get_optional_dir("core"), "boards"),
             join(self.get_dir(), "boards"),
         ]
 
@@ -649,28 +662,28 @@ class PlatformBase(  # pylint: disable=too-many-public-methods
                 continue
             _pkg_name = self.frameworks[framework].get("package")
             if _pkg_name:
-                self.packages[_pkg_name]['optional'] = False
+                self.packages[_pkg_name]["optional"] = False
 
         # enable upload tools for upload targets
         if any(["upload" in t for t in targets] + ["program" in targets]):
             for name, opts in self.packages.items():
                 if opts.get("type") == "uploader":
-                    self.packages[name]['optional'] = False
+                    self.packages[name]["optional"] = False
                 # skip all packages in "nobuild" mode
                 # allow only upload tools and frameworks
                 elif "nobuild" in targets and opts.get("type") != "framework":
-                    self.packages[name]['optional'] = True
+                    self.packages[name]["optional"] = True
 
     def get_lib_storages(self):
-        storages = []
+        storages = {}
         for opts in (self.frameworks or {}).values():
             if "package" not in opts:
                 continue
-            pkg_dir = self.get_package_dir(opts['package'])
+            pkg_dir = self.get_package_dir(opts["package"])
             if not pkg_dir or not isdir(join(pkg_dir, "libraries")):
                 continue
             libs_dir = join(pkg_dir, "libraries")
-            storages.append({"name": opts['package'], "path": libs_dir})
+            storages[libs_dir] = opts["package"]
             libcores_dir = join(libs_dir, "__cores__")
             if not isdir(libcores_dir):
                 continue
@@ -678,16 +691,12 @@ class PlatformBase(  # pylint: disable=too-many-public-methods
                 libcore_dir = join(libcores_dir, item)
                 if not isdir(libcore_dir):
                     continue
-                storages.append({
-                    "name": "%s-core-%s" % (opts['package'], item),
-                    "path": libcore_dir
-                })
+                storages[libcore_dir] = "%s-core-%s" % (opts["package"], item)
 
-        return storages
+        return [dict(name=name, path=path) for path, name in storages.items()]
 
 
 class PlatformBoardConfig(object):
-
     def __init__(self, manifest_path):
         self._id = basename(manifest_path)[:-5]
         assert isfile(manifest_path)
@@ -698,8 +707,8 @@ class PlatformBoardConfig(object):
             raise exception.InvalidBoardManifest(manifest_path)
         if not set(["name", "url", "vendor"]) <= set(self._manifest):
             raise exception.PlatformioException(
-                "Please specify name, url and vendor fields for " +
-                manifest_path)
+                "Please specify name, url and vendor fields for " + manifest_path
+            )
 
     def get(self, path, default=None):
         try:
@@ -751,41 +760,33 @@ class PlatformBoardConfig(object):
 
     def get_brief_data(self):
         return {
-            "id":
-            self.id,
-            "name":
-            self._manifest['name'],
-            "platform":
-            self._manifest.get("platform"),
-            "mcu":
-            self._manifest.get("build", {}).get("mcu", "").upper(),
-            "fcpu":
-            int("".join([
-                c for c in str(
-                    self._manifest.get("build", {}).get("f_cpu", "0L"))
-                if c.isdigit()
-            ])),
-            "ram":
-            self._manifest.get("upload", {}).get("maximum_ram_size", 0),
-            "rom":
-            self._manifest.get("upload", {}).get("maximum_size", 0),
-            "connectivity":
-            self._manifest.get("connectivity"),
-            "frameworks":
-            self._manifest.get("frameworks"),
-            "debug":
-            self.get_debug_data(),
-            "vendor":
-            self._manifest['vendor'],
-            "url":
-            self._manifest['url']
+            "id": self.id,
+            "name": self._manifest["name"],
+            "platform": self._manifest.get("platform"),
+            "mcu": self._manifest.get("build", {}).get("mcu", "").upper(),
+            "fcpu": int(
+                "".join(
+                    [
+                        c
+                        for c in str(self._manifest.get("build", {}).get("f_cpu", "0L"))
+                        if c.isdigit()
+                    ]
+                )
+            ),
+            "ram": self._manifest.get("upload", {}).get("maximum_ram_size", 0),
+            "rom": self._manifest.get("upload", {}).get("maximum_size", 0),
+            "connectivity": self._manifest.get("connectivity"),
+            "frameworks": self._manifest.get("frameworks"),
+            "debug": self.get_debug_data(),
+            "vendor": self._manifest["vendor"],
+            "url": self._manifest["url"],
         }
 
     def get_debug_data(self):
         if not self._manifest.get("debug", {}).get("tools"):
             return None
         tools = {}
-        for name, options in self._manifest['debug']['tools'].items():
+        for name, options in self._manifest["debug"]["tools"].items():
             tools[name] = {}
             for key, value in options.items():
                 if key in ("default", "onboard"):
@@ -798,22 +799,23 @@ class PlatformBoardConfig(object):
         if tool_name == "custom":
             return tool_name
         if not debug_tools:
-            raise exception.DebugSupportError(self._manifest['name'])
+            raise exception.DebugSupportError(self._manifest["name"])
         if tool_name:
             if tool_name in debug_tools:
                 return tool_name
             raise exception.DebugInvalidOptions(
-                "Unknown debug tool `%s`. Please use one of `%s` or `custom`" %
-                (tool_name, ", ".join(sorted(list(debug_tools)))))
+                "Unknown debug tool `%s`. Please use one of `%s` or `custom`"
+                % (tool_name, ", ".join(sorted(list(debug_tools))))
+            )
 
         # automatically select best tool
         data = {"default": [], "onboard": [], "external": []}
         for key, value in debug_tools.items():
             if value.get("default"):
-                data['default'].append(key)
+                data["default"].append(key)
             elif value.get("onboard"):
-                data['onboard'].append(key)
-            data['external'].append(key)
+                data["onboard"].append(key)
+            data["external"].append(key)
 
         for key, value in data.items():
             if not value:

platformio/package/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/package/exception.py

@@ -0,0 +1,36 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from platformio.exception import PlatformioException
+
+
+class ManifestException(PlatformioException):
+    pass
+
+
+class ManifestParserError(ManifestException):
+    pass
+
+
+class ManifestValidationError(ManifestException):
+    def __init__(self, error, data):
+        super(ManifestValidationError, self).__init__()
+        self.error = error
+        self.data = data
+
+    def __str__(self):
+        return (
+            "Invalid manifest fields: %s. \nPlease check specification -> "
+            "http://docs.platformio.org/page/librarymanager/config.html" % self.error
+        )

platformio/package/manifest/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/package/manifest/parser.py

@@ -0,0 +1,553 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import re
+
+import requests
+
+from platformio.compat import get_class_attributes, string_types
+from platformio.fs import get_file_contents
+from platformio.package.exception import ManifestParserError
+from platformio.project.helpers import is_platformio_project
+
+try:
+    from urllib.parse import urlparse
+except ImportError:
+    from urlparse import urlparse
+
+
+class ManifestFileType(object):
+    PLATFORM_JSON = "platform.json"
+    LIBRARY_JSON = "library.json"
+    LIBRARY_PROPERTIES = "library.properties"
+    MODULE_JSON = "module.json"
+    PACKAGE_JSON = "package.json"
+
+    @classmethod
+    def from_uri(cls, uri):
+        if uri.endswith(".properties"):
+            return ManifestFileType.LIBRARY_PROPERTIES
+        if uri.endswith("platform.json"):
+            return ManifestFileType.PLATFORM_JSON
+        if uri.endswith("module.json"):
+            return ManifestFileType.MODULE_JSON
+        if uri.endswith("package.json"):
+            return ManifestFileType.PACKAGE_JSON
+        if uri.endswith("library.json"):
+            return ManifestFileType.LIBRARY_JSON
+        return None
+
+
+class ManifestParserFactory(object):
+    @staticmethod
+    def type_to_clsname(t):
+        t = t.replace(".", " ")
+        t = t.title()
+        return "%sManifestParser" % t.replace(" ", "")
+
+    @staticmethod
+    def new_from_file(path, remote_url=False):
+        if not path or not os.path.isfile(path):
+            raise ManifestParserError("Manifest file does not exist %s" % path)
+        for t in get_class_attributes(ManifestFileType).values():
+            if path.endswith(t):
+                return ManifestParserFactory.new(get_file_contents(path), t, remote_url)
+        raise ManifestParserError("Unknown manifest file type %s" % path)
+
+    @staticmethod
+    def new_from_dir(path, remote_url=None):
+        assert os.path.isdir(path), "Invalid directory %s" % path
+
+        type_from_uri = ManifestFileType.from_uri(remote_url) if remote_url else None
+        if type_from_uri and os.path.isfile(os.path.join(path, type_from_uri)):
+            return ManifestParserFactory.new(
+                get_file_contents(os.path.join(path, type_from_uri)),
+                type_from_uri,
+                remote_url=remote_url,
+                package_dir=path,
+            )
+
+        file_order = [
+            ManifestFileType.PLATFORM_JSON,
+            ManifestFileType.LIBRARY_JSON,
+            ManifestFileType.LIBRARY_PROPERTIES,
+            ManifestFileType.MODULE_JSON,
+            ManifestFileType.PACKAGE_JSON,
+        ]
+        for t in file_order:
+            if not os.path.isfile(os.path.join(path, t)):
+                continue
+            return ManifestParserFactory.new(
+                get_file_contents(os.path.join(path, t)),
+                t,
+                remote_url=remote_url,
+                package_dir=path,
+            )
+        raise ManifestParserError("Unknown manifest file type in %s directory" % path)
+
+    @staticmethod
+    def new_from_url(remote_url):
+        r = requests.get(remote_url)
+        r.raise_for_status()
+        return ManifestParserFactory.new(
+            r.text,
+            ManifestFileType.from_uri(remote_url) or ManifestFileType.LIBRARY_JSON,
+            remote_url,
+        )
+
+    @staticmethod
+    def new(contents, type, remote_url=None, package_dir=None):
+        # pylint: disable=redefined-builtin
+        clsname = ManifestParserFactory.type_to_clsname(type)
+        if clsname not in globals():
+            raise ManifestParserError("Unknown manifest file type %s" % clsname)
+        return globals()[clsname](contents, remote_url, package_dir)
+
+
+class BaseManifestParser(object):
+    def __init__(self, contents, remote_url=None, package_dir=None):
+        self.remote_url = remote_url
+        self.package_dir = package_dir
+        try:
+            self._data = self.parse(contents)
+        except Exception as e:
+            raise ManifestParserError("Could not parse manifest -> %s" % e)
+        self._data = self.parse_examples(self._data)
+
+        # remove None fields
+        for key in list(self._data.keys()):
+            if self._data[key] is None:
+                del self._data[key]
+
+    def parse(self, contents):
+        raise NotImplementedError
+
+    def as_dict(self):
+        return self._data
+
+    @staticmethod
+    def cleanup_author(author):
+        assert isinstance(author, dict)
+        if author.get("email"):
+            author["email"] = re.sub(r"\s+[aA][tT]\s+", "@", author["email"])
+        for key in list(author.keys()):
+            if author[key] is None:
+                del author[key]
+        return author
+
+    @staticmethod
+    def parse_author_name_and_email(raw):
+        if raw == "None" or "://" in raw:
+            return (None, None)
+        name = raw
+        email = None
+        for ldel, rdel in [("<", ">"), ("(", ")")]:
+            if ldel in raw and rdel in raw:
+                name = raw[: raw.index(ldel)]
+                email = raw[raw.index(ldel) + 1 : raw.index(rdel)]
+        return (name.strip(), email.strip() if email else None)
+
+    def parse_examples(self, data):
+        examples = data.get("examples")
+        if (
+            not examples
+            or not isinstance(examples, list)
+            or not all(isinstance(v, dict) for v in examples)
+        ):
+            examples = None
+        if not examples and self.package_dir:
+            data["examples"] = self.parse_examples_from_dir(self.package_dir)
+        if "examples" in data and not data["examples"]:
+            del data["examples"]
+        return data
+
+    @staticmethod
+    def parse_examples_from_dir(package_dir):
+        assert os.path.isdir(package_dir)
+        examples_dir = os.path.join(package_dir, "examples")
+        if not os.path.isdir(examples_dir):
+            examples_dir = os.path.join(package_dir, "Examples")
+            if not os.path.isdir(examples_dir):
+                return None
+
+        allowed_exts = (
+            ".c",
+            ".cc",
+            ".cpp",
+            ".h",
+            ".hpp",
+            ".asm",
+            ".ASM",
+            ".s",
+            ".S",
+            ".ino",
+            ".pde",
+        )
+
+        result = {}
+        last_pio_project = None
+        for root, _, files in os.walk(examples_dir):
+            # skip hidden files, symlinks, and folders
+            files = [
+                f
+                for f in files
+                if not f.startswith(".") and not os.path.islink(os.path.join(root, f))
+            ]
+            if os.path.basename(root).startswith(".") or not files:
+                continue
+
+            if is_platformio_project(root):
+                last_pio_project = root
+                result[last_pio_project] = dict(
+                    name=os.path.relpath(root, examples_dir),
+                    base=os.path.relpath(root, package_dir),
+                    files=files,
+                )
+                continue
+            if last_pio_project:
+                if root.startswith(last_pio_project):
+                    result[last_pio_project]["files"].extend(
+                        [
+                            os.path.relpath(os.path.join(root, f), last_pio_project)
+                            for f in files
+                        ]
+                    )
+                    continue
+                last_pio_project = None
+
+            matched_files = [f for f in files if f.endswith(allowed_exts)]
+            if not matched_files:
+                continue
+            result[root] = dict(
+                name="Examples"
+                if root == examples_dir
+                else os.path.relpath(root, examples_dir),
+                base=os.path.relpath(root, package_dir),
+                files=matched_files,
+            )
+
+        result = list(result.values())
+
+        # normalize example names
+        for item in result:
+            item["name"] = item["name"].replace(os.path.sep, "/")
+            item["name"] = re.sub(r"[^a-z\d\d\-\_/]+", "_", item["name"], flags=re.I)
+
+        return result or None
+
+
+class LibraryJsonManifestParser(BaseManifestParser):
+    def parse(self, contents):
+        data = json.loads(contents)
+        data = self._process_renamed_fields(data)
+
+        # normalize Union[str, list] fields
+        for k in ("keywords", "platforms", "frameworks"):
+            if k in data:
+                data[k] = self._str_to_list(data[k], sep=",")
+
+        if "authors" in data:
+            data["authors"] = self._parse_authors(data["authors"])
+        if "platforms" in data:
+            data["platforms"] = self._parse_platforms(data["platforms"]) or None
+        if "export" in data:
+            data["export"] = self._parse_export(data["export"])
+
+        return data
+
+    @staticmethod
+    def _str_to_list(value, sep=",", lowercase=True):
+        if isinstance(value, string_types):
+            value = value.split(sep)
+        assert isinstance(value, list)
+        result = []
+        for item in value:
+            item = item.strip()
+            if not item:
+                continue
+            if lowercase:
+                item = item.lower()
+            result.append(item)
+        return result
+
+    @staticmethod
+    def _process_renamed_fields(data):
+        if "url" in data:
+            data["homepage"] = data["url"]
+            del data["url"]
+
+        for key in ("include", "exclude"):
+            if key not in data:
+                continue
+            if "export" not in data:
+                data["export"] = {}
+            data["export"][key] = data[key]
+            del data[key]
+
+        return data
+
+    def _parse_authors(self, raw):
+        if not raw:
+            return None
+        # normalize Union[dict, list] fields
+        if not isinstance(raw, list):
+            raw = [raw]
+        return [self.cleanup_author(author) for author in raw]
+
+    @staticmethod
+    def _parse_platforms(raw):
+        assert isinstance(raw, list)
+        result = []
+        # renamed platforms
+        for item in raw:
+            if item == "espressif":
+                item = "espressif8266"
+            result.append(item)
+        return result
+
+    @staticmethod
+    def _parse_export(raw):
+        if not isinstance(raw, dict):
+            return None
+        result = {}
+        for k in ("include", "exclude"):
+            if k not in raw:
+                continue
+            result[k] = raw[k] if isinstance(raw[k], list) else [raw[k]]
+        return result
+
+
+class ModuleJsonManifestParser(BaseManifestParser):
+    def parse(self, contents):
+        data = json.loads(contents)
+        data["frameworks"] = ["mbed"]
+        data["platforms"] = ["*"]
+        data["export"] = {"exclude": ["tests", "test", "*.doxyfile", "*.pdf"]}
+        if "author" in data:
+            data["authors"] = self._parse_authors(data.get("author"))
+            del data["author"]
+        if "licenses" in data:
+            data["license"] = self._parse_license(data.get("licenses"))
+            del data["licenses"]
+        return data
+
+    def _parse_authors(self, raw):
+        if not raw:
+            return None
+        result = []
+        for author in raw.split(","):
+            name, email = self.parse_author_name_and_email(author)
+            if not name:
+                continue
+            result.append(self.cleanup_author(dict(name=name, email=email)))
+        return result
+
+    @staticmethod
+    def _parse_license(raw):
+        if not raw or not isinstance(raw, list):
+            return None
+        return raw[0].get("type")
+
+
+class LibraryPropertiesManifestParser(BaseManifestParser):
+    def parse(self, contents):
+        data = self._parse_properties(contents)
+        repository = self._parse_repository(data)
+        homepage = data.get("url")
+        if repository and repository["url"] == homepage:
+            homepage = None
+        data.update(
+            dict(
+                frameworks=["arduino"],
+                homepage=homepage,
+                repository=repository or None,
+                description=self._parse_description(data),
+                platforms=self._parse_platforms(data) or ["*"],
+                keywords=self._parse_keywords(data),
+                export=self._parse_export(),
+            )
+        )
+        if "author" in data:
+            data["authors"] = self._parse_authors(data)
+            del data["author"]
+        return data
+
+    @staticmethod
+    def _parse_properties(contents):
+        data = {}
+        for line in contents.splitlines():
+            line = line.strip()
+            if not line or "=" not in line:
+                continue
+            # skip comments
+            if line.startswith("#"):
+                continue
+            key, value = line.split("=", 1)
+            data[key.strip()] = value.strip()
+        return data
+
+    @staticmethod
+    def _parse_description(properties):
+        lines = []
+        for k in ("sentence", "paragraph"):
+            if k in properties and properties[k] not in lines:
+                lines.append(properties[k])
+        if len(lines) == 2 and not lines[0].endswith("."):
+            lines[0] += "."
+        return " ".join(lines)
+
+    @staticmethod
+    def _parse_keywords(properties):
+        result = []
+        for item in re.split(r"[\s/]+", properties.get("category", "uncategorized")):
+            item = item.strip()
+            if not item:
+                continue
+            result.append(item.lower())
+        return result
+
+    @staticmethod
+    def _parse_platforms(properties):
+        result = []
+        platforms_map = {
+            "avr": "atmelavr",
+            "sam": "atmelsam",
+            "samd": "atmelsam",
+            "esp8266": "espressif8266",
+            "esp32": "espressif32",
+            "arc32": "intel_arc32",
+            "stm32": "ststm32",
+        }
+        for arch in properties.get("architectures", "").split(","):
+            if "particle-" in arch:
+                raise ManifestParserError("Particle is not supported yet")
+            arch = arch.strip()
+            if not arch:
+                continue
+            if arch == "*":
+                return ["*"]
+            if arch in platforms_map:
+                result.append(platforms_map[arch])
+        return result
+
+    def _parse_authors(self, properties):
+        if "author" not in properties:
+            return None
+        authors = []
+        for author in properties["author"].split(","):
+            name, email = self.parse_author_name_and_email(author)
+            if not name:
+                continue
+            authors.append(self.cleanup_author(dict(name=name, email=email)))
+        for author in properties.get("maintainer", "").split(","):
+            name, email = self.parse_author_name_and_email(author)
+            if not name:
+                continue
+            found = False
+            for item in authors:
+                if item.get("name", "").lower() != name.lower():
+                    continue
+                found = True
+                item["maintainer"] = True
+                if not item.get("email"):
+                    item["email"] = email
+            if not found:
+                authors.append(
+                    self.cleanup_author(dict(name=name, email=email, maintainer=True))
+                )
+        return authors
+
+    def _parse_repository(self, properties):
+        if self.remote_url:
+            repo_parse = urlparse(self.remote_url)
+            repo_path_tokens = repo_parse.path[1:].split("/")[:-1]
+            if "github" in repo_parse.netloc:
+                return dict(
+                    type="git",
+                    url="%s://github.com/%s"
+                    % (repo_parse.scheme, "/".join(repo_path_tokens[:2])),
+                )
+            if "raw" in repo_path_tokens:
+                return dict(
+                    type="git",
+                    url="%s://%s/%s"
+                    % (
+                        repo_parse.scheme,
+                        repo_parse.netloc,
+                        "/".join(repo_path_tokens[: repo_path_tokens.index("raw")]),
+                    ),
+                )
+        if properties.get("url", "").startswith("https://github.com"):
+            return dict(type="git", url=properties["url"])
+        return None
+
+    def _parse_export(self):
+        result = {"exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"]}
+        include = None
+        if self.remote_url:
+            repo_parse = urlparse(self.remote_url)
+            repo_path_tokens = repo_parse.path[1:].split("/")[:-1]
+            if "github" in repo_parse.netloc:
+                include = "/".join(repo_path_tokens[3:]) or None
+            elif "raw" in repo_path_tokens:
+                include = (
+                    "/".join(repo_path_tokens[repo_path_tokens.index("raw") + 2 :])
+                    or None
+                )
+        if include:
+            result["include"] = [include]
+        return result
+
+
+class PlatformJsonManifestParser(BaseManifestParser):
+    def parse(self, contents):
+        data = json.loads(contents)
+        if "frameworks" in data:
+            data["frameworks"] = self._parse_frameworks(data["frameworks"])
+        return data
+
+    @staticmethod
+    def _parse_frameworks(raw):
+        if not isinstance(raw, dict):
+            return None
+        return [name.lower() for name in raw.keys()]
+
+
+class PackageJsonManifestParser(BaseManifestParser):
+    def parse(self, contents):
+        data = json.loads(contents)
+        data = self._parse_system(data)
+        data = self._parse_homepage(data)
+        return data
+
+    @staticmethod
+    def _parse_system(data):
+        if "system" not in data:
+            return data
+        if data["system"] in ("*", ["*"], "all"):
+            del data["system"]
+            return data
+        if not isinstance(data["system"], list):
+            data["system"] = [data["system"]]
+        data["system"] = [s.strip().lower() for s in data["system"]]
+        return data
+
+    @staticmethod
+    def _parse_homepage(data):
+        if "url" in data:
+            data["homepage"] = data["url"]
+            del data["url"]
+        return data

platformio/package/manifest/schema.py

@@ -0,0 +1,182 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import requests
+import semantic_version
+from marshmallow import Schema, ValidationError, fields, validate, validates
+
+from platformio.package.exception import ManifestValidationError
+from platformio.util import memoized
+
+
+class StrictSchema(Schema):
+    def handle_error(self, error, data):
+        # skip broken records
+        if self.many:
+            error.data = [
+                item for idx, item in enumerate(data) if idx not in error.messages
+            ]
+        else:
+            error.data = None
+        raise error
+
+
+class StrictListField(fields.List):
+    def _deserialize(self, value, attr, data):
+        try:
+            return super(StrictListField, self)._deserialize(value, attr, data)
+        except ValidationError as exc:
+            if exc.data:
+                exc.data = [item for item in exc.data if item is not None]
+            raise exc
+
+
+class AuthorSchema(StrictSchema):
+    name = fields.Str(required=True, validate=validate.Length(min=1, max=50))
+    email = fields.Email(validate=validate.Length(min=1, max=50))
+    maintainer = fields.Bool(default=False)
+    url = fields.Url(validate=validate.Length(min=1, max=255))
+
+
+class RepositorySchema(StrictSchema):
+    type = fields.Str(
+        required=True,
+        validate=validate.OneOf(
+            ["git", "hg", "svn"],
+            error="Invalid repository type, please use one of [git, hg, svn]",
+        ),
+    )
+    url = fields.Str(required=True, validate=validate.Length(min=1, max=255))
+    branch = fields.Str(validate=validate.Length(min=1, max=50))
+
+
+class ExportSchema(Schema):
+    include = StrictListField(fields.Str)
+    exclude = StrictListField(fields.Str)
+
+
+class ExampleSchema(StrictSchema):
+    name = fields.Str(
+        required=True,
+        validate=[
+            validate.Length(min=1, max=100),
+            validate.Regexp(
+                r"^[a-zA-Z\d\-\_/]+$", error="Only [a-zA-Z0-9-_/] chars are allowed"
+            ),
+        ],
+    )
+    base = fields.Str(required=True)
+    files = StrictListField(fields.Str, required=True)
+
+
+class ManifestSchema(Schema):
+    # Required fields
+    name = fields.Str(required=True, validate=validate.Length(min=1, max=100))
+    version = fields.Str(required=True, validate=validate.Length(min=1, max=50))
+
+    # Optional fields
+
+    authors = fields.Nested(AuthorSchema, many=True)
+    description = fields.Str(validate=validate.Length(min=1, max=1000))
+    homepage = fields.Url(validate=validate.Length(min=1, max=255))
+    license = fields.Str(validate=validate.Length(min=1, max=255))
+    repository = fields.Nested(RepositorySchema)
+    export = fields.Nested(ExportSchema)
+    examples = fields.Nested(ExampleSchema, many=True)
+
+    keywords = StrictListField(
+        fields.Str(
+            validate=[
+                validate.Length(min=1, max=50),
+                validate.Regexp(
+                    r"^[a-z\d\-\+\. ]+$", error="Only [a-z0-9-+. ] chars are allowed"
+                ),
+            ]
+        )
+    )
+
+    platforms = StrictListField(
+        fields.Str(
+            validate=[
+                validate.Length(min=1, max=50),
+                validate.Regexp(
+                    r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed"
+                ),
+            ]
+        )
+    )
+    frameworks = StrictListField(
+        fields.Str(
+            validate=[
+                validate.Length(min=1, max=50),
+                validate.Regexp(
+                    r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed"
+                ),
+            ]
+        )
+    )
+
+    # platform.json specific
+    title = fields.Str(validate=validate.Length(min=1, max=100))
+
+    # package.json specific
+    system = StrictListField(
+        fields.Str(
+            validate=[
+                validate.Length(min=1, max=50),
+                validate.Regexp(
+                    r"^[a-z\d\-_]+$", error="Only [a-z0-9-_] chars are allowed"
+                ),
+            ]
+        )
+    )
+
+    def handle_error(self, error, data):
+        if self.strict:
+            raise ManifestValidationError(error, data)
+
+    @validates("version")
+    def validate_version(self, value):  # pylint: disable=no-self-use
+        try:
+            value = str(value)
+            assert "." in value
+            semantic_version.Version.coerce(value)
+        except (AssertionError, ValueError):
+            raise ValidationError(
+                "Invalid semantic versioning format, see https://semver.org/"
+            )
+
+    @validates("license")
+    def validate_license(self, value):
+        try:
+            spdx = self.load_spdx_licenses()
+        except requests.exceptions.RequestException:
+            raise ValidationError("Could not load SPDX licenses for validation")
+        for item in spdx.get("licenses", []):
+            if item.get("licenseId") == value:
+                return
+        raise ValidationError(
+            "Invalid SPDX license identifier. See valid identifiers at "
+            "https://spdx.org/licenses/"
+        )
+
+    @staticmethod
+    @memoized(expire="1h")
+    def load_spdx_licenses():
+        r = requests.get(
+            "https://raw.githubusercontent.com/spdx/license-list-data"
+            "/v3.6/json/licenses.json"
+        )
+        r.raise_for_status()
+        return r.json()

platformio/proc.py

@@ -19,11 +19,15 @@ from os.path import isdir, isfile, join, normpath
 from threading import Thread
 
 from platformio import exception
-from platformio.compat import WINDOWS, string_types
+from platformio.compat import (
+    WINDOWS,
+    get_filesystem_encoding,
+    get_locale_encoding,
+    string_types,
+)
 
 
 class AsyncPipeBase(object):
-
     def __init__(self):
         self._fd_read, self._fd_write = os.pipe()
         self._pipe_reader = os.fdopen(self._fd_read)
@@ -53,7 +57,6 @@ class AsyncPipeBase(object):
 
 
 class BuildAsyncPipe(AsyncPipeBase):
-
     def __init__(self, line_callback, data_callback):
         self.line_callback = line_callback
         self.data_callback = data_callback
@@ -88,7 +91,6 @@ class BuildAsyncPipe(AsyncPipeBase):
 
 
 class LineBufferedAsyncPipe(AsyncPipeBase):
-
     def __init__(self, line_callback):
         self.line_callback = line_callback
         super(LineBufferedAsyncPipe, self).__init__()
@@ -109,8 +111,8 @@ def exec_command(*args, **kwargs):
 
     p = subprocess.Popen(*args, **kwargs)
     try:
-        result['out'], result['err'] = p.communicate()
-        result['returncode'] = p.returncode
+        result["out"], result["err"] = p.communicate()
+        result["returncode"] = p.returncode
     except KeyboardInterrupt:
         raise exception.AbortedByUser()
     finally:
@@ -125,7 +127,9 @@ def exec_command(*args, **kwargs):
     for k, v in result.items():
         if isinstance(result[k], bytes):
             try:
-                result[k] = result[k].decode(sys.getdefaultencoding())
+                result[k] = result[k].decode(
+                    get_locale_encoding() or get_filesystem_encoding()
+                )
             except UnicodeDecodeError:
                 result[k] = result[k].decode("latin-1")
         if v and isinstance(v, string_types):
@@ -160,24 +164,22 @@ def copy_pythonpath_to_osenv():
     for p in os.sys.path:
         conditions = [p not in _PYTHONPATH]
         if not WINDOWS:
-            conditions.append(
-                isdir(join(p, "click")) or isdir(join(p, "platformio")))
+            conditions.append(isdir(join(p, "click")) or isdir(join(p, "platformio")))
         if all(conditions):
             _PYTHONPATH.append(p)
-    os.environ['PYTHONPATH'] = os.pathsep.join(_PYTHONPATH)
+    os.environ["PYTHONPATH"] = os.pathsep.join(_PYTHONPATH)
 
 
 def where_is_program(program, envpath=None):
     env = os.environ
     if envpath:
-        env['PATH'] = envpath
+        env["PATH"] = envpath
 
     # try OS's built-in commands
     try:
-        result = exec_command(["where" if WINDOWS else "which", program],
-                              env=env)
-        if result['returncode'] == 0 and isfile(result['out'].strip()):
-            return result['out'].strip()
+        result = exec_command(["where" if WINDOWS else "which", program], env=env)
+        if result["returncode"] == 0 and isfile(result["out"].strip()):
+            return result["out"].strip()
     except OSError:
         pass
 

platformio/project/config.py

@@ -16,11 +16,12 @@ import glob
 import json
 import os
 import re
-from os.path import expanduser, getmtime, isfile
+from hashlib import sha1
 
 import click
 
-from platformio import exception
+from platformio import exception, fs
+from platformio.compat import PY2, WINDOWS, hashlib_encode_data
 from platformio.project.options import ProjectOptions
 
 try:
@@ -41,7 +42,7 @@ CONFIG_HEADER = """;PlatformIO Project Configuration File
 """
 
 
-class ProjectConfig(object):
+class ProjectConfigBase(object):
 
     INLINE_COMMENT_RE = re.compile(r"\s+;.*$")
     VARTPL_RE = re.compile(r"\$\{([^\.\}]+)\.([^\}]+)\}")
@@ -49,7 +50,6 @@ class ProjectConfig(object):
     expand_interpolations = True
     warnings = []
 
-    _instances = {}
     _parser = None
     _parsed = []
 
@@ -66,30 +66,34 @@ class ProjectConfig(object):
             if not item or item.startswith((";", "#")):
                 continue
             if ";" in item:
-                item = ProjectConfig.INLINE_COMMENT_RE.sub("", item).strip()
+                item = ProjectConfigBase.INLINE_COMMENT_RE.sub("", item).strip()
             result.append(item)
         return result
 
     @staticmethod
-    def get_instance(path):
-        mtime = getmtime(path) if isfile(path) else 0
-        instance = ProjectConfig._instances.get(path)
-        if instance and instance["mtime"] != mtime:
-            instance = None
-        if not instance:
-            instance = {"mtime": mtime, "config": ProjectConfig(path)}
-            ProjectConfig._instances[path] = instance
-        return instance["config"]
+    def get_default_path():
+        from platformio import app  # pylint: disable=import-outside-toplevel
 
-    def __init__(self, path, parse_extra=True, expand_interpolations=True):
+        return app.get_session_var("custom_project_conf") or os.path.join(
+            os.getcwd(), "platformio.ini"
+        )
+
+    def __init__(self, path=None, parse_extra=True, expand_interpolations=True):
+        path = self.get_default_path() if path is None else path
         self.path = path
         self.expand_interpolations = expand_interpolations
         self.warnings = []
         self._parsed = []
-        self._parser = ConfigParser.ConfigParser()
-        if isfile(path):
+        self._parser = (
+            ConfigParser.ConfigParser()
+            if PY2
+            else ConfigParser.ConfigParser(inline_comment_prefixes=("#", ";"))
+        )
+        if path and os.path.isfile(path):
             self.read(path, parse_extra)
 
+        self._maintain_renaimed_options()
+
     def __getattr__(self, name):
         return getattr(self._parser, name)
 
@@ -108,31 +112,32 @@ class ProjectConfig(object):
         # load extra configs
         for pattern in self.get("platformio", "extra_configs", []):
             if pattern.startswith("~"):
-                pattern = expanduser(pattern)
+                pattern = fs.expanduser(pattern)
             for item in glob.glob(pattern):
                 self.read(item)
 
-        self._maintain_renaimed_options()
-
     def _maintain_renaimed_options(self):
         # legacy `lib_extra_dirs` in [platformio]
-        if (self._parser.has_section("platformio")
-                and self._parser.has_option("platformio", "lib_extra_dirs")):
+        if self._parser.has_section("platformio") and self._parser.has_option(
+            "platformio", "lib_extra_dirs"
+        ):
             if not self._parser.has_section("env"):
                 self._parser.add_section("env")
-            self._parser.set("env", "lib_extra_dirs",
-                             self._parser.get("platformio", "lib_extra_dirs"))
+            self._parser.set(
+                "env",
+                "lib_extra_dirs",
+                self._parser.get("platformio", "lib_extra_dirs"),
+            )
             self._parser.remove_option("platformio", "lib_extra_dirs")
             self.warnings.append(
                 "`lib_extra_dirs` configuration option is deprecated in "
-                "section [platformio]! Please move it to global `env` section")
+                "section [platformio]! Please move it to global `env` section"
+            )
 
         renamed_options = {}
         for option in ProjectOptions.values():
             if option.oldnames:
-                renamed_options.update(
-                    {name: option.name
-                     for name in option.oldnames})
+                renamed_options.update({name: option.name for name in option.oldnames})
 
         for section in self._parser.sections():
             scope = section.split(":", 1)[0]
@@ -143,54 +148,74 @@ class ProjectConfig(object):
                     self.warnings.append(
                         "`%s` configuration option in section [%s] is "
                         "deprecated and will be removed in the next release! "
-                        "Please use `%s` instead" %
-                        (option, section, renamed_options[option]))
+                        "Please use `%s` instead"
+                        % (option, section, renamed_options[option])
+                    )
                     # rename on-the-fly
-                    self._parser.set(section, renamed_options[option],
-                                     self._parser.get(section, option))
+                    self._parser.set(
+                        section,
+                        renamed_options[option],
+                        self._parser.get(section, option),
+                    )
                     self._parser.remove_option(section, option)
                     continue
 
                 # unknown
                 unknown_conditions = [
                     ("%s.%s" % (scope, option)) not in ProjectOptions,
-                    scope != "env" or
-                    not option.startswith(("custom_", "board_"))
-                ]  # yapf: disable
+                    scope != "env" or not option.startswith(("custom_", "board_")),
+                ]
                 if all(unknown_conditions):
                     self.warnings.append(
                         "Ignore unknown configuration option `%s` "
-                        "in section [%s]" % (option, section))
+                        "in section [%s]" % (option, section)
+                    )
         return True
 
+    def walk_options(self, root_section):
+        extends_queue = (
+            ["env", root_section] if root_section.startswith("env:") else [root_section]
+        )
+        extends_done = []
+        while extends_queue:
+            section = extends_queue.pop()
+            extends_done.append(section)
+            if not self._parser.has_section(section):
+                continue
+            for option in self._parser.options(section):
+                yield (section, option)
+            if self._parser.has_option(section, "extends"):
+                extends_queue.extend(
+                    self.parse_multi_values(self._parser.get(section, "extends"))[::-1]
+                )
+
     def options(self, section=None, env=None):
+        result = []
         assert section or env
         if not section:
             section = "env:" + env
-        options = self._parser.options(section)
 
-        # handle global options from [env]
-        if ((env or section.startswith("env:"))
-                and self._parser.has_section("env")):
-            for option in self._parser.options("env"):
-                if option not in options:
-                    options.append(option)
+        if not self.expand_interpolations:
+            return self._parser.options(section)
+
+        for _, option in self.walk_options(section):
+            if option not in result:
+                result.append(option)
 
         # handle system environment variables
         scope = section.split(":", 1)[0]
         for option_meta in ProjectOptions.values():
-            if option_meta.scope != scope or option_meta.name in options:
+            if option_meta.scope != scope or option_meta.name in result:
                 continue
             if option_meta.sysenvvar and option_meta.sysenvvar in os.environ:
-                options.append(option_meta.name)
+                result.append(option_meta.name)
 
-        return options
+        return result
 
     def has_option(self, section, option):
         if self._parser.has_option(section, option):
             return True
-        return (section.startswith("env:") and self._parser.has_section("env")
-                and self._parser.has_option("env", option))
+        return option in self.options(section)
 
     def items(self, section=None, env=None, as_dict=False):
         assert section or env
@@ -198,29 +223,36 @@ class ProjectConfig(object):
             section = "env:" + env
         if as_dict:
             return {
-                option: self.get(section, option)
-                for option in self.options(section)
+                option: self.get(section, option) for option in self.options(section)
             }
-        return [(option, self.get(section, option))
-                for option in self.options(section)]
+        return [(option, self.get(section, option)) for option in self.options(section)]
 
     def set(self, section, option, value):
         if isinstance(value, (list, tuple)):
             value = "\n".join(value)
-            if value:
-                value = "\n" + value  # start from a new line
+        elif isinstance(value, bool):
+            value = "yes" if value else "no"
+        elif isinstance(value, (int, float)):
+            value = str(value)
+        # start multi-line value from a new line
+        if "\n" in value and not value.startswith("\n"):
+            value = "\n" + value
         self._parser.set(section, option, value)
 
     def getraw(self, section, option):
         if not self.expand_interpolations:
             return self._parser.get(section, option)
 
-        try:
+        value = None
+        found = False
+        for sec, opt in self.walk_options(section):
+            if opt == option:
+                value = self._parser.get(sec, option)
+                found = True
+                break
+
+        if not found:
             value = self._parser.get(section, option)
-        except ConfigParser.NoOptionError as e:
-            if not section.startswith("env:"):
-                raise e
-            value = self._parser.get("env", option)
 
         if "${" not in value or "}" not in value:
             return value
@@ -232,7 +264,7 @@ class ProjectConfig(object):
             return os.getenv(option)
         return self.getraw(section, option)
 
-    def get(self, section, option, default=None):
+    def get(self, section, option, default=None):  # pylint: disable=too-many-branches
         value = None
         try:
             value = self.getraw(section, option)
@@ -241,8 +273,7 @@ class ProjectConfig(object):
         except ConfigParser.Error as e:
             raise exception.InvalidProjectConf(self.path, str(e))
 
-        option_meta = ProjectOptions.get("%s.%s" %
-                                         (section.split(":", 1)[0], option))
+        option_meta = ProjectOptions.get("%s.%s" % (section.split(":", 1)[0], option))
         if not option_meta:
             return value or default
 
@@ -263,17 +294,20 @@ class ProjectConfig(object):
                 value = envvar_value
 
         # option is not specified by user
-        if value is None:
-            return default
+        if value is None or (
+            option_meta.multiple and value == [] and option_meta.default
+        ):
+            return default if default is not None else option_meta.default
 
         try:
-            return self._covert_value(value, option_meta.type)
+            return self.cast_to(value, option_meta.type)
         except click.BadParameter as e:
-            raise exception.ProjectOptionValueError(e.format_message(), option,
-                                                    section)
+            if not self.expand_interpolations:
+                return value
+            raise exception.ProjectOptionValueError(e.format_message(), option, section)
 
     @staticmethod
-    def _covert_value(value, to_type):
+    def cast_to(value, to_type):
         items = value
         if not isinstance(value, (list, tuple)):
             items = [value]
@@ -290,7 +324,7 @@ class ProjectConfig(object):
         return self.get("platformio", "default_envs", [])
 
     def validate(self, envs=None, silent=False):
-        if not isfile(self.path):
+        if not os.path.isfile(self.path):
             raise exception.NotPlatformIOProject(self.path)
         # check envs
         known = set(self.envs())
@@ -298,18 +332,114 @@ class ProjectConfig(object):
             raise exception.ProjectEnvsNotAvailable()
         unknown = set(list(envs or []) + self.default_envs()) - known
         if unknown:
-            raise exception.UnknownEnvNames(", ".join(unknown),
-                                            ", ".join(known))
+            raise exception.UnknownEnvNames(", ".join(unknown), ", ".join(known))
         if not silent:
             for warning in self.warnings:
                 click.secho("Warning! %s" % warning, fg="yellow")
         return True
 
+
+class ProjectConfigDirsMixin(object):
+    def _get_core_dir(self, exists=False):
+        default = ProjectOptions["platformio.core_dir"].default
+        core_dir = self.get("platformio", "core_dir")
+        win_core_dir = None
+        if WINDOWS and core_dir == default:
+            win_core_dir = os.path.splitdrive(core_dir)[0] + "\\.platformio"
+            if os.path.isdir(win_core_dir):
+                core_dir = win_core_dir
+
+        if exists and not os.path.isdir(core_dir):
+            try:
+                os.makedirs(core_dir)
+            except OSError as e:
+                if win_core_dir:
+                    os.makedirs(win_core_dir)
+                    core_dir = win_core_dir
+                else:
+                    raise e
+
+        return core_dir
+
+    def get_optional_dir(self, name, exists=False):
+        if not ProjectOptions.get("platformio.%s_dir" % name):
+            raise ValueError("Unknown optional directory -> " + name)
+
+        if name == "core":
+            result = self._get_core_dir(exists)
+        else:
+            result = self.get("platformio", name + "_dir")
+
+        if result is None:
+            return None
+
+        project_dir = os.getcwd()
+
+        # patterns
+        if "$PROJECT_HASH" in result:
+            result = result.replace(
+                "$PROJECT_HASH",
+                "%s-%s"
+                % (
+                    os.path.basename(project_dir),
+                    sha1(hashlib_encode_data(project_dir)).hexdigest()[:10],
+                ),
+            )
+
+        if "$PROJECT_DIR" in result:
+            result = result.replace("$PROJECT_DIR", project_dir)
+        if "$PROJECT_CORE_DIR" in result:
+            result = result.replace("$PROJECT_CORE_DIR", self.get_optional_dir("core"))
+        if "$PROJECT_WORKSPACE_DIR" in result:
+            result = result.replace(
+                "$PROJECT_WORKSPACE_DIR", self.get_optional_dir("workspace")
+            )
+
+        if result.startswith("~"):
+            result = fs.expanduser(result)
+
+        result = os.path.realpath(result)
+
+        if exists and not os.path.isdir(result):
+            os.makedirs(result)
+
+        return result
+
+
+class ProjectConfig(ProjectConfigBase, ProjectConfigDirsMixin):
+
+    _instances = {}
+
+    @staticmethod
+    def get_instance(path=None):
+        path = ProjectConfig.get_default_path() if path is None else path
+        mtime = os.path.getmtime(path) if os.path.isfile(path) else 0
+        instance = ProjectConfig._instances.get(path)
+        if instance and instance["mtime"] != mtime:
+            instance = None
+        if not instance:
+            instance = {"mtime": mtime, "config": ProjectConfig(path)}
+            ProjectConfig._instances[path] = instance
+        return instance["config"]
+
+    def __repr__(self):
+        return "<ProjectConfig %s>" % (self.path or "in-memory")
+
+    def as_tuple(self):
+        return [(s, self.items(s)) for s in self.sections()]
+
     def to_json(self):
-        result = {}
-        for section in self.sections():
-            result[section] = self.items(section, as_dict=True)
-        return json.dumps(result)
+        return json.dumps(self.as_tuple())
+
+    def update(self, data, clear=False):
+        assert isinstance(data, list)
+        if clear:
+            self._parser = ConfigParser.ConfigParser()
+        for section, options in data:
+            if not self._parser.has_section(section):
+                self._parser.add_section(section)
+            for option, value in options:
+                self.set(section, option, value)
 
     def save(self, path=None):
         path = path or self.path
@@ -318,3 +448,4 @@ class ProjectConfig(object):
         with open(path or self.path, "w") as fp:
             fp.write(CONFIG_HEADER)
             self._parser.write(fp)
+        return True

platformio/project/helpers.py

@@ -16,12 +16,11 @@ import json
 import os
 from hashlib import sha1
 from os import walk
-from os.path import (basename, dirname, expanduser, isdir, isfile, join,
-                     realpath, splitdrive)
+from os.path import dirname, isdir, isfile, join
 
 from click.testing import CliRunner
 
-from platformio import __version__, exception
+from platformio import __version__, exception, fs
 from platformio.compat import WINDOWS, hashlib_encode_data
 from platformio.project.config import ProjectConfig
 
@@ -46,123 +45,62 @@ def find_project_dir_above(path):
     return None
 
 
-def get_project_optional_dir(name, default=None):
-    project_dir = get_project_dir()
-    config = ProjectConfig.get_instance(join(project_dir, "platformio.ini"))
-    optional_dir = config.get("platformio", name)
-
-    if not optional_dir:
-        return default
-
-    if "$PROJECT_HASH" in optional_dir:
-        optional_dir = optional_dir.replace(
-            "$PROJECT_HASH", "%s-%s" %
-            (basename(project_dir), sha1(
-                hashlib_encode_data(project_dir)).hexdigest()[:10]))
-
-    if optional_dir.startswith("~"):
-        optional_dir = expanduser(optional_dir)
-
-    return realpath(optional_dir)
-
-
 def get_project_core_dir():
-    default = join(expanduser("~"), ".platformio")
-    core_dir = get_project_optional_dir(
-        "core_dir", get_project_optional_dir("home_dir", default))
-    win_core_dir = None
-    if WINDOWS and core_dir == default:
-        win_core_dir = splitdrive(core_dir)[0] + "\\.platformio"
-        if isdir(win_core_dir):
-            core_dir = win_core_dir
-
-    if not isdir(core_dir):
-        try:
-            os.makedirs(core_dir)
-        except OSError as e:
-            if win_core_dir:
-                os.makedirs(win_core_dir)
-                core_dir = win_core_dir
-            else:
-                raise e
-
-    assert isdir(core_dir)
-    return core_dir
-
-
-def get_project_global_lib_dir():
-    return get_project_optional_dir("globallib_dir",
-                                    join(get_project_core_dir(), "lib"))
-
-
-def get_project_platforms_dir():
-    return get_project_optional_dir("platforms_dir",
-                                    join(get_project_core_dir(), "platforms"))
-
-
-def get_project_packages_dir():
-    return get_project_optional_dir("packages_dir",
-                                    join(get_project_core_dir(), "packages"))
+    """ Deprecated, use ProjectConfig.get_optional_dir("core") instead """
+    return ProjectConfig.get_instance(
+        join(get_project_dir(), "platformio.ini")
+    ).get_optional_dir("core", exists=True)
 
 
 def get_project_cache_dir():
-    return get_project_optional_dir("cache_dir",
-                                    join(get_project_core_dir(), ".cache"))
+    """ Deprecated, use ProjectConfig.get_optional_dir("cache") instead """
+    return ProjectConfig.get_instance(
+        join(get_project_dir(), "platformio.ini")
+    ).get_optional_dir("cache")
 
 
-def get_project_workspace_dir():
-    return get_project_optional_dir("workspace_dir",
-                                    join(get_project_dir(), ".pio"))
-
-
-def get_project_build_dir(force=False):
-    path = get_project_optional_dir("build_dir",
-                                    join(get_project_workspace_dir(), "build"))
-    try:
-        if not isdir(path):
-            os.makedirs(path)
-    except Exception as e:  # pylint: disable=broad-except
-        if not force:
-            raise Exception(e)
-    return path
-
-
-def get_project_libdeps_dir():
-    return get_project_optional_dir(
-        "libdeps_dir", join(get_project_workspace_dir(), "libdeps"))
+def get_project_global_lib_dir():
+    """
+    Deprecated, use ProjectConfig.get_optional_dir("globallib") instead
+    "platformio-node-helpers" depends on it
+    """
+    return ProjectConfig.get_instance(
+        join(get_project_dir(), "platformio.ini")
+    ).get_optional_dir("globallib")
 
 
 def get_project_lib_dir():
-    return get_project_optional_dir("lib_dir", join(get_project_dir(), "lib"))
+    """
+    Deprecated, use ProjectConfig.get_optional_dir("lib") instead
+    "platformio-node-helpers" depends on it
+    """
+    return ProjectConfig.get_instance(
+        join(get_project_dir(), "platformio.ini")
+    ).get_optional_dir("lib")
 
 
-def get_project_include_dir():
-    return get_project_optional_dir("include_dir",
-                                    join(get_project_dir(), "include"))
+def get_project_libdeps_dir():
+    """
+    Deprecated, use ProjectConfig.get_optional_dir("libdeps") instead
+    "platformio-node-helpers" depends on it
+    """
+    return ProjectConfig.get_instance(
+        join(get_project_dir(), "platformio.ini")
+    ).get_optional_dir("libdeps")
 
 
-def get_project_src_dir():
-    return get_project_optional_dir("src_dir", join(get_project_dir(), "src"))
+def get_default_projects_dir():
+    docs_dir = join(fs.expanduser("~"), "Documents")
+    try:
+        assert WINDOWS
+        import ctypes.wintypes  # pylint: disable=import-outside-toplevel
 
-
-def get_project_test_dir():
-    return get_project_optional_dir("test_dir", join(get_project_dir(),
-                                                     "test"))
-
-
-def get_project_boards_dir():
-    return get_project_optional_dir("boards_dir",
-                                    join(get_project_dir(), "boards"))
-
-
-def get_project_data_dir():
-    return get_project_optional_dir("data_dir", join(get_project_dir(),
-                                                     "data"))
-
-
-def get_project_shared_dir():
-    return get_project_optional_dir("shared_dir",
-                                    join(get_project_dir(), "shared"))
+        buf = ctypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH)
+        ctypes.windll.shell32.SHGetFolderPathW(None, 5, None, 0, buf)
+        docs_dir = buf.value
+    except:  # pylint: disable=bare-except
+        pass
+    return join(docs_dir, "PlatformIO", "Projects")
 
 
 def compute_project_checksum(config):
@@ -174,8 +112,11 @@ def compute_project_checksum(config):
 
     # project file structure
     check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S")
-    for d in (get_project_include_dir(), get_project_src_dir(),
-              get_project_lib_dir()):
+    for d in (
+        config.get_optional_dir("include"),
+        config.get_optional_dir("src"),
+        config.get_optional_dir("lib"),
+    ):
         if not isdir(d):
             continue
         chunks = []
@@ -194,17 +135,21 @@ def compute_project_checksum(config):
     return checksum.hexdigest()
 
 
-def load_project_ide_data(project_dir, envs):
-    from platformio.commands.run import cli as cmd_run
-    assert envs
-    if not isinstance(envs, (list, tuple, set)):
+def load_project_ide_data(project_dir, env_or_envs):
+    # pylint: disable=import-outside-toplevel
+    from platformio.commands.run.command import cli as cmd_run
+
+    assert env_or_envs
+    envs = env_or_envs
+    if not isinstance(envs, list):
         envs = [envs]
     args = ["--project-dir", project_dir, "--target", "idedata"]
     for env in envs:
         args.extend(["-e", env])
     result = CliRunner().invoke(cmd_run, args)
-    if result.exit_code != 0 and not isinstance(result.exception,
-                                                exception.ReturnErrorCode):
+    if result.exit_code != 0 and not isinstance(
+        result.exception, exception.ReturnErrorCode
+    ):
         raise result.exception
     if '"includes":' not in result.output:
         raise exception.PlatformioException(result.output)
@@ -212,11 +157,10 @@ def load_project_ide_data(project_dir, envs):
     data = {}
     for line in result.output.split("\n"):
         line = line.strip()
-        if (line.startswith('{"') and line.endswith("}")
-                and "env_name" in line):
+        if line.startswith('{"') and line.endswith("}") and "env_name" in line:
             _data = json.loads(line)
             if "env_name" in _data:
-                data[_data['env_name']] = _data
-    if len(envs) == 1 and envs[0] in data:
-        return data[envs[0]]
+                data[_data["env_name"]] = _data
+    if not isinstance(env_or_envs, list) and env_or_envs in data:
+        return data[env_or_envs]
     return data or None

platformio/project/options.py

@@ -14,24 +14,60 @@
 
 # pylint: disable=redefined-builtin, too-many-arguments
 
-from collections import OrderedDict, namedtuple
+import os
+from collections import OrderedDict
 
 import click
 
-ConfigOptionClass = namedtuple("ConfigOption", [
-    "scope", "name", "type", "multiple", "sysenvvar", "buildenvvar", "oldnames"
-])
+from platformio import fs
 
 
-def ConfigOption(scope,
-                 name,
-                 type=str,
-                 multiple=False,
-                 sysenvvar=None,
-                 buildenvvar=None,
-                 oldnames=None):
-    return ConfigOptionClass(scope, name, type, multiple, sysenvvar,
-                             buildenvvar, oldnames)
+class ConfigOption(object):  # pylint: disable=too-many-instance-attributes
+    def __init__(
+        self,
+        scope,
+        group,
+        name,
+        description,
+        type=str,
+        multiple=False,
+        sysenvvar=None,
+        buildenvvar=None,
+        oldnames=None,
+        default=None,
+    ):
+        self.scope = scope
+        self.group = group
+        self.name = name
+        self.description = description
+        self.type = type
+        self.multiple = multiple
+        self.sysenvvar = sysenvvar
+        self.buildenvvar = buildenvvar
+        self.oldnames = oldnames
+        self.default = default
+
+    def as_dict(self):
+        result = dict(
+            scope=self.scope,
+            group=self.group,
+            name=self.name,
+            description=self.description,
+            type="string",
+            multiple=self.multiple,
+            sysenvvar=self.sysenvvar,
+            default=self.default,
+        )
+        if isinstance(self.type, click.ParamType):
+            result["type"] = self.type.name
+
+        if isinstance(self.type, (click.IntRange, click.FloatRange)):
+            result["min"] = self.type.min
+            result["max"] = self.type.max
+        if isinstance(self.type, click.Choice):
+            result["choices"] = self.type.choices
+
+        return result
 
 
 def ConfigPlatformioOption(*args, **kwargs):
@@ -42,162 +78,610 @@ def ConfigEnvOption(*args, **kwargs):
     return ConfigOption("env", *args, **kwargs)
 
 
-ProjectOptions = OrderedDict([
-    ("%s.%s" % (option.scope, option.name), option) for option in [
-        #
-        # [platformio]
-        #
-        ConfigPlatformioOption(name="description"),
-        ConfigPlatformioOption(name="default_envs",
-                               oldnames=["env_default"],
-                               multiple=True,
-                               sysenvvar="PLATFORMIO_DEFAULT_ENVS"),
-        ConfigPlatformioOption(name="extra_configs", multiple=True),
-
-        # Dirs
-        ConfigPlatformioOption(name="core_dir",
-                               oldnames=["home_dir"],
-                               sysenvvar="PLATFORMIO_CORE_DIR"),
-        ConfigPlatformioOption(name="globallib_dir",
-                               sysenvvar="PLATFORMIO_GLOBALLIB_DIR"),
-        ConfigPlatformioOption(name="platforms_dir",
-                               sysenvvar="PLATFORMIO_PLATFORMS_DIR"),
-        ConfigPlatformioOption(name="packages_dir",
-                               sysenvvar="PLATFORMIO_PACKAGES_DIR"),
-        ConfigPlatformioOption(name="cache_dir",
-                               sysenvvar="PLATFORMIO_CACHE_DIR"),
-        ConfigPlatformioOption(name="build_cache_dir",
-                               sysenvvar="PLATFORMIO_BUILD_CACHE_DIR"),
-        ConfigPlatformioOption(name="workspace_dir",
-                               sysenvvar="PLATFORMIO_WORKSPACE_DIR"),
-        ConfigPlatformioOption(name="build_dir",
-                               sysenvvar="PLATFORMIO_BUILD_DIR"),
-        ConfigPlatformioOption(name="libdeps_dir",
-                               sysenvvar="PLATFORMIO_LIBDEPS_DIR"),
-        ConfigPlatformioOption(name="lib_dir", sysenvvar="PLATFORMIO_LIB_DIR"),
-        ConfigPlatformioOption(name="include_dir",
-                               sysenvvar="PLATFORMIO_INCLUDE_DIR"),
-        ConfigPlatformioOption(name="src_dir", sysenvvar="PLATFORMIO_SRC_DIR"),
-        ConfigPlatformioOption(name="test_dir",
-                               sysenvvar="PLATFORMIO_TEST_DIR"),
-        ConfigPlatformioOption(name="boards_dir",
-                               sysenvvar="PLATFORMIO_BOARDS_DIR"),
-        ConfigPlatformioOption(name="data_dir",
-                               sysenvvar="PLATFORMIO_DATA_DIR"),
-        ConfigPlatformioOption(name="shared_dir",
-                               sysenvvar="PLATFORMIO_SHARED_DIR"),
-
-        #
-        # [env]
-        #
-
-        # Generic
-        ConfigEnvOption(name="platform", buildenvvar="PIOPLATFORM"),
-        ConfigEnvOption(name="platform_packages", multiple=True),
-        ConfigEnvOption(
-            name="framework", multiple=True, buildenvvar="PIOFRAMEWORK"),
-
-        # Board
-        ConfigEnvOption(name="board", buildenvvar="BOARD"),
-        ConfigEnvOption(name="board_build.mcu",
-                        oldnames=["board_mcu"],
-                        buildenvvar="BOARD_MCU"),
-        ConfigEnvOption(name="board_build.f_cpu",
-                        oldnames=["board_f_cpu"],
-                        buildenvvar="BOARD_F_CPU"),
-        ConfigEnvOption(name="board_build.f_flash",
-                        oldnames=["board_f_flash"],
-                        buildenvvar="BOARD_F_FLASH"),
-        ConfigEnvOption(name="board_build.flash_mode",
-                        oldnames=["board_flash_mode"],
-                        buildenvvar="BOARD_FLASH_MODE"),
-
-        # Build
-        ConfigEnvOption(name="build_type",
-                        type=click.Choice(["release", "debug"])),
-        ConfigEnvOption(name="build_flags",
-                        multiple=True,
-                        sysenvvar="PLATFORMIO_BUILD_FLAGS",
-                        buildenvvar="BUILD_FLAGS"),
-        ConfigEnvOption(name="src_build_flags",
-                        multiple=True,
-                        sysenvvar="PLATFORMIO_SRC_BUILD_FLAGS",
-                        buildenvvar="SRC_BUILD_FLAGS"),
-        ConfigEnvOption(name="build_unflags",
-                        multiple=True,
-                        sysenvvar="PLATFORMIO_BUILD_UNFLAGS",
-                        buildenvvar="BUILD_UNFLAGS"),
-        ConfigEnvOption(name="src_filter",
-                        multiple=True,
-                        sysenvvar="PLATFORMIO_SRC_FILTER",
-                        buildenvvar="SRC_FILTER"),
-        ConfigEnvOption(name="targets", multiple=True),
-
-        # Upload
-        ConfigEnvOption(name="upload_port",
-                        sysenvvar="PLATFORMIO_UPLOAD_PORT",
-                        buildenvvar="UPLOAD_PORT"),
-        ConfigEnvOption(name="upload_protocol", buildenvvar="UPLOAD_PROTOCOL"),
-        ConfigEnvOption(
-            name="upload_speed", type=click.INT, buildenvvar="UPLOAD_SPEED"),
-        ConfigEnvOption(name="upload_flags",
-                        multiple=True,
-                        sysenvvar="PLATFORMIO_UPLOAD_FLAGS",
-                        buildenvvar="UPLOAD_FLAGS"),
-        ConfigEnvOption(name="upload_resetmethod",
-                        buildenvvar="UPLOAD_RESETMETHOD"),
-        ConfigEnvOption(name="upload_command", buildenvvar="UPLOADCMD"),
-
-        # Monitor
-        ConfigEnvOption(name="monitor_port"),
-        ConfigEnvOption(name="monitor_speed", oldnames=["monitor_baud"]),
-        ConfigEnvOption(name="monitor_rts", type=click.IntRange(0, 1)),
-        ConfigEnvOption(name="monitor_dtr", type=click.IntRange(0, 1)),
-        ConfigEnvOption(name="monitor_flags", multiple=True),
-
-        # Library
-        ConfigEnvOption(name="lib_deps",
-                        oldnames=["lib_use", "lib_force", "lib_install"],
-                        multiple=True),
-        ConfigEnvOption(name="lib_ignore", multiple=True),
-        ConfigEnvOption(name="lib_extra_dirs",
-                        multiple=True,
-                        sysenvvar="PLATFORMIO_LIB_EXTRA_DIRS"),
-        ConfigEnvOption(name="lib_ldf_mode",
-                        type=click.Choice(
-                            ["off", "chain", "deep", "chain+", "deep+"])),
-        ConfigEnvOption(name="lib_compat_mode",
-                        type=click.Choice(["off", "soft", "strict"])),
-        ConfigEnvOption(name="lib_archive", type=click.BOOL),
-
-        # Test
-        ConfigEnvOption(name="test_filter", multiple=True),
-        ConfigEnvOption(name="test_ignore", multiple=True),
-        ConfigEnvOption(name="test_port"),
-        ConfigEnvOption(name="test_speed", type=click.INT),
-        ConfigEnvOption(name="test_transport"),
-        ConfigEnvOption(name="test_build_project_src", type=click.BOOL),
-
-        # Debug
-        ConfigEnvOption(name="debug_tool"),
-        ConfigEnvOption(name="debug_init_break"),
-        ConfigEnvOption(name="debug_init_cmds", multiple=True),
-        ConfigEnvOption(name="debug_extra_cmds", multiple=True),
-        ConfigEnvOption(name="debug_load_cmds",
-                        oldnames=["debug_load_cmd"],
-                        multiple=True),
-        ConfigEnvOption(name="debug_load_mode",
-                        type=click.Choice(["always", "modified", "manual"])),
-        ConfigEnvOption(name="debug_server", multiple=True),
-        ConfigEnvOption(name="debug_port"),
-        ConfigEnvOption(name="debug_svd_path",
-                        type=click.Path(
-                            exists=True, file_okay=True, dir_okay=False)),
-
-        # Other
-        ConfigEnvOption(name="extra_scripts",
-                        oldnames=["extra_script"],
-                        multiple=True,
-                        sysenvvar="PLATFORMIO_EXTRA_SCRIPTS")
+ProjectOptions = OrderedDict(
+    [
+        ("%s.%s" % (option.scope, option.name), option)
+        for option in [
+            #
+            # [platformio]
+            #
+            ConfigPlatformioOption(
+                group="generic",
+                name="description",
+                description="Describe a project with a short information",
+            ),
+            ConfigPlatformioOption(
+                group="generic",
+                name="default_envs",
+                description=(
+                    "Configure a list with environments which PlatformIO should "
+                    "process by default"
+                ),
+                oldnames=["env_default"],
+                multiple=True,
+                sysenvvar="PLATFORMIO_DEFAULT_ENVS",
+            ),
+            ConfigPlatformioOption(
+                group="generic",
+                name="extra_configs",
+                description=(
+                    "Extend main configuration with the extra configuration files"
+                ),
+                multiple=True,
+            ),
+            # Dirs
+            ConfigPlatformioOption(
+                group="directory",
+                name="core_dir",
+                description=(
+                    "PlatformIO Core location where it keeps installed development "
+                    "platforms, packages, global libraries, "
+                    "and other internal information"
+                ),
+                oldnames=["home_dir"],
+                sysenvvar="PLATFORMIO_CORE_DIR",
+                default=os.path.join(fs.expanduser("~"), ".platformio"),
+            ),
+            ConfigPlatformioOption(
+                group="directory",
+                name="globallib_dir",
+                description=(
+                    "A library folder/storage where PlatformIO Library Dependency "
+                    "Finder (LDF) looks for global libraries"
+                ),
+                sysenvvar="PLATFORMIO_GLOBALLIB_DIR",
+                default=os.path.join("$PROJECT_CORE_DIR", "lib"),
+            ),
+            ConfigPlatformioOption(
+                group="directory",
+                name="platforms_dir",
+                description=(
+                    "A location where PlatformIO Core keeps installed development "
+                    "platforms"
+                ),
+                sysenvvar="PLATFORMIO_PLATFORMS_DIR",
+                default=os.path.join("$PROJECT_CORE_DIR", "platforms"),
+            ),
+            ConfigPlatformioOption(
+                group="directory",
+                name="packages_dir",
+                description=(
+                    "A location where PlatformIO Core keeps installed packages"
+                ),
+                sysenvvar="PLATFORMIO_PACKAGES_DIR",
+                default=os.path.join("$PROJECT_CORE_DIR", "packages"),
+            ),
+            ConfigPlatformioOption(
+                group="directory",
+                name="cache_dir",
+                description=(
+                    "A location where PlatformIO Core stores caching information "
+                    "(requests to PlatformIO Registry, downloaded packages and "
+                    "other service information)"
+                ),
+                sysenvvar="PLATFORMIO_CACHE_DIR",
+                default=os.path.join("$PROJECT_CORE_DIR", ".cache"),
+            ),
+            ConfigPlatformioOption(
+                group="directory",
+                name="build_cache_dir",
+                description=(
+                    "A location where PlatformIO Core keeps derived files from a "
+                    "build system (objects, firmwares, ELFs) and caches them between "
+                    "build environments"
+                ),
+                sysenvvar="PLATFORMIO_BUILD_CACHE_DIR",
+            ),
+            ConfigPlatformioOption(
+                group="directory",
+                name="workspace_dir",
+                description=(
+                    "A path to a project workspace directory where PlatformIO keeps "
+                    "by default compiled objects, static libraries, firmwares, and "
+                    "external library dependencies"
+                ),
+                sysenvvar="PLATFORMIO_WORKSPACE_DIR",
+                default=os.path.join("$PROJECT_DIR", ".pio"),
+            ),
+            ConfigPlatformioOption(
+                group="directory",
+                name="build_dir",
+                description=(
+                    "PlatformIO Build System uses this folder for project environments"
+                    " to store compiled object files, static libraries, firmwares, "
+                    "and other cached information"
+                ),
+                sysenvvar="PLATFORMIO_BUILD_DIR",
+                default=os.path.join("$PROJECT_WORKSPACE_DIR", "build"),
+            ),
+            ConfigPlatformioOption(
+                group="directory",
+                name="libdeps_dir",
+                description=(
+                    "Internal storage where Library Manager will install project "
+                    "dependencies declared via `lib_deps` option"
+                ),
+                sysenvvar="PLATFORMIO_LIBDEPS_DIR",
+                default=os.path.join("$PROJECT_WORKSPACE_DIR", "libdeps"),
+            ),
+            ConfigPlatformioOption(
+                group="directory",
+                name="include_dir",
+                description=(
+                    "A default location for project header files. PlatformIO Build "
+                    "System automatically adds this path to CPPPATH scope"
+                ),
+                sysenvvar="PLATFORMIO_INCLUDE_DIR",
+                default=os.path.join("$PROJECT_DIR", "include"),
+            ),
+            ConfigPlatformioOption(
+                group="directory",
+                name="src_dir",
+                description=(
+                    "A default location where PlatformIO Build System looks for the "
+                    "project C/C++ source files"
+                ),
+                sysenvvar="PLATFORMIO_SRC_DIR",
+                default=os.path.join("$PROJECT_DIR", "src"),
+            ),
+            ConfigPlatformioOption(
+                group="directory",
+                name="lib_dir",
+                description="A storage for the custom/private project libraries",
+                sysenvvar="PLATFORMIO_LIB_DIR",
+                default=os.path.join("$PROJECT_DIR", "lib"),
+            ),
+            ConfigPlatformioOption(
+                group="directory",
+                name="data_dir",
+                description=(
+                    "A data directory to store contents which can be uploaded to "
+                    "file system (SPIFFS, etc.)"
+                ),
+                sysenvvar="PLATFORMIO_DATA_DIR",
+                default=os.path.join("$PROJECT_DIR", "data"),
+            ),
+            ConfigPlatformioOption(
+                group="directory",
+                name="test_dir",
+                description=(
+                    "A location where PIO Unit Testing engine looks for "
+                    "test source files"
+                ),
+                sysenvvar="PLATFORMIO_TEST_DIR",
+                default=os.path.join("$PROJECT_DIR", "test"),
+            ),
+            ConfigPlatformioOption(
+                group="directory",
+                name="boards_dir",
+                description="A global storage for custom board manifests",
+                sysenvvar="PLATFORMIO_BOARDS_DIR",
+                default=os.path.join("$PROJECT_DIR", "boards"),
+            ),
+            ConfigPlatformioOption(
+                group="directory",
+                name="shared_dir",
+                description=(
+                    "A location which PIO Remote uses to synchronize extra files "
+                    "between remote machines"
+                ),
+                sysenvvar="PLATFORMIO_SHARED_DIR",
+                default=os.path.join("$PROJECT_DIR", "shared"),
+            ),
+            #
+            # [env]
+            #
+            # Platform
+            ConfigEnvOption(
+                group="platform",
+                name="platform",
+                description="A name or specification of development platform",
+                buildenvvar="PIOPLATFORM",
+            ),
+            ConfigEnvOption(
+                group="platform",
+                name="platform_packages",
+                description="Custom packages and specifications",
+                multiple=True,
+            ),
+            ConfigEnvOption(
+                group="platform",
+                name="framework",
+                description="A list of project dependent frameworks",
+                multiple=True,
+                buildenvvar="PIOFRAMEWORK",
+            ),
+            # Board
+            ConfigEnvOption(
+                group="board",
+                name="board",
+                description="A board ID",
+                buildenvvar="BOARD",
+            ),
+            ConfigEnvOption(
+                group="board",
+                name="board_build.mcu",
+                description="A custom board MCU",
+                oldnames=["board_mcu"],
+                buildenvvar="BOARD_MCU",
+            ),
+            ConfigEnvOption(
+                group="board",
+                name="board_build.f_cpu",
+                description="A custom MCU frequency",
+                oldnames=["board_f_cpu"],
+                buildenvvar="BOARD_F_CPU",
+            ),
+            ConfigEnvOption(
+                group="board",
+                name="board_build.f_flash",
+                description="A custom flash frequency",
+                oldnames=["board_f_flash"],
+                buildenvvar="BOARD_F_FLASH",
+            ),
+            ConfigEnvOption(
+                group="board",
+                name="board_build.flash_mode",
+                description="A custom flash mode",
+                oldnames=["board_flash_mode"],
+                buildenvvar="BOARD_FLASH_MODE",
+            ),
+            # Build
+            ConfigEnvOption(
+                group="build",
+                name="build_type",
+                description="Project build configuration",
+                type=click.Choice(["release", "debug"]),
+                default="release",
+            ),
+            ConfigEnvOption(
+                group="build",
+                name="build_flags",
+                description=(
+                    "Custom build flags/options for preprocessing, compilation, "
+                    "assembly, and linking processes"
+                ),
+                multiple=True,
+                sysenvvar="PLATFORMIO_BUILD_FLAGS",
+                buildenvvar="BUILD_FLAGS",
+            ),
+            ConfigEnvOption(
+                group="build",
+                name="src_build_flags",
+                description=(
+                    "The same as `build_flags` but configures flags the only for "
+                    "project source files (`src` folder)"
+                ),
+                multiple=True,
+                sysenvvar="PLATFORMIO_SRC_BUILD_FLAGS",
+                buildenvvar="SRC_BUILD_FLAGS",
+            ),
+            ConfigEnvOption(
+                group="build",
+                name="build_unflags",
+                description="A list with flags/option which should be removed",
+                multiple=True,
+                sysenvvar="PLATFORMIO_BUILD_UNFLAGS",
+                buildenvvar="BUILD_UNFLAGS",
+            ),
+            ConfigEnvOption(
+                group="build",
+                name="src_filter",
+                description=(
+                    "Control which source files should be included/excluded from a "
+                    "build process"
+                ),
+                multiple=True,
+                sysenvvar="PLATFORMIO_SRC_FILTER",
+                buildenvvar="SRC_FILTER",
+                default="+<*> -<.git/> -<.svn/>",
+            ),
+            ConfigEnvOption(
+                group="build",
+                name="targets",
+                description="A custom list of targets for PlatformIO Build System",
+                multiple=True,
+            ),
+            # Upload
+            ConfigEnvOption(
+                group="upload",
+                name="upload_port",
+                description=(
+                    "An upload port which `uploader` tool uses for a firmware flashing"
+                ),
+                sysenvvar="PLATFORMIO_UPLOAD_PORT",
+                buildenvvar="UPLOAD_PORT",
+            ),
+            ConfigEnvOption(
+                group="upload",
+                name="upload_protocol",
+                description="A protocol that `uploader` tool uses to talk to a board",
+                buildenvvar="UPLOAD_PROTOCOL",
+            ),
+            ConfigEnvOption(
+                group="upload",
+                name="upload_speed",
+                description=(
+                    "A connection speed (baud rate) which `uploader` tool uses when "
+                    "sending firmware to a board"
+                ),
+                type=click.INT,
+                buildenvvar="UPLOAD_SPEED",
+            ),
+            ConfigEnvOption(
+                group="upload",
+                name="upload_flags",
+                description="An extra flags for `uploader` tool",
+                multiple=True,
+                sysenvvar="PLATFORMIO_UPLOAD_FLAGS",
+                buildenvvar="UPLOAD_FLAGS",
+            ),
+            ConfigEnvOption(
+                group="upload",
+                name="upload_resetmethod",
+                description="A custom reset method",
+                buildenvvar="UPLOAD_RESETMETHOD",
+            ),
+            ConfigEnvOption(
+                group="upload",
+                name="upload_command",
+                description=(
+                    "A custom upload command which overwrites a default from "
+                    "development platform"
+                ),
+                buildenvvar="UPLOADCMD",
+            ),
+            # Monitor
+            ConfigEnvOption(
+                group="monitor",
+                name="monitor_port",
+                description="A port, a number or a device name",
+            ),
+            ConfigEnvOption(
+                group="monitor",
+                name="monitor_speed",
+                description="A monitor speed (baud rate)",
+                type=click.INT,
+                oldnames=["monitor_baud"],
+                default=9600,
+            ),
+            ConfigEnvOption(
+                group="monitor",
+                name="monitor_rts",
+                description="A monitor initial RTS line state",
+                type=click.IntRange(0, 1),
+            ),
+            ConfigEnvOption(
+                group="monitor",
+                name="monitor_dtr",
+                description="A monitor initial DTR line state",
+                type=click.IntRange(0, 1),
+            ),
+            ConfigEnvOption(
+                group="monitor",
+                name="monitor_flags",
+                description=(
+                    "The extra flags and options for `platformio device monitor` "
+                    "command"
+                ),
+                multiple=True,
+            ),
+            # Library
+            ConfigEnvOption(
+                group="library",
+                name="lib_deps",
+                description=(
+                    "A list of project library dependencies which should be installed "
+                    "automatically before a build process"
+                ),
+                oldnames=["lib_use", "lib_force", "lib_install"],
+                multiple=True,
+            ),
+            ConfigEnvOption(
+                group="library",
+                name="lib_ignore",
+                description=(
+                    "A list of library names which should be ignored by "
+                    "Library Dependency Finder (LDF)"
+                ),
+                multiple=True,
+            ),
+            ConfigEnvOption(
+                group="library",
+                name="lib_extra_dirs",
+                description=(
+                    "A list of extra directories/storages where Library Dependency "
+                    "Finder (LDF) will look for dependencies"
+                ),
+                multiple=True,
+                sysenvvar="PLATFORMIO_LIB_EXTRA_DIRS",
+            ),
+            ConfigEnvOption(
+                group="library",
+                name="lib_ldf_mode",
+                description=(
+                    "Control how Library Dependency Finder (LDF) should analyze "
+                    "dependencies (`#include` directives)"
+                ),
+                type=click.Choice(["off", "chain", "deep", "chain+", "deep+"]),
+                default="chain",
+            ),
+            ConfigEnvOption(
+                group="library",
+                name="lib_compat_mode",
+                description=(
+                    "Configure a strictness (compatibility mode by frameworks, "
+                    "development platforms) of Library Dependency Finder (LDF)"
+                ),
+                type=click.Choice(["off", "soft", "strict"]),
+                default="soft",
+            ),
+            ConfigEnvOption(
+                group="library",
+                name="lib_archive",
+                description=(
+                    "Create an archive (`*.a`, static library) from the object files "
+                    "and link it into a firmware (program)"
+                ),
+                type=click.BOOL,
+                default=True,
+            ),
+            # Check
+            ConfigEnvOption(
+                group="check",
+                name="check_tool",
+                description="A list of check tools used for analysis",
+                type=click.Choice(["cppcheck", "clangtidy"]),
+                multiple=True,
+                default=["cppcheck"],
+            ),
+            ConfigEnvOption(
+                group="check",
+                name="check_patterns",
+                description=(
+                    "Configure a list of target files or directories for checking "
+                    "(Unix shell-style wildcards)"
+                ),
+                multiple=True,
+            ),
+            ConfigEnvOption(
+                group="check",
+                name="check_flags",
+                description="An extra flags to be passed to a check tool",
+                multiple=True,
+            ),
+            ConfigEnvOption(
+                group="check",
+                name="check_severity",
+                description="List of defect severity types for analysis",
+                multiple=True,
+                type=click.Choice(["low", "medium", "high"]),
+                default=["low", "medium", "high"],
+            ),
+            # Test
+            ConfigEnvOption(
+                group="test",
+                name="test_filter",
+                description="Process tests where the name matches specified patterns",
+                multiple=True,
+            ),
+            ConfigEnvOption(
+                group="test",
+                name="test_ignore",
+                description="Ignore tests where the name matches specified patterns",
+                multiple=True,
+            ),
+            ConfigEnvOption(
+                group="test",
+                name="test_port",
+                description="A serial port to communicate with a target device",
+            ),
+            ConfigEnvOption(
+                group="test",
+                name="test_speed",
+                description="A connection speed (baud rate) to communicate with a target device",
+                type=click.INT,
+            ),
+            ConfigEnvOption(group="test", name="test_transport", description="",),
+            ConfigEnvOption(
+                group="test",
+                name="test_build_project_src",
+                description="",
+                type=click.BOOL,
+                default=False,
+            ),
+            # Debug
+            ConfigEnvOption(
+                group="debug",
+                name="debug_tool",
+                description="A name of debugging tool",
+            ),
+            ConfigEnvOption(
+                group="debug",
+                name="debug_init_break",
+                description=(
+                    "An initial breakpoint that makes program stop whenever a "
+                    "certain point in the program is reached"
+                ),
+                default="tbreak main",
+            ),
+            ConfigEnvOption(
+                group="debug",
+                name="debug_init_cmds",
+                description="Initial commands to be passed to a back-end debugger",
+                multiple=True,
+            ),
+            ConfigEnvOption(
+                group="debug",
+                name="debug_extra_cmds",
+                description="An extra commands to be passed to a back-end debugger",
+                multiple=True,
+            ),
+            ConfigEnvOption(
+                group="debug",
+                name="debug_load_cmds",
+                description=(
+                    "A list of commands to be used to load program/firmware "
+                    "to a target device"
+                ),
+                oldnames=["debug_load_cmd"],
+                multiple=True,
+                default=["load"],
+            ),
+            ConfigEnvOption(
+                group="debug",
+                name="debug_load_mode",
+                description=(
+                    "Allows one to control when PlatformIO should load debugging "
+                    "firmware to the end target"
+                ),
+                type=click.Choice(["always", "modified", "manual"]),
+                default="always",
+            ),
+            ConfigEnvOption(
+                group="debug",
+                name="debug_server",
+                description="Allows one to setup a custom debugging server",
+                multiple=True,
+            ),
+            ConfigEnvOption(
+                group="debug",
+                name="debug_port",
+                description=(
+                    "A debugging port of a remote target (a serial device or "
+                    "network address)"
+                ),
+            ),
+            ConfigEnvOption(
+                group="debug",
+                name="debug_svd_path",
+                description=(
+                    "A custom path to SVD file which contains information about "
+                    "device peripherals"
+                ),
+                type=click.Path(exists=True, file_okay=True, dir_okay=False),
+            ),
+            # Advanced
+            ConfigEnvOption(
+                group="advanced",
+                name="extends",
+                description=(
+                    "Inherit configuration from other sections or build environments"
+                ),
+                multiple=True,
+            ),
+            ConfigEnvOption(
+                group="advanced",
+                name="extra_scripts",
+                description="A list of PRE and POST extra scripts",
+                oldnames=["extra_script"],
+                multiple=True,
+                sysenvvar="PLATFORMIO_EXTRA_SCRIPTS",
+            ),
+        ]
     ]
-])
+)
+
+
+def get_config_options_schema():
+    return [opt.as_dict() for opt in ProjectOptions.values()]

platformio/telemetry.py

@@ -38,7 +38,6 @@ except ImportError:
 
 
 class TelemetryBase(object):
-
     def __init__(self):
         self._params = {}
 
@@ -64,17 +63,17 @@ class MeasurementProtocol(TelemetryBase):
         "event_category": "ec",
         "event_action": "ea",
         "event_label": "el",
-        "event_value": "ev"
+        "event_value": "ev",
     }
 
     def __init__(self):
         super(MeasurementProtocol, self).__init__()
-        self['v'] = 1
-        self['tid'] = self.TID
-        self['cid'] = app.get_cid()
+        self["v"] = 1
+        self["tid"] = self.TID
+        self["cid"] = app.get_cid()
 
         try:
-            self['sr'] = "%dx%d" % click.get_terminal_size()
+            self["sr"] = "%dx%d" % click.get_terminal_size()
         except ValueError:
             pass
 
@@ -93,7 +92,7 @@ class MeasurementProtocol(TelemetryBase):
         super(MeasurementProtocol, self).__setitem__(name, value)
 
     def _prefill_appinfo(self):
-        self['av'] = __version__
+        self["av"] = __version__
 
         # gather dependent packages
         dpdata = []
@@ -102,10 +101,9 @@ class MeasurementProtocol(TelemetryBase):
             dpdata.append("Caller/%s" % app.get_session_var("caller_id"))
         if getenv("PLATFORMIO_IDE"):
             dpdata.append("IDE/%s" % getenv("PLATFORMIO_IDE"))
-        self['an'] = " ".join(dpdata)
+        self["an"] = " ".join(dpdata)
 
     def _prefill_custom_data(self):
-
         def _filter_args(items):
             result = []
             stop = False
@@ -119,17 +117,16 @@ class MeasurementProtocol(TelemetryBase):
             return result
 
         caller_id = str(app.get_session_var("caller_id"))
-        self['cd1'] = util.get_systype()
-        self['cd2'] = "Python/%s %s" % (platform.python_version(),
-                                        platform.platform())
+        self["cd1"] = util.get_systype()
+        self["cd2"] = "Python/%s %s" % (platform.python_version(), platform.platform())
         # self['cd3'] = " ".join(_filter_args(sys.argv[1:]))
-        self['cd4'] = 1 if (not util.is_ci() and
-                            (caller_id or not is_container())) else 0
+        self["cd4"] = (
+            1 if (not util.is_ci() and (caller_id or not is_container())) else 0
+        )
         if caller_id:
-            self['cd5'] = caller_id.lower()
+            self["cd5"] = caller_id.lower()
 
     def _prefill_screen_name(self):
-
         def _first_arg_from_list(args_, list_):
             for _arg in args_:
                 if _arg in list_:
@@ -146,12 +143,27 @@ class MeasurementProtocol(TelemetryBase):
             return
 
         cmd_path = args[:1]
-        if args[0] in ("platform", "platforms", "serialports", "device",
-                       "settings", "account"):
+        if args[0] in (
+            "platform",
+            "platforms",
+            "serialports",
+            "device",
+            "settings",
+            "account",
+        ):
             cmd_path = args[:2]
         if args[0] == "lib" and len(args) > 1:
-            lib_subcmds = ("builtin", "install", "list", "register", "search",
-                           "show", "stats", "uninstall", "update")
+            lib_subcmds = (
+                "builtin",
+                "install",
+                "list",
+                "register",
+                "search",
+                "show",
+                "stats",
+                "uninstall",
+                "update",
+            )
             sub_cmd = _first_arg_from_list(args[1:], lib_subcmds)
             if sub_cmd:
                 cmd_path.append(sub_cmd)
@@ -165,24 +177,25 @@ class MeasurementProtocol(TelemetryBase):
                     sub_cmd = _first_arg_from_list(args[2:], remote2_subcmds)
                     if sub_cmd:
                         cmd_path.append(sub_cmd)
-        self['screen_name'] = " ".join([p.title() for p in cmd_path])
+        self["screen_name"] = " ".join([p.title() for p in cmd_path])
 
     @staticmethod
     def _ignore_hit():
         if not app.get_setting("enable_telemetry"):
             return True
-        if app.get_session_var("caller_id") and \
-                all(c in sys.argv for c in ("run", "idedata")):
+        if app.get_session_var("caller_id") and all(
+            c in sys.argv for c in ("run", "idedata")
+        ):
             return True
         return False
 
     def send(self, hittype):
         if self._ignore_hit():
             return
-        self['t'] = hittype
+        self["t"] = hittype
         # correct queue time
-        if "qt" in self._params and isinstance(self['qt'], float):
-            self['qt'] = int((time() - self['qt']) * 1000)
+        if "qt" in self._params and isinstance(self["qt"], float):
+            self["qt"] = int((time() - self["qt"]) * 1000)
         MPDataPusher().push(self._params)
 
 
@@ -202,7 +215,7 @@ class MPDataPusher(object):
         # if network is off-line
         if self._http_offline:
             if "qt" not in item:
-                item['qt'] = time()
+                item["qt"] = time()
             self._failedque.append(item)
             return
 
@@ -243,7 +256,7 @@ class MPDataPusher(object):
                 item = self._queue.get()
                 _item = item.copy()
                 if "qt" not in _item:
-                    _item['qt'] = time()
+                    _item["qt"] = time()
                 self._failedque.append(_item)
                 if self._send_data(item):
                     self._failedque.remove(_item)
@@ -259,7 +272,8 @@ class MPDataPusher(object):
                 "https://ssl.google-analytics.com/collect",
                 data=data,
                 headers=util.get_request_defheaders(),
-                timeout=1)
+                timeout=1,
+            )
             r.raise_for_status()
             return True
         except requests.exceptions.HTTPError as e:
@@ -284,11 +298,10 @@ def on_command():
 
 def measure_ci():
     event = {"category": "CI", "action": "NoName", "label": None}
-    known_cis = ("TRAVIS", "APPVEYOR", "GITLAB_CI", "CIRCLECI", "SHIPPABLE",
-                 "DRONE")
+    known_cis = ("TRAVIS", "APPVEYOR", "GITLAB_CI", "CIRCLECI", "SHIPPABLE", "DRONE")
     for name in known_cis:
         if getenv(name, "false").lower() == "true":
-            event['action'] = name
+            event["action"] = name
             break
     on_event(**event)
 
@@ -307,32 +320,37 @@ def on_run_environment(options, targets):
 
 def on_event(category, action, label=None, value=None, screen_name=None):
     mp = MeasurementProtocol()
-    mp['event_category'] = category[:150]
-    mp['event_action'] = action[:500]
+    mp["event_category"] = category[:150]
+    mp["event_action"] = action[:500]
     if label:
-        mp['event_label'] = label[:500]
+        mp["event_label"] = label[:500]
     if value:
-        mp['event_value'] = int(value)
+        mp["event_value"] = int(value)
     if screen_name:
-        mp['screen_name'] = screen_name[:2048]
+        mp["screen_name"] = screen_name[:2048]
     mp.send("event")
 
 
 def on_exception(e):
-
     def _cleanup_description(text):
         text = text.replace("Traceback (most recent call last):", "")
-        text = re.sub(r'File "([^"]+)"',
-                      lambda m: join(*m.group(1).split(sep)[-2:]),
-                      text,
-                      flags=re.M)
+        text = re.sub(
+            r'File "([^"]+)"',
+            lambda m: join(*m.group(1).split(sep)[-2:]),
+            text,
+            flags=re.M,
+        )
         text = re.sub(r"\s+", " ", text, flags=re.M)
         return text.strip()
 
     skip_conditions = [
-        isinstance(e, cls) for cls in (IOError, exception.ReturnErrorCode,
-                                       exception.UserSideException,
-                                       exception.PlatformIOProjectException)
+        isinstance(e, cls)
+        for cls in (
+            IOError,
+            exception.ReturnErrorCode,
+            exception.UserSideException,
+            exception.PlatformIOProjectException,
+        )
     ]
     try:
         skip_conditions.append("[API] Account: " in str(e))
@@ -340,14 +358,16 @@ def on_exception(e):
         e = ue
     if any(skip_conditions):
         return
-    is_crash = any([
-        not isinstance(e, exception.PlatformioException),
-        "Error" in e.__class__.__name__
-    ])
+    is_crash = any(
+        [
+            not isinstance(e, exception.PlatformioException),
+            "Error" in e.__class__.__name__,
+        ]
+    )
     mp = MeasurementProtocol()
     description = _cleanup_description(format_exc() if is_crash else str(e))
-    mp['exd'] = ("%s: %s" % (type(e).__name__, description))[:2048]
-    mp['exf'] = 1 if is_crash else 0
+    mp["exd"] = ("%s: %s" % (type(e).__name__, description))[:2048]
+    mp["exf"] = 1 if is_crash else 0
     mp.send("exception")
 
 
@@ -373,7 +393,7 @@ def backup_reports(items):
     KEEP_MAX_REPORTS = 100
     tm = app.get_state_item("telemetry", {})
     if "backup" not in tm:
-        tm['backup'] = []
+        tm["backup"] = []
 
     for params in items:
         # skip static options
@@ -383,28 +403,28 @@ def backup_reports(items):
 
         # store time in UNIX format
         if "qt" not in params:
-            params['qt'] = time()
-        elif not isinstance(params['qt'], float):
-            params['qt'] = time() - (params['qt'] / 1000)
+            params["qt"] = time()
+        elif not isinstance(params["qt"], float):
+            params["qt"] = time() - (params["qt"] / 1000)
 
-        tm['backup'].append(params)
+        tm["backup"].append(params)
 
-    tm['backup'] = tm['backup'][KEEP_MAX_REPORTS * -1:]
+    tm["backup"] = tm["backup"][KEEP_MAX_REPORTS * -1 :]
     app.set_state_item("telemetry", tm)
 
 
 def resend_backuped_reports():
     tm = app.get_state_item("telemetry", {})
-    if "backup" not in tm or not tm['backup']:
+    if "backup" not in tm or not tm["backup"]:
         return False
 
-    for report in tm['backup']:
+    for report in tm["backup"]:
         mp = MeasurementProtocol()
         for key, value in report.items():
             mp[key] = value
-        mp.send(report['t'])
+        mp.send(report["t"])
 
     # clean
-    tm['backup'] = []
+    tm["backup"] = []
     app.set_state_item("telemetry", tm)
     return True

platformio/unpacker.py

@@ -12,8 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from os import chmod
-from os.path import exists, join
+import os
 from tarfile import open as tarfile_open
 from time import mktime
 from zipfile import ZipFile
@@ -24,7 +23,6 @@ from platformio import exception, util
 
 
 class ArchiveBase(object):
-
     def __init__(self, arhfileobj):
         self._afo = arhfileobj
 
@@ -34,6 +32,9 @@ class ArchiveBase(object):
     def get_item_filename(self, item):
         raise NotImplementedError()
 
+    def is_link(self, item):
+        raise NotImplementedError()
+
     def extract_item(self, item, dest_dir):
         self._afo.extract(item, dest_dir)
         self.after_extract(item, dest_dir)
@@ -46,7 +47,6 @@ class ArchiveBase(object):
 
 
 class TARArchive(ArchiveBase):
-
     def __init__(self, archpath):
         super(TARArchive, self).__init__(tarfile_open(archpath))
 
@@ -57,12 +57,37 @@ class TARArchive(ArchiveBase):
         return item.name
 
     @staticmethod
-    def islink(item):
+    def is_link(item):
         return item.islnk() or item.issym()
 
+    @staticmethod
+    def resolve_path(path):
+        return os.path.realpath(os.path.abspath(path))
+
+    def is_bad_path(self, path, base):
+        return not self.resolve_path(os.path.join(base, path)).startswith(base)
+
+    def is_bad_link(self, item, base):
+        return not self.resolve_path(
+            os.path.join(os.path.join(base, os.path.dirname(item.name)), item.linkname)
+        ).startswith(base)
+
+    def extract_item(self, item, dest_dir):
+        bad_conds = [
+            self.is_bad_path(item.name, dest_dir),
+            self.is_link(item) and self.is_bad_link(item, dest_dir),
+        ]
+        if not any(bad_conds):
+            super(TARArchive, self).extract_item(item, dest_dir)
+        else:
+            click.secho(
+                "Blocked insecure item `%s` from TAR archive" % item.name,
+                fg="red",
+                err=True,
+            )
+
 
 class ZIPArchive(ArchiveBase):
-
     def __init__(self, archpath):
         super(ZIPArchive, self).__init__(ZipFile(archpath))
 
@@ -70,12 +95,18 @@ class ZIPArchive(ArchiveBase):
     def preserve_permissions(item, dest_dir):
         attrs = item.external_attr >> 16
         if attrs:
-            chmod(join(dest_dir, item.filename), attrs)
+            os.chmod(os.path.join(dest_dir, item.filename), attrs)
 
     @staticmethod
     def preserve_mtime(item, dest_dir):
-        util.change_filemtime(join(dest_dir, item.filename),
-                              mktime(tuple(item.date_time) + tuple([0, 0, 0])))
+        util.change_filemtime(
+            os.path.join(dest_dir, item.filename),
+            mktime(tuple(item.date_time) + tuple([0, 0, 0])),
+        )
+
+    @staticmethod
+    def is_link(_):
+        return False
 
     def get_items(self):
         return self._afo.infolist()
@@ -83,22 +114,18 @@ class ZIPArchive(ArchiveBase):
     def get_item_filename(self, item):
         return item.filename
 
-    def islink(self, item):
-        raise NotImplementedError()
-
     def after_extract(self, item, dest_dir):
         self.preserve_permissions(item, dest_dir)
         self.preserve_mtime(item, dest_dir)
 
 
 class FileUnpacker(object):
-
     def __init__(self, archpath):
         self.archpath = archpath
         self._unpacker = None
 
     def __enter__(self):
-        if self.archpath.lower().endswith((".gz", ".bz2")):
+        if self.archpath.lower().endswith((".gz", ".bz2", ".tar")):
             self._unpacker = TARArchive(self.archpath)
         elif self.archpath.lower().endswith(".zip"):
             self._unpacker = ZIPArchive(self.archpath)
@@ -110,7 +137,7 @@ class FileUnpacker(object):
         if self._unpacker:
             self._unpacker.close()
 
-    def unpack(self, dest_dir=".", with_progress=True):
+    def unpack(self, dest_dir=".", with_progress=True, check_unpacked=True):
         assert self._unpacker
         if not with_progress:
             click.echo("Unpacking...")
@@ -122,12 +149,15 @@ class FileUnpacker(object):
                 for item in pb:
                     self._unpacker.extract_item(item, dest_dir)
 
+        if not check_unpacked:
+            return True
+
         # check on disk
         for item in self._unpacker.get_items():
             filename = self._unpacker.get_item_filename(item)
-            item_path = join(dest_dir, filename)
+            item_path = os.path.join(dest_dir, filename)
             try:
-                if not self._unpacker.islink(item) and not exists(item_path):
+                if not self._unpacker.is_link(item) and not os.path.exists(item_path):
                     raise exception.ExtractArchiveItemError(filename, dest_dir)
             except NotImplementedError:
                 pass

platformio/util.py

@@ -40,7 +40,6 @@ from platformio.proc import is_ci  # pylint: disable=unused-import
 
 
 class memoized(object):
-
     def __init__(self, expire=0):
         expire = str(expire)
         if expire.isdigit():
@@ -51,13 +50,12 @@ class memoized(object):
         self.cache = {}
 
     def __call__(self, func):
-
         @wraps(func)
         def wrapper(*args, **kwargs):
             key = str(args) + str(kwargs)
-            if (key not in self.cache
-                    or (self.expire > 0
-                        and self.cache[key][0] < time.time() - self.expire)):
+            if key not in self.cache or (
+                self.expire > 0 and self.cache[key][0] < time.time() - self.expire
+            ):
                 self.cache[key] = (time.time(), func(*args, **kwargs))
             return self.cache[key][1]
 
@@ -69,13 +67,11 @@ class memoized(object):
 
 
 class throttle(object):
-
     def __init__(self, threshhold):
         self.threshhold = threshhold  # milliseconds
         self.last = 0
 
     def __call__(self, func):
-
         @wraps(func)
         def wrapper(*args, **kwargs):
             diff = int(round((time.time() - self.last) * 1000))
@@ -130,6 +126,7 @@ def change_filemtime(path, mtime):
 
 def get_serial_ports(filter_hwid=False):
     try:
+        # pylint: disable=import-outside-toplevel
         from serial.tools.list_ports import comports
     except ImportError:
         raise exception.GetSerialPortsError(os.name)
@@ -166,17 +163,14 @@ def get_logical_devices():
     if WINDOWS:
         try:
             result = exec_command(
-                ["wmic", "logicaldisk", "get",
-                 "name,VolumeName"]).get("out", "")
+                ["wmic", "logicaldisk", "get", "name,VolumeName"]
+            ).get("out", "")
             devicenamere = re.compile(r"^([A-Z]{1}\:)\s*(\S+)?")
             for line in result.split("\n"):
                 match = devicenamere.match(line.strip())
                 if not match:
                     continue
-                items.append({
-                    "path": match.group(1) + "\\",
-                    "name": match.group(2)
-                })
+                items.append({"path": match.group(1) + "\\", "name": match.group(2)})
             return items
         except WindowsError:  # pylint: disable=undefined-variable
             pass
@@ -192,35 +186,31 @@ def get_logical_devices():
         match = devicenamere.match(line.strip())
         if not match:
             continue
-        items.append({
-            "path": match.group(1),
-            "name": os.path.basename(match.group(1))
-        })
+        items.append({"path": match.group(1), "name": os.path.basename(match.group(1))})
     return items
 
 
 def get_mdns_services():
+    # pylint: disable=import-outside-toplevel
     try:
         import zeroconf
     except ImportError:
         from site import addsitedir
         from platformio.managers.core import get_core_package_dir
+
         contrib_pysite_dir = get_core_package_dir("contrib-pysite")
         addsitedir(contrib_pysite_dir)
         sys.path.insert(0, contrib_pysite_dir)
-        import zeroconf
+        import zeroconf  # pylint: disable=import-outside-toplevel
 
     class mDNSListener(object):
-
         def __init__(self):
-            self._zc = zeroconf.Zeroconf(
-                interfaces=zeroconf.InterfaceChoice.All)
+            self._zc = zeroconf.Zeroconf(interfaces=zeroconf.InterfaceChoice.All)
             self._found_types = []
             self._found_services = []
 
         def __enter__(self):
-            zeroconf.ServiceBrowser(self._zc, "_services._dns-sd._udp.local.",
-                                    self)
+            zeroconf.ServiceBrowser(self._zc, "_services._dns-sd._udp.local.", self)
             return self
 
         def __exit__(self, etype, value, traceback):
@@ -233,8 +223,7 @@ def get_mdns_services():
             try:
                 assert zeroconf.service_type_name(name)
                 assert str(name)
-            except (AssertionError, UnicodeError,
-                    zeroconf.BadTypeInNameException):
+            except (AssertionError, UnicodeError, zeroconf.BadTypeInNameException):
                 return
             if name not in self._found_types:
                 self._found_types.append(name)
@@ -255,29 +244,29 @@ def get_mdns_services():
             if service.properties:
                 try:
                     properties = {
-                        k.decode("utf8"):
-                        v.decode("utf8") if isinstance(v, bytes) else v
+                        k.decode("utf8"): v.decode("utf8")
+                        if isinstance(v, bytes)
+                        else v
                         for k, v in service.properties.items()
                     }
                     json.dumps(properties)
                 except UnicodeDecodeError:
                     properties = None
 
-            items.append({
-                "type":
-                service.type,
-                "name":
-                service.name,
-                "ip":
-                ".".join([
-                    str(c if isinstance(c, int) else ord(c))
-                    for c in service.address
-                ]),
-                "port":
-                service.port,
-                "properties":
-                properties
-            })
+            items.append(
+                {
+                    "type": service.type,
+                    "name": service.name,
+                    "ip": ".".join(
+                        [
+                            str(c if isinstance(c, int) else ord(c))
+                            for c in service.address
+                        ]
+                    ),
+                    "port": service.port,
+                    "properties": properties,
+                }
+            )
     return items
 
 
@@ -293,11 +282,9 @@ def _api_request_session():
 
 @throttle(500)
 def _get_api_result(
-        url,  # pylint: disable=too-many-branches
-        params=None,
-        data=None,
-        auth=None):
-    from platformio.app import get_setting
+    url, params=None, data=None, auth=None  # pylint: disable=too-many-branches
+):
+    from platformio.app import get_setting  # pylint: disable=import-outside-toplevel
 
     result = {}
     r = None
@@ -311,30 +298,29 @@ def _get_api_result(
 
     try:
         if data:
-            r = _api_request_session().post(url,
-                                            params=params,
-                                            data=data,
-                                            headers=headers,
-                                            auth=auth,
-                                            verify=verify_ssl)
+            r = _api_request_session().post(
+                url,
+                params=params,
+                data=data,
+                headers=headers,
+                auth=auth,
+                verify=verify_ssl,
+            )
         else:
-            r = _api_request_session().get(url,
-                                           params=params,
-                                           headers=headers,
-                                           auth=auth,
-                                           verify=verify_ssl)
+            r = _api_request_session().get(
+                url, params=params, headers=headers, auth=auth, verify=verify_ssl
+            )
         result = r.json()
         r.raise_for_status()
         return r.text
     except requests.exceptions.HTTPError as e:
         if result and "message" in result:
-            raise exception.APIRequestError(result['message'])
+            raise exception.APIRequestError(result["message"])
         if result and "errors" in result:
-            raise exception.APIRequestError(result['errors'][0]['title'])
+            raise exception.APIRequestError(result["errors"][0]["title"])
         raise exception.APIRequestError(e)
     except ValueError:
-        raise exception.APIRequestError("Invalid response: %s" %
-                                        r.text.encode("utf-8"))
+        raise exception.APIRequestError("Invalid response: %s" % r.text.encode("utf-8"))
     finally:
         if r:
             r.close()
@@ -342,11 +328,13 @@ def _get_api_result(
 
 
 def get_api_result(url, params=None, data=None, auth=None, cache_valid=None):
-    from platformio.app import ContentCache
+    from platformio.app import ContentCache  # pylint: disable=import-outside-toplevel
+
     total = 0
     max_retries = 5
-    cache_key = (ContentCache.key_from_args(url, params, data, auth)
-                 if cache_valid else None)
+    cache_key = (
+        ContentCache.key_from_args(url, params, data, auth) if cache_valid else None
+    )
     while total < max_retries:
         try:
             with ContentCache() as cc:
@@ -363,24 +351,25 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None):
                 with ContentCache() as cc:
                     cc.set(cache_key, result, cache_valid)
             return json.loads(result)
-        except (requests.exceptions.ConnectionError,
-                requests.exceptions.Timeout) as e:
+        except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e:
             total += 1
             if not PlatformioCLI.in_silence():
                 click.secho(
                     "[API] ConnectionError: {0} (incremented retry: max={1}, "
                     "total={2})".format(e, max_retries, total),
-                    fg="yellow")
+                    fg="yellow",
+                )
             time.sleep(2 * total)
 
     raise exception.APIRequestError(
-        "Could not connect to PlatformIO API Service. "
-        "Please try later.")
+        "Could not connect to PlatformIO API Service. Please try later."
+    )
 
 
 PING_INTERNET_IPS = [
     "192.30.253.113",  # github.com
-    "193.222.52.25"  # dl.platformio.org
+    "31.28.1.238",  # dl.platformio.org
+    "193.222.52.25",  # dl.platformio.org
 ]
 
 
@@ -391,12 +380,9 @@ def _internet_on():
     for ip in PING_INTERNET_IPS:
         try:
             if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")):
-                requests.get("http://%s" % ip,
-                             allow_redirects=False,
-                             timeout=timeout)
+                requests.get("http://%s" % ip, allow_redirects=False, timeout=timeout)
             else:
-                socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect(
-                    (ip, 80))
+                socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((ip, 80))
             return True
         except:  # pylint: disable=bare-except
             pass
@@ -438,8 +424,7 @@ def merge_dicts(d1, d2, path=None):
     if path is None:
         path = []
     for key in d2:
-        if (key in d1 and isinstance(d1[key], dict)
-                and isinstance(d2[key], dict)):
+        if key in d1 and isinstance(d1[key], dict) and isinstance(d2[key], dict):
             merge_dicts(d1[key], d2[key], path + [str(key)])
         else:
             d1[key] = d2[key]
@@ -450,9 +435,7 @@ def print_labeled_bar(label, is_error=False, fg=None):
     terminal_width, _ = click.get_terminal_size()
     width = len(click.unstyle(label))
     half_line = "=" * int((terminal_width - width - 2) / 2)
-    click.secho("%s %s %s" % (half_line, label, half_line),
-                fg=fg,
-                err=is_error)
+    click.secho("%s %s %s" % (half_line, label, half_line), fg=fg, err=is_error)
 
 
 def humanize_duration_time(duration):

platformio/vcsclient.py

@@ -27,7 +27,6 @@ except ImportError:
 
 
 class VCSClientFactory(object):
-
     @staticmethod
     def newClient(src_dir, remote_url, silent=False):
         result = urlparse(remote_url)
@@ -38,15 +37,14 @@ class VCSClientFactory(object):
             remote_url = remote_url[4:]
         elif "+" in result.scheme:
             type_, _ = result.scheme.split("+", 1)
-            remote_url = remote_url[len(type_) + 1:]
+            remote_url = remote_url[len(type_) + 1 :]
         if "#" in remote_url:
             remote_url, tag = remote_url.rsplit("#", 1)
         if not type_:
-            raise PlatformioException("VCS: Unknown repository type %s" %
-                                      remote_url)
-        obj = getattr(modules[__name__],
-                      "%sClient" % type_.title())(src_dir, remote_url, tag,
-                                                  silent)
+            raise PlatformioException("VCS: Unknown repository type %s" % remote_url)
+        obj = getattr(modules[__name__], "%sClient" % type_.title())(
+            src_dir, remote_url, tag, silent
+        )
         assert isinstance(obj, VCSClientBase)
         return obj
 
@@ -71,8 +69,8 @@ class VCSClientBase(object):
                 assert self.run_cmd(["--version"])
         except (AssertionError, OSError, PlatformioException):
             raise UserSideException(
-                "VCS: `%s` client is not installed in your system" %
-                self.command)
+                "VCS: `%s` client is not installed in your system" % self.command
+            )
         return True
 
     @property
@@ -98,24 +96,23 @@ class VCSClientBase(object):
     def run_cmd(self, args, **kwargs):
         args = [self.command] + args
         if "cwd" not in kwargs:
-            kwargs['cwd'] = self.src_dir
+            kwargs["cwd"] = self.src_dir
         try:
             check_call(args, **kwargs)
             return True
         except CalledProcessError as e:
-            raise PlatformioException("VCS: Could not process command %s" %
-                                      e.cmd)
+            raise PlatformioException("VCS: Could not process command %s" % e.cmd)
 
     def get_cmd_output(self, args, **kwargs):
         args = [self.command] + args
         if "cwd" not in kwargs:
-            kwargs['cwd'] = self.src_dir
+            kwargs["cwd"] = self.src_dir
         result = exec_command(args, **kwargs)
-        if result['returncode'] == 0:
-            return result['out'].strip()
+        if result["returncode"] == 0:
+            return result["out"].strip()
         raise PlatformioException(
-            "VCS: Could not receive an output from `%s` command (%s)" %
-            (args, result))
+            "VCS: Could not receive an output from `%s` command (%s)" % (args, result)
+        )
 
 
 class GitClient(VCSClientBase):
@@ -127,7 +124,8 @@ class GitClient(VCSClientBase):
             return VCSClientBase.check_client(self)
         except UserSideException:
             raise UserSideException(
-                "Please install Git client from https://git-scm.com/downloads")
+                "Please install Git client from https://git-scm.com/downloads"
+            )
 
     def get_branches(self):
         output = self.get_cmd_output(["branch"])
@@ -166,7 +164,10 @@ class GitClient(VCSClientBase):
         args += [self.remote_url, self.src_dir]
         assert self.run_cmd(args)
         if is_commit:
-            return self.run_cmd(["reset", "--hard", self.tag])
+            assert self.run_cmd(["reset", "--hard", self.tag])
+            return self.run_cmd(
+                ["submodule", "update", "--init", "--recursive", "--force"]
+            )
         return True
 
     def update(self):
@@ -232,7 +233,8 @@ class SvnClient(VCSClientBase):
 
     def get_current_revision(self):
         output = self.get_cmd_output(
-            ["info", "--non-interactive", "--trust-server-cert", "-r", "HEAD"])
+            ["info", "--non-interactive", "--trust-server-cert", "-r", "HEAD"]
+        )
         for line in output.split("\n"):
             line = line.strip()
             if line.startswith("Revision:"):

scripts/99-platformio-udev.rules

@@ -25,45 +25,35 @@
 #
 
 # CP210X USB UART
-SUBSYSTEMS=="usb", ATTRS{idVendor}=="10c4", ATTRS{idProduct}=="ea60", MODE:="0666"
-
-# FT232R USB UART
-SUBSYSTEMS=="usb", ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6001", MODE:="0666"
+ATTRS{idVendor}=="10c4", ATTRS{idProduct}=="ea60", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # FT231XS USB UART
-SUBSYSTEMS=="usb", ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6015", MODE:="0666"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6015", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # Prolific Technology, Inc. PL2303 Serial Port
-SUBSYSTEMS=="usb", ATTRS{idVendor}=="067b", ATTRS{idProduct}=="2303", MODE:="0666"
+ATTRS{idVendor}=="067b", ATTRS{idProduct}=="2303", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # QinHeng Electronics HL-340 USB-Serial adapter
-SUBSYSTEMS=="usb", ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="7523", MODE:="0666"
+ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="7523", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # Arduino boards
-SUBSYSTEMS=="usb", ATTRS{idVendor}=="2341", ATTRS{idProduct}=="[08][02]*", MODE:="0666"
-SUBSYSTEMS=="usb", ATTRS{idVendor}=="2a03", ATTRS{idProduct}=="[08][02]*", MODE:="0666"
+ATTRS{idVendor}=="2341", ATTRS{idProduct}=="[08][02]*", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
+ATTRS{idVendor}=="2a03", ATTRS{idProduct}=="[08][02]*", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # Arduino SAM-BA
-ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="6124", ENV{ID_MM_DEVICE_IGNORE}="1"
-ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="6124", ENV{MTP_NO_PROBE}="1"
-SUBSYSTEMS=="usb", ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="6124", MODE:="0666"
-KERNEL=="ttyACM*", ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="6124", MODE:="0666"
+ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="6124", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{MTP_NO_PROBE}="1"
 
 # Digistump boards
-SUBSYSTEMS=="usb", ATTRS{idVendor}=="16d0", ATTRS{idProduct}=="0753", MODE:="0666"
-KERNEL=="ttyACM*", ATTRS{idVendor}=="16d0", ATTRS{idProduct}=="0753", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1"
-
-# STM32 discovery boards, with onboard st/linkv2
-SUBSYSTEMS=="usb", ATTRS{idVendor}=="0483", ATTRS{idProduct}=="374?", MODE:="0666"
+ATTRS{idVendor}=="16d0", ATTRS{idProduct}=="0753", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # Maple with DFU
-SUBSYSTEMS=="usb", ATTRS{idVendor}=="1eaf", ATTRS{idProduct}=="000[34]", MODE:="0666"
+ATTRS{idVendor}=="1eaf", ATTRS{idProduct}=="000[34]", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # USBtiny
-SUBSYSTEMS=="usb", ATTRS{idProduct}=="0c9f", ATTRS{idVendor}=="1781", MODE="0666"
+ATTRS{idProduct}=="0c9f", ATTRS{idVendor}=="1781", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # USBasp V2.0
-SUBSYSTEMS=="usb", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="05dc", MODE:="0666"
+ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="05dc", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # Teensy boards
 ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789B]?", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
@@ -72,191 +62,108 @@ SUBSYSTEMS=="usb", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789ABCD]?", MO
 KERNEL=="ttyACM*", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789B]?", MODE:="0666"
 
 #TI Stellaris Launchpad
-SUBSYSTEMS=="usb", ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="0666"
+ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 #TI MSP430 Launchpad
-SUBSYSTEMS=="usb", ATTRS{idVendor}=="0451", ATTRS{idProduct}=="f432", MODE="0666"
+ATTRS{idVendor}=="0451", ATTRS{idProduct}=="f432", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
+#GD32V DFU Bootloader
+ATTRS{idVendor}=="28e9", ATTRS{idProduct}=="0189", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 #
 # Debuggers
 #
 
 # Black Magic Probe
-SUBSYSTEM=="tty", ATTRS{interface}=="Black Magic GDB Server"
-SUBSYSTEM=="tty", ATTRS{interface}=="Black Magic UART Port"
+SUBSYSTEM=="tty", ATTRS{interface}=="Black Magic GDB Server", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
+SUBSYSTEM=="tty", ATTRS{interface}=="Black Magic UART Port", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # opendous and estick
-ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="204f", MODE="0666"
+ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="204f", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # Original FT232/FT245 VID:PID
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6001", MODE="0666"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6001", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # Original FT2232 VID:PID
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6010", MODE="0666"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6010", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # Original FT4232 VID:PID
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6011", MODE="0666"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6011", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # Original FT232H VID:PID
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6014", MODE="0666"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6014", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # DISTORTEC JTAG-lock-pick Tiny 2
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8220", MODE="0666"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8220", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # TUMPA, TUMPA Lite
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a98", MODE="0666"
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a99", MODE="0666"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a98", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a99", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # XDS100v2
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="a6d0", MODE="0666"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="a6d0", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # Xverve Signalyzer Tool (DT-USB-ST), Signalyzer LITE (DT-USB-SLITE)
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca0", MODE="0666"
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca1", MODE="0666"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca0", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca1", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # TI/Luminary Stellaris Evaluation Board FTDI (several)
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcd9", MODE="0666"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcd9", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # TI/Luminary Stellaris In-Circuit Debug Interface FTDI (ICDI) Board
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcda", MODE="0666"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcda", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # egnite Turtelizer 2
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bdc8", MODE="0666"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bdc8", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # Section5 ICEbear
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c140", MODE="0666"
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c141", MODE="0666"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c140", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c141", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # Amontec JTAGkey and JTAGkey-tiny
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="cff8", MODE="0666"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="cff8", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # TI ICDI
-ATTRS{idVendor}=="0451", ATTRS{idProduct}=="c32a", MODE="0666"
+ATTRS{idVendor}=="0451", ATTRS{idProduct}=="c32a", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
-# STLink v1
-ATTRS{idVendor}=="0483", ATTRS{idProduct}=="3744", MODE="0666"
-
-# STLink v2
-ATTRS{idVendor}=="0483", ATTRS{idProduct}=="3748", MODE="0666"
-
-# STLink v2-1
-ATTRS{idVendor}=="0483", ATTRS{idProduct}=="374b", MODE="0666"
+# STLink probes
+ATTRS{idVendor}=="0483", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # Hilscher NXHX Boards
-ATTRS{idVendor}=="0640", ATTRS{idProduct}=="0028", MODE="0666"
+ATTRS{idVendor}=="0640", ATTRS{idProduct}=="0028", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
-# Hitex STR9-comStick
-ATTRS{idVendor}=="0640", ATTRS{idProduct}=="002c", MODE="0666"
-
-# Hitex STM32-PerformanceStick
-ATTRS{idVendor}=="0640", ATTRS{idProduct}=="002d", MODE="0666"
+# Hitex probes
+ATTRS{idVendor}=="0640", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # Altera USB Blaster
-ATTRS{idVendor}=="09fb", ATTRS{idProduct}=="6001", MODE="0666"
+ATTRS{idVendor}=="09fb", ATTRS{idProduct}=="6001", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # Amontec JTAGkey-HiSpeed
-ATTRS{idVendor}=="0fbb", ATTRS{idProduct}=="1000", MODE="0666"
+ATTRS{idVendor}=="0fbb", ATTRS{idProduct}=="1000", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # SEGGER J-Link
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0101", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0102", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0103", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0104", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0105", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0107", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0108", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1010", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1011", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1012", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1013", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1014", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1015", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1016", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1017", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1018", MODE="0666"
+ATTRS{idVendor}=="1366", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # Raisonance RLink
-ATTRS{idVendor}=="138e", ATTRS{idProduct}=="9000", MODE="0666"
+ATTRS{idVendor}=="138e", ATTRS{idProduct}=="9000", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # Debug Board for Neo1973
-ATTRS{idVendor}=="1457", ATTRS{idProduct}=="5118", MODE="0666"
+ATTRS{idVendor}=="1457", ATTRS{idProduct}=="5118", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
-# Olimex ARM-USB-OCD
-ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="0003", MODE="0666"
-
-# Olimex ARM-USB-OCD-TINY
-ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="0004", MODE="0666"
-
-# Olimex ARM-JTAG-EW
-ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="001e", MODE="0666"
-
-# Olimex ARM-USB-OCD-TINY-H
-ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="002a", MODE="0666"
-
-# Olimex ARM-USB-OCD-H
-ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="002b", MODE="0666"
+# Olimex probes
+ATTRS{idVendor}=="15ba", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # USBprog with OpenOCD firmware
-ATTRS{idVendor}=="1781", ATTRS{idProduct}=="0c63", MODE="0666"
+ATTRS{idVendor}=="1781", ATTRS{idProduct}=="0c63", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # TI/Luminary Stellaris In-Circuit Debug Interface (ICDI) Board
-ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="0666"
+ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # Marvell Sheevaplug
-ATTRS{idVendor}=="9e88", ATTRS{idProduct}=="9e8f", MODE="0666"
+ATTRS{idVendor}=="9e88", ATTRS{idProduct}=="9e8f", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # Keil Software, Inc. ULink
-ATTRS{idVendor}=="c251", ATTRS{idProduct}=="2710", MODE="0666"
+ATTRS{idVendor}=="c251", ATTRS{idProduct}=="2710", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # CMSIS-DAP compatible adapters
-ATTRS{product}=="*CMSIS-DAP*", MODE="0666"
-
-#SEGGER J-LIK
-ATTR{idProduct}=="1001", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1002", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1003", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1004", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1005", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1006", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1007", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1008", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1009", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="100a", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="100b", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="100c", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="100d", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="100e", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="100f", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1010", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1011", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1012", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1013", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1014", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1015", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1016", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1017", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1018", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1019", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="101a", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="101b", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="101c", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="101d", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="101e", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="101f", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1020", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1021", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1022", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1023", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1024", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1025", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1026", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1027", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1028", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1029", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="102a", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="102b", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="102c", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="102d", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="102e", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="102f", ATTR{idVendor}=="1366", MODE="0666"
+ATTRS{product}=="*CMSIS-DAP*", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"

scripts/docspregen.py

@@ -13,16 +13,22 @@
 # limitations under the License.
 
 import os
-import urlparse
 from os.path import dirname, isdir, isfile, join, realpath
 from sys import exit as sys_exit
 from sys import path
 
 path.append("..")
 
+import click
+
 from platformio import fs, util
 from platformio.managers.platform import PlatformFactory, PlatformManager
 
+try:
+    from urlparse import ParseResult, urlparse, urlunparse
+except ImportError:
+    from urllib.parse import ParseResult, urlparse, urlunparse
+
 RST_COPYRIGHT = """..  Copyright (c) 2014-present PlatformIO <contact@platformio.org>
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -48,14 +54,14 @@ def is_compat_platform_and_framework(platform, framework):
 
 
 def campaign_url(url, source="platformio", medium="docs"):
-    data = urlparse.urlparse(url)
+    data = urlparse(url)
     query = data.query
     if query:
         query += "&"
     query += "utm_source=%s&utm_medium=%s" % (source, medium)
-    return urlparse.urlunparse(
-        urlparse.ParseResult(data.scheme, data.netloc, data.path, data.params,
-                             query, data.fragment))
+    return urlunparse(
+        ParseResult(data.scheme, data.netloc, data.path, data.params, query,
+                    data.fragment))
 
 
 def generate_boards_table(boards, skip_columns=None):
@@ -64,7 +70,7 @@ def generate_boards_table(boards, skip_columns=None):
         ("Platform", ":ref:`platform_{platform}`"),
         ("Debug", "{debug}"),
         ("MCU", "{mcu}"),
-        ("Frequency", "{f_cpu:d}MHz"),
+        ("Frequency", "{f_cpu}MHz"),
         ("Flash", "{rom}"),
         ("RAM", "{ram}"),
     ]
@@ -90,15 +96,14 @@ def generate_boards_table(boards, skip_columns=None):
         elif data['debug']:
             debug = "External"
 
-        variables = dict(
-            id=data['id'],
-            name=data['name'],
-            platform=data['platform'],
-            debug=debug,
-            mcu=data['mcu'].upper(),
-            f_cpu=int(data['fcpu']) / 1000000,
-            ram=fs.format_filesize(data['ram']),
-            rom=fs.format_filesize(data['rom']))
+        variables = dict(id=data['id'],
+                         name=data['name'],
+                         platform=data['platform'],
+                         debug=debug,
+                         mcu=data['mcu'].upper(),
+                         f_cpu=int(data['fcpu'] / 1000000.0),
+                         ram=fs.format_filesize(data['ram']),
+                         rom=fs.format_filesize(data['rom']))
 
         for (name, template) in columns:
             if skip_columns and name in skip_columns:
@@ -132,8 +137,9 @@ Frameworks
         lines.append("""
     * - :ref:`framework_{name}`
       - {description}""".format(**framework))
-    assert known >= set(frameworks), "Unknown frameworks %s " % (
-        set(frameworks) - known)
+    if set(frameworks) - known:
+        click.secho("Unknown frameworks %s " % (
+        set(frameworks) - known), fg="red")
     return lines
 
 
@@ -208,8 +214,8 @@ Boards listed below have on-board debug probe and **ARE READY** for debugging!
 You do not need to use/buy external debug probe.
 """)
         lines.extend(
-            generate_boards_table(
-                onboard_debug, skip_columns=skip_board_columns))
+            generate_boards_table(onboard_debug,
+                                  skip_columns=skip_board_columns))
     if external_debug:
         lines.append("""
 External Debug Tools
@@ -220,8 +226,8 @@ external debug probe. They **ARE NOT READY** for debugging.
 Please click on board name for the further details.
 """)
         lines.extend(
-            generate_boards_table(
-                external_debug, skip_columns=skip_board_columns))
+            generate_boards_table(external_debug,
+                                  skip_columns=skip_board_columns))
     return lines
 
 
@@ -239,13 +245,18 @@ Packages
     * - Name
       - Description""")
     for name in sorted(packagenames):
-        assert name in API_PACKAGES, name
-        lines.append("""
+        if name not in API_PACKAGES:
+            click.secho("Unknown package `%s`" % name, fg="red")
+            lines.append("""
+    * - {name}
+      -
+                """.format(name=name))
+        else:
+            lines.append("""
     * - `{name} <{url}>`__
-      - {description}""".format(
-            name=name,
-            url=campaign_url(API_PACKAGES[name]['url']),
-            description=API_PACKAGES[name]['description']))
+      - {description}""".format(name=name,
+                                url=campaign_url(API_PACKAGES[name]['url']),
+                                description=API_PACKAGES[name]['description']))
 
     if is_embedded:
         lines.append("""
@@ -344,8 +355,9 @@ Examples are listed from `%s development platform repository <%s>`_:
             generate_debug_contents(
                 compatible_boards,
                 skip_board_columns=["Platform"],
-                extra_rst="%s_debug.rst" % name if isfile(
-                    join(rst_dir, "%s_debug.rst" % name)) else None))
+                extra_rst="%s_debug.rst" %
+                name if isfile(join(rst_dir, "%s_debug.rst" %
+                                    name)) else None))
 
     #
     # Development version of dev/platform
@@ -483,8 +495,9 @@ For more detailed information please visit `vendor site <%s>`_.
         lines.extend(
             generate_debug_contents(
                 compatible_boards,
-                extra_rst="%s_debug.rst" % type_ if isfile(
-                    join(rst_dir, "%s_debug.rst" % type_)) else None))
+                extra_rst="%s_debug.rst" %
+                type_ if isfile(join(rst_dir, "%s_debug.rst" %
+                                     type_)) else None))
 
     if compatible_platforms:
         # examples
@@ -494,11 +507,10 @@ Examples
 """)
         for manifest in compatible_platforms:
             p = PlatformFactory.newPlatform(manifest['name'])
-            lines.append(
-                "* `%s for %s <%s>`_" %
-                (data['title'], manifest['title'],
-                 campaign_url(
-                     "%s/tree/master/examples" % p.repository_url[:-4])))
+            lines.append("* `%s for %s <%s>`_" %
+                         (data['title'], manifest['title'],
+                          campaign_url("%s/tree/master/examples" %
+                                       p.repository_url[:-4])))
 
         # Platforms
         lines.extend(
@@ -568,7 +580,7 @@ popular embedded boards and IDE.
         else:
             platforms[platform] = [data]
 
-    for platform, boards in sorted(platforms.iteritems()):
+    for platform, boards in sorted(platforms.items()):
         p = PlatformFactory.newPlatform(platform)
         lines.append(p.title)
         lines.append("-" * len(p.title))
@@ -605,21 +617,20 @@ def update_embedded_board(rst_path, board):
         board_manifest_url = board_manifest_url[:-4]
     board_manifest_url += "/blob/master/boards/%s.json" % board['id']
 
-    variables = dict(
-        id=board['id'],
-        name=board['name'],
-        platform=board['platform'],
-        platform_description=platform.description,
-        url=campaign_url(board['url']),
-        mcu=board_config.get("build", {}).get("mcu", ""),
-        mcu_upper=board['mcu'].upper(),
-        f_cpu=board['fcpu'],
-        f_cpu_mhz=int(board['fcpu']) / 1000000,
-        ram=fs.format_filesize(board['ram']),
-        rom=fs.format_filesize(board['rom']),
-        vendor=board['vendor'],
-        board_manifest_url=board_manifest_url,
-        upload_protocol=board_config.get("upload.protocol", ""))
+    variables = dict(id=board['id'],
+                     name=board['name'],
+                     platform=board['platform'],
+                     platform_description=platform.description,
+                     url=campaign_url(board['url']),
+                     mcu=board_config.get("build", {}).get("mcu", ""),
+                     mcu_upper=board['mcu'].upper(),
+                     f_cpu=board['fcpu'],
+                     f_cpu_mhz=int(int(board['fcpu']) / 1000000),
+                     ram=fs.format_filesize(board['ram']),
+                     rom=fs.format_filesize(board['rom']),
+                     vendor=board['vendor'],
+                     board_manifest_url=board_manifest_url,
+                     upload_protocol=board_config.get("upload.protocol", ""))
 
     lines = [RST_COPYRIGHT]
     lines.append(".. _board_{platform}_{id}:".format(**variables))
@@ -639,7 +650,7 @@ Platform :ref:`platform_{platform}`: {platform_description}
   * - **Microcontroller**
     - {mcu_upper}
   * - **Frequency**
-    - {f_cpu_mhz}MHz
+    - {f_cpu_mhz:d}MHz
   * - **Flash**
     - {rom}
   * - **RAM**
@@ -805,15 +816,14 @@ Boards
 .. note::
     For more detailed ``board`` information please scroll tables below by horizontal.
 """)
-    for vendor, boards in sorted(vendors.iteritems()):
+    for vendor, boards in sorted(vendors.items()):
         lines.append(str(vendor))
         lines.append("~" * len(vendor))
         lines.extend(generate_boards_table(boards))
 
     # save
-    with open(
-            join(fs.get_source_dir(), "..", "docs", "plus", "debugging.rst"),
-            "r+") as fp:
+    with open(join(fs.get_source_dir(), "..", "docs", "plus", "debugging.rst"),
+              "r+") as fp:
         content = fp.read()
         fp.seek(0)
         fp.truncate()
@@ -823,7 +833,9 @@ Boards
     # Debug tools
     for tool, platforms in tool_to_platforms.items():
         tool_path = join(DOCS_ROOT_DIR, "plus", "debug-tools", "%s.rst" % tool)
-        assert isfile(tool_path), tool
+        if not isfile(tool_path):
+            click.secho("Unknown debug tool `%s`" % tool, fg="red")
+            continue
         platforms = sorted(set(platforms))
 
         lines = [".. begin_platforms"]